Current section

Files

Jump to
jsv lib jsv resolver.ex
Raw

lib/jsv/resolver.ex

defmodule JSV.Resolver do
alias JSV.Helpers.EnumExt
alias JSV.Key
alias JSV.Ref
alias JSV.RNS
@moduledoc """
A behaviour describing the implementation of a [guides/build/custom resolver.
Resolves remote resources when building a JSON schema.
"""
defmodule Resolved do
@moduledoc """
Metadata gathered from a remote schema or a sub-schema.
"""
# TODO(Draft7-removal) drop parent_ns once we do not support draft-7
@enforce_keys [:raw, :meta, :vocabularies, :ns, :parent_ns, :rev_path]
defstruct @enforce_keys
@type t :: %__MODULE__{
raw: JSV.normal_schema(),
meta: binary,
vocabularies: term,
ns: binary,
parent_ns: binary
}
end
defmodule Descriptor do
@enforce_keys [:raw, :meta, :aliases, :ns, :parent_ns, :rev_path]
defstruct @enforce_keys
@moduledoc false
end
@doc """
Receives an URI and the options passed in the resolver tuple to `JSV.build/2`
and returns a result tuple for a raw JSON schema map.
Returning boolean schemas from resolvers is not supported. You may wrap the
boolean value in a `$defs` or any other pointer as a workaround.
Schemas will be normalized using `JSV.Schema.normalize/1`. If the resolver
returns schema that are already in JSON-decoded form (like a response body
from an HTTP call) without atoms, module names or structs, the resolver
implementation can return `{:normal, map}` instead to skip the normalization.
"""
@callback resolve(uri :: String.t(), opts :: term) :: {:ok, map} | {:normal, map} | {:error, term}
@derive {Inspect, Application.compile_env(:jsv, :resolver_inspect_derive, except: [:fetch_cache])}
defstruct chain: [],
default_meta: nil,
# fetch_cache is a local cache for the resolver instance. Actual
# caching of remote resources should be done in each resolver
# implementation.
fetch_cache: %{},
resolved: %{}
@opaque t :: %__MODULE__{}
@type resolvable :: Key.ns() | Key.pointer() | Ref.t()
@doc """
Returns a new resolver, with the given behaviour implementations, and a
default meta-schema URL to use with schemas that do not declare a `$schema`
property.
"""
@spec chain_of([{module, term}], binary) :: t
def chain_of([_ | _] = resolvers, default_meta) do
%__MODULE__{chain: resolvers, default_meta: default_meta}
end
@doc false
@spec put_cached(t, binary | :root, JSV.normal_schema()) :: {:ok, t} | {:error, {:key_exists, term}}
def put_cached(%__MODULE__{} = rsv, ext_id, raw_schema)
when is_map(raw_schema) and (is_binary(ext_id) or :root == ext_id) do
case rsv.fetch_cache do
%{^ext_id => _} -> {:error, {:key_exists, ext_id}}
fetch_cache -> {:ok, %{rsv | fetch_cache: Map.put(fetch_cache, ext_id, raw_schema)}}
end
end
@doc """
Fetches the remote resource into the internal resolver cache and returns a new
resolver with that updated cache.
"""
@spec resolve(t, resolvable) :: {:ok, t} | {:error, term}
def resolve(rsv, resolvable) do
case check_resolved(rsv, resolvable) do
:unresolved -> do_resolve(rsv, resolvable)
:already_resolved -> {:ok, rsv}
end
end
defp do_resolve(rsv, resolvable) do
with {:ok, raw_schema, rsv} <- ensure_fetched(rsv, resolvable),
{:ok, identified_schemas} <- scan_schema(raw_schema, external_id(resolvable), rsv.default_meta),
{:ok, cache_entries} <- create_cache_entries(identified_schemas),
{:ok, rsv} <- insert_cache_entries(rsv, cache_entries) do
resolve_meta_loop(rsv, metas_of(cache_entries))
else
{:error, _} = err -> err
end
end
defp external_id(:root) do
:root
end
defp external_id(ns) when is_binary(ns) do
ns
end
defp external_id(%Ref{ns: ns}) do
ns
end
defp metas_of(cache_entries) do
cache_entries
|> Enum.flat_map(fn
{_, {:alias_of, _}} -> []
{_, %{meta: meta}} -> [meta]
end)
|> Enum.uniq()
end
defp resolve_meta_loop(rsv, []) do
{:ok, rsv}
end
defp resolve_meta_loop(rsv, [nil | tail]) do
resolve_meta_loop(rsv, tail)
end
defp resolve_meta_loop(rsv, [meta | tail]) when is_binary(meta) do
with :unresolved <- check_resolved(rsv, {:meta, meta}),
{:ok, raw_schema, rsv} <- ensure_fetched(rsv, meta),
{:ok, cache_entry} <- create_meta_entry(raw_schema, meta),
{:ok, rsv} <- insert_cache_entries(rsv, [{{:meta, meta}, cache_entry}]) do
resolve_meta_loop(rsv, [cache_entry.meta | tail])
else
:already_resolved -> resolve_meta_loop(rsv, tail)
{:error, _} = err -> err
end
end
defp check_resolved(rsv, id) when is_binary(id) or :root == id do
case rsv do
%{resolved: %{^id => _}} -> :already_resolved
_ -> :unresolved
end
end
defp check_resolved(rsv, {:meta, id}) when is_binary(id) do
case rsv do
%{resolved: %{{:meta, ^id} => _}} -> :already_resolved
_ -> :unresolved
end
end
defp check_resolved(rsv, %Ref{ns: ns}) do
check_resolved(rsv, ns)
end
# Extract all $ids and achors. We receive the top schema
defp scan_schema(top_schema, external_id, default_meta) when not is_nil(external_id) do
{id, anchor, dynamic_anchor} = extract_keys(top_schema)
# For self references that target "#" or "#some/path" in the document, when
# the document does not have an id, we will force it. This is for the root
# document only.
ns =
case id do
nil -> external_id
_ -> id
end
nss = [id, external_id] |> Enum.reject(&is_nil/1) |> Enum.uniq()
# Anchor needs to be resolved from the $id or the external ID (an URL) if
# set.
anchors =
case anchor do
nil -> []
_ -> Enum.map(nss, &Key.for_anchor(&1, anchor))
end
dynamic_anchors =
case dynamic_anchor do
# a dynamic anchor is also adressable as a regular anchor for the given namespace
nil -> []
_ -> Enum.flat_map(nss, &[Key.for_dynamic_anchor(&1, dynamic_anchor), Key.for_anchor(&1, dynamic_anchor)])
end
# The schema will be findable by its $id or external id.
id_aliases = nss
aliases = id_aliases ++ anchors ++ dynamic_anchors
# If no metaschema is defined we will use the default draft as a fallback.
# We normalize it because many schemas use
# "http://json-schema.org/draft-07/schema#" with a trailing "#".
meta = normalize_meta(Map.get(top_schema, "$schema", default_meta))
top_descriptor = %Descriptor{
raw: top_schema,
meta: meta,
aliases: aliases,
ns: ns,
parent_ns: nil,
rev_path: [external_id]
}
acc = [top_descriptor]
scan_schema_pairs(top_schema, id, nss, meta, [ns], acc)
end
defp scan_subschema(raw_schema, ns, nss, meta, path, acc) when is_map(raw_schema) do
# If the subschema defines an id, we will discard the current namespaces, as
# the sibling or nested anchors will now only relate to this id
parent_ns = ns
{id, anchors, dynamic_anchor} =
case extract_keys(raw_schema) do
# ID that is only a fragment is replaced as an anchor
{"#" <> frag_id, anchor, dynamic_anchor} -> {nil, [frag_id | List.wrap(anchor)], dynamic_anchor}
{id, anchor, dynamic_anchor} -> {id, List.wrap(anchor), dynamic_anchor}
end
{id_aliases, ns, nss} =
with true <- is_binary(id),
{:ok, full_id} <- merge_id(ns, id) do
{[full_id], full_id, [full_id]}
else
_ -> {[], ns, nss}
end
anchors =
for new_ns <- nss, a <- anchors do
Key.for_anchor(new_ns, a)
end
dynamic_anchors =
case dynamic_anchor do
nil -> []
# a dynamic anchor is also adressable as a regular anchor for the given namespace
da -> Enum.flat_map(nss, &[Key.for_dynamic_anchor(&1, da), Key.for_anchor(&1, da)])
end
# We do not check for the meta $schema is subschemas, we only add the
# parent_one to the descriptor.
acc =
case(id_aliases ++ anchors ++ dynamic_anchors) do
[] ->
acc
aliases ->
descriptor =
%Descriptor{
raw: raw_schema,
meta: meta,
aliases: aliases,
ns: ns,
parent_ns: parent_ns,
rev_path: path
}
[descriptor | acc]
end
scan_schema_pairs(raw_schema, ns, nss, meta, path, acc)
end
defp scan_subschema(scalar, _parent_id, _nss, _meta, _path, acc)
when is_binary(scalar)
when is_atom(scalar)
when is_number(scalar) do
{:ok, acc}
end
defp scan_subschema(list, parent_id, nss, meta, path, acc) when is_list(list) do
list
|> Enum.with_index()
|> EnumExt.reduce_ok(acc, fn {item, index}, acc ->
scan_subschema(item, parent_id, nss, meta, [index | path], acc)
end)
end
defp extract_keys(schema) do
id =
case Map.fetch(schema, "$id") do
{:ok, id} -> id
:error -> nil
end
anchor =
case Map.fetch(schema, "$anchor") do
{:ok, anchor} -> anchor
:error -> nil
end
dynamic_anchor =
case Map.fetch(schema, "$dynamicAnchor") do
{:ok, dynamic_anchor} -> dynamic_anchor
:error -> nil
end
{id, anchor, dynamic_anchor}
end
# Keywords whose value is a map from user-defined names (or patterns) to
# subschemas, arrays or other generic maps.
@generic_map_keywords [
"$defs",
"definitions",
"dependencies",
"dependentSchemas",
"patternProperties",
"properties"
]
defp scan_schema_pairs(schema, parent_id, nss, meta, path, acc) do
EnumExt.reduce_ok(schema, acc, fn
{k, v}, acc when k in @generic_map_keywords and is_map(v) ->
scan_generic_map(v, parent_id, nss, meta, [k | path], acc)
{ignored, _}, acc when ignored in ["enum", "const", "examples"] ->
{:ok, acc}
{k, v}, acc ->
scan_subschema(v, parent_id, nss, meta, [k | path], acc)
end)
end
defp scan_generic_map(map, parent_id, nss, meta, path, acc) do
EnumExt.reduce_ok(map, acc, fn {name, subschema}, acc ->
scan_subschema(subschema, parent_id, nss, meta, [name | path], acc)
end)
end
defp create_cache_entries(identified_schemas) do
{:ok, Enum.flat_map(identified_schemas, &to_cache_entries/1)}
end
defp to_cache_entries(descriptor) do
%Descriptor{aliases: aliases, meta: meta, raw: raw, ns: ns, parent_ns: parent_ns, rev_path: rev_path} = descriptor
resolved =
%Resolved{meta: meta, raw: raw, ns: ns, parent_ns: parent_ns, vocabularies: nil, rev_path: rev_path}
case aliases do
[single] -> [{single, resolved}]
[first | aliases] -> [{first, resolved} | Enum.map(aliases, &{&1, {:alias_of, first}})]
end
end
defp insert_cache_entries(rsv, entries) do
%__MODULE__{resolved: cache} = rsv
cache_result =
EnumExt.reduce_ok(entries, cache, fn {k, resolved}, cache ->
case cache do
%{^k => existing} ->
# Allow a duplicate resolution that is the exact same value as the
# preexisting copy. This allows a root schema with an $id to reference
# itself with an external id such as `jsv:module:MODULE`.
check_duplicated_cache_entry(k, resolved, existing, cache)
_ ->
{:ok, Map.put(cache, k, resolved)}
end
end)
with {:ok, cache} <- cache_result do
{:ok, %{rsv | resolved: cache}}
end
end
defp check_duplicated_cache_entry(k, resolved, existing, cache) do
case {resolved, existing} do
{%Resolved{raw: same}, %Resolved{raw: same}} -> {:ok, cache}
_ -> {:error, {:duplicate_resolution, k}}
end
end
defp create_meta_entry(raw_schema, ext_id) when not is_struct(raw_schema) do
case fetch_vocabulary_from_raw(raw_schema, ext_id) do
{:ok, vocabulary} ->
# Meta entries are only identified by they external URL so the :ns and
# :raw value should not be used anywhere. We will just put :__meta__ in
# here so it's easier to debug.
resolved = %Resolved{
vocabularies: vocabulary,
meta: nil,
ns: :__meta__,
parent_ns: nil,
raw: :__meta__,
rev_path: [ext_id]
}
{:ok, resolved}
:error ->
{:error, {:undefined_vocabulary, ext_id}}
end
end
defp fetch_vocabulary_from_raw(raw_schema, ext_id) do
case Map.fetch(raw_schema, "$vocabulary") do
{:ok, vocab} when is_map(vocab) -> {:ok, vocab}
:error -> vocabulary_fallback(ext_id)
end
end
defp vocabulary_fallback("http://json-schema.org/draft-07/schema") do
vocab = %{
"https://json-schema.org/draft-07/--fallback--vocab/core" => true,
"https://json-schema.org/draft-07/--fallback--vocab/validation" => true,
"https://json-schema.org/draft-07/--fallback--vocab/applicator" => true,
"https://json-schema.org/draft-07/--fallback--vocab/content" => true,
"https://json-schema.org/draft-07/--fallback--vocab/format-annotation" => true,
"https://json-schema.org/draft-07/--fallback--vocab/meta-data" => true
# We do not declare format assertion to have the same behaviour as 2020-12
# "https://json-schema.org/draft-07/--fallback--vocab/format-assertion" => true,
}
{:ok, vocab}
end
defp vocabulary_fallback(_) do
:error
end
defp ensure_fetched(rsv, fetchable) do
with :unfetched <- check_fetched(rsv, fetchable),
{:ok, ext_id, raw_schema} <- fetch_raw_schema(rsv, fetchable),
{:ok, rsv} <- put_cached(rsv, ext_id, raw_schema) do
{:ok, raw_schema, rsv}
else
{:already_fetched, raw_schema} -> {:ok, raw_schema, rsv}
{:error, _} = err -> err
end
end
defp check_fetched(rsv, %Ref{ns: ns}) do
check_fetched(rsv, ns)
end
defp check_fetched(rsv, id) when is_binary(id) when :root == id do
case rsv do
%{fetch_cache: %{^id => fetched}} -> {:already_fetched, fetched}
_ -> :unfetched
end
end
@spec fetch_raw_schema(t, binary | {:meta, binary} | Ref.t()) :: {:ok, binary, JSV.normal_schema()} | {:error, term}
defp fetch_raw_schema(rsv, {:meta, url}) do
fetch_raw_schema(rsv, url)
end
defp fetch_raw_schema(rsv, url) when is_binary(url) do
call_chain(rsv.chain, url)
end
defp fetch_raw_schema(rsv, %Ref{ns: ns}) do
fetch_raw_schema(rsv, ns)
end
defp call_chain(chain, url) do
call_chain(chain, url, _err_acc = [])
end
defp call_chain([{module, opts} | chain], url, err_acc) do
case module.resolve(url, opts) do
{:ok, resolved} when is_map(resolved) ->
{:ok, url, normalize_resolved(resolved)}
{:normal, resolved} when is_map(resolved) ->
{:ok, url, resolved}
{:error, reason} ->
call_chain(chain, url, [{module, reason} | err_acc])
other ->
raise "invalid return from #{inspect(module)}.resolve/2, expected {:ok, map} or {:error, reason}, got: #{inspect(other)}"
end
end
defp call_chain([], _url, err_acc) do
{:error, {:resolver_error, :lists.reverse(err_acc)}}
end
defp normalize_resolved(map) when is_map(map) do
JSV.Schema.normalize(map)
end
defp merge_id(nil, child) do
RNS.derive(child, "")
end
defp merge_id(parent, child) do
RNS.derive(parent, child)
end
# Removes the fragment from the given URL. Accepts nil values
defp normalize_meta(nil) do
nil
end
defp normalize_meta(meta) do
case URI.parse(meta) do
%{fragment: nil} -> meta
uri -> URI.to_string(%{uri | fragment: nil})
end
end
@doc """
Returns the $vocabulary property of a schema identified by its namespace.
The schema must have been resolved previously as a meta-schema (_i.e._ found
in an $schema property of a resolved schema).
"""
@spec fetch_vocabulary(t, binary) :: {:ok, %{optional(binary) => boolean}} | {:error, term}
def fetch_vocabulary(rsv, meta) do
case fetch_resolved(rsv, {:meta, meta}) do
{:ok, %Resolved{vocabularies: vocabularies}} -> {:ok, vocabularies}
{:error, _} = err -> err
end
end
@doc """
Returns the raw schema identified by the given key if was previously resolved.
"""
@spec fetch_resolved(t(), resolvable | {:meta, resolvable}) ::
{:ok, Resolved.t() | {:alias_of, Key.t()}} | {:error, term}
def fetch_resolved(rsv, {:pointer, _, _} = pointer) do
fetch_pointer(rsv.resolved, pointer)
end
def fetch_resolved(rsv, key) do
fetch_local(rsv.resolved, key)
end
defp fetch_pointer(cache, {:pointer, ns, docpath}) do
with {:ok, %Resolved{raw: raw, meta: meta, ns: ns, parent_ns: parent_ns, rev_path: rev_path}} <-
fetch_local(cache, ns, :dealias),
{:ok, [sub | _] = parent_chain} <- fetch_docpath(raw, docpath),
{:ok, ns, parent_ns} <- derive_docpath_ns(parent_chain, ns, parent_ns) do
{:ok,
%Resolved{
raw: sub,
meta: meta,
vocabularies: nil,
ns: ns,
parent_ns: parent_ns,
rev_path: :lists.reverse(docpath, rev_path)
}}
else
{:error, _} = err -> err
end
end
defp fetch_local(cache, key, aliases \\ nil) do
case Map.fetch(cache, key) do
{:ok, {:alias_of, key}} when aliases == :dealias -> fetch_local(cache, key)
{:ok, {:alias_of, key}} -> {:ok, {:alias_of, key}}
{:ok, cached} -> {:ok, cached}
:error -> {:error, {:unresolved, key}}
end
end
defp fetch_docpath(raw_schema, docpath) do
case do_fetch_docpath(raw_schema, docpath, []) do
{:ok, sub} -> {:ok, sub}
{:error, reason} -> {:error, {:invalid_docpath, docpath, raw_schema, reason}}
end
end
# When fetching a docpath we will create a list of all parents up to the
# fetched subschema. The top parent is the last item in the list, the fetched
# subschema is the head.
#
# TODO(Draft7-removal) This is to support Draft 7 to define the correct NS for
# the subschema. We can remove that list building once Draft 7 is not
# supported anymore.
defp do_fetch_docpath(list, [h | t], parents) when is_list(list) and is_integer(h) do
case Enum.fetch(list, h) do
{:ok, item} -> do_fetch_docpath(item, t, [list | parents])
:error -> {:error, {:pointer_error, h, list}}
end
end
defp do_fetch_docpath(list, [h | t], parents) when is_list(list) and is_binary(h) do
case Integer.parse(h) do
{n, ""} when n >= 0 ->
case Enum.fetch(list, n) do
{:ok, item} -> do_fetch_docpath(item, t, [list | parents])
:error -> {:error, {:pointer_error, h, list}}
end
_ ->
{:error, {:pointer_error, h, list}}
end
end
defp do_fetch_docpath(raw_schema, [h | t], parents) when is_map(raw_schema) and is_binary(h) do
case Map.fetch(raw_schema, h) do
{:ok, sub} -> do_fetch_docpath(sub, t, [raw_schema | parents])
:error -> {:error, {:pointer_error, h, raw_schema}}
end
end
defp do_fetch_docpath(raw_schema, [], parents) do
{:ok, [raw_schema | parents]}
end
# TODO(Draft7-removal) remove derive_docpath_ns/3, this is only to support Draft7 where we
# must keep the parent_ns around in a %Resolved{}
defp derive_docpath_ns([%{"$id" => id} | [_ | _] = tail], parent_ns, parent_parent_ns) do
# Recursion first to go back to the top schema of the docpath
with {:ok, parent_ns, _parent_parent_ns} <- derive_docpath_ns(tail, parent_ns, parent_parent_ns),
{:ok, new_ns} <- RNS.derive(parent_ns, id) do
{:ok, new_ns, parent_ns}
end
end
defp derive_docpath_ns([_sub_no_id | [_ | _] = tail], parent_ns, parent_parent_ns) do
derive_docpath_ns(tail, parent_ns, parent_parent_ns)
end
defp derive_docpath_ns([_single], ns, parent_ns) do
# Do not derive from the last schema in the list, as `ns, parent_ns` represent that schema itself
{:ok, ns, parent_ns}
end
end