Current section

Files

Jump to
rock_solid lib rock_solid transformation.ex
Raw

lib/rock_solid/transformation.ex

defmodule RockSolid.Transformation do
@moduledoc false
alias RockSolid.Combinatorics
alias RockSolid.Context
alias RockSolid.Intersection
alias RockSolid.Schemas
alias RockSolid.Schemas.Schema
import RockSolid.Traversal, only: [property?: 1, is_atomic: 1]
@doc """
Simplifies a JSON schema to a canonical format
"""
def simplify(%{"$id" => uri} = schema) do
Context.put_schema(schema)
schema
|> simplify(_root = ["#"])
|> tap(&Context.put_simplified(uri, &1))
end
def simplify(schema, rev_path) do
cond do
property?(rev_path) or hd(rev_path) == "$defs" ->
Map.new(schema, fn {k, v} -> {k, simplify(v, [k | rev_path])} end)
is_atomic(schema) ->
schema
hd(rev_path) in ["dependentRequired", "required"] ->
schema
is_list(schema) ->
for {val, idx} <- Enum.with_index(schema), do: simplify(val, [to_string(idx) | rev_path])
Map.has_key?(schema, "$ref") ->
simplify_ref(schema, rev_path)
Map.has_key?(schema, "const") ->
{:ok, enums} = enums_matching_all([schema["const"]], [schema])
drop_non_keywords(enums)
Map.has_key?(schema, "enum") ->
{:ok, enums} = enums_matching_all(schema["enum"], [schema])
drop_non_keywords(enums)
schema == %{"not" => true} ->
false
true ->
schema |> drop_non_keywords() |> postorder_simplify(rev_path)
end
end
defp drop_non_keywords(schema) do
schema |> Map.drop(non_keywords()) |> Map.reject(&custom_key_value_pair?/1)
end
defp non_keywords do
["description", "example", "examples", "title", "default"]
end
defp custom_key_value_pair?({"x-" <> _, _}), do: true
defp custom_key_value_pair?(_), do: false
defp postorder_simplify(schema, rev_path) do
simplified = Map.new(schema, fn {k, v} -> {k, simplify(v, [k | rev_path])} end)
simplify_funcs()
|> Enum.reduce_while([simplified], fn func, accs ->
case Enum.flat_map(accs, func) |> Enum.reject(&(&1 == false)) do
[] -> {:halt, []}
other when is_list(other) -> {:cont, other}
end
end)
# This is a bit inefficient because we are generating a bunch of extra schemas.
# The problem comes because `$ref` are expanded to every possible type. Instead
# what we should do is extract them into a "$ref" type and perform intersection
# with `"$ref"` types. The problem is that `common_types` might be empty in
# `all_of_to_any_of` and we have to consider it. Refine the algorithm later.
# So far this will take longer and might generate additional redundant `anyOf`
# if enums are merged out of order. For example:
# %{"anyOf" => [%{"enum" => [1, 2]}, %{"enum" => [2,1]}]}
|> Enum.uniq()
|> case do
[] -> raise "Empty simplification for #{inspect(schema)}"
[one_schema] -> one_schema
schemas when is_list(schemas) -> %{"anyOf" => schemas}
end
end
defp simplify_funcs do
[
&to_schemas_by_types/1,
&expand_dependent_schemas/1,
&expand_case_schema/1,
&expand_if_then_else/1,
&merge_boolean_schemas/1
]
end
@doc """
Converts a schema that indirectly supports muliple types to an equivalent `anyOf`
schema
"""
@spec to_any_of(Schema.t()) :: Schema.t()
def to_any_of(schema) when is_map(schema) do
schema
|> split_by_types()
|> Map.values()
|> case do
[single_type] -> single_type
many_types when length(many_types) > 1 -> %{"anyOf" => many_types}
end
end
def to_any_of(schema) when is_boolean(schema), do: schema
defp to_schemas_by_types(schema), do: Map.values(split_by_types(schema))
@spec split_by_types(Schema.t()) :: %{String.t() => Schema.t()}
defp split_by_types(true) do
Map.new(Schema.base_types(), fn type -> {type, %{"type" => type}} end)
end
defp split_by_types(false), do: %{}
defp split_by_types(schema) when is_map(schema) do
Enum.reduce(schema, Map.new(), fn {key, value}, acc ->
Enum.reduce(types(key, value), acc, fn type, acc ->
new_acc = Map.put_new(acc, type, default(type))
case key do
# Do not override "type" key, otherwise we can end up
# with %{"type" => ["number", "array"]} again because
# the value overwrites the default inserted by `put_new`
"type" ->
# The only edge case is if we have "integer" in types but not
# "number", in that case we have to replace it. If we have
# ["number", "integer"] then "number" wins
if value == "integer" or
(is_list(value) and "integer" in value and "number" not in value) do
# Because we might reach this part before 'number' type is parsed
# we had to add an extra `Map.put_new` just in case
new_acc
|> Map.put_new("number", %{"type" => "integer"})
|> put_in(["number", "type"], "integer")
else
new_acc
end
# In case of enum we have to append if the value already exists
# We might not need `enum` and `const` at all if we always convert
# from schema with `enum` or `const` to single enum list
"enum" ->
# Fix this later, we are traversing everything multiple times for each
# enum
insert_enums(new_acc, value)
# Other regular keys are simply inserted
other ->
put_in(new_acc, [type, other], value)
end
end)
end)
|> add_unspecified_types()
|> keep_explicit_types(Map.take(schema, ["const", "type", "enum"]))
end
defp insert_enums(schemas_by_type, enums) do
Enum.reduce(enums, schemas_by_type, fn enum, acc ->
[type] = types("const", enum)
acc
|> Map.put_new(type, default(type))
|> update_in([type, "enum"], fn
nil -> [enum]
values when is_list(values) -> Enum.uniq(values ++ [enum])
end)
end)
end
defp types("type", "integer"), do: ["number"]
defp types("type", type) when is_binary(type), do: [type]
defp types("type", types) when is_list(types), do: Enum.flat_map(types, &types("type", &1))
defp types("const", val), do: [enum_type(val)]
defp types("enum", values), do: values |> Enum.map(&enum_type/1) |> Enum.uniq()
defp types("format", format) do
Enum.find_value(all_schemas(), fn {type, module} ->
if Enum.member?(Schema.formats(module), format) do
type
end
end)
|> case do
nil -> []
type -> [type]
end
end
defp types(key, _value) do
Enum.reduce(all_schemas(), MapSet.new(), fn {type, module}, acc ->
if key in Schema.fields(module), do: MapSet.put(acc, type), else: acc
end)
end
defp all_schemas do
[
{"boolean", Schemas.Boolean},
{"null", Schemas.Null},
{"number", Schemas.Number},
{"array", Schemas.Array},
{"string", Schemas.String},
{"object", Schemas.Object}
]
end
defp enum_type(t) when is_boolean(t), do: "boolean"
defp enum_type(t) when is_number(t), do: "number"
defp enum_type(t) when is_list(t), do: "array"
defp enum_type(t) when is_binary(t), do: "string"
defp enum_type(t) when is_map(t), do: "object"
defp enum_type(nil), do: "null"
defp default(type), do: %{"type" => type}
# If `const`, `enum` or `types` was specified then the types are explicit, keep
# only the possible types.
#
# Validate if `const` is present then the value matches the original schema
# If `enum` is present (and `const` isn't) then keep only the values that match
# the schema.
# We should do this before in the pre-simplification step
defp keep_explicit_types(types_map, %{"const" => c}), do: Map.take(types_map, types("const", c))
defp keep_explicit_types(types_map, %{"enum" => e}), do: Map.take(types_map, types("enum", e))
defp keep_explicit_types(types_map, %{"type" => t}), do: Map.take(types_map, types("type", t))
defp keep_explicit_types(types_map, _), do: types_map
defp add_unspecified_types(types_map) do
Enum.reduce(Schema.base_types(), types_map, fn type, acc ->
Map.put_new(acc, type, default(type))
end)
end
@doc """
Converts an `allOf` list of schemas to the equivalent `anyOf`
"""
@spec all_of_to_any_of(list(Schema.t())) :: {:ok, Schema.t()} | {:error, any()}
def all_of_to_any_of(schemas) when is_list(schemas) do
case collect_enums(schemas) do
[] -> all_of_to_any_of_schemas(schemas)
enums -> enums_matching_all(enums, schemas)
end
end
defp collect_enums(schemas) when is_list(schemas) do
schemas
|> Enum.flat_map(fn schema ->
case schema do
%{"const" => const} -> [const]
%{"enum" => enums} -> enums
_ -> []
end
end)
|> Enum.uniq()
end
defp enums_matching_all(enums, schemas) when is_list(enums) and is_list(schemas) do
j_schemas = Enum.map(schemas, &Context.build!/1)
enums
|> Enum.filter(fn enum ->
Enum.all?(j_schemas, fn j_schema -> match?({:ok, _}, JSV.validate(enum, j_schema)) end)
end)
|> case do
[] -> {:error, "no enum in #{inspect(enums)} matches schemas: #{inspect(schemas)}"}
other when is_list(other) -> {:ok, %{"enum" => other}}
end
end
defp all_of_to_any_of_schemas(schemas) when is_list(schemas) do
by_types = Enum.map(schemas, &split_by_types/1)
common_types =
by_types
|> Enum.map(&Map.keys/1)
|> Enum.reduce(fn k1, k2 -> MapSet.intersection(MapSet.new(k1), MapSet.new(k2)) end)
|> Enum.to_list()
by_types
|> Enum.map(&Map.take(&1, common_types))
|> Enum.reduce(fn map1, map2 ->
Map.merge(map1, map2, fn _k, v1, v2 -> Intersection.safe_intersection(v1, v2) end)
end)
|> Map.values()
|> discard_impossible_intersections()
|> case do
[] -> {:error, "empty anyOf"}
[value] -> {:ok, value}
values when is_list(values) -> {:ok, %{"anyOf" => values}}
end
end
@doc """
Converts a list of `oneOf` to the equivalent `anyOf`
"""
@spec one_of_to_any_of(list(Schema.t())) :: {:ok, Schema.t()} | {:error, any()}
def one_of_to_any_of(schemas) when is_list(schemas) do
schemas
|> Enum.with_index()
|> Enum.map(fn {schema, i} -> to_mutually_exclusive(schema, List.delete_at(schemas, i)) end)
|> Enum.reject(&Intersection.impossible?/1)
|> case do
[] -> {:error, "impossible oneOf condition"}
[value] -> {:ok, value}
values when is_list(values) -> {:ok, %{"anyOf" => values}}
end
end
def one_of_to_any_of(%{"oneOf" => schemas}), do: one_of_to_any_of(schemas)
def one_of_to_any_of(schema) when is_map(schema), do: {:ok, true}
defp to_mutually_exclusive(schema, others) do
Intersection.Not.add_clauses(
schema,
Enum.reject(others, &Intersection.mutually_exclusive?(&1, schema))
)
end
@doc """
Expands an `if/then/else` clause into multiple clauses and returns a list
of possible schemas
"""
@spec expand_if_then_else(Schema.t()) :: list(Schema.t())
def expand_if_then_else(%{"if" => if_, "then" => then_} = schema) do
else_ = Map.get(schema, "else", true)
schema = Map.drop(schema, ["if", "then", "else"])
case Intersection.safe_intersection(schema, if_) do
# Consider special case where if never matches, no need to add the `not` clause
# to the else case
false ->
[Intersection.safe_intersection(schema, else_)]
if_intersection ->
[
Intersection.safe_intersection(if_intersection, then_),
schema |> Intersection.Not.add_clause(if_) |> Intersection.safe_intersection(else_)
]
end
|> discard_impossible_intersections()
end
def expand_if_then_else(schema), do: [schema]
@doc """
Converts a schema with `anyOf`, `oneOf` and/or `allOf` into a list of `anyOf` schemas
"""
@spec merge_boolean_schemas(list(Schema.t())) :: list(Schema.t())
def merge_boolean_schemas(schema) when is_map(schema) do
{bools, schema} = Map.split(schema, ["anyOf", "oneOf", "allOf"])
schema = if(map_size(schema) == 0, do: true, else: schema)
any_of = if(Map.has_key?(bools, "anyOf"), do: Map.take(bools, ["anyOf"]), else: true)
with {:ok, all_of} <- all_of_to_any_of(Map.get(bools, "allOf", [true])),
{:ok, one_of} <- one_of_to_any_of(bools) do
case Enum.reduce([schema, any_of, one_of, all_of], &Intersection.safe_intersection(&1, &2)) do
%{"anyOf" => values} -> values
false -> []
value -> [value]
end
else
{:error, _} -> []
end
end
@doc """
Expands a schema with `dependentSchemas` to a list of all the combinations
of properties. Impossible cases are discarded
"""
@spec expand_dependent_schemas(Schema.t()) :: [Schema.t()]
def expand_dependent_schemas(%{"dependentSchemas" => _} = schema) do
schema
|> Map.get("dependentSchemas")
|> Map.keys()
|> Combinatorics.power_set()
|> Enum.map(fn req_props -> intersect_dependent_schemas(schema, req_props) end)
|> discard_impossible_intersections()
end
def expand_dependent_schemas(schema), do: [schema]
defp intersect_dependent_schemas(schema, properties) do
forbidden_props =
schema |> Map.get("dependentSchemas", %{}) |> Map.keys() |> Enum.reject(&(&1 in properties))
new_schema =
schema
|> Map.update("required", properties, fn req -> Enum.uniq(req ++ properties) end)
|> add_forbidden_properties(forbidden_props)
|> Map.delete("dependentSchemas")
Enum.reduce(properties, new_schema, fn prop, acc ->
Intersection.safe_intersection(acc, schema["dependentSchemas"][prop])
end)
end
@doc """
Adds a list of banned properties to the schema, that should never be added
because they conflict with a previous intersection. Returns the updated schema,
or false if one of the banned properties is required
"""
def add_forbidden_properties(schema, []), do: schema
def add_forbidden_properties(schema, forbidden_properties) when is_list(forbidden_properties) do
if Enum.any?(Map.get(schema, "required") || [], &(&1 in forbidden_properties)) do
false
else
no_props = Map.from_keys(forbidden_properties, false)
Map.update(schema, "properties", no_props, &Map.merge(&1, no_props))
end
end
@doc """
Expands a case schema into a list of mutually exclusive schemas. A case
schema is written as an %{"allOf": [%{"if" => ..., "else" => ...}, %{"if" => ...}]}
"""
def expand_case_schema(schema) do
if(case_schema?(schema), do: do_expand_case_schema(schema), else: [schema])
end
defp do_expand_case_schema(schema) do
{all_of, base_schema} = Map.pop(schema, "allOf")
{if_cases, rest} = split_if_then_clauses(all_of)
base_schema = Enum.reduce(rest, base_schema, &Intersection.safe_intersection/2)
Enum.reduce(if_cases, [base_schema], fn if_case, schemas ->
Enum.flat_map(schemas, fn schema ->
schema
|> Map.merge(Map.take(if_case, ["if", "then", "else"]))
|> expand_if_then_else()
end)
end)
|> discard_impossible_intersections()
end
defp case_schema?(%{"allOf" => all_of}), do: Enum.any?(all_of, &if_then_case?/1)
defp case_schema?(_), do: false
defp split_if_then_clauses(clauses), do: Enum.split_with(clauses, &if_then_case?/1)
defp if_then_case?(%{"if" => _, "then" => _}), do: true
defp if_then_case?(_), do: false
defp discard_impossible_intersections(schemas), do: Enum.reject(schemas, &(&1 == false))
@doc """
Simplifies "required" and "dependentRequired" by including in "required" all
the properties that depend on another "required" property and removes them
from dependentRequired list.
For example an object containing
```elixir
%{
"required" => ["name"],
"dependentRequired" => %{"name" => ["age"], "birthDate" => ["passportNumber"]}
}
```
is converted to
```elixir
%{
"required" => ["name", "age"],
"dependentRequired" => %{"birthDate" => ["passportNumber"]}
}
```
"""
def simplify_dependent_required(%{"required" => req, "dependentRequired" => dep_req} = schema) do
{required, dependent_required} =
simplify_dependent_required(:queue.from_list(req), dep_req, MapSet.new())
schema = Map.put(schema, "required", required)
if map_size(dependent_required) == 0 do
Map.delete(schema, "dependentRequired")
else
Map.put(schema, "dependentRequired", dependent_required)
end
end
def simplify_dependent_required(schema), do: schema
defp simplify_dependent_required(queue, dependent_required, required) do
case :queue.out(queue) do
{:empty, _} ->
{
MapSet.to_list(required),
Map.filter(dependent_required, fn {_k, v} -> not Enum.empty?(v) end)
}
{{:value, property_name}, updated_queue} ->
if MapSet.member?(required, property_name) do
simplify_dependent_required(updated_queue, dependent_required, required)
else
{deps, new_dependent_required} = Map.pop(dependent_required, property_name, [])
new_dependent_required =
Map.new(new_dependent_required, fn {prop_name, deps} ->
{prop_name, Enum.reject(deps, &(&1 == property_name))}
end)
new_required = MapSet.put(required, property_name)
new_queue = :queue.join(updated_queue, :queue.from_list(deps))
simplify_dependent_required(new_queue, new_dependent_required, new_required)
end
end
end
defp simplify_ref(%{"$ref" => ref_ptr, "x-rocksolid-refbehaviour" => "ignore"}, _) do
%{"$ref" => ref_ptr}
end
defp simplify_ref(%{"$ref" => _, "x-rocksolid-refbehaviour" => "merge"} = ref_schema, rev_path) do
{ref, rest} = Map.split(ref_schema, ["$ref"])
rest = drop_non_keywords(rest)
{ref_value, rest} = extract_additional_properties(ref, rest)
if rest == %{} do
ref_value
else
{:ok, intersection} = Intersection.intersection(ref_value, simplify(rest, rev_path))
intersection
end
end
# Edge case. `additionalProperties` is excluded if `$ref` already contains it, otherwise
# it's put inside `$ref`
defp extract_additional_properties(%{"$ref" => ref_ptr}, %{"additionalProperties" => _} = rest) do
{additional_props, rest_no_props} = Map.pop!(rest, "additionalProperties")
ref_val = Context.get_ref(ref_ptr)
{Map.put_new(ref_val, "additionalProperties", additional_props), rest_no_props}
end
defp extract_additional_properties(ref, rest), do: {ref, rest}
end