Packages
smart_city
5.4.7
6.0.0
5.4.7
5.4.6
5.4.5
5.4.4
5.4.3
5.4.2
5.4.1
5.4.0
5.3.0
5.2.8
5.2.6
5.2.5
5.2.4
5.2.3
5.2.2
5.2.1
5.2.0
5.1.1
5.1.0
5.0.7
5.0.6
5.0.5
5.0.4
5.0.3
5.0.2
5.0.1
5.0.0
4.0.1
4.0.0
3.25.0
3.24.0
3.23.0
3.22.0
3.21.0
3.20.1
3.20.0
3.19.1
3.19.0
3.18.3
3.18.2
3.18.1
3.18.0
3.17.0
3.16.0
3.15.1
3.15.0
3.14.1
3.14.0
3.13.0
3.12.0
3.11.0
3.10.0
3.9.0
3.8.0
3.7.0
3.6.0
3.5.1
3.5.0
3.4.0
3.3.0
3.2.4
3.2.3
3.2.1
3.2.0
3.1.0
3.0.0
2.9.0
2.8.2
2.8.0
2.7.1
2.7.0
2.6.1
2.5.0
2.4.2
retired
2.4.1
2.4.0
2.3.0
2.2.0
2.1.6
2.1.5
2.1.4
2.1.3
2.1.2
A library for shared helper modules in the Smart Cities Data project.
Current section
Files
Jump to
Current section
Files
lib/smart_city/schema_generator.ex
defmodule SmartCity.SchemaGenerator do
@moduledoc """
Take the first row of data from a datasample and generates an implied schema from that with all field types being strings. Lists and nested maps are generated with their subschemas where applicable.
"""
@base_schema %{"biased" => "No", "demographic" => "None", "description" => "", "masked" => "N/A", "pii" => "None"}
def generate_schema(data) do
extract_field({:top, data |> hd()}) |> Map.get("subSchema")
end
def extract_field({key, value}) when is_list(value) do
item_type =
value
|> List.first()
|> infer_type()
case item_type do
"map" ->
schema = List.first(value) |> Enum.map(&extract_field/1)
schema_field_list("list", key, item_type, schema)
"list" ->
subSchema = [extract_field({"child_of_list", List.first(value)})]
schema_field_list("list", key, item_type, subSchema)
_ ->
schema_field_list("list", key, item_type)
end
end
def extract_field({key, value}) when is_map(value) do
schema = Enum.map(value, &extract_field/1)
infer_type(value) |> schema_field(key, schema)
end
def extract_field({key, value}) do
infer_type(value) |> schema_field(key)
end
def extract_field(list) do
List.first(list)
|> infer_type
|> schema_field("child_of_list")
end
defp infer_type(value) when is_list(value), do: "list"
defp infer_type(value) when is_map(value), do: "map"
defp infer_type(value) when is_integer(value), do: "integer"
defp infer_type(value) when is_float(value), do: "float"
defp infer_type(value) when is_boolean(value), do: "boolean"
defp infer_type(value) do
cond do
is_timestamp(value) -> "timestamp"
is_date(value) -> "date"
true -> "string"
end
end
defp is_timestamp(value) do
case Timex.parse(value, "{ISO:Extended}") do
{:ok, _} -> true
_ -> false
end
end
defp is_date(value) do
case Timex.parse(value, "{ISOdate}") do
{:ok, _} -> true
_ -> false
end
end
defp schema_field_list(type, name, item_type) when type == "list" do
%{
"name" => name,
"type" => type,
"itemType" => item_type,
"ingestion_field_selector" => name
}
|> Map.merge(@base_schema)
end
defp schema_field_list(type, name, item_type, sub_schema) when type == "list" and item_type == "list" do
%{
"name" => name,
"type" => type,
"itemType" => item_type,
"subSchema" => sub_schema,
"ingestion_field_selector" => name
}
|> Map.merge(@base_schema)
end
defp schema_field_list(type, name, item_type, schema) when type == "list" and item_type == "map" do
%{
"name" => name,
"type" => type,
"itemType" => item_type,
"subSchema" => schema,
"ingestion_field_selector" => name
}
|> Map.merge(@base_schema)
end
defp schema_field(type, name, schema) when type == "map" do
%{
"name" => name,
"type" => type,
"subSchema" => schema,
"ingestion_field_selector" => name
}
|> Map.merge(@base_schema)
end
defp schema_field(type, name) do
%{
"biased" => "No",
"demographic" => "None",
"description" => "",
"masked" => "N/A",
"name" => name,
"pii" => "None",
"type" => type,
"ingestion_field_selector" => name
}
end
end