Packages
Unified schema definition, validation, and JSON generation for Elixir. Sinter is a focused, high-performance schema validation library designed specifically for dynamic frameworks like DSPy. It distills complex validation APIs into a single, powerful, and unified engine.
Current section
Files
Jump to
Current section
Files
lib/sinter.ex
defmodule Sinter do
@moduledoc """
Unified schema definition, validation, and JSON generation for Elixir.
Sinter provides a focused, high-performance schema validation library designed
specifically for dynamic frameworks. It follows the "One True Way" principle:
- **One way** to define schemas (unified core engine)
- **One way** to validate data (single validation pipeline)
- **One way** to generate JSON Schema (unified generator)
## Quick Start
# Define a schema
fields = [
{:name, :string, [required: true]},
{:age, :integer, [optional: true, gt: 0]}
]
schema = Sinter.Schema.define(fields)
# Validate data
{:ok, validated} = Sinter.Validator.validate(schema, %{
name: "Alice",
age: 30
})
# Generate JSON Schema
json_schema = Sinter.JsonSchema.generate(schema)
## Dynamic Schema Creation
Sinter supports dynamic schema creation for teleprompters and runtime optimization:
# Infer schema from examples (perfect for MIPRO teleprompter)
examples = [
%{"name" => "Alice", "age" => 30},
%{"name" => "Bob", "age" => 25}
]
schema = Sinter.infer_schema(examples)
# Merge schemas for signature composition
input_schema = Sinter.Schema.define([{:query, :string, [required: true]}])
output_schema = Sinter.Schema.define([{:answer, :string, [required: true]}])
program_schema = Sinter.merge_schemas([input_schema, output_schema])
## Convenience Helpers
This module provides convenient helper functions for common validation tasks
that internally use the core unified engine.
## Design Philosophy
Sinter distills data validation to its pure essence, extracting the essential
power from complex systems while eliminating unnecessary abstraction. It follows
three key principles:
1. **Validation, Not Transformation** - Sinter validates data structure and constraints
but does not perform business logic transformations
2. **Runtime-First Design** - Schemas are data structures that can be created and
modified at runtime, perfect for dynamic frameworks
3. **Unified Core Engine** - All validation flows through a single, well-tested
pipeline for consistency and reliability
"""
alias Sinter.{Schema, Validator}
@type schema :: Schema.t()
@type validation_opts :: Validator.validation_opts()
@type validation_result :: {:ok, term()} | {:error, [Sinter.Error.t()]}
@doc """
Validates a single value against a type specification.
This is a convenient helper for one-off type validation that creates a
temporary schema internally and uses the unified validation engine.
## Parameters
* `type_spec` - The type specification to validate against
* `value` - The value to validate
* `opts` - Validation options
## Options
* `:coerce` - Enable type coercion (default: false)
* `:constraints` - Additional constraints to apply
## Returns
* `{:ok, validated_value}` on success
* `{:error, errors}` on validation failure
## Examples
iex> Sinter.validate_type(:integer, "42", coerce: true)
{:ok, 42}
iex> Sinter.validate_type({:array, :string}, ["hello", "world"])
{:ok, ["hello", "world"]}
iex> {:error, [error]} = Sinter.validate_type(:string, 123)
iex> error.code
:type
"""
@spec validate_type(Sinter.Types.type_spec(), term(), validation_opts()) :: validation_result()
def validate_type(type_spec, value, opts \\ []) do
# Extract explicit constraints or treat all non-validation options as constraints
explicit_constraints = Keyword.get(opts, :constraints, [])
validation_option_keys = [:coerce, :strict, :constraints]
{validation_only_opts, constraint_opts} = Keyword.split(opts, validation_option_keys)
# Combine explicit constraints with direct constraint options
constraints = explicit_constraints ++ constraint_opts
temp_field = "__temp__"
# Create temporary single-field schema
temp_schema = Schema.define([{temp_field, type_spec, constraints}])
case Validator.validate(
temp_schema,
%{temp_field => value},
Keyword.delete(validation_only_opts, :constraints)
) do
{:ok, %{^temp_field => validated_value}} ->
{:ok, validated_value}
{:error, errors} ->
# Strip the temporary field name from error paths
fixed_errors =
Enum.map(errors, fn error ->
case error.path do
[^temp_field | rest] -> %{error | path: rest}
path -> %{error | path: path}
end
end)
{:error, fixed_errors}
end
end
@doc """
Validates a single named value against a type specification.
This is a convenient helper for single field validation that creates a
temporary schema internally.
## Parameters
* `field_name` - Name for the field (used in error messages)
* `type_spec` - The type specification to validate against
* `value` - The value to validate
* `opts` - Validation options and constraints
## Examples
iex> Sinter.validate_value(:email, :string, "test@example.com",
...> constraints: [format: ~r/@/])
{:ok, "test@example.com"}
iex> Sinter.validate_value(:score, :integer, "95",
...> coerce: true, constraints: [gteq: 0, lteq: 100])
{:ok, 95}
"""
@spec validate_value(atom(), Sinter.Types.type_spec(), term(), validation_opts()) ::
validation_result()
def validate_value(field_name, type_spec, value, opts \\ []) do
# Extract explicit constraints or treat all non-validation options as constraints
explicit_constraints = Keyword.get(opts, :constraints, [])
validation_option_keys = [:coerce, :strict, :constraints]
{validation_only_opts, constraint_opts} = Keyword.split(opts, validation_option_keys)
# Combine explicit constraints with direct constraint options
constraints = explicit_constraints ++ constraint_opts
field_key = normalize_field_name(field_name)
# Create temporary schema with named field
temp_schema = Schema.define([{field_key, type_spec, constraints}])
case Validator.validate(
temp_schema,
%{field_key => value},
Keyword.delete(validation_only_opts, :constraints)
) do
{:ok, validated_map} -> {:ok, Map.get(validated_map, field_key)}
{:error, errors} -> {:error, errors}
end
end
@doc """
Validates multiple values efficiently against their respective type specifications.
## Parameters
* `type_value_pairs` - List of `{type_spec, value}` or `{field_name, type_spec, value}` tuples
* `opts` - Validation options
## Examples
iex> Sinter.validate_many([
...> {:string, "hello"},
...> {:integer, 42},
...> {:email, :string, "test@example.com", [format: ~r/@/]}
...> ])
{:ok, ["hello", 42, "test@example.com"]}
"""
@spec validate_many(
[
{Sinter.Types.type_spec(), term()}
| {atom(), Sinter.Types.type_spec(), term()}
| {atom(), Sinter.Types.type_spec(), term(), keyword()}
],
validation_opts()
) :: {:ok, [term()]} | {:error, %{integer() => [Sinter.Error.t()]}}
def validate_many(type_value_pairs, opts \\ []) do
results =
type_value_pairs
|> Enum.with_index()
|> Enum.map(fn {spec, index} ->
case spec do
{type_spec, value} ->
validate_type(type_spec, value, opts)
{field_name, type_spec, value} ->
validate_value(field_name, type_spec, value, opts)
{field_name, type_spec, value, field_opts} ->
merged_opts = Keyword.merge(opts, field_opts)
validate_value(field_name, type_spec, value, merged_opts)
end
|> case do
{:ok, validated} -> {:ok, {index, validated}}
{:error, errors} -> {:error, {index, errors}}
end
end)
case Enum.split_with(results, &match?({:ok, _}, &1)) do
{oks, []} ->
validated_values =
oks
|> Enum.map(fn {:ok, {_index, value}} -> value end)
{:ok, validated_values}
{_, errors} ->
error_map =
errors
|> Enum.map(fn {:error, {index, errs}} -> {index, errs} end)
|> Map.new()
{:error, error_map}
end
end
@doc """
Creates a reusable validation function for a specific type and constraints.
Returns a function that can be used to validate multiple values against
the same specification efficiently.
## Parameters
* `type_spec` - The type specification
* `base_opts` - Base validation options and constraints
## Examples
iex> email_validator = Sinter.validator_for(:string,
...> constraints: [format: ~r/@/])
iex> email_validator.("test@example.com")
{:ok, "test@example.com"}
iex> {:error, [error]} = email_validator.("invalid")
iex> error.code
:format
"""
@spec validator_for(Sinter.Types.type_spec(), validation_opts()) ::
(term() -> validation_result())
def validator_for(type_spec, base_opts \\ []) do
fn value ->
validate_type(type_spec, value, base_opts)
end
end
@doc """
Creates a batch validator function for multiple type specifications.
## Parameters
* `field_specs` - List of field specifications or `{name, type_spec}` tuples
* `base_opts` - Base validation options
## Examples
iex> batch_validator = Sinter.batch_validator_for([
...> {:name, :string},
...> {:age, :integer}
...> ])
iex> batch_validator.(%{name: "Alice", age: 30})
{:ok, %{name: "Alice", age: 30}}
"""
@spec batch_validator_for([{atom(), Sinter.Types.type_spec()}], validation_opts()) ::
(map() -> {:ok, map()} | {:error, [Sinter.Error.t()]})
def batch_validator_for(field_specs, base_opts \\ []) do
# Create schema once for reuse
fields =
Enum.map(field_specs, fn
{name, type_spec} -> {name, type_spec, []}
{name, type_spec, constraints} -> {name, type_spec, constraints}
end)
schema = Schema.define(fields)
fn data ->
Validator.validate(schema, data, base_opts)
end
end
# ============================================================================
# PHASE 2: DYNAMIC SCHEMA CREATION FOR DSPEX
# ============================================================================
@doc """
Creates a schema by analyzing examples to infer field types and requirements.
This function is essential for DSPEx teleprompters like MIPRO that need to
dynamically optimize schemas based on program execution examples.
## Parameters
* `examples` - List of maps representing example data
* `opts` - Schema creation options
## Options
* `:title` - Schema title
* `:description` - Schema description
* `:strict` - Whether to reject unknown fields (default: false)
* `:min_occurrence_ratio` - Minimum ratio for field to be considered required (default: 0.8)
## Returns
* `Schema.t()` - A schema inferred from the examples
## Examples
iex> examples = [
...> %{"name" => "Alice", "age" => 30},
...> %{"name" => "Bob", "age" => 25},
...> %{"name" => "Charlie", "age" => 35}
...> ]
iex> schema = Sinter.infer_schema(examples)
iex> fields = Sinter.Schema.fields(schema)
iex> fields[:name].type
:string
iex> fields[:age].type
:integer
## Algorithm
1. **Field Discovery**: Find all unique field names across examples
2. **Type Inference**: Determine the most common type for each field
3. **Requirement Analysis**: Fields present in >= min_occurrence_ratio are required
4. **Constraint Inference**: Infer basic constraints from value patterns
"""
@spec infer_schema([map()], keyword()) :: Schema.t()
def infer_schema(examples, opts \\ []) when is_list(examples) do
if Enum.empty?(examples) do
raise ArgumentError, "Cannot infer schema from empty examples list"
end
# Validate all examples are maps
unless Enum.all?(examples, &is_map/1) do
raise ArgumentError, "All examples must be maps"
end
min_occurrence_ratio = Keyword.get(opts, :min_occurrence_ratio, 0.8)
example_count = length(examples)
normalized_examples = Enum.map(examples, &normalize_example/1)
# Step 1: Discover all field names
all_field_names =
normalized_examples
|> Enum.flat_map(&Map.keys/1)
|> Enum.uniq()
# Step 2: Analyze each field
field_specs =
Enum.map(all_field_names, fn field_name ->
analyze_field(field_name, normalized_examples, example_count, min_occurrence_ratio)
end)
# Step 3: Create schema with inferred fields
schema_opts = Keyword.drop(opts, [:min_occurrence_ratio])
Schema.define(field_specs, schema_opts)
end
@doc """
Merges multiple schemas into a single schema.
This is useful for DSPEx signature composition where you need to combine
input and output schemas or merge component signatures.
## Parameters
* `schemas` - List of Schema.t() to merge
* `opts` - Optional schema configuration overrides
## Merge Rules
1. **Fields**: All fields from all schemas are included
2. **Conflicts**: Later schemas override earlier ones for conflicting field definitions
3. **Configuration**: First non-nil configuration value wins, except for `:strict` where last wins
## Examples
iex> input_schema = Sinter.Schema.define([
...> {:query, :string, [required: true]}
...> ])
iex> output_schema = Sinter.Schema.define([
...> {:answer, :string, [required: true]},
...> {:confidence, :float, [optional: true]}
...> ])
iex> merged = Sinter.merge_schemas([input_schema, output_schema])
iex> fields = Sinter.Schema.fields(merged)
iex> Map.keys(fields)
[:query, :answer, :confidence]
"""
@spec merge_schemas([Schema.t()], keyword()) :: Schema.t()
def merge_schemas(schemas, opts \\ []) when is_list(schemas) do
if Enum.empty?(schemas) do
raise ArgumentError, "Cannot merge empty schemas list"
end
# Collect all fields from all schemas
all_fields =
schemas
|> Enum.flat_map(fn schema ->
Schema.fields(schema)
|> Enum.map(fn {name, field_def} ->
# Convert field definition back to field spec format
{name, field_def.type, build_field_options(field_def)}
end)
end)
# Handle field conflicts - later definitions override earlier ones
unique_fields =
all_fields
# Reverse so later entries have precedence
|> Enum.reverse()
|> Enum.uniq_by(fn {name, _type, _opts} -> name end)
# Restore original order
|> Enum.reverse()
# Merge configurations
merged_config = merge_schema_configs(schemas)
final_opts = Keyword.merge(merged_config, opts)
Schema.define(unique_fields, final_opts)
end
# ============================================================================
# PRIVATE HELPER FUNCTIONS FOR SCHEMA INFERENCE
# ============================================================================
# Analyzes a single field across all examples to determine its specification
@spec analyze_field(String.t(), [map()], integer(), float()) ::
{String.t(), Sinter.Types.type_spec(), keyword()}
defp analyze_field(field_name, examples, example_count, min_occurrence_ratio) do
# Extract all values for this field
field_values =
examples
|> Enum.map(&Map.get(&1, field_name))
|> Enum.reject(&is_nil/1)
occurrence_count = length(field_values)
occurrence_ratio = occurrence_count / example_count
# Determine if field is required
required = occurrence_ratio >= min_occurrence_ratio
# Infer type from values
inferred_type = infer_field_type(field_values)
# Build field options
field_opts = [required: required]
{field_name, inferred_type, field_opts}
end
# Infers the type of a field from its values
@spec infer_field_type([term()]) :: Sinter.Types.type_spec()
defp infer_field_type([]), do: :any
defp infer_field_type(values) do
# Group values by their Elixir type
type_frequencies =
values
|> Enum.map(&get_elixir_type/1)
|> Enum.frequencies()
# Get the most common type
{most_common_type, _frequency} =
type_frequencies
|> Enum.max_by(fn {_type, count} -> count end)
most_common_type
end
# Maps an Elixir value to its Sinter type specification
@spec get_elixir_type(term()) :: Sinter.Types.type_spec()
defp get_elixir_type(value) when is_binary(value), do: :string
defp get_elixir_type(value) when is_integer(value), do: :integer
defp get_elixir_type(value) when is_float(value), do: :float
defp get_elixir_type(value) when is_boolean(value), do: :boolean
defp get_elixir_type(value) when is_atom(value), do: :atom
defp get_elixir_type(value) when is_map(value), do: :map
defp get_elixir_type(value) when is_list(value) do
case infer_array_type(value) do
:mixed -> {:array, :any}
inner_type -> {:array, inner_type}
end
end
defp get_elixir_type(_value), do: :any
# Infers the inner type of an array
@spec infer_array_type([term()]) :: Sinter.Types.type_spec() | :mixed
defp infer_array_type([]), do: :any
defp infer_array_type(list) do
# Get types of all elements
element_types =
list
|> Enum.map(&get_elixir_type/1)
|> Enum.uniq()
case element_types do
# All elements same type
[single_type] -> single_type
# Mixed types, use :any
_multiple -> :mixed
end
end
# Builds field options from a field definition (for schema merging)
@spec build_field_options(Schema.field_definition()) :: keyword()
defp build_field_options(field_def) do
opts = [required: field_def.required]
opts = if field_def.description, do: [description: field_def.description] ++ opts, else: opts
opts = if field_def.example, do: [example: field_def.example] ++ opts, else: opts
opts =
case field_def.default do
nil -> opts
default -> [default: default] ++ opts
end
# Add constraints
field_def.constraints ++ opts
end
# Merges configuration from multiple schemas
@spec merge_schema_configs([Schema.t()]) :: keyword()
defp merge_schema_configs(schemas) do
configs = Enum.map(schemas, &Schema.config/1)
# Merge with specific rules
merged = %{
title: find_first_non_nil(configs, :title),
description: find_first_non_nil(configs, :description),
strict: find_last_non_nil(configs, :strict, false),
post_validate: find_first_non_nil(configs, :post_validate)
}
# Convert to keyword list, filtering out nil values
merged
|> Enum.reject(fn {_key, value} -> is_nil(value) end)
|> Enum.into([])
end
# Helper to find first non-nil value for a key across configs
@spec find_first_non_nil([map()], atom()) :: term() | nil
defp find_first_non_nil(configs, key) do
configs
|> Enum.map(&Map.get(&1, key))
|> Enum.find(&(not is_nil(&1)))
end
# Helper to find last non-nil value for a key across configs, with default
@spec find_last_non_nil([map()], atom(), term()) :: term()
defp find_last_non_nil(configs, key, default) do
configs
|> Enum.map(&Map.get(&1, key))
|> Enum.reverse()
|> Enum.find(&(not is_nil(&1)))
|> case do
nil -> default
value -> value
end
end
defp normalize_example(%_{} = struct) do
struct
|> Map.from_struct()
|> normalize_example()
end
defp normalize_example(example) when is_map(example) do
Enum.reduce(example, %{}, fn {key, value}, acc ->
Map.put(acc, normalize_field_name(key), value)
end)
end
defp normalize_example(example), do: example
defp normalize_field_name(name) when is_atom(name), do: Atom.to_string(name)
defp normalize_field_name(name) when is_binary(name), do: name
defp normalize_field_name(name), do: to_string(name)
end