Current section
Files
Jump to
Current section
Files
lib/toon/encoder/core.ex
defmodule ExToon.Encoder.Core do
@moduledoc false
# Main encoding pipeline. Converts a normalized value tree into a TOON document string.
#
# Design decisions:
# - normalize/1 returns ordered [{String.t(), term()}] pairs for objects and [term()] for
# arrays. The encoder distinguishes them by checking if the head element is a
# {binary_key, _} tuple.
# - encode_value/3 is the external entry point called by ExToon.encode/2.
# - All internal helpers return String.t() directly (the iodata() wrapping is a thin
# layer — the public API converts via IO.iodata_to_binary/1).
# - Empty objects (empty pair-list) encode to "" (no output); empty arrays encode to
# an inline form with zero length.
alias ExToon.StringUtils
alias ExToon.Encoder.{Normalize, Primitives, Replacer, Folding}
@type opts :: keyword()
# ---------------------------------------------------------------------------
# Public entry point
# ---------------------------------------------------------------------------
@spec encode_value(term(), opts(), [String.t() | integer()]) :: iodata()
def encode_value(input, opts, path) do
replacer = Keyword.get(opts, :replacer)
# Apply replacer to the root value (key is "" for root) before normalization
# so the replacer receives the original Elixir term.
replaced_input =
case Replacer.apply(input, replacer, "", path) do
{:keep, v} -> v
# Skipping the root is a no-op — there's nothing to omit it from
:skip -> input
end
# For plain maps: use lazy normalization so the replacer receives the original
# Elixir values (not normalized pairs) and uses child_path (path including current key).
case replaced_input do
map when is_map(map) and not is_struct(map) ->
if map_size(map) == 0 do
""
else
sorted_pairs =
map
|> Enum.sort_by(fn {k, _} -> raw_key_to_string(k) end)
|> Enum.map(fn {k, v} -> {raw_key_to_string(k), v} end)
encode_object_with_original_values(sorted_pairs, opts, path, 0)
end
_ ->
final = Normalize.normalize(replaced_input)
do_encode(final, opts, path, 0)
end
end
# Convert atom or binary key to string (without recursively normalizing the value).
defp raw_key_to_string(k) when is_atom(k), do: Atom.to_string(k)
defp raw_key_to_string(k) when is_binary(k), do: k
# Encode an object from a list of {string_key, original_elixir_value} pairs.
# Unlike encode_object/4 (which expects normalized pairs), this function:
# - passes the original (pre-normalization) value to the replacer
# - uses child_path (path ++ [k]) for the replacer call
# - detects key-folding collisions before folding
# - normalizes each value AFTER the replacer has run
defp encode_object_with_original_values(pairs, opts, path, depth) do
indent_size = Keyword.get(opts, :indent, 2)
key_folding = Keyword.get(opts, :key_folding, :off)
flatten_depth = Keyword.get(opts, :flatten_depth, :infinity)
replacer = Keyword.get(opts, :replacer)
doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma))
# Build the set of ALL original sibling keys for collision detection.
all_sibling_keys = MapSet.new(pairs, fn {k, _} -> k end)
pairs
|> Enum.flat_map(fn {k, v_original} ->
child_path = path ++ [k]
# Normalize the value to allow Folding.fold (which requires normalized pairs lists).
v_for_fold = Normalize.normalize(v_original)
{folded_key, folded_value_norm} = Folding.fold(k, v_for_fold, key_folding, flatten_depth, 0)
# Collision detection: skip folding if the resulting key conflicts with a sibling.
collision? = folded_key != k and MapSet.member?(all_sibling_keys, folded_key)
{use_key, use_opts, precomputed_norm} =
if collision? do
# Disable key folding in the subtree to prevent partial folding that
# could recreate the same colliding path at a nested level.
{k, Keyword.put(opts, :key_folding, :off), v_for_fold}
else
{folded_key, opts, folded_value_norm}
end
# Apply replacer with the ORIGINAL value and the CHILD PATH.
case Replacer.apply(v_original, replacer, k, child_path) do
:skip ->
[]
{:keep, final_v} ->
# When the replacer returns the same object (identity), reuse the pre-computed
# normalized/folded value to avoid re-normalizing the whole subtree.
final_norm =
if final_v === v_original do
precomputed_norm
else
Normalize.normalize(final_v)
end
line = encode_kv(use_key, final_norm, use_opts, child_path, depth, doc_delim, indent_size)
if line == "" do
[]
else
[line]
end
end
end)
|> Enum.join("\n")
end
# encode_lines/2: returns a Stream of line binaries.
# NOTE: unlike encode/2, encoding failures raise EncodeError during enumeration.
@spec encode_lines(term(), opts()) :: Enumerable.t()
def encode_lines(input, opts) do
Stream.resource(
fn ->
result = encode_value(input, opts, [])
binary = IO.iodata_to_binary(result)
# An empty binary (e.g. empty map) must produce zero lines, not [""].
if binary == "", do: [], else: String.split(binary, "\n")
end,
fn
[] -> {:halt, []}
[line | rest] -> {[line], rest}
end,
fn _ -> :ok end
)
end
# ---------------------------------------------------------------------------
# Core dispatch
# ---------------------------------------------------------------------------
# Primitives are delegated directly to Primitives module with the document delimiter
defp do_encode(nil, _opts, _path, _depth), do: "null"
defp do_encode(true, _opts, _path, _depth), do: "true"
defp do_encode(false, _opts, _path, _depth), do: "false"
defp do_encode(n, _opts, _path, _depth) when is_integer(n), do: Integer.to_string(n)
defp do_encode(f, opts, _path, _depth) when is_float(f) do
doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma))
Primitives.encode_primitive(f, doc_delim)
end
defp do_encode(s, opts, _path, _depth) when is_binary(s) do
doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma))
Primitives.encode_primitive(s, doc_delim)
end
# Empty object sentinel — produced by normalize/1 for empty plain maps.
# Encodes to "" (no output, invisible in the document).
defp do_encode(:empty_object, _opts, _path, _depth), do: ""
defp do_encode(pairs, opts, path, depth) when is_list(pairs) do
case pairs do
[] ->
# Empty list (from Encodable.to_toon returning [] or normalize of empty keyword list).
# This is treated as an empty ARRAY (encoded as [0]:).
# Empty MAPS go through :empty_object sentinel instead — see normalize/1.
doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma))
prefix = String.duplicate(" ", depth * Keyword.get(opts, :indent, 2))
delim_sym = delimiter_symbol(doc_delim)
"#{prefix}[0#{delim_sym}]:"
[{k, _} | _] when is_binary(k) ->
# Object: ordered key-value pairs
encode_object(pairs, opts, path, depth)
_ ->
# Array
active_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma))
encode_array_standalone(pairs, opts, path, depth, active_delim)
end
end
# ---------------------------------------------------------------------------
# Object encoding
# ---------------------------------------------------------------------------
defp encode_object(pairs, opts, path, depth) do
indent_size = Keyword.get(opts, :indent, 2)
key_folding = Keyword.get(opts, :key_folding, :off)
flatten_depth = Keyword.get(opts, :flatten_depth, :infinity)
replacer = Keyword.get(opts, :replacer)
doc_delim = resolve_delimiter(Keyword.get(opts, :delimiter, :comma))
# Build sibling key set for collision detection (same as in encode_object_with_original_values).
all_sibling_keys = MapSet.new(pairs, fn {k, _} -> k end)
pairs
|> Enum.flat_map(fn {k, v} ->
{folded_key, folded_value} = Folding.fold(k, v, key_folding, flatten_depth, depth)
child_path = path ++ [k]
# Collision: if folding produced a key that already exists as a sibling, skip fold
# and disable folding for the subtree to prevent partial-fold collisions at deeper levels.
{use_key, use_value, use_opts} =
if folded_key != k and MapSet.member?(all_sibling_keys, folded_key) do
{k, v, Keyword.put(opts, :key_folding, :off)}
else
{folded_key, folded_value, opts}
end
case Replacer.apply(use_value, replacer, k, path) do
:skip ->
[]
{:keep, final_v} ->
normalized_v = Normalize.normalize(final_v)
line = encode_kv(use_key, normalized_v, use_opts, child_path, depth, doc_delim, indent_size)
if line == "" do
[]
else
[line]
end
end
end)
|> Enum.join("\n")
end
# ---------------------------------------------------------------------------
# Key-value line encoding
# ---------------------------------------------------------------------------
defp encode_kv(key, value, opts, path, depth, doc_delim, indent_size) do
prefix = String.duplicate(" ", depth * indent_size)
encoded_key = encode_key(key)
case value do
nil ->
"#{prefix}#{encoded_key}: null"
true ->
"#{prefix}#{encoded_key}: true"
false ->
"#{prefix}#{encoded_key}: false"
n when is_integer(n) ->
"#{prefix}#{encoded_key}: #{n}"
f when is_float(f) ->
"#{prefix}#{encoded_key}: #{Primitives.encode_primitive(f, doc_delim)}"
s when is_binary(s) ->
encoded_val = Primitives.encode_primitive(s, doc_delim)
"#{prefix}#{encoded_key}: #{encoded_val}"
:empty_object ->
# Empty nested object — emit bare key with no value
"#{prefix}#{encoded_key}:"
[] ->
# Empty array — use zero-length inline form
delim_sym = delimiter_symbol(doc_delim)
"#{prefix}#{encoded_key}[0#{delim_sym}]:"
[{_, _} | _] = sub_pairs ->
# Nested object: recurse with increased depth
child_block = encode_object(sub_pairs, opts, path, depth + 1)
if child_block == "" do
"#{prefix}#{encoded_key}:"
else
"#{prefix}#{encoded_key}:\n#{child_block}"
end
list when is_list(list) ->
encode_array_kv(encoded_key, list, opts, path, depth, doc_delim, indent_size, prefix)
end
end
# ---------------------------------------------------------------------------
# Array encoding (as a key-value entry)
# ---------------------------------------------------------------------------
defp encode_array_kv(encoded_key, list, opts, path, depth, doc_delim, indent_size, prefix) do
replacer = Keyword.get(opts, :replacer)
delim_sym = delimiter_symbol(doc_delim)
# Apply replacer to each array element before deciding encoding format.
filtered_list =
list
|> Enum.with_index()
|> Enum.flat_map(fn {item, idx} ->
key = Integer.to_string(idx)
item_path = path ++ [idx]
case Replacer.apply(item, replacer, key, item_path) do
:skip -> []
{:keep, final_item} -> [Normalize.normalize(final_item)]
end
end)
n = length(filtered_list)
cond do
all_primitives?(filtered_list) ->
# Primitive array: key[N]: v1,v2,v3
values = Enum.map(filtered_list, &Primitives.encode_primitive(&1, doc_delim))
"#{prefix}#{encoded_key}[#{n}#{delim_sym}]: #{Enum.join(values, <<doc_delim>>)}"
# Tabular format is skipped when a replacer is present because tabular
# encoding bypasses per-field replacer application. Fall through to the
# expanded list format so each object's fields are individually processed.
replacer == nil and tabular?(filtered_list) ->
encode_tabular_kv(
encoded_key,
filtered_list,
doc_delim,
indent_size,
prefix,
depth,
n,
delim_sym
)
true ->
# Expanded list: key[N]:\n - item (replacer already applied above)
encode_expanded_kv_normalized(
encoded_key,
filtered_list,
opts,
path,
depth,
doc_delim,
indent_size,
prefix,
n,
delim_sym
)
end
end
defp encode_tabular_kv(
encoded_key,
list,
doc_delim,
indent_size,
prefix,
depth,
n,
delim_sym
) do
# Extract field order from the first row
[{_, _} | _] = first = hd(list)
fields = Enum.map(first, fn {k, _} -> k end)
encoded_fields = Enum.map(fields, &encode_key/1)
fields_str = Enum.join(encoded_fields, <<doc_delim>>)
row_prefix = String.duplicate(" ", (depth + 1) * indent_size)
rows =
Enum.map(list, fn pairs ->
values =
Enum.map(fields, fn f ->
v = find_value(pairs, f)
Primitives.encode_primitive(v, doc_delim)
end)
"#{row_prefix}#{Enum.join(values, <<doc_delim>>)}"
end)
"#{prefix}#{encoded_key}[#{n}#{delim_sym}]{#{fields_str}}:\n#{Enum.join(rows, "\n")}"
end
defp encode_expanded_kv(
encoded_key,
list,
opts,
path,
depth,
doc_delim,
indent_size,
prefix,
n,
delim_sym
) do
# List is NOT yet replacer-filtered — apply replacer and normalize here.
replacer = Keyword.get(opts, :replacer)
item_prefix = String.duplicate(" ", (depth + 1) * indent_size)
items =
list
|> Enum.with_index()
|> Enum.flat_map(fn {item, idx} ->
item_path = path ++ [idx]
key = Integer.to_string(idx)
case Replacer.apply(item, replacer, key, item_path) do
:skip -> []
{:keep, final_item} ->
normalized_item = Normalize.normalize(final_item)
[encode_list_item(normalized_item, opts, item_path, depth + 1, doc_delim, indent_size, item_prefix)]
end
end)
actual_n = length(items)
"#{prefix}#{encoded_key}[#{actual_n}#{delim_sym}]:\n#{Enum.join(items, "\n")}"
end
# Like encode_expanded_kv but for already-normalized, already-filtered lists
# (called from encode_array_kv which has already applied the replacer).
defp encode_expanded_kv_normalized(
encoded_key,
filtered_list,
opts,
path,
depth,
doc_delim,
indent_size,
prefix,
n,
delim_sym
) do
item_prefix = String.duplicate(" ", (depth + 1) * indent_size)
items =
filtered_list
|> Enum.with_index()
|> Enum.map(fn {item, idx} ->
item_path = path ++ [idx]
encode_list_item(item, opts, item_path, depth + 1, doc_delim, indent_size, item_prefix)
end)
"#{prefix}#{encoded_key}[#{n}#{delim_sym}]:\n#{Enum.join(items, "\n")}"
end
# ---------------------------------------------------------------------------
# Array encoding (standalone, no key context)
# ---------------------------------------------------------------------------
defp encode_array_standalone(list, opts, path, depth, doc_delim) do
indent_size = Keyword.get(opts, :indent, 2)
replacer = Keyword.get(opts, :replacer)
delim_sym = delimiter_symbol(doc_delim)
prefix = String.duplicate(" ", depth * indent_size)
# Apply replacer to each element (using index as string key), filtering :skip items.
# NOTE: we re-normalize after replacer so the final list is always normalized.
filtered_list =
list
|> Enum.with_index()
|> Enum.flat_map(fn {item, idx} ->
key = Integer.to_string(idx)
item_path = path ++ [idx]
case Replacer.apply(item, replacer, key, item_path) do
:skip -> []
{:keep, final_item} -> [Normalize.normalize(final_item)]
end
end)
n = length(filtered_list)
cond do
all_primitives?(filtered_list) ->
values = Enum.map(filtered_list, &Primitives.encode_primitive(&1, doc_delim))
"#{prefix}[#{n}#{delim_sym}]: #{Enum.join(values, <<doc_delim>>)}"
tabular?(filtered_list) ->
[{_, _} | _] = first = hd(filtered_list)
fields = Enum.map(first, fn {k, _} -> k end)
encoded_fields = Enum.map(fields, &encode_key/1)
fields_str = Enum.join(encoded_fields, <<doc_delim>>)
row_prefix = String.duplicate(" ", (depth + 1) * indent_size)
rows =
Enum.map(filtered_list, fn pairs ->
values =
Enum.map(fields, fn f ->
v = find_value(pairs, f)
Primitives.encode_primitive(v, doc_delim)
end)
"#{row_prefix}#{Enum.join(values, <<doc_delim>>)}"
end)
"#{prefix}[#{n}#{delim_sym}]{#{fields_str}}:\n#{Enum.join(rows, "\n")}"
true ->
item_prefix = String.duplicate(" ", (depth + 1) * indent_size)
items =
filtered_list
|> Enum.with_index()
|> Enum.map(fn {item, idx} ->
encode_list_item(
item,
opts,
path ++ [idx],
depth + 1,
doc_delim,
indent_size,
item_prefix
)
end)
"#{prefix}[#{n}#{delim_sym}]:\n#{Enum.join(items, "\n")}"
end
end
# ---------------------------------------------------------------------------
# List item encoding (prefixed with "- ")
# ---------------------------------------------------------------------------
defp encode_list_item(item, opts, path, depth, doc_delim, indent_size, item_prefix) do
case item do
nil -> "#{item_prefix}- null"
true -> "#{item_prefix}- true"
false -> "#{item_prefix}- false"
n when is_integer(n) -> "#{item_prefix}- #{n}"
f when is_float(f) -> "#{item_prefix}- #{Primitives.encode_primitive(f, doc_delim)}"
s when is_binary(s) -> "#{item_prefix}- #{Primitives.encode_primitive(s, doc_delim)}"
:empty_object ->
# Empty object as list item — bare hyphen
"#{item_prefix}-"
[] ->
delim_sym = delimiter_symbol(doc_delim)
"#{item_prefix}- [0#{delim_sym}]:"
[{_, _} | _] = pairs ->
# Object as list item: first field on the hyphen line (YAML-style inline).
# Remaining fields are at depth+1 with normal indentation.
encode_object_as_list_item(pairs, opts, path, depth, doc_delim, indent_size, item_prefix)
list when is_list(list) ->
sub_n = length(list)
delim_sym = delimiter_symbol(doc_delim)
if all_primitives?(list) do
values = Enum.map(list, &Primitives.encode_primitive(&1, doc_delim))
"#{item_prefix}- [#{sub_n}#{delim_sym}]: #{Enum.join(values, <<doc_delim>>)}"
else
sub_prefix = String.duplicate(" ", (depth + 1) * indent_size)
sub_items =
list
|> Enum.with_index()
|> Enum.map(fn {sub, idx} ->
encode_list_item(
sub,
opts,
path ++ [idx],
depth + 1,
doc_delim,
indent_size,
sub_prefix
)
end)
"#{item_prefix}- [#{sub_n}#{delim_sym}]:\n#{Enum.join(sub_items, "\n")}"
end
end
end
# ---------------------------------------------------------------------------
# List-item object encoding (YAML-style: first field on the hyphen line)
# ---------------------------------------------------------------------------
# Encodes an object as a list item. The first rendered field is placed on the
# same line as the "- " prefix; remaining fields are at depth+1 with their
# normal indentation. This produces the canonical TOON list-item-object format:
#
# - firstKey: value
# secondKey: value
#
# or, when the first field is an array with a block body:
#
# - arr[N]{f}:
# row1
# sibling: val
#
defp encode_object_as_list_item(pairs, opts, path, depth, doc_delim, indent_size, item_prefix) do
key_folding = Keyword.get(opts, :key_folding, :off)
flatten_depth = Keyword.get(opts, :flatten_depth, :infinity)
replacer = Keyword.get(opts, :replacer)
# child_depth is the depth at which fields of this object are written
child_depth = depth + 1
child_prefix = String.duplicate(" ", child_depth * indent_size)
# Render all visible fields (same logic as encode_object).
# Replacer receives `path` (the path to this object) and `k` (the key).
field_lines =
pairs
|> Enum.flat_map(fn {k, v} ->
{folded_key, folded_value} = Folding.fold(k, v, key_folding, flatten_depth, child_depth)
child_path = path ++ [k]
case Replacer.apply(folded_value, replacer, k, path) do
:skip ->
[]
{:keep, final_v} ->
normalized_v = Normalize.normalize(final_v)
line = encode_kv(folded_key, normalized_v, opts, child_path, child_depth, doc_delim, indent_size)
if line == "" do
[]
else
[line]
end
end
end)
case field_lines do
[] ->
# All fields were skipped — emit a bare hyphen
"#{item_prefix}-"
[only_line] ->
# Single field: replace its child_prefix with "item_prefix- "
inline = String.replace_prefix(only_line, child_prefix, "#{item_prefix}- ")
inline
[first_line | rest_lines] ->
# Multiple fields: first on hyphen line, rest unchanged
inline_first = String.replace_prefix(first_line, child_prefix, "#{item_prefix}- ")
([inline_first] ++ rest_lines) |> Enum.join("\n")
end
end
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
defp encode_key(key) do
if StringUtils.key_needs_quoting?(key) do
"\"#{StringUtils.escape(key)}\""
else
key
end
end
# Returns true when every element is a TOON primitive (nil, boolean, number, string)
defp all_primitives?(list) do
Enum.all?(list, fn
v when is_binary(v) or is_number(v) or is_boolean(v) or is_nil(v) -> true
_ -> false
end)
end
# Tabular: all elements are objects (ordered pairs) with the same keys, all values primitive.
# Key order is taken from the first element; rows must have the same key set (order-insensitive).
defp tabular?([]), do: false
defp tabular?(list) do
case hd(list) do
[{_, _} | _] = first ->
first_keys = Enum.map(first, fn {k, _} -> k end) |> Enum.sort()
Enum.all?(list, fn
[{_, _} | _] = pairs ->
row_keys = Enum.map(pairs, fn {k, _} -> k end) |> Enum.sort()
row_keys == first_keys and
Enum.all?(pairs, fn {_, v} ->
is_binary(v) or is_number(v) or is_boolean(v) or is_nil(v)
end)
_ ->
false
end)
_ ->
false
end
end
# Lookup a value in an ordered pairs list by key (used for tabular row rendering)
defp find_value(pairs, key) do
case List.keyfind(pairs, key, 0) do
{_, v} -> v
nil -> nil
end
end
defp resolve_delimiter(:comma), do: ?,
defp resolve_delimiter(:tab), do: ?\t
defp resolve_delimiter(:pipe), do: ?|
defp resolve_delimiter(c) when is_integer(c), do: c
# Returns the suffix appended to array length in headers:
# comma (default) is omitted; tab and pipe are written explicitly.
defp delimiter_symbol(?,), do: ""
defp delimiter_symbol(?\t), do: "\t"
defp delimiter_symbol(?|), do: "|"
end