Current section
Files
Jump to
Current section
Files
lib/toon/decoder/parser.ex
defmodule ExToon.Decoder.Parser do
@moduledoc false
# Structural parser for TOON documents.
#
# Consumes a list of %Scanner{} structs and produces a flat list of stream events:
# %{type: :start_object}
# %{type: :end_object}
# %{type: :start_array, length: n}
# %{type: :end_array}
# %{type: :key, key: str}
# %{type: :primitive, value: val}
#
# Design:
# - Recursive descent over the line list, tracking expected depth.
# - Root form detection per §5: first depth-0 line determines array vs object vs primitive.
# - Array forms: inline primitive, tabular rows, expanded list items.
# - Object forms: key:value pairs at a given depth.
alias ExToon.Decoder.Scanner
alias ExToon.{DecodeError, LiteralUtils, StringUtils}
@type event :: map()
# Parse a list of Scanner structs into stream events.
# Returns {:ok, [event()]} | {:error, term()}
@spec parse([Scanner.t()], pos_integer(), boolean()) ::
{:ok, [event()]} | {:error, term()}
def parse(parsed_lines, indent_size, strict) do
non_blank = Enum.reject(parsed_lines, fn l -> String.trim(l.raw) == "" end)
# Strict-mode: blank lines inside arrays (depth > 0 context) are invalid.
# We detect them by looking for blank lines that occur between two non-blank
# lines where at least one surrounding non-blank line is at depth > 0.
if strict do
validate_no_blank_lines_in_arrays(parsed_lines)
end
# Strict-mode: multiple depth-0 non-kv lines with no structure = ambiguous root.
if strict do
depth0_unstructured = Enum.filter(non_blank, fn l ->
l.depth == 0 and
not match?({:array_header, _, _, _, _, _}, Scanner.parse_header(l.content)) and
not is_kv_line?(l)
end)
if length(depth0_unstructured) > 1 do
first_extra = Enum.at(depth0_unstructured, 1)
throw({:decode_error, %DecodeError{
line: first_extra.line_number,
reason: :multiple_root_primitives,
message:
"multiple root-level primitives not allowed in strict mode at line #{first_extra.line_number}"
}})
end
end
case detect_root_form(non_blank) do
:empty ->
{:ok, [%{type: :start_object}, %{type: :end_object}]}
{:root_primitive, line} ->
{:ok, [%{type: :primitive, value: parse_primitive_token(line.content)}]}
{:root_array, _line} ->
parse_root_array(non_blank, indent_size, strict)
:root_object ->
parse_root_object(non_blank, indent_size, strict)
end
end
# ---------------------------------------------------------------------------
# Root form detection (§5)
# ---------------------------------------------------------------------------
defp detect_root_form([]), do: :empty
defp detect_root_form(lines) do
first = hd(lines)
depth0_lines = Enum.filter(lines, &(&1.depth == 0))
case Scanner.parse_header(first.content) do
{:array_header, nil, _, _, _, _} ->
# Keyless array header at root depth → root array
{:root_array, first}
{:array_header, _key, _, _, _, _} ->
# Keyed array header at root → this is an object with an array-valued field
:root_object
_ ->
if length(depth0_lines) == 1 and not is_kv_line?(first) and
not String.starts_with?(first.content, "- ") do
{:root_primitive, first}
else
:root_object
end
end
end
defp is_kv_line?(line) do
Scanner.parse_kv(line.content) != :not_kv
end
# ---------------------------------------------------------------------------
# Root array
# ---------------------------------------------------------------------------
defp parse_root_array([header_line | rest], indent_size, strict) do
{:array_header, _nil_key, length, delimiter, fields, inline} =
Scanner.parse_header(header_line.content)
{body_events, _remaining} =
parse_array_body(rest, 1, length, delimiter, fields, inline, indent_size, strict)
{:ok,
[%{type: :start_array, length: length}] ++ body_events ++ [%{type: :end_array}]}
end
# ---------------------------------------------------------------------------
# Root object
# ---------------------------------------------------------------------------
defp parse_root_object(lines, indent_size, strict) do
{events, _remaining} = parse_object_body(lines, 0, indent_size, strict)
{:ok, [%{type: :start_object}] ++ events ++ [%{type: :end_object}]}
end
# ---------------------------------------------------------------------------
# Object body: parse key-value pairs at the given depth
# ---------------------------------------------------------------------------
defp parse_object_body([], _depth, _indent_size, _strict), do: {[], []}
defp parse_object_body([line | rest] = lines, depth, indent_size, strict) do
if line.depth < depth do
# Depth decreased — end of this object scope
{[], lines}
else
case Scanner.parse_header(line.content) do
{:array_header, key, length, delimiter, fields, inline} when key != nil ->
# Array-valued field
key_event = %{type: :key, key: key}
start_event = %{type: :start_array, length: length}
{body_events, remaining} =
parse_array_body(rest, depth + 1, length, delimiter, fields, inline, indent_size, strict)
end_event = %{type: :end_array}
{more_events, final_remaining} =
parse_object_body(remaining, depth, indent_size, strict)
events =
[key_event, start_event] ++
body_events ++
[end_event] ++
more_events
{events, final_remaining}
_ ->
case Scanner.parse_kv(line.content) do
{:kv, key, value_str, was_quoted} ->
key_event =
if was_quoted,
do: %{type: :key, key: key, literal: true},
else: %{type: :key, key: key}
{value_events, remaining} =
parse_value(value_str, rest, depth, indent_size, strict)
{more_events, final_remaining} =
parse_object_body(remaining, depth, indent_size, strict)
{[key_event] ++ value_events ++ more_events, final_remaining}
:not_kv ->
# In strict mode, an unrecognized line AT the exact object depth is an error.
# Lines at deeper depth are silently skipped (they may belong to nested contexts
# that the current pass doesn't own). In lenient mode, always skip.
if strict and depth > 0 and line.depth == depth do
throw({:decode_error, %DecodeError{
line: line.line_number,
reason: :syntax_error,
message:
"expected key:value or array header at line #{line.line_number}, got: #{inspect(line.content)}"
}})
else
parse_object_body(rest, depth, indent_size, strict)
end
end
end
end
end
# ---------------------------------------------------------------------------
# Value parsing (after "key:")
# ---------------------------------------------------------------------------
# Empty after colon — nested object or nested array
defp parse_value("", rest, depth, indent_size, strict) do
case rest do
[] ->
{[%{type: :start_object}, %{type: :end_object}], []}
[next | _] ->
if next.depth > depth do
# Peek to decide: if next line is an array header without key, it's a nested array.
# Otherwise it's a nested object.
case Scanner.parse_header(next.content) do
{:array_header, nil, length, delimiter, fields, inline} ->
# Anonymous array header at child depth — nested array value
[_header | after_header] = rest
{body_events, remaining} =
parse_array_body(
after_header,
depth + 2,
length,
delimiter,
fields,
inline,
indent_size,
strict
)
{[%{type: :start_array, length: length}] ++ body_events ++ [%{type: :end_array}],
remaining}
_ ->
# Nested object
{child_events, remaining} =
parse_object_body(rest, depth + 1, indent_size, strict)
{[%{type: :start_object}] ++ child_events ++ [%{type: :end_object}], remaining}
end
else
{[%{type: :start_object}, %{type: :end_object}], rest}
end
end
end
# Inline primitive value after colon
defp parse_value(value_str, rest, _depth, _indent_size, _strict) do
value = parse_primitive_token(value_str)
{[%{type: :primitive, value: value}], rest}
end
# ---------------------------------------------------------------------------
# Array body dispatch
# ---------------------------------------------------------------------------
defp parse_array_body(rest, _depth, length, delimiter, nil, inline, _indent_size, strict)
when inline != nil do
# Inline primitive array — all values on the header line itself.
# The first clause (fields == nil, inline != nil) handles cases where the
# header had no field spec and all values are on the same line.
values = parse_inline_values(inline, delimiter)
if strict and length(values) != length do
throw({:decode_error, %DecodeError{
reason: :length_mismatch,
message: "array declared length #{length} but found #{length(values)} inline values"
}})
end
events = Enum.map(values, &%{type: :primitive, value: &1})
{events, rest}
end
defp parse_array_body(rest, depth, length, delimiter, fields, inline, indent_size, strict) do
cond do
inline != nil ->
values = parse_inline_values(inline, delimiter)
if strict and length(values) != length do
throw({:decode_error, %DecodeError{
reason: :length_mismatch,
message: "array declared length #{length} but found #{length(values)} inline values"
}})
end
events = Enum.map(values, &%{type: :primitive, value: &1})
{events, rest}
fields != nil ->
parse_tabular_rows(rest, depth, length, delimiter, fields, indent_size, strict)
true ->
parse_list_items(rest, depth, length, delimiter, indent_size, strict)
end
end
# ---------------------------------------------------------------------------
# Inline values: split by delimiter, parse each as primitive
# ---------------------------------------------------------------------------
defp parse_inline_values(str, delimiter) do
str
|> Scanner.split_by_delimiter(delimiter)
|> Enum.map(fn token ->
token
|> String.trim()
|> parse_primitive_token()
end)
end
# ---------------------------------------------------------------------------
# Tabular rows
# ---------------------------------------------------------------------------
defp parse_tabular_rows([], _depth, _length, _delimiter, _fields, _indent_size, _strict) do
{[], []}
end
defp parse_tabular_rows(
[line | rest] = lines,
depth,
length,
delimiter,
fields,
indent_size,
strict
) do
# Blank line inside tabular array: strict mode throws, lenient mode skips.
if String.trim(line.raw) == "" do
if strict do
throw({:decode_error, %DecodeError{
line: line.line_number,
reason: :blank_line_in_array,
message: "blank line inside tabular array at line #{line.line_number}"
}})
else
parse_tabular_rows(rest, depth, length, delimiter, fields, indent_size, strict)
end
else
if line.depth != depth do
# If we're in strict mode and there are MORE rows at this depth than declared,
# we've already consumed what was expected; this line at a different depth ends the array.
{[], lines}
else
# Strict mode: if we have already consumed the declared number of rows (length == 0),
# but there is still a valid data row at the correct depth, that is a row-count mismatch.
if strict and length <= 0 do
throw({:decode_error, %DecodeError{
line: line.line_number,
reason: :row_count_mismatch,
message:
"tabular array has more rows than the declared count at line #{line.line_number}"
}})
end
raw_values =
line.content
|> Scanner.split_by_delimiter(delimiter)
|> Enum.map(&String.trim/1)
values = Enum.map(raw_values, &parse_primitive_token/1)
# Strict mode: validate that each row has exactly the right number of fields.
if strict and length(values) != length(fields) do
throw({:decode_error, %DecodeError{
line: line.line_number,
reason: :field_count_mismatch,
message:
"tabular row has #{length(values)} values but header declares #{length(fields)} fields at line #{line.line_number}"
}})
end
field_events =
fields
|> Enum.zip(values)
|> Enum.flat_map(fn {field, value} ->
[%{type: :key, key: field}, %{type: :primitive, value: value}]
end)
row_events =
[%{type: :start_object}] ++ field_events ++ [%{type: :end_object}]
{more_events, remaining} =
parse_tabular_rows(rest, depth, length - 1, delimiter, fields, indent_size, strict)
{row_events ++ more_events, remaining}
end
end
end
# ---------------------------------------------------------------------------
# Expanded list items
# ---------------------------------------------------------------------------
defp parse_list_items(lines, depth, expected_length, delimiter, indent_size, strict) do
{events, remaining} = collect_items(lines, depth, delimiter, indent_size, strict, [])
# Count items by counting :start_object/:primitive events at the item level
# (before any nested object/array content). We approximate by counting parse results
# that would come from the item-level loop. Instead, count by tracking in collect.
# We pass expected_length to collect_items for strict validation.
if strict do
actual = count_items(events)
if actual != expected_length do
throw({:decode_error, %DecodeError{
reason: :length_mismatch,
message: "list array declared length #{expected_length} but found #{actual} items"
}})
end
end
{events, remaining}
end
# Count top-level items in the flattened event list.
# Items are: start_object at depth 0, start_array at depth 0, or bare primitives at depth 0.
defp count_items(events) do
{count, _depth} =
Enum.reduce(events, {0, 0}, fn event, {count, depth} ->
case event do
%{type: :start_object} when depth == 0 -> {count + 1, depth + 1}
%{type: :start_object} -> {count, depth + 1}
%{type: :end_object} -> {count, depth - 1}
%{type: :start_array} when depth == 0 -> {count + 1, depth + 1}
%{type: :start_array} -> {count, depth + 1}
%{type: :end_array} -> {count, depth - 1}
%{type: :primitive} when depth == 0 -> {count + 1, depth}
_ -> {count, depth}
end
end)
count
end
defp collect_items([], _depth, _delimiter, _indent_size, _strict, acc) do
{acc |> Enum.reverse() |> List.flatten(), []}
end
defp collect_items([line | rest] = lines, depth, delimiter, indent_size, strict, acc) do
# Blank line (or whitespace-only line) inside an expanded list array.
if String.trim(line.raw) == "" do
if strict and acc != [] do
# Only error on blank lines that appear INSIDE the array (after at least one item).
throw({:decode_error, %DecodeError{
line: line.line_number,
reason: :blank_line_in_array,
message: "blank line inside list array at line #{line.line_number}"
}})
else
collect_items(rest, depth, delimiter, indent_size, strict, acc)
end
else
if line.depth < depth do
{acc |> Enum.reverse() |> List.flatten(), lines}
else
content = line.content
if String.starts_with?(content, "- ") or content == "-" do
item_content =
if content == "-", do: "", else: String.slice(content, 2, String.length(content))
{item_events, remaining} =
parse_list_item_content(item_content, rest, depth, delimiter, indent_size, strict)
collect_items(remaining, depth, delimiter, indent_size, strict, [item_events | acc])
else
# Not a list item — stop collecting
{acc |> Enum.reverse() |> List.flatten(), lines}
end
end
end
end
# ---------------------------------------------------------------------------
# List item content dispatch
# ---------------------------------------------------------------------------
# Bare hyphen — check for nested object at depth+1
defp parse_list_item_content("", rest, depth, _delimiter, indent_size, strict) do
case rest do
[] ->
{[%{type: :start_object}, %{type: :end_object}], []}
[next | _] ->
if next.depth > depth do
{child_events, remaining} = parse_object_body(rest, depth + 1, indent_size, strict)
{[%{type: :start_object}] ++ child_events ++ [%{type: :end_object}], remaining}
else
{[%{type: :start_object}, %{type: :end_object}], rest}
end
end
end
defp parse_list_item_content(item_content, rest, depth, _delimiter, indent_size, strict) do
# Try array header first (for "- [N]:" or "- key[N]:" patterns)
case Scanner.parse_header(item_content) do
{:array_header, key, length, delim, fields, inline} when key != nil ->
# Object with an array field on the hyphen line: "- key[N]{...}:"
# Per spec: tabular/list body is at depth+2 (one for the item scope, one for the
# array body level), while sibling object fields are at depth+1.
key_event = %{type: :key, key: key}
start_event = %{type: :start_array, length: length}
{array_body, after_array} =
parse_array_body(rest, depth + 2, length, delim, fields, inline, indent_size, strict)
{more_obj, final_remaining} =
parse_object_body(after_array, depth + 1, indent_size, strict)
events =
[%{type: :start_object}, key_event, start_event] ++
array_body ++
[%{type: :end_array}] ++
more_obj ++
[%{type: :end_object}]
{events, final_remaining}
{:array_header, nil, length, delim, fields, inline} ->
# Root-style array as a list item: "- [N]:"
start_event = %{type: :start_array, length: length}
{array_body, remaining} =
parse_array_body(rest, depth + 1, length, delim, fields, inline, indent_size, strict)
{[start_event] ++ array_body ++ [%{type: :end_array}], remaining}
:not_header ->
# Try key:value (object starting on the hyphen line)
case Scanner.parse_kv(item_content) do
{:kv, key, value_str, was_quoted} ->
key_event =
if was_quoted,
do: %{type: :key, key: key, literal: true},
else: %{type: :key, key: key}
{value_events, after_value} =
parse_value(value_str, rest, depth, indent_size, strict)
{more_events, remaining} =
parse_object_body(after_value, depth + 1, indent_size, strict)
events =
[%{type: :start_object}, key_event] ++
value_events ++
more_events ++
[%{type: :end_object}]
{events, remaining}
:not_kv ->
# Plain primitive list item
value = parse_primitive_token(String.trim(item_content))
{[%{type: :primitive, value: value}], rest}
end
end
end
# ---------------------------------------------------------------------------
# Primitive token parsing
# ---------------------------------------------------------------------------
# Quoted string — unescape inner content
defp parse_primitive_token("\"" <> _ = quoted) do
len = String.length(quoted)
if len >= 2 and String.last(quoted) == "\"" do
inner = String.slice(quoted, 1, len - 2)
case StringUtils.unescape(inner) do
{:ok, s} -> s
# Return raw on bad escape — strict mode catches this separately
{:error, _} -> quoted
end
else
quoted
end
end
defp parse_primitive_token(token), do: LiteralUtils.parse_primitive(token)
# ---------------------------------------------------------------------------
# Blank line validation (strict mode)
# ---------------------------------------------------------------------------
# In strict mode, blank lines inside arrays (between list items or tabular rows)
# are forbidden. Blank lines between object fields are allowed.
# Detection: a blank line is inside an array if it is preceded by a line that
# looks like an array body line (starts with "- " or "- " indicator OR is a
# tabular row at depth > 0 — i.e., not a kv pair and not an array header).
defp validate_no_blank_lines_in_arrays(parsed_lines) do
do_validate_blank_lines(parsed_lines, nil)
end
defp do_validate_blank_lines([], _prev_line), do: :ok
defp do_validate_blank_lines([line | rest], prev_line) do
if String.trim(line.raw) == "" do
# Check surrounding non-blank context to decide if we're inside an array.
next_line = find_next_non_blank(rest)
# Only throw if BOTH surrounding lines indicate we're inside an array scope.
# If the next line is at depth 0 (or nil), the blank line is outside the array.
prev_in_array = is_array_body_line?(prev_line)
next_in_array = is_array_body_line?(next_line)
next_depth = if next_line, do: next_line.depth, else: 0
prev_depth = if prev_line, do: prev_line.depth, else: 0
if prev_in_array and next_in_array and next_depth == prev_depth do
throw({:decode_error, %DecodeError{
line: line.line_number,
reason: :blank_line_in_array,
message: "blank line inside array at line #{line.line_number}"
}})
else
do_validate_blank_lines(rest, prev_line)
end
else
do_validate_blank_lines(rest, line)
end
end
defp find_next_non_blank([]), do: nil
defp find_next_non_blank([line | rest]) do
if String.trim(line.raw) == "", do: find_next_non_blank(rest), else: line
end
# An array body line is a list item ("- " prefix or bare "-") OR
# a tabular row (at depth > 0, not a kv pair, not an array header).
defp is_array_body_line?(nil), do: false
defp is_array_body_line?(line) do
cond do
String.starts_with?(line.content, "- ") or line.content == "-" -> true
line.depth > 0 and
Scanner.parse_kv(line.content) == :not_kv and
Scanner.parse_header(line.content) == :not_header -> true
true -> false
end
end
end