Packages
toon_codec
1.0.0
TOON (Token-Oriented Object Notation) encoder/decoder - a compact, token-efficient format for LLM input
Current section
Files
Jump to
Current section
Files
src/toon_codec@decode.erl
-module(toon_codec@decode).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/toon_codec/decode.gleam").
-export([decode_value/2]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(
" Main decoding module for converting TOON format to JSON values.\n"
"\n"
" This module provides the primary decoding function that coordinates\n"
" scanning, parsing, and reconstruction of JSON values from TOON format.\n"
" Decode TOON format string to JSON value.\n"
"\n"
" This is the main entry point for decoding. It handles:\n"
" - Root primitives\n"
" - Root arrays\n"
" - Root objects\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" // Simple object\n"
" decode_value(\"name: Alice\\nage: 30\", default_decode_options())\n"
" // -> Ok(Object([#(\"name\", String(\"Alice\")), #(\"age\", Number(30.0))]))\n"
"\n"
" // Root array\n"
" decode_value(\"[3]: 1,2,3\", default_decode_options())\n"
" // -> Ok(Array([Number(1.0), Number(2.0), Number(3.0)]))\n"
" ```\n"
" Detect the type of root value.\n"
" Decode based on detected root form.\n"
" Decode an object from the cursor at the given depth.\n"
" Decode a key-value pair from a line.\n"
" Decode an array from its header.\n"
" Decode an inline primitive array.\n"
" Decode a tabular array.\n"
" Decode a list array (expanded form).\n"
" Decode a single list item.\n"
).
-file("src/toon_codec/decode.gleam", 72).
-spec detect_root_form(toon_codec@decode@scanner:line_cursor()) -> {ok,
toon_codec@types:root_form()} |
{error, toon_codec@error:toon_error()}.
detect_root_form(Cursor) ->
case toon_codec@decode@scanner:cursor_peek(Cursor) of
none ->
{error, empty_input};
{some, First} ->
Content = toon_codec@types:parsed_line_content(First),
case toon_codec@decode@parser:is_array_header(Content) of
true ->
{ok, root_array};
false ->
case toon_codec@decode@scanner:cursor_length(Cursor) =:= 1 of
true ->
case toon_codec@decode@parser:is_key_value_line(
Content
) of
false ->
case toon_codec@decode@parser:parse_primitive(
Content
) of
{ok, Value} ->
{ok, {root_primitive, Value}};
{error, Err} ->
{error, Err}
end;
true ->
{ok, root_object}
end;
false ->
{ok, root_object}
end
end
end.
-file("src/toon_codec/decode.gleam", 333).
-spec parse_tokens_to_values(
list(binary()),
list(toon_codec@types:json_value())
) -> {ok, list(toon_codec@types:json_value())} |
{error, toon_codec@error:toon_error()}.
parse_tokens_to_values(Tokens, Acc) ->
case Tokens of
[] ->
{ok, lists:reverse(Acc)};
[Token | Rest] ->
case toon_codec@decode@parser:parse_primitive(Token) of
{ok, Value} ->
parse_tokens_to_values(Rest, [Value | Acc]);
{error, Err} ->
{error, Err}
end
end.
-file("src/toon_codec/decode.gleam", 324).
-spec decode_inline_array(binary(), toon_codec@types:delimiter(), integer()) -> {ok,
list(toon_codec@types:json_value())} |
{error, toon_codec@error:toon_error()}.
decode_inline_array(Values_str, Delimiter, _) ->
Tokens = toon_codec@decode@parser:parse_delimited_values(
Values_str,
Delimiter
),
parse_tokens_to_values(Tokens, []).
-file("src/toon_codec/decode.gleam", 370).
-spec decode_tabular_rows(
toon_codec@decode@scanner:line_cursor(),
integer(),
list(binary()),
toon_codec@types:delimiter(),
integer(),
toon_codec@types:decode_options(),
list(toon_codec@types:json_value()),
integer(),
integer()
) -> {ok,
{list(toon_codec@types:json_value()),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_tabular_rows(
Cursor,
Row_depth,
Fields,
Delimiter,
Expected_width,
Options,
Acc,
Row_count,
Expected_length
) ->
case toon_codec@decode@scanner:cursor_peek(Cursor) of
none ->
case erlang:element(3, Options) of
true ->
case toon_codec@decode@validation:validate_count(
Expected_length,
Row_count,
<<"tabular rows"/utf8>>
) of
{ok, _} ->
{ok, {lists:reverse(Acc), Cursor}};
{error, Err} ->
{error, Err}
end;
false ->
{ok, {lists:reverse(Acc), Cursor}}
end;
{some, Line} ->
Depth = toon_codec@types:parsed_line_depth(Line),
case Depth < Row_depth of
true ->
case erlang:element(3, Options) of
true ->
case toon_codec@decode@validation:validate_count(
Expected_length,
Row_count,
<<"tabular rows"/utf8>>
) of
{ok, _} ->
{ok, {lists:reverse(Acc), Cursor}};
{error, Err@1} ->
{error, Err@1}
end;
false ->
{ok, {lists:reverse(Acc), Cursor}}
end;
false ->
case Depth =:= Row_depth of
true ->
Content = toon_codec@types:parsed_line_content(Line),
Values = toon_codec@decode@parser:parse_delimited_values(
Content,
Delimiter
),
case erlang:element(3, Options) of
true ->
case toon_codec@decode@validation:validate_row_width(
Expected_width,
erlang:length(Values),
Row_count + 1
) of
{ok, _} ->
case parse_tokens_to_values(
Values,
[]
) of
{ok, Parsed_values} ->
Obj = begin
_pipe = gleam@list:zip(
Fields,
Parsed_values
),
{object, _pipe}
end,
decode_tabular_rows(
toon_codec@decode@scanner:cursor_advance(
Cursor
),
Row_depth,
Fields,
Delimiter,
Expected_width,
Options,
[Obj | Acc],
Row_count + 1,
Expected_length
);
{error, Err@2} ->
{error, Err@2}
end;
{error, Err@3} ->
{error, Err@3}
end;
false ->
case parse_tokens_to_values(Values, []) of
{ok, Parsed_values@1} ->
Obj@1 = begin
_pipe@1 = gleam@list:zip(
Fields,
Parsed_values@1
),
{object, _pipe@1}
end,
decode_tabular_rows(
toon_codec@decode@scanner:cursor_advance(
Cursor
),
Row_depth,
Fields,
Delimiter,
Expected_width,
Options,
[Obj@1 | Acc],
Row_count + 1,
Expected_length
);
{error, Err@4} ->
{error, Err@4}
end
end;
false ->
{error,
toon_codec@error:structure_error(
<<"Unexpected indentation in tabular array"/utf8>>
)}
end
end
end.
-file("src/toon_codec/decode.gleam", 348).
-spec decode_tabular_array(
integer(),
list(binary()),
toon_codec@types:delimiter(),
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options()
) -> {ok,
{list(toon_codec@types:json_value()),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_tabular_array(
Expected_length,
Fields,
Delimiter,
Cursor,
Base_depth,
Options
) ->
Expected_width = erlang:length(Fields),
decode_tabular_rows(
Cursor,
Base_depth + 1,
Fields,
Delimiter,
Expected_width,
Options,
[],
0,
Expected_length
).
-file("src/toon_codec/decode.gleam", 197).
-spec decode_key_value_pair(
toon_codec@types:parsed_line(),
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options()
) -> {ok,
{binary(),
toon_codec@types:json_value(),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_key_value_pair(Line, Cursor, Base_depth, Options) ->
Content = toon_codec@types:parsed_line_content(Line),
case toon_codec@decode@parser:parse_array_header(Content) of
{ok, {Header, Inline_values}} ->
case erlang:element(2, Header) of
{some, Key} ->
Cursor@1 = toon_codec@decode@scanner:cursor_advance(Cursor),
case decode_array_from_header(
Header,
Inline_values,
Cursor@1,
Base_depth,
Options
) of
{ok, {Items, New_cursor}} ->
{ok, {Key, {array, Items}, New_cursor}};
{error, Err} ->
{error, Err}
end;
none ->
{error,
toon_codec@error:structure_error(
<<"Array header without key in object context"/utf8>>
)}
end;
{error, _} ->
case toon_codec@decode@parser:parse_key(Content) of
{ok, {Key@1, Pos}} ->
Rest = begin
_pipe = gleam@string:slice(Content, Pos, 1000),
gleam@string:trim_start(_pipe)
end,
case Rest of
<<""/utf8>> ->
Cursor@2 = toon_codec@decode@scanner:cursor_advance(
Cursor
),
case decode_object(
Cursor@2,
Base_depth + 1,
Options
) of
{ok, {Nested_fields, New_cursor@1}} ->
{ok,
{Key@1,
{object, Nested_fields},
New_cursor@1}};
{error, Err@1} ->
{error, Err@1}
end;
_ ->
case toon_codec@decode@parser:parse_primitive(Rest) of
{ok, Value} ->
Cursor@3 = toon_codec@decode@scanner:cursor_advance(
Cursor
),
{ok, {Key@1, Value, Cursor@3}};
{error, Err@2} ->
{error, Err@2}
end
end;
{error, Err@3} ->
{error, Err@3}
end
end.
-file("src/toon_codec/decode.gleam", 156).
-spec decode_object(
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options()
) -> {ok,
{list({binary(), toon_codec@types:json_value()}),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_object(Cursor, Base_depth, Options) ->
decode_object_loop(Cursor, Base_depth, Options, []).
-file("src/toon_codec/decode.gleam", 164).
-spec decode_object_loop(
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options(),
list({binary(), toon_codec@types:json_value()})
) -> {ok,
{list({binary(), toon_codec@types:json_value()}),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_object_loop(Cursor, Base_depth, Options, Acc) ->
case toon_codec@decode@scanner:cursor_peek(Cursor) of
none ->
{ok, {lists:reverse(Acc), Cursor}};
{some, Line} ->
Depth = toon_codec@types:parsed_line_depth(Line),
case Depth < Base_depth of
true ->
{ok, {lists:reverse(Acc), Cursor}};
false ->
case Depth =:= Base_depth of
true ->
case decode_key_value_pair(
Line,
Cursor,
Base_depth,
Options
) of
{ok, {Key, Value, New_cursor}} ->
decode_object_loop(
New_cursor,
Base_depth,
Options,
[{Key, Value} | Acc]
);
{error, Err} ->
{error, Err}
end;
false ->
{ok, {lists:reverse(Acc), Cursor}}
end
end
end.
-file("src/toon_codec/decode.gleam", 144).
-spec decode_root_object(
toon_codec@decode@scanner:line_cursor(),
toon_codec@types:decode_options()
) -> {ok, toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
decode_root_object(Cursor, Options) ->
case decode_object(Cursor, 0, Options) of
{ok, {Fields, _}} ->
{ok, {object, Fields}};
{error, Err} ->
{error, Err}
end.
-file("src/toon_codec/decode.gleam", 587).
-spec decode_list_item(
binary(),
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options()
) -> {ok,
{toon_codec@types:json_value(), toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_list_item(After_hyphen, Cursor, Item_depth, Options) ->
case toon_codec@decode@parser:parse_array_header(After_hyphen) of
{ok, {Header, Inline_values}} ->
Cursor@1 = toon_codec@decode@scanner:cursor_advance(Cursor),
case decode_array_from_header(
Header,
Inline_values,
Cursor@1,
Item_depth,
Options
) of
{ok, {Items, New_cursor}} ->
{ok, {{array, Items}, New_cursor}};
{error, Err} ->
{error, Err}
end;
{error, _} ->
case toon_codec@decode@parser:is_key_value_line(After_hyphen) of
true ->
case toon_codec@decode@parser:parse_key(After_hyphen) of
{ok, {Key, Pos}} ->
Rest = begin
_pipe = gleam@string:slice(
After_hyphen,
Pos,
1000
),
gleam@string:trim_start(_pipe)
end,
case Rest of
<<""/utf8>> ->
Cursor@2 = toon_codec@decode@scanner:cursor_advance(
Cursor
),
case decode_object(
Cursor@2,
Item_depth + 2,
Options
) of
{ok, {Nested_fields, Cursor2}} ->
case decode_object(
Cursor2,
Item_depth + 1,
Options
) of
{ok,
{Sibling_fields,
New_cursor@1}} ->
All_fields = lists:append(
[{Key,
{object,
Nested_fields}}],
Sibling_fields
),
{ok,
{{object, All_fields},
New_cursor@1}};
{error, Err@1} ->
{error, Err@1}
end;
{error, Err@2} ->
{error, Err@2}
end;
_ ->
case toon_codec@decode@parser:parse_primitive(
Rest
) of
{ok, Value} ->
Cursor@3 = toon_codec@decode@scanner:cursor_advance(
Cursor
),
case decode_object(
Cursor@3,
Item_depth + 1,
Options
) of
{ok,
{Other_fields, New_cursor@2}} ->
All_fields@1 = [{Key, Value} |
Other_fields],
{ok,
{{object, All_fields@1},
New_cursor@2}};
{error, Err@3} ->
{error, Err@3}
end;
{error, Err@4} ->
{error, Err@4}
end
end;
{error, Err@5} ->
{error, Err@5}
end;
false ->
case toon_codec@decode@parser:parse_primitive(After_hyphen) of
{ok, Value@1} ->
Cursor@4 = toon_codec@decode@scanner:cursor_advance(
Cursor
),
{ok, {Value@1, Cursor@4}};
{error, Err@6} ->
{error, Err@6}
end
end
end.
-file("src/toon_codec/decode.gleam", 266).
-spec decode_array_from_header(
toon_codec@types:array_header(),
gleam@option:option(binary()),
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options()
) -> {ok,
{list(toon_codec@types:json_value()),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_array_from_header(Header, Inline_values, Cursor, Base_depth, Options) ->
case Inline_values of
{some, Values_str} ->
case decode_inline_array(
Values_str,
erlang:element(4, Header),
erlang:element(3, Header)
) of
{ok, Values} ->
case erlang:element(3, Options) of
true ->
case toon_codec@decode@validation:validate_count(
erlang:element(3, Header),
erlang:length(Values),
<<"inline array"/utf8>>
) of
{ok, _} ->
{ok, {Values, Cursor}};
{error, Err} ->
{error, Err}
end;
false ->
{ok, {Values, Cursor}}
end;
{error, Err@1} ->
{error, Err@1}
end;
none ->
case erlang:element(5, Header) of
{some, Fields} ->
decode_tabular_array(
erlang:element(3, Header),
Fields,
erlang:element(4, Header),
Cursor,
Base_depth,
Options
);
none ->
decode_list_array(
erlang:element(3, Header),
erlang:element(4, Header),
Cursor,
Base_depth,
Options
)
end
end.
-file("src/toon_codec/decode.gleam", 494).
-spec decode_list_array(
integer(),
toon_codec@types:delimiter(),
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options()
) -> {ok,
{list(toon_codec@types:json_value()),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_list_array(Expected_length, _, Cursor, Base_depth, Options) ->
decode_list_items(Cursor, Base_depth + 1, Options, [], 0, Expected_length).
-file("src/toon_codec/decode.gleam", 504).
-spec decode_list_items(
toon_codec@decode@scanner:line_cursor(),
integer(),
toon_codec@types:decode_options(),
list(toon_codec@types:json_value()),
integer(),
integer()
) -> {ok,
{list(toon_codec@types:json_value()),
toon_codec@decode@scanner:line_cursor()}} |
{error, toon_codec@error:toon_error()}.
decode_list_items(Cursor, Item_depth, Options, Acc, Item_count, Expected_length) ->
case toon_codec@decode@scanner:cursor_peek(Cursor) of
none ->
case erlang:element(3, Options) of
true ->
case toon_codec@decode@validation:validate_count(
Expected_length,
Item_count,
<<"list items"/utf8>>
) of
{ok, _} ->
{ok, {lists:reverse(Acc), Cursor}};
{error, Err} ->
{error, Err}
end;
false ->
{ok, {lists:reverse(Acc), Cursor}}
end;
{some, Line} ->
Depth = toon_codec@types:parsed_line_depth(Line),
case Depth < Item_depth of
true ->
case erlang:element(3, Options) of
true ->
case toon_codec@decode@validation:validate_count(
Expected_length,
Item_count,
<<"list items"/utf8>>
) of
{ok, _} ->
{ok, {lists:reverse(Acc), Cursor}};
{error, Err@1} ->
{error, Err@1}
end;
false ->
{ok, {lists:reverse(Acc), Cursor}}
end;
false ->
case Depth =:= Item_depth of
true ->
Content = toon_codec@types:parsed_line_content(Line),
case gleam_stdlib:string_starts_with(
Content,
<<"- "/utf8>>
) of
true ->
After_hyphen = begin
_pipe = gleam@string:drop_start(
Content,
2
),
gleam@string:trim_start(_pipe)
end,
case decode_list_item(
After_hyphen,
Cursor,
Item_depth,
Options
) of
{ok, {Item, New_cursor}} ->
decode_list_items(
New_cursor,
Item_depth,
Options,
[Item | Acc],
Item_count + 1,
Expected_length
);
{error, Err@2} ->
{error, Err@2}
end;
false ->
{error,
toon_codec@error:structure_error(
<<"Expected list item (line starting with '- ')"/utf8>>
)}
end;
false ->
{error,
toon_codec@error:structure_error(
<<"Unexpected indentation in list array"/utf8>>
)}
end
end
end.
-file("src/toon_codec/decode.gleam", 118).
-spec decode_root_array(
toon_codec@decode@scanner:line_cursor(),
toon_codec@types:decode_options()
) -> {ok, toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
decode_root_array(Cursor, Options) ->
case toon_codec@decode@scanner:cursor_peek(Cursor) of
{some, Line} ->
Content = toon_codec@types:parsed_line_content(Line),
case toon_codec@decode@parser:parse_array_header(Content) of
{ok, {Header, Inline_values}} ->
Cursor@1 = toon_codec@decode@scanner:cursor_advance(Cursor),
case decode_array_from_header(
Header,
Inline_values,
Cursor@1,
0,
Options
) of
{ok, {Items, _}} ->
{ok, {array, Items}};
{error, Err} ->
{error, Err}
end;
{error, Err@1} ->
{error, Err@1}
end;
none ->
{error, empty_input}
end.
-file("src/toon_codec/decode.gleam", 104).
-spec decode_by_form(
toon_codec@types:root_form(),
toon_codec@decode@scanner:line_cursor(),
toon_codec@types:decode_options()
) -> {ok, toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
decode_by_form(Form, Cursor, Options) ->
case Form of
{root_primitive, Value} ->
{ok, Value};
root_array ->
decode_root_array(Cursor, Options);
root_object ->
decode_root_object(Cursor, Options)
end.
-file("src/toon_codec/decode.gleam", 48).
-spec decode_value(binary(), toon_codec@types:decode_options()) -> {ok,
toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
decode_value(Input, Options) ->
case toon_codec@decode@scanner:scan_lines(
Input,
erlang:element(2, Options),
erlang:element(3, Options)
) of
{ok, Lines} ->
case Lines of
[] ->
{error, empty_input};
_ ->
Cursor = toon_codec@decode@scanner:cursor_new(Lines),
case detect_root_form(Cursor) of
{ok, Form} ->
decode_by_form(Form, Cursor, Options);
{error, Err} ->
{error, Err}
end
end;
{error, Err@1} ->
{error, Err@1}
end.