Packages
toon_codec
1.0.0
TOON (Token-Oriented Object Notation) encoder/decoder - a compact, token-efficient format for LLM input
Current section
Files
Jump to
Current section
Files
src/toon_codec@decode@parser.erl
-module(toon_codec@decode@parser).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/toon_codec/decode/parser.gleam").
-export([parse_primitive/1, parse_delimited_values/2, parse_key/1, parse_array_header/1, is_array_header/1, is_key_value_line/1]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(
" Parsing functions for TOON format elements.\n"
"\n"
" This module provides functions to parse array headers, keys, values,\n"
" and delimited data from TOON format strings.\n"
" Parse a primitive token (string, number, boolean, or null).\n"
"\n"
" Quoted strings remain strings even if they look like numbers.\n"
" Parse a quoted string token.\n"
" Parse an unquoted value (number or string).\n"
" Parse a numeric token.\n"
" Check if a string has a forbidden leading zero.\n"
" Parse delimited values from a string.\n"
"\n"
" Splits on the delimiter while respecting quoted sections.\n"
" Split a string on delimiter, respecting quotes.\n"
" Parse a key from the beginning of a line.\n"
"\n"
" Returns the key and the position after the colon.\n"
" Parse a quoted key.\n"
" Parse an unquoted key.\n"
" Parse an array header line.\n"
"\n"
" Returns the header info and any inline values after the colon.\n"
" Parse the bracket segment [#?N delim?]\n"
" Parse length and delimiter from bracket content.\n"
" Parse fields segment {field1,field2,...}\n"
" Unescape field names (handle quoted field names).\n"
" Extract inline values after the colon.\n"
" Check if a line looks like an array header.\n"
" Check if content is a key-value line (has unquoted colon).\n"
).
-file("src/toon_codec/decode/parser.gleam", 65).
-spec parse_quoted_string(binary()) -> {ok, toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
parse_quoted_string(Token) ->
case toon_codec@internal@string:unquote_string(Token) of
{ok, Unescaped} ->
{ok, {string, Unescaped}};
{error, Err} ->
{error, Err}
end.
-file("src/toon_codec/decode/parser.gleam", 96).
-spec has_forbidden_leading_zero(binary()) -> boolean().
has_forbidden_leading_zero(S) ->
case gleam@string:to_graphemes(S) of
[<<"0"/utf8>>, Second | _] ->
toon_codec@internal@char:is_digit(Second);
_ ->
false
end.
-file("src/toon_codec/decode/parser.gleam", 83).
-spec parse_number(binary()) -> {ok, toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
parse_number(Token) ->
case has_forbidden_leading_zero(Token) of
true ->
{error,
toon_codec@error:validation_error(
<<"Invalid number with leading zero"/utf8>>
)};
false ->
case gleam_stdlib:parse_float(Token) of
{ok, N} ->
{ok, {number, N}};
{error, _} ->
{error,
toon_codec@error:validation_error(
<<"Invalid number: "/utf8, Token/binary>>
)}
end
end.
-file("src/toon_codec/decode/parser.gleam", 72).
-spec parse_unquoted_value(binary()) -> {ok, toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
parse_unquoted_value(Token) ->
case parse_number(Token) of
{ok, Num} ->
{ok, Num};
{error, _} ->
{ok, {string, Token}}
end.
-file("src/toon_codec/decode/parser.gleam", 48).
?DOC(" Unquoted tokens are interpreted based on their content.\n").
-spec parse_primitive(binary()) -> {ok, toon_codec@types:json_value()} |
{error, toon_codec@error:toon_error()}.
parse_primitive(Token) ->
Trimmed = gleam@string:trim(Token),
case Trimmed of
<<""/utf8>> ->
{error, toon_codec@error:validation_error(<<"Empty token"/utf8>>)};
<<"null"/utf8>> ->
{ok, null};
<<"true"/utf8>> ->
{ok, {bool, true}};
<<"false"/utf8>> ->
{ok, {bool, false}};
_ ->
case gleam_stdlib:string_starts_with(Trimmed, <<"\""/utf8>>) of
true ->
parse_quoted_string(Trimmed);
false ->
parse_unquoted_value(Trimmed)
end
end.
-file("src/toon_codec/decode/parser.gleam", 118).
-spec split_loop(
list(binary()),
binary(),
list(binary()),
list(binary()),
boolean()
) -> list(binary()).
split_loop(Chars, Delimiter, Current_token, Acc, In_quotes) ->
case Chars of
[] ->
Token = gleam@string:join(lists:reverse(Current_token), <<""/utf8>>),
lists:reverse([Token | Acc]);
[<<"\\"/utf8>>, Next | Rest] when In_quotes ->
split_loop(
Rest,
Delimiter,
[Next, <<"\\"/utf8>> | Current_token],
Acc,
In_quotes
);
[<<"\""/utf8>> | Rest@1] ->
split_loop(
Rest@1,
Delimiter,
[<<"\""/utf8>> | Current_token],
Acc,
not In_quotes
);
[Char | Rest@2] when (Char =:= Delimiter) andalso not In_quotes ->
Token@1 = gleam@string:join(
lists:reverse(Current_token),
<<""/utf8>>
),
split_loop(Rest@2, Delimiter, [], [Token@1 | Acc], In_quotes);
[Char@1 | Rest@3] ->
split_loop(
Rest@3,
Delimiter,
[Char@1 | Current_token],
Acc,
In_quotes
)
end.
-file("src/toon_codec/decode/parser.gleam", 113).
-spec split_respecting_quotes(binary(), binary()) -> list(binary()).
split_respecting_quotes(Text, Delimiter) ->
Chars = gleam@string:to_graphemes(Text),
split_loop(Chars, Delimiter, [], [], false).
-file("src/toon_codec/decode/parser.gleam", 105).
-spec parse_delimited_values(binary(), toon_codec@types:delimiter()) -> list(binary()).
parse_delimited_values(Text, Delimiter) ->
Delim_str = toon_codec@types:delimiter_to_string(Delimiter),
split_respecting_quotes(Text, Delim_str).
-file("src/toon_codec/decode/parser.gleam", 164).
-spec parse_quoted_key(binary()) -> {ok, {binary(), integer()}} |
{error, toon_codec@error:toon_error()}.
parse_quoted_key(Content) ->
case toon_codec@internal@string:find_closing_quote(Content, 0) of
{ok, Close_pos} ->
Key_with_quotes = gleam@string:slice(Content, 0, Close_pos + 1),
case toon_codec@internal@string:unquote_string(Key_with_quotes) of
{ok, Key} ->
After_quote = gleam@string:slice(
Content,
Close_pos + 1,
1000
),
case gleam@string:trim_start(After_quote) of
<<":"/utf8, Rest/binary>> ->
{ok,
{Key,
((Close_pos + 1) + string:length(
After_quote
))
- string:length(Rest)}};
_ ->
{error,
toon_codec@error:validation_error(
<<"Missing colon after quoted key"/utf8>>
)}
end;
{error, Err} ->
{error, Err}
end;
{error, Err@1} ->
{error, Err@1}
end.
-file("src/toon_codec/decode/parser.gleam", 189).
-spec parse_unquoted_key(binary()) -> {ok, {binary(), integer()}} |
{error, toon_codec@error:toon_error()}.
parse_unquoted_key(Content) ->
case toon_codec@internal@string:find_unquoted_char(Content, <<":"/utf8>>, 0) of
{some, Colon_pos} ->
Key = begin
_pipe = gleam@string:slice(Content, 0, Colon_pos),
gleam@string:trim(_pipe)
end,
{ok, {Key, Colon_pos + 1}};
none ->
{error,
toon_codec@error:validation_error(
<<"Missing colon after key"/utf8>>
)}
end.
-file("src/toon_codec/decode/parser.gleam", 157).
-spec parse_key(binary()) -> {ok, {binary(), integer()}} |
{error, toon_codec@error:toon_error()}.
parse_key(Content) ->
case gleam_stdlib:string_starts_with(Content, <<"\""/utf8>>) of
true ->
parse_quoted_key(Content);
false ->
parse_unquoted_key(Content)
end.
-file("src/toon_codec/decode/parser.gleam", 265).
-spec parse_length_and_delimiter(binary(), boolean(), binary()) -> {ok,
{integer(), toon_codec@types:delimiter(), boolean(), binary()}} |
{error, toon_codec@error:toon_error()}.
parse_length_and_delimiter(Content, Has_marker, Rest) ->
case gleam_stdlib:string_ends_with(Content, <<"\t"/utf8>>) of
true ->
Len_str = begin
_pipe = gleam@string:slice(
Content,
0,
string:length(Content) - 1
),
gleam@string:trim(_pipe)
end,
case gleam_stdlib:parse_int(Len_str) of
{ok, Length} ->
{ok, {Length, tab, Has_marker, Rest}};
{error, _} ->
{error,
toon_codec@error:validation_error(
<<"Invalid array length: "/utf8, Len_str/binary>>
)}
end;
false ->
case gleam_stdlib:string_ends_with(Content, <<"|"/utf8>>) of
true ->
Len_str@1 = begin
_pipe@1 = gleam@string:slice(
Content,
0,
string:length(Content) - 1
),
gleam@string:trim(_pipe@1)
end,
case gleam_stdlib:parse_int(Len_str@1) of
{ok, Length@1} ->
{ok, {Length@1, pipe, Has_marker, Rest}};
{error, _} ->
{error,
toon_codec@error:validation_error(
<<"Invalid array length: "/utf8,
Len_str@1/binary>>
)}
end;
false ->
Len_str@2 = gleam@string:trim(Content),
case gleam_stdlib:parse_int(Len_str@2) of
{ok, Length@2} ->
{ok, {Length@2, comma, Has_marker, Rest}};
{error, _} ->
{error,
toon_codec@error:validation_error(
<<"Invalid array length: "/utf8,
Len_str@2/binary>>
)}
end
end
end.
-file("src/toon_codec/decode/parser.gleam", 243).
-spec parse_bracket_segment(binary()) -> {ok,
{integer(), toon_codec@types:delimiter(), boolean(), binary()}} |
{error, toon_codec@error:toon_error()}.
parse_bracket_segment(After_bracket) ->
case gleam@string:split_once(After_bracket, <<"]"/utf8>>) of
{ok, {Bracket_content, Rest}} ->
{Has_marker, Content_after_marker} = case gleam_stdlib:string_starts_with(
Bracket_content,
<<"#"/utf8>>
) of
true ->
{true, gleam@string:drop_start(Bracket_content, 1)};
false ->
{false, Bracket_content}
end,
parse_length_and_delimiter(Content_after_marker, Has_marker, Rest);
{error, _} ->
{error,
toon_codec@error:validation_error(
<<"Invalid array header: no closing bracket"/utf8>>
)}
end.
-file("src/toon_codec/decode/parser.gleam", 338).
-spec unescape_names_loop(list(binary()), list(binary())) -> {ok,
list(binary())} |
{error, toon_codec@error:toon_error()}.
unescape_names_loop(Names, Acc) ->
case Names of
[] ->
{ok, lists:reverse(Acc)};
[Name | Rest] ->
Trimmed = gleam@string:trim(Name),
case gleam_stdlib:string_starts_with(Trimmed, <<"\""/utf8>>) of
true ->
case toon_codec@internal@string:unquote_string(Trimmed) of
{ok, Unescaped} ->
unescape_names_loop(Rest, [Unescaped | Acc]);
{error, Err} ->
{error, Err}
end;
false ->
unescape_names_loop(Rest, [Trimmed | Acc])
end
end.
-file("src/toon_codec/decode/parser.gleam", 334).
-spec unescape_field_names(list(binary())) -> {ok, list(binary())} |
{error, toon_codec@error:toon_error()}.
unescape_field_names(Names) ->
unescape_names_loop(Names, []).
-file("src/toon_codec/decode/parser.gleam", 306).
-spec parse_fields_segment(binary(), toon_codec@types:delimiter()) -> {ok,
{gleam@option:option(list(binary())), binary()}} |
{error, toon_codec@error:toon_error()}.
parse_fields_segment(After_bracket, Delimiter) ->
Trimmed = gleam@string:trim_start(After_bracket),
case gleam_stdlib:string_starts_with(Trimmed, <<"{"/utf8>>) of
true ->
case gleam@string:split_once(Trimmed, <<"}"/utf8>>) of
{ok, {Fields_content, Rest}} ->
Fields_str = gleam@string:drop_start(Fields_content, 1),
Field_names = parse_delimited_values(Fields_str, Delimiter),
case unescape_field_names(Field_names) of
{ok, Unescaped_fields} ->
{ok, {{some, Unescaped_fields}, Rest}};
{error, Err} ->
{error, Err}
end;
{error, _} ->
{error,
toon_codec@error:validation_error(
<<"Unclosed fields segment: missing }"/utf8>>
)}
end;
false ->
{ok, {none, Trimmed}}
end.
-file("src/toon_codec/decode/parser.gleam", 359).
-spec extract_inline_values(binary()) -> gleam@option:option(binary()).
extract_inline_values(After_fields) ->
Trimmed = gleam@string:trim_start(After_fields),
case gleam_stdlib:string_starts_with(Trimmed, <<":"/utf8>>) of
true ->
After_colon = begin
_pipe = gleam@string:drop_start(Trimmed, 1),
gleam@string:trim_start(_pipe)
end,
case After_colon of
<<""/utf8>> ->
none;
Values ->
{some, Values}
end;
false ->
none
end.
-file("src/toon_codec/decode/parser.gleam", 201).
-spec parse_array_header(binary()) -> {ok,
{toon_codec@types:array_header(), gleam@option:option(binary())}} |
{error, toon_codec@error:toon_error()}.
parse_array_header(Line) ->
case gleam@string:split_once(Line, <<"["/utf8>>) of
{ok, {Before_bracket, After_bracket}} ->
Key = case gleam@string:trim(Before_bracket) of
<<""/utf8>> ->
none;
K ->
{some, K}
end,
case parse_bracket_segment(After_bracket) of
{ok, {Length, Delimiter, Has_marker, Rest_after_bracket}} ->
case parse_fields_segment(Rest_after_bracket, Delimiter) of
{ok, {Fields, Rest_after_fields}} ->
Inline_values = extract_inline_values(
Rest_after_fields
),
Header = {array_header,
Key,
Length,
Delimiter,
Fields,
Has_marker},
{ok, {Header, Inline_values}};
{error, Err} ->
{error, Err}
end;
{error, Err@1} ->
{error, Err@1}
end;
{error, _} ->
{error,
toon_codec@error:validation_error(
<<"Invalid array header: no opening bracket"/utf8>>
)}
end.
-file("src/toon_codec/decode/parser.gleam", 376).
-spec is_array_header(binary()) -> boolean().
is_array_header(Content) ->
gleam_stdlib:contains_string(Content, <<"["/utf8>>) andalso gleam_stdlib:contains_string(
Content,
<<"]"/utf8>>
).
-file("src/toon_codec/decode/parser.gleam", 380).
-spec is_key_value_line(binary()) -> boolean().
is_key_value_line(Content) ->
case toon_codec@internal@string:find_unquoted_char(Content, <<":"/utf8>>, 0) of
{some, _} ->
true;
none ->
false
end.