Packages

TOON (Token-Oriented Object Notation) encoder/decoder - a compact, token-efficient format for LLM input

Current section

Files

Jump to
toon_codec src toon_codec@encode@primitives.erl
Raw

src/toon_codec@encode@primitives.erl

-module(toon_codec@encode@primitives).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/toon_codec/encode/primitives.gleam").
-export([encode_primitive/2, encode_key/1, format_header/4, format_tabular_header/5, encode_and_join_primitives/2]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(
" Primitive value encoding and header formatting.\n"
"\n"
" This module handles encoding of primitive JSON values (strings, numbers,\n"
" booleans, null) as well as formatting array headers.\n"
" Encode a primitive JSON value to its TOON string representation.\n"
"\n"
" Applies quoting rules based on the active delimiter context.\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" encode_primitive(Null, Comma)\n"
" // -> \"null\"\n"
"\n"
" encode_primitive(Bool(True), Comma)\n"
" // -> \"true\"\n"
"\n"
" encode_primitive(Number(42.0), Comma)\n"
" // -> \"42\"\n"
"\n"
" encode_primitive(JsonString(\"hello\"), Comma)\n"
" // -> \"hello\"\n"
" ```\n"
" Encode a number, removing unnecessary decimals and normalizing -0 to 0.\n"
" Encode a string value, applying quoting rules based on delimiter.\n"
" Encode a key for use in key-value pairs.\n"
"\n"
" Applies quoting rules for keys: keys must be quoted unless they\n"
" match the pattern ^[A-Za-z_][\\\\w.]*$\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" encode_key(\"name\")\n"
" // -> \"name\"\n"
"\n"
" encode_key(\"user-id\")\n"
" // -> \"\\\"user-id\\\"\"\n"
" ```\n"
" Format an array header line.\n"
"\n"
" ## Arguments\n"
"\n"
" * `length` - The array length\n"
" * `key` - Optional key name for the array\n"
" * `delimiter` - The delimiter for this array's scope\n"
" * `marker` - Whether to include the \"#\" length marker\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" format_header(3, None, Comma, NoMarker)\n"
" // -> \"[3]:\"\n"
"\n"
" format_header(3, Some(\"tags\"), Comma, HashMarker)\n"
" // -> \"tags[#3]:\"\n"
"\n"
" format_header(2, Some(\"items\"), Tab, NoMarker)\n"
" // -> \"items[2\\t]:\"\n"
" ```\n"
" Format a tabular array header with field names.\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" format_tabular_header(2, Some(\"users\"), [\"id\", \"name\"], Comma, NoMarker)\n"
" // -> \"users[2]{id,name}:\"\n"
" ```\n"
" Convert delimiter to its header representation.\n"
"\n"
" Comma has no representation (empty string).\n"
" Tab and Pipe appear as their literal characters.\n"
" Encode and join primitive values with a delimiter.\n"
"\n"
" Used for inline primitive arrays.\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" let values = [Number(1.0), Number(2.0), Number(3.0)]\n"
" encode_and_join_primitives(values, Comma)\n"
" // -> \"1,2,3\"\n"
" ```\n"
).
-file("src/toon_codec/encode/primitives.gleam", 110).
-spec encode_number(float()) -> binary().
encode_number(N) ->
case N =:= +0.0 of
true ->
<<"0"/utf8>>;
false ->
Floored = math:floor(N),
case Floored =:= N of
true ->
Truncated = erlang:trunc(N),
erlang:integer_to_binary(Truncated);
false ->
gleam_stdlib:float_to_string(N)
end
end.
-file("src/toon_codec/encode/primitives.gleam", 128).
-spec encode_string_value(binary(), toon_codec@types:delimiter()) -> binary().
encode_string_value(Value, Delimiter) ->
case toon_codec@internal@string:needs_quoting(Value, Delimiter) of
true ->
toon_codec@internal@string:quote_string(Value);
false ->
Value
end.
-file("src/toon_codec/encode/primitives.gleam", 99).
-spec encode_primitive(
toon_codec@types:json_value(),
toon_codec@types:delimiter()
) -> binary().
encode_primitive(Value, Delimiter) ->
case Value of
null ->
<<"null"/utf8>>;
{bool, true} ->
<<"true"/utf8>>;
{bool, false} ->
<<"false"/utf8>>;
{number, N} ->
encode_number(N);
{string, S} ->
encode_string_value(S, Delimiter);
_ ->
erlang:error(#{gleam_error => panic,
message => <<"encode_primitive called with non-primitive value"/utf8>>,
file => <<?FILEPATH/utf8>>,
module => <<"toon_codec/encode/primitives"/utf8>>,
function => <<"encode_primitive"/utf8>>,
line => 106})
end.
-file("src/toon_codec/encode/primitives.gleam", 137).
-spec encode_key(binary()) -> binary().
encode_key(Key) ->
case toon_codec@internal@string:key_needs_quoting(Key) of
true ->
toon_codec@internal@string:quote_string(Key);
false ->
Key
end.
-file("src/toon_codec/encode/primitives.gleam", 211).
-spec delimiter_to_header_string(toon_codec@types:delimiter()) -> binary().
delimiter_to_header_string(Delimiter) ->
case Delimiter of
comma ->
<<""/utf8>>;
tab ->
<<"\t"/utf8>>;
pipe ->
<<"|"/utf8>>
end.
-file("src/toon_codec/encode/primitives.gleam", 146).
-spec format_header(
integer(),
gleam@option:option(binary()),
toon_codec@types:delimiter(),
toon_codec@types:length_marker()
) -> binary().
format_header(Length, Key, Delimiter, Marker) ->
Key_part = case Key of
{some, K} ->
encode_key(K);
none ->
<<""/utf8>>
end,
Marker_part = case Marker of
hash_marker ->
<<"#"/utf8>>;
no_marker ->
<<""/utf8>>
end,
Delimiter_part = delimiter_to_header_string(Delimiter),
Bracket_part = <<<<<<<<(<<"["/utf8>>)/binary, Marker_part/binary>>/binary,
(erlang:integer_to_binary(Length))/binary>>/binary,
Delimiter_part/binary>>/binary,
(<<"]"/utf8>>)/binary>>,
<<<<Key_part/binary, Bracket_part/binary>>/binary, (<<":"/utf8>>)/binary>>.
-file("src/toon_codec/encode/primitives.gleam", 174).
-spec format_tabular_header(
integer(),
gleam@option:option(binary()),
list(binary()),
toon_codec@types:delimiter(),
toon_codec@types:length_marker()
) -> binary().
format_tabular_header(Length, Key, Fields, Delimiter, Marker) ->
Key_part = case Key of
{some, K} ->
encode_key(K);
none ->
<<""/utf8>>
end,
Marker_part = case Marker of
hash_marker ->
<<"#"/utf8>>;
no_marker ->
<<""/utf8>>
end,
Delimiter_part = delimiter_to_header_string(Delimiter),
Bracket_part = <<<<<<<<(<<"["/utf8>>)/binary, Marker_part/binary>>/binary,
(erlang:integer_to_binary(Length))/binary>>/binary,
Delimiter_part/binary>>/binary,
(<<"]"/utf8>>)/binary>>,
Delim_str = toon_codec@types:delimiter_to_string(Delimiter),
Encoded_fields = gleam@list:map(Fields, fun encode_key/1),
Fields_part = <<<<(<<"{"/utf8>>)/binary,
(gleam@string:join(Encoded_fields, Delim_str))/binary>>/binary,
(<<"}"/utf8>>)/binary>>,
<<<<<<Key_part/binary, Bracket_part/binary>>/binary, Fields_part/binary>>/binary,
(<<":"/utf8>>)/binary>>.
-file("src/toon_codec/encode/primitives.gleam", 221).
-spec encode_and_join_primitives(
list(toon_codec@types:json_value()),
toon_codec@types:delimiter()
) -> binary().
encode_and_join_primitives(Values, Delimiter) ->
Delim_str = toon_codec@types:delimiter_to_string(Delimiter),
_pipe = Values,
_pipe@1 = gleam@list:map(
_pipe,
fun(_capture) -> encode_primitive(_capture, Delimiter) end
),
gleam@string:join(_pipe@1, Delim_str).