Packages

TOON (Token-Oriented Object Notation) encoder/decoder - a compact, token-efficient format for LLM input

Current section

Files

Jump to
toon_codec src toon_codec@encode@internal.erl
Raw

src/toon_codec@encode@internal.erl

-module(toon_codec@encode@internal).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/toon_codec/encode/internal.gleam").
-export([is_primitive/1, is_object/1, is_array/1, is_array_of_primitives/1, is_array_of_arrays/1, is_array_of_objects/1, all_arrays_of_primitives/1, all_values_primitive/1, detect_tabular/1, extract_field_values/2]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(
" Internal utilities for array structure detection.\n"
"\n"
" This module provides functions to analyze arrays and determine\n"
" whether they should be encoded as inline, tabular, or expanded form.\n"
" Check if a JSON value is a primitive (not array or object).\n"
" Check if a JSON value is an object.\n"
" Check if a JSON value is an array.\n"
" Check if all values in a list are primitives.\n"
"\n"
" Returns True if the list is empty or all elements are primitives.\n"
" Check if all values in a list are arrays.\n"
" Check if all values in a list are objects.\n"
" Check if all arrays in a list contain only primitives.\n"
"\n"
" Used to detect arrays of primitive arrays.\n"
" Detect if an array of objects can be encoded in tabular form.\n"
"\n"
" Returns Some(fields) if tabular encoding is possible, where fields\n"
" is the ordered list of field names. Returns None otherwise.\n"
"\n"
" Tabular encoding requires:\n"
" 1. All elements are objects\n"
" 2. All objects have the same set of keys\n"
" 3. All values in all objects are primitives (no nested structures)\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" let objects = [\n"
" Object([#(\"id\", Number(1.0)), #(\"name\", JsonString(\"Alice\"))]),\n"
" Object([#(\"id\", Number(2.0)), #(\"name\", JsonString(\"Bob\"))]),\n"
" ]\n"
" detect_tabular(objects)\n"
" // -> Some([\"id\", \"name\"])\n"
" ```\n"
" Check if all values in an object are primitives.\n"
" Check if all objects in a list have the same keys and all primitive values.\n"
" Extract field values from an object in a specified order.\n"
"\n"
" Returns the values corresponding to the given field names,\n"
" in the same order as the field names list.\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" let obj = Object([#(\"name\", JsonString(\"Alice\")), #(\"id\", Number(1.0))])\n"
" extract_field_values(obj, [\"id\", \"name\"])\n"
" // -> [Number(1.0), JsonString(\"Alice\")]\n"
" ```\n"
).
-file("src/toon_codec/encode/internal.gleam", 60).
-spec is_primitive(toon_codec@types:json_value()) -> boolean().
is_primitive(Value) ->
case Value of
null ->
true;
{bool, _} ->
true;
{number, _} ->
true;
{string, _} ->
true;
{array, _} ->
false;
{object, _} ->
false
end.
-file("src/toon_codec/encode/internal.gleam", 67).
-spec is_object(toon_codec@types:json_value()) -> boolean().
is_object(Value) ->
case Value of
{object, _} ->
true;
_ ->
false
end.
-file("src/toon_codec/encode/internal.gleam", 74).
-spec is_array(toon_codec@types:json_value()) -> boolean().
is_array(Value) ->
case Value of
{array, _} ->
true;
_ ->
false
end.
-file("src/toon_codec/encode/internal.gleam", 83).
-spec is_array_of_primitives(list(toon_codec@types:json_value())) -> boolean().
is_array_of_primitives(Values) ->
gleam@list:all(Values, fun is_primitive/1).
-file("src/toon_codec/encode/internal.gleam", 87).
-spec is_array_of_arrays(list(toon_codec@types:json_value())) -> boolean().
is_array_of_arrays(Values) ->
case Values of
[] ->
false;
_ ->
gleam@list:all(Values, fun is_array/1)
end.
-file("src/toon_codec/encode/internal.gleam", 94).
-spec is_array_of_objects(list(toon_codec@types:json_value())) -> boolean().
is_array_of_objects(Values) ->
case Values of
[] ->
false;
_ ->
gleam@list:all(Values, fun is_object/1)
end.
-file("src/toon_codec/encode/internal.gleam", 101).
-spec all_arrays_of_primitives(list(toon_codec@types:json_value())) -> boolean().
all_arrays_of_primitives(Values) ->
gleam@list:all(Values, fun(Value) -> case Value of
{array, Items} ->
is_array_of_primitives(Items);
_ ->
false
end end).
-file("src/toon_codec/encode/internal.gleam", 135).
-spec all_values_primitive(list({binary(), toon_codec@types:json_value()})) -> boolean().
all_values_primitive(Fields) ->
gleam@list:all(
Fields,
fun(Pair) -> is_primitive(erlang:element(2, Pair)) end
).
-file("src/toon_codec/encode/internal.gleam", 139).
-spec check_uniform_objects(list(toon_codec@types:json_value()), list(binary())) -> boolean().
check_uniform_objects(Objects, Expected_keys) ->
gleam@list:all(Objects, fun(Obj) -> case Obj of
{object, Fields} ->
Keys = gleam@list:map(
Fields,
fun(Pair) -> erlang:element(1, Pair) end
),
Keys_set = gleam@set:from_list(Keys),
Expected_set = gleam@set:from_list(Expected_keys),
(Keys_set =:= Expected_set) andalso all_values_primitive(
Fields
);
_ ->
false
end end).
-file("src/toon_codec/encode/internal.gleam", 112).
-spec detect_tabular(list(toon_codec@types:json_value())) -> gleam@option:option(list(binary())).
detect_tabular(Values) ->
case Values of
[] ->
none;
[{object, First_fields} | Rest] ->
First_keys = gleam@list:map(
First_fields,
fun(Pair) -> erlang:element(1, Pair) end
),
case all_values_primitive(First_fields) of
false ->
none;
true ->
case check_uniform_objects(Rest, First_keys) of
true ->
{some, First_keys};
false ->
none
end
end;
_ ->
none
end.
-file("src/toon_codec/encode/internal.gleam", 160).
-spec extract_field_values(toon_codec@types:json_value(), list(binary())) -> list(toon_codec@types:json_value()).
extract_field_values(Obj, Field_names) ->
case Obj of
{object, Fields} ->
gleam@list:map(
Field_names,
fun(Name) -> case gleam@list:key_find(Fields, Name) of
{ok, Value} ->
Value;
{error, _} ->
null
end end
);
_ ->
[]
end.