Packages

TOON (Token-Oriented Object Notation) encoder/decoder - a compact, token-efficient format for LLM input

Current section

Files

Jump to
toon_codec src toon_codec@internal@string.erl
Raw

src/toon_codec@internal@string.erl

-module(toon_codec@internal@string).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/toon_codec/internal/string.gleam").
-export([escape_string/1, unescape_string/1, find_closing_quote/2, find_unquoted_char/3, needs_quoting/2, key_needs_quoting/1, quote_string/1, unquote_string/1]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(false).
-file("src/toon_codec/internal/string.gleam", 65).
?DOC(false).
-spec escape_string(binary()) -> binary().
escape_string(Value) ->
_pipe = Value,
_pipe@1 = gleam@string:replace(_pipe, <<"\\"/utf8>>, <<"\\\\"/utf8>>),
_pipe@2 = gleam@string:replace(_pipe@1, <<"\""/utf8>>, <<"\\\""/utf8>>),
_pipe@3 = gleam@string:replace(_pipe@2, <<"\n"/utf8>>, <<"\\n"/utf8>>),
_pipe@4 = gleam@string:replace(_pipe@3, <<"\r"/utf8>>, <<"\\r"/utf8>>),
gleam@string:replace(_pipe@4, <<"\t"/utf8>>, <<"\\t"/utf8>>).
-file("src/toon_codec/internal/string.gleam", 81).
?DOC(false).
-spec unescape_chars(list(binary()), list(binary()), integer()) -> {ok,
binary()} |
{error, toon_codec@error:toon_error()}.
unescape_chars(Chars, Acc, Position) ->
case Chars of
[] ->
{ok, gleam@string:join(lists:reverse(Acc), <<""/utf8>>)};
[<<"\\"/utf8>>, Next | Rest] ->
case Next of
<<"\\"/utf8>> ->
unescape_chars(Rest, [<<"\\"/utf8>> | Acc], Position + 2);
<<"\""/utf8>> ->
unescape_chars(Rest, [<<"\""/utf8>> | Acc], Position + 2);
<<"n"/utf8>> ->
unescape_chars(Rest, [<<"\n"/utf8>> | Acc], Position + 2);
<<"r"/utf8>> ->
unescape_chars(Rest, [<<"\r"/utf8>> | Acc], Position + 2);
<<"t"/utf8>> ->
unescape_chars(Rest, [<<"\t"/utf8>> | Acc], Position + 2);
_ ->
{error, toon_codec@error:invalid_escape(Next, Position + 1)}
end;
[<<"\\"/utf8>>] ->
{error, toon_codec@error:invalid_escape(<<""/utf8>>, Position)};
[Char | Rest@1] ->
unescape_chars(Rest@1, [Char | Acc], Position + 1)
end.
-file("src/toon_codec/internal/string.gleam", 76).
?DOC(false).
-spec unescape_string(binary()) -> {ok, binary()} |
{error, toon_codec@error:toon_error()}.
unescape_string(Value) ->
Chars = gleam@string:to_graphemes(Value),
unescape_chars(Chars, [], 0).
-file("src/toon_codec/internal/string.gleam", 113).
?DOC(false).
-spec find_closing_quote_loop(list(binary()), integer(), integer(), boolean()) -> {ok,
integer()} |
{error, toon_codec@error:toon_error()}.
find_closing_quote_loop(Chars, Position, Search_pos, Escaped) ->
case gleam@list:drop(Chars, Search_pos) of
[] ->
{error, toon_codec@error:unterminated_string(Position)};
[<<"\\"/utf8>>, _ | _] when not Escaped ->
find_closing_quote_loop(Chars, Position, Search_pos + 2, false);
[<<"\""/utf8>> | _] when not Escaped ->
{ok, Search_pos};
[_ | _] ->
find_closing_quote_loop(Chars, Position, Search_pos + 1, false)
end.
-file("src/toon_codec/internal/string.gleam", 108).
?DOC(false).
-spec find_closing_quote(binary(), integer()) -> {ok, integer()} |
{error, toon_codec@error:toon_error()}.
find_closing_quote(Content, Start) ->
Chars = gleam@string:to_graphemes(Content),
find_closing_quote_loop(Chars, Start + 1, Start + 1, false).
-file("src/toon_codec/internal/string.gleam", 142).
?DOC(false).
-spec find_unquoted_char_loop(
list(binary()),
binary(),
integer(),
boolean(),
integer()
) -> gleam@option:option(integer()).
find_unquoted_char_loop(Chars, Target, Position, In_quotes, Current) ->
case Chars of
[] ->
none;
[<<"\\"/utf8>>, _ | Rest] when In_quotes ->
find_unquoted_char_loop(
Rest,
Target,
Position,
In_quotes,
Current + 2
);
[<<"\""/utf8>> | Rest@1] ->
New_in_quotes = not In_quotes,
find_unquoted_char_loop(
Rest@1,
Target,
Position,
New_in_quotes,
Current + 1
);
[C | _] when ((C =:= Target) andalso not In_quotes) andalso (Current >= Position) ->
{some, Current};
[_ | Rest@2] ->
find_unquoted_char_loop(
Rest@2,
Target,
Position,
In_quotes,
Current + 1
)
end.
-file("src/toon_codec/internal/string.gleam", 133).
?DOC(false).
-spec find_unquoted_char(binary(), binary(), integer()) -> gleam@option:option(integer()).
find_unquoted_char(Content, Char, Start) ->
Chars = gleam@string:to_graphemes(Content),
find_unquoted_char_loop(Chars, Char, Start, false, 0).
-file("src/toon_codec/internal/string.gleam", 185).
?DOC(false).
-spec has_leading_or_trailing_whitespace(binary()) -> boolean().
has_leading_or_trailing_whitespace(Value) ->
case gleam@string:to_graphemes(Value) of
[] ->
false;
[First | _] ->
case toon_codec@internal@char:is_whitespace(First) of
true ->
true;
false ->
Last = begin
_pipe = Value,
_pipe@1 = gleam@string:to_graphemes(_pipe),
gleam@list:last(_pipe@1)
end,
case Last of
{ok, L} ->
toon_codec@internal@char:is_whitespace(L);
_ ->
false
end
end
end.
-file("src/toon_codec/internal/string.gleam", 206).
?DOC(false).
-spec is_literal(binary()) -> boolean().
is_literal(Value) ->
((Value =:= <<"true"/utf8>>) orelse (Value =:= <<"false"/utf8>>)) orelse (Value
=:= <<"null"/utf8>>).
-file("src/toon_codec/internal/string.gleam", 221).
?DOC(false).
-spec is_numeric_pattern(list(binary())) -> boolean().
is_numeric_pattern(Chars) ->
case Chars of
[] ->
false;
[First | _] ->
toon_codec@internal@char:is_digit(First) andalso gleam@list:all(
Chars,
fun(C) ->
((((toon_codec@internal@char:is_digit(C) orelse (C =:= <<"."/utf8>>))
orelse (C =:= <<"e"/utf8>>))
orelse (C =:= <<"E"/utf8>>))
orelse (C =:= <<"+"/utf8>>))
orelse (C =:= <<"-"/utf8>>)
end
)
end.
-file("src/toon_codec/internal/string.gleam", 210).
?DOC(false).
-spec is_numeric_like(binary()) -> boolean().
is_numeric_like(Value) ->
case gleam@string:to_graphemes(Value) of
[] ->
false;
[<<"-"/utf8>> | Rest] ->
is_numeric_pattern(Rest);
Chars ->
is_numeric_pattern(Chars)
end.
-file("src/toon_codec/internal/string.gleam", 237).
?DOC(false).
-spec contains_special_chars(binary()) -> boolean().
contains_special_chars(Value) ->
((((((((gleam_stdlib:contains_string(Value, <<":"/utf8>>) orelse gleam_stdlib:contains_string(
Value,
<<"\""/utf8>>
))
orelse gleam_stdlib:contains_string(Value, <<"\\"/utf8>>))
orelse gleam_stdlib:contains_string(Value, <<"["/utf8>>))
orelse gleam_stdlib:contains_string(Value, <<"]"/utf8>>))
orelse gleam_stdlib:contains_string(Value, <<"{"/utf8>>))
orelse gleam_stdlib:contains_string(Value, <<"}"/utf8>>))
orelse gleam_stdlib:contains_string(Value, <<"\n"/utf8>>))
orelse gleam_stdlib:contains_string(Value, <<"\r"/utf8>>))
orelse gleam_stdlib:contains_string(Value, <<"\t"/utf8>>).
-file("src/toon_codec/internal/string.gleam", 250).
?DOC(false).
-spec contains_delimiter(binary(), toon_codec@types:delimiter()) -> boolean().
contains_delimiter(Value, Delimiter) ->
Delim_str = toon_codec@types:delimiter_to_string(Delimiter),
gleam_stdlib:contains_string(Value, Delim_str).
-file("src/toon_codec/internal/string.gleam", 255).
?DOC(false).
-spec starts_with_hyphen(binary()) -> boolean().
starts_with_hyphen(Value) ->
gleam_stdlib:string_starts_with(Value, <<"-"/utf8>>).
-file("src/toon_codec/internal/string.gleam", 175).
?DOC(false).
-spec needs_quoting(binary(), toon_codec@types:delimiter()) -> boolean().
needs_quoting(Value, Delimiter) ->
((((((Value =:= <<""/utf8>>) orelse has_leading_or_trailing_whitespace(
Value
))
orelse is_literal(Value))
orelse is_numeric_like(Value))
orelse contains_special_chars(Value))
orelse contains_delimiter(Value, Delimiter))
orelse starts_with_hyphen(Value).
-file("src/toon_codec/internal/string.gleam", 259).
?DOC(false).
-spec key_needs_quoting(binary()) -> boolean().
key_needs_quoting(Key) ->
case gleam@string:to_graphemes(Key) of
[] ->
true;
[First | Rest] ->
case toon_codec@internal@char:is_key_start(First) of
false ->
true;
true ->
not gleam@list:all(
Rest,
fun toon_codec@internal@char:is_key_char/1
)
end
end.
-file("src/toon_codec/internal/string.gleam", 272).
?DOC(false).
-spec quote_string(binary()) -> binary().
quote_string(Value) ->
<<<<"\""/utf8, (escape_string(Value))/binary>>/binary, "\""/utf8>>.
-file("src/toon_codec/internal/string.gleam", 276).
?DOC(false).
-spec unquote_string(binary()) -> {ok, binary()} |
{error, toon_codec@error:toon_error()}.
unquote_string(Value) ->
case gleam_stdlib:string_starts_with(Value, <<"\""/utf8>>) andalso gleam_stdlib:string_ends_with(
Value,
<<"\""/utf8>>
) of
true ->
Len = string:length(Value),
Content = gleam@string:slice(Value, 1, Len - 2),
unescape_string(Content);
false ->
{error,
toon_codec@error:validation_error(
<<"Not a quoted string"/utf8>>
)}
end.