Packages

TOON (Token-Oriented Object Notation) encoder/decoder - a compact, token-efficient format for LLM input

Current section

Files

Jump to
toon_codec src toon_codec@decode@scanner.erl
Raw

src/toon_codec@decode@scanner.erl

-module(toon_codec@decode@scanner).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/toon_codec/decode/scanner.gleam").
-export([scan_lines/3, cursor_new/1, cursor_peek/1, cursor_advance/1, cursor_at_end/1, cursor_position/1, cursor_length/1, cursor_peek_ahead/2, cursor_remaining_lines/1]).
-export_type([line_cursor/0]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(
" Line scanner and cursor for TOON decoding.\n"
"\n"
" This module provides functions to scan input text into parsed lines\n"
" with depth information, and a cursor type for traversing those lines.\n"
" Scan input into a list of parsed lines with depth information.\n"
"\n"
" Each line is analyzed for indentation, depth is calculated,\n"
" and blank lines are tracked.\n"
"\n"
" ## Arguments\n"
"\n"
" * `input` - The TOON format string to scan\n"
" * `indent_size` - Number of spaces per indentation level\n"
" * `strict` - Whether to enforce strict indentation rules\n"
" Parse a single line to extract depth and content.\n"
" Count leading spaces in a string.\n"
" Calculate indentation depth from leading spaces.\n"
" Opaque type for traversing parsed lines.\n"
" Create a new cursor from parsed lines.\n"
" Peek at the current line without advancing.\n"
" Advance the cursor to the next line.\n"
" Check if the cursor is at the end (no more lines).\n"
" Get the current position of the cursor.\n"
" Get the total number of lines.\n"
" Peek ahead at a line N positions from current.\n"
" Get all remaining lines from the current position.\n"
).
-opaque line_cursor() :: {line_cursor,
list(toon_codec@types:parsed_line()),
integer()}.
-file("src/toon_codec/decode/scanner.gleam", 103).
-spec count_leading_spaces_loop(list(binary()), integer()) -> integer().
count_leading_spaces_loop(Chars, Count) ->
case Chars of
[<<" "/utf8>> | Rest] ->
count_leading_spaces_loop(Rest, Count + 1);
_ ->
Count
end.
-file("src/toon_codec/decode/scanner.gleam", 98).
-spec count_leading_spaces(binary()) -> integer().
count_leading_spaces(S) ->
Chars = gleam@string:to_graphemes(S),
count_leading_spaces_loop(Chars, 0).
-file("src/toon_codec/decode/scanner.gleam", 110).
-spec calculate_depth(integer(), integer(), boolean(), integer()) -> {ok,
integer()} |
{error, toon_codec@error:toon_error()}.
calculate_depth(Indent, Indent_size, Strict, Line_number) ->
case Strict of
true ->
case case Indent_size of
0 -> 0;
Gleam@denominator -> Indent rem Gleam@denominator
end of
0 ->
{ok, case Indent_size of
0 -> 0;
Gleam@denominator@1 -> Indent div Gleam@denominator@1
end};
_ ->
{error,
toon_codec@error:indentation_error(
<<<<"Indentation must be a multiple of "/utf8,
(erlang:integer_to_binary(Indent_size))/binary>>/binary,
" spaces"/utf8>>,
Line_number
)}
end;
false ->
{ok, case Indent_size of
0 -> 0;
Gleam@denominator@2 -> Indent div Gleam@denominator@2
end}
end.
-file("src/toon_codec/decode/scanner.gleam", 79).
-spec parse_line(binary(), integer(), integer(), boolean()) -> {ok,
toon_codec@types:parsed_line()} |
{error, toon_codec@error:toon_error()}.
parse_line(Raw, Line_number, Indent_size, Strict) ->
Indent = count_leading_spaces(Raw),
case calculate_depth(Indent, Indent_size, Strict, Line_number) of
{ok, Depth} ->
Content = gleam@string:trim_start(Raw),
{ok,
toon_codec@types:new_parsed_line(
Raw,
Depth,
Indent,
Content,
Line_number
)};
{error, Err} ->
{error, Err}
end.
-file("src/toon_codec/decode/scanner.gleam", 47).
-spec scan_lines_loop(
list(binary()),
list(toon_codec@types:parsed_line()),
integer(),
boolean(),
integer()
) -> {ok, list(toon_codec@types:parsed_line())} |
{error, toon_codec@error:toon_error()}.
scan_lines_loop(Lines, Acc, Indent_size, Strict, Line_number) ->
case Lines of
[] ->
{ok, lists:reverse(Acc)};
[Line | Rest] ->
case gleam@string:trim(Line) of
<<""/utf8>> ->
scan_lines_loop(
Rest,
Acc,
Indent_size,
Strict,
Line_number + 1
);
_ ->
case parse_line(Line, Line_number, Indent_size, Strict) of
{ok, Parsed} ->
scan_lines_loop(
Rest,
[Parsed | Acc],
Indent_size,
Strict,
Line_number + 1
);
{error, Err} ->
{error, Err}
end
end
end.
-file("src/toon_codec/decode/scanner.gleam", 37).
-spec scan_lines(binary(), integer(), boolean()) -> {ok,
list(toon_codec@types:parsed_line())} |
{error, toon_codec@error:toon_error()}.
scan_lines(Input, Indent_size, Strict) ->
Lines = gleam@string:split(Input, <<"\n"/utf8>>),
scan_lines_loop(Lines, [], Indent_size, Strict, 1).
-file("src/toon_codec/decode/scanner.gleam", 143).
-spec cursor_new(list(toon_codec@types:parsed_line())) -> line_cursor().
cursor_new(Lines) ->
{line_cursor, Lines, 0}.
-file("src/toon_codec/decode/scanner.gleam", 147).
-spec cursor_peek(line_cursor()) -> gleam@option:option(toon_codec@types:parsed_line()).
cursor_peek(Cursor) ->
_pipe = erlang:element(2, Cursor),
_pipe@1 = gleam@list:drop(_pipe, erlang:element(3, Cursor)),
_pipe@2 = gleam@list:first(_pipe@1),
gleam@option:from_result(_pipe@2).
-file("src/toon_codec/decode/scanner.gleam", 154).
-spec cursor_advance(line_cursor()) -> line_cursor().
cursor_advance(Cursor) ->
{line_cursor, erlang:element(2, Cursor), erlang:element(3, Cursor) + 1}.
-file("src/toon_codec/decode/scanner.gleam", 158).
-spec cursor_at_end(line_cursor()) -> boolean().
cursor_at_end(Cursor) ->
erlang:element(3, Cursor) >= erlang:length(erlang:element(2, Cursor)).
-file("src/toon_codec/decode/scanner.gleam", 162).
-spec cursor_position(line_cursor()) -> integer().
cursor_position(Cursor) ->
erlang:element(3, Cursor).
-file("src/toon_codec/decode/scanner.gleam", 166).
-spec cursor_length(line_cursor()) -> integer().
cursor_length(Cursor) ->
erlang:length(erlang:element(2, Cursor)).
-file("src/toon_codec/decode/scanner.gleam", 170).
-spec cursor_peek_ahead(line_cursor(), integer()) -> gleam@option:option(toon_codec@types:parsed_line()).
cursor_peek_ahead(Cursor, Offset) ->
_pipe = erlang:element(2, Cursor),
_pipe@1 = gleam@list:drop(_pipe, erlang:element(3, Cursor) + Offset),
_pipe@2 = gleam@list:first(_pipe@1),
gleam@option:from_result(_pipe@2).
-file("src/toon_codec/decode/scanner.gleam", 177).
-spec cursor_remaining_lines(line_cursor()) -> list(toon_codec@types:parsed_line()).
cursor_remaining_lines(Cursor) ->
gleam@list:drop(erlang:element(2, Cursor), erlang:element(3, Cursor)).