Packages

A markup language to define ER schemas

Current section

Files

Jump to
prequel src prequel@internals@token.erl
Raw

src/prequel@internals@token.erl

-module(prequel@internals@token).
-compile([no_auto_import, nowarn_unused_vars]).
-export([to_string/1, scan/1]).
-export_type([token/0]).
-type token() :: open_bracket |
close_bracket |
open_parens |
close_parens |
colon |
circle_lollipop |
star_lollipop |
arrow_lollipop |
minus |
ampersand |
{simple_comment, binary()} |
{number, binary()} |
{word, binary()}.
-spec to_string(token()) -> binary().
to_string(Token) ->
case Token of
open_bracket ->
<<"{"/utf8>>;
close_bracket ->
<<"}"/utf8>>;
open_parens ->
<<"("/utf8>>;
close_parens ->
<<")"/utf8>>;
colon ->
<<":"/utf8>>;
circle_lollipop ->
<<"-o"/utf8>>;
star_lollipop ->
<<"-*"/utf8>>;
arrow_lollipop ->
<<"->"/utf8>>;
minus ->
<<"-"/utf8>>;
ampersand ->
<<"&"/utf8>>;
{simple_comment, Comment} ->
<<"//"/utf8, Comment/binary>>;
{number, Number} ->
Number;
{word, Word} ->
Word
end.
-spec scan_comment_body(
gleam@string_builder:string_builder(),
integer(),
binary()
) -> {binary(), integer(), binary()}.
scan_comment_body(Acc, Size, Source) ->
case Source of
<<"\r\n"/utf8, Rest/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Rest};
<<"\n"/utf8, Rest/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Rest};
Rest@1 ->
case gleam@string:pop_grapheme(Rest@1) of
{error, nil} ->
{gleam@string_builder:to_string(Acc), Size, Rest@1};
{ok, {Grapheme, Rest@2}} ->
scan_comment_body(
gleam@string_builder:append(Acc, Grapheme),
Size + 1,
Rest@2
)
end
end.
-spec scan_word(gleam@string_builder:string_builder(), integer(), binary()) -> {binary(),
integer(),
binary()}.
scan_word(Acc, Size, Source) ->
case Source of
<<"//"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"///"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"////"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<":"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"("/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<")"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"{"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"}"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<" "/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"\n"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"\r"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
<<"\t"/utf8, _/binary>> ->
{gleam@string_builder:to_string(Acc), Size, Source};
Rest ->
case gleam@string:pop_grapheme(Rest) of
{error, nil} ->
{gleam@string_builder:to_string(Acc), Size, Rest};
{ok, {Grapheme, Rest@1}} ->
scan_word(
gleam@string_builder:append(Acc, Grapheme),
Size + 1,
Rest@1
)
end
end.
-spec do_scan(
binary(),
integer(),
integer(),
list({token(), prequel@span:span()})
) -> list({token(), prequel@span:span()}).
do_scan(Source, Line, Column, Acc) ->
case Source of
<<""/utf8>> ->
gleam@list:reverse(Acc);
<<"\n"/utf8, Rest/binary>> ->
do_scan(Rest, Line + 1, 1, Acc);
<<" "/utf8, Rest@1/binary>> ->
do_scan(Rest@1, Line, Column + 1, Acc);
<<"\t"/utf8, Rest@1/binary>> ->
do_scan(Rest@1, Line, Column + 1, Acc);
<<"\r"/utf8, Rest@1/binary>> ->
do_scan(Rest@1, Line, Column + 1, Acc);
<<"{"/utf8, Rest@2/binary>> ->
Lexeme = {open_bracket, prequel@span:point(Line, Column)},
do_scan(Rest@2, Line, Column + 1, [Lexeme | Acc]);
<<"}"/utf8, Rest@3/binary>> ->
Lexeme@1 = {close_bracket, prequel@span:point(Line, Column)},
do_scan(Rest@3, Line, Column + 1, [Lexeme@1 | Acc]);
<<"("/utf8, Rest@4/binary>> ->
Lexeme@2 = {open_parens, prequel@span:point(Line, Column)},
do_scan(Rest@4, Line, Column + 1, [Lexeme@2 | Acc]);
<<")"/utf8, Rest@5/binary>> ->
Lexeme@3 = {close_parens, prequel@span:point(Line, Column)},
do_scan(Rest@5, Line, Column + 1, [Lexeme@3 | Acc]);
<<":"/utf8, Rest@6/binary>> ->
Lexeme@4 = {colon, prequel@span:point(Line, Column)},
do_scan(Rest@6, Line, Column + 1, [Lexeme@4 | Acc]);
<<"-o"/utf8, Rest@7/binary>> ->
Lexeme@5 = {circle_lollipop,
prequel@span:segment(Line, Column, Column + 1)},
do_scan(Rest@7, Line, Column + 2, [Lexeme@5 | Acc]);
<<"-*"/utf8, Rest@8/binary>> ->
Lexeme@6 = {star_lollipop,
prequel@span:segment(Line, Column, Column + 1)},
do_scan(Rest@8, Line, Column + 2, [Lexeme@6 | Acc]);
<<"->"/utf8, Rest@9/binary>> ->
Lexeme@7 = {arrow_lollipop,
prequel@span:segment(Line, Column, Column + 1)},
do_scan(Rest@9, Line, Column + 2, [Lexeme@7 | Acc]);
<<"-"/utf8, Rest@10/binary>> ->
Lexeme@8 = {minus, prequel@span:point(Line, Column)},
do_scan(Rest@10, Line, Column + 1, [Lexeme@8 | Acc]);
<<"&"/utf8, Rest@11/binary>> ->
Lexeme@9 = {ampersand, prequel@span:point(Line, Column)},
do_scan(Rest@11, Line, Column + 1, [Lexeme@9 | Acc]);
<<"//"/utf8, Rest@12/binary>> ->
{Comment_body, Size, Rest@13} = scan_comment_body(
gleam@string_builder:new(),
2,
Rest@12
),
Span = prequel@span:segment(Line, Column, (Column + Size) - 1),
Lexeme@10 = {{simple_comment, Comment_body}, Span},
do_scan(Rest@13, Line + 1, 1, [Lexeme@10 | Acc]);
<<"0"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"1"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"2"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"3"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"4"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"5"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"6"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"7"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"8"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
<<"9"/utf8, _/binary>> ->
{Number, Size@1, Rest@14} = scan_number(
gleam@string_builder:new(),
0,
Source
),
Span@1 = prequel@span:segment(Line, Column, (Column + Size@1) - 1),
Lexeme@11 = {{number, Number}, Span@1},
do_scan(Rest@14, Line, Column + Size@1, [Lexeme@11 | Acc]);
Rest@15 ->
{Word, Size@2, Rest@16} = scan_word(
gleam@string_builder:new(),
0,
Rest@15
),
Lexeme@12 = {{word, Word},
prequel@span:segment(Line, Column, (Column + Size@2) - 1)},
do_scan(Rest@16, Line, Column + Size@2, [Lexeme@12 | Acc])
end.
-spec scan(binary()) -> list({token(), prequel@span:span()}).
scan(Source) ->
do_scan(Source, 1, 1, []).
-spec scan_number(gleam@string_builder:string_builder(), integer(), binary()) -> {binary(),
integer(),
binary()}.
scan_number(Acc, Size, Content) ->
case Content of
<<"0"/utf8, Rest/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"0"/utf8>>),
Size + 1,
Rest
);
<<"1"/utf8, Rest@1/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"1"/utf8>>),
Size + 1,
Rest@1
);
<<"2"/utf8, Rest@2/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"2"/utf8>>),
Size + 1,
Rest@2
);
<<"3"/utf8, Rest@3/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"3"/utf8>>),
Size + 1,
Rest@3
);
<<"4"/utf8, Rest@4/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"4"/utf8>>),
Size + 1,
Rest@4
);
<<"5"/utf8, Rest@5/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"5"/utf8>>),
Size + 1,
Rest@5
);
<<"6"/utf8, Rest@6/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"6"/utf8>>),
Size + 1,
Rest@6
);
<<"7"/utf8, Rest@7/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"7"/utf8>>),
Size + 1,
Rest@7
);
<<"8"/utf8, Rest@8/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"8"/utf8>>),
Size + 1,
Rest@8
);
<<"9"/utf8, Rest@9/binary>> ->
scan_number(
gleam@string_builder:append(Acc, <<"9"/utf8>>),
Size + 1,
Rest@9
);
Rest@10 ->
{gleam@string_builder:to_string(Acc), Size, Rest@10}
end.