Packages
caffeine_lang
6.2.2
6.3.1
6.3.0
6.2.2
6.2.1
6.2.0
6.1.2
6.1.1
6.1.0
6.0.0
5.6.0
5.5.0
5.4.4
5.4.3
5.4.2
5.4.1
5.4.0
5.3.0
5.2.0
5.1.1
5.1.0
5.0.12
5.0.11
5.0.10
5.0.8
5.0.7
5.0.6
5.0.5
5.0.4
5.0.1
5.0.0
4.10.0
4.9.0
4.8.3
4.8.2
4.8.1
4.8.0
4.7.9
4.7.8
4.7.7
4.7.6
4.7.5
4.6.7
4.6.6
4.6.5
4.6.4
4.6.3
4.6.2
4.6.0
4.5.1
4.5.0
4.4.4
4.4.3
4.4.1
4.4.0
4.3.7
4.3.6
3.0.6
3.0.5
3.0.4
3.0.3
3.0.2
3.0.1
3.0.0
2.0.5
2.0.4
2.0.3
2.0.2
2.0.1
2.0.0
1.0.2
1.0.1
0.1.0
0.0.24
0.0.23
0.0.22
0.0.21
0.0.20
0.0.19
0.0.18
0.0.17
0.0.16
0.0.15
0.0.14
0.0.13
0.0.12
0.0.11
0.0.10
0.0.9
0.0.8
0.0.7
0.0.6
0.0.5
0.0.4
0.0.2
0.0.1
A compiler for generating reliability artifacts from service expectation definitions.
Current section
Files
Jump to
Current section
Files
src/caffeine_lang@frontend@tokenizer.erl
-module(caffeine_lang@frontend@tokenizer).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/caffeine_lang/frontend/tokenizer.gleam").
-export([tokenize/1]).
-export_type([tokenizer_state/0]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
-type tokenizer_state() :: {tokenizer_state,
binary(),
integer(),
integer(),
boolean()}.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 20).
-spec pop_codepoint(binary()) -> {ok, {binary(), binary()}} | {error, nil}.
pop_codepoint(S) ->
case tokenizer_ffi:pop_codepoint(S) of
{<<""/utf8>>, _} ->
{error, nil};
Pair ->
{ok, Pair}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 526).
-spec keyword_or_identifier(binary()) -> caffeine_lang@frontend@token:token().
keyword_or_identifier(Word) ->
case Word of
<<"success_rate"/utf8>> ->
keyword_success_rate;
<<"time_slice"/utf8>> ->
keyword_time_slice;
<<"Assumes"/utf8>> ->
keyword_assumes;
<<"Guarantees"/utf8>> ->
keyword_guarantees;
<<"over"/utf8>> ->
keyword_over;
<<"window"/utf8>> ->
keyword_window;
<<"as"/utf8>> ->
keyword_as;
<<"measured"/utf8>> ->
keyword_measured;
<<"by"/utf8>> ->
keyword_by;
<<"with"/utf8>> ->
keyword_with;
<<"below"/utf8>> ->
keyword_below;
<<"hard"/utf8>> ->
keyword_hard;
<<"soft"/utf8>> ->
keyword_soft;
<<"dependency"/utf8>> ->
keyword_dependency;
<<"on"/utf8>> ->
keyword_on;
<<"extends"/utf8>> ->
keyword_extends;
<<"Requires"/utf8>> ->
keyword_requires;
<<"Provides"/utf8>> ->
keyword_provides;
<<"in"/utf8>> ->
keyword_in;
<<"x"/utf8>> ->
keyword_x;
<<"String"/utf8>> ->
keyword_string;
<<"Integer"/utf8>> ->
keyword_integer;
<<"Float"/utf8>> ->
keyword_float;
<<"Boolean"/utf8>> ->
keyword_boolean;
<<"List"/utf8>> ->
keyword_list;
<<"Dict"/utf8>> ->
keyword_dict;
<<"Optional"/utf8>> ->
keyword_optional;
<<"Defaulted"/utf8>> ->
keyword_defaulted;
<<"Type"/utf8>> ->
keyword_type;
<<"URL"/utf8>> ->
keyword_u_r_l;
<<"Percentage"/utf8>> ->
keyword_percentage;
<<"from"/utf8>> ->
keyword_from;
<<"where"/utf8>> ->
keyword_where;
<<"and"/utf8>> ->
keyword_and;
<<"true"/utf8>> ->
literal_true;
<<"false"/utf8>> ->
literal_false;
_ ->
{identifier, Word}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 273).
-spec advance(tokenizer_state(), binary(), integer()) -> tokenizer_state().
advance(State, Source, Len) ->
{tokenizer_state,
Source,
erlang:element(3, State),
erlang:element(4, State) + Len,
false}.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 489).
-spec is_digit(binary()) -> boolean().
is_digit(Char) ->
Code = tokenizer_ffi:code_unit_at(Char, 0),
(Code >= 48) andalso (Code =< 57).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 502).
-spec is_letter(binary()) -> boolean().
is_letter(Char) ->
Code = tokenizer_ffi:code_unit_at(Char, 0),
((Code >= 65) andalso (Code =< 90)) orelse ((Code >= 97) andalso (Code =< 122)).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 498).
-spec is_identifier_char(binary()) -> boolean().
is_identifier_char(Char) ->
(is_letter(Char) orelse is_digit(Char)) orelse (Char =:= <<"_"/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 511).
-spec read_identifier_loop(binary(), list(binary())) -> {binary(), binary()}.
read_identifier_loop(Source, Acc) ->
case pop_codepoint(Source) of
{ok, {Char, Rest}} ->
case is_identifier_char(Char) of
true ->
read_identifier_loop(Rest, [Char | Acc]);
false ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end;
{error, nil} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 507).
-spec read_identifier(binary()) -> {binary(), binary()}.
read_identifier(Source) ->
read_identifier_loop(Source, []).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 494).
-spec is_identifier_start(binary()) -> boolean().
is_identifier_start(Char) ->
is_letter(Char) orelse (Char =:= <<"_"/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 417).
?DOC(" Checks if a number token is followed by `%`, converting it to LiteralPercentage.\n").
-spec maybe_percentage(
caffeine_lang@frontend@token:token(),
binary(),
integer()
) -> {caffeine_lang@frontend@token:token(), binary(), integer()}.
maybe_percentage(Tok, Remaining, Len) ->
case pop_codepoint(Remaining) of
{ok, {<<"%"/utf8>>, After_percent}} ->
Float_val = case Tok of
{literal_integer, N} ->
erlang:float(N);
{literal_float, F} ->
F;
_ ->
+0.0
end,
{{literal_percentage, Float_val}, After_percent, Len + 1};
_ ->
{Tok, Remaining, Len}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 469).
-spec try_unit(binary(), binary()) -> {ok, {binary(), binary(), integer()}} |
{error, nil}.
try_unit(Source, Unit) ->
case gleam_stdlib:string_starts_with(Source, Unit) of
false ->
{error, nil};
true ->
After = gleam@string:drop_start(
Source,
tokenizer_ffi:code_unit_length(Unit)
),
case pop_codepoint(After) of
{error, nil} ->
{ok, {Unit, After, tokenizer_ffi:code_unit_length(Unit)}};
{ok, {Next, _}} ->
case is_identifier_char(Next) of
true ->
{error, nil};
false ->
{ok,
{Unit,
After,
tokenizer_ffi:code_unit_length(Unit)}}
end
end
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 460).
?DOC(
" Tries to match a duration unit suffix at the start of `source`. The suffix\n"
" must be followed by a non-identifier character (or end of input) to count.\n"
" Longest match wins: `ms` is tried before `m`.\n"
).
-spec match_duration_unit(binary()) -> {ok, {binary(), binary(), integer()}} |
{error, nil}.
match_duration_unit(Source) ->
_pipe = try_unit(Source, <<"ms"/utf8>>),
_pipe@1 = gleam@result:lazy_or(
_pipe,
fun() -> try_unit(Source, <<"s"/utf8>>) end
),
_pipe@2 = gleam@result:lazy_or(
_pipe@1,
fun() -> try_unit(Source, <<"m"/utf8>>) end
),
_pipe@3 = gleam@result:lazy_or(
_pipe@2,
fun() -> try_unit(Source, <<"h"/utf8>>) end
),
gleam@result:lazy_or(_pipe@3, fun() -> try_unit(Source, <<"d"/utf8>>) end).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 439).
?DOC(
" Tries to attach a duration unit suffix to a numeric token; falls through to\n"
" `maybe_percentage` if no unit matches. A duration unit only matches when the\n"
" character after the suffix is NOT an identifier character — so `10d` is a\n"
" duration but `10days` is `10` followed by identifier `days`.\n"
).
-spec maybe_duration_or_percentage(
caffeine_lang@frontend@token:token(),
binary(),
integer()
) -> {caffeine_lang@frontend@token:token(), binary(), integer()}.
maybe_duration_or_percentage(Tok, Remaining, Len) ->
case match_duration_unit(Remaining) of
{ok, {Unit, After_unit, Unit_len}} ->
Float_val = case Tok of
{literal_integer, N} ->
erlang:float(N);
{literal_float, F} ->
F;
_ ->
+0.0
end,
{{literal_duration, Float_val, Unit}, After_unit, Len + Unit_len};
{error, nil} ->
maybe_percentage(Tok, Remaining, Len)
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 395).
-spec parse_integer(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
parse_integer(Digits, Remaining) ->
case gleam_stdlib:parse_int(Digits) of
{ok, N} ->
{ok,
{{literal_integer, N},
Remaining,
tokenizer_ffi:code_unit_length(Digits)}};
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 405).
-spec parse_float(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
parse_float(Float_str, Remaining) ->
case gleam_stdlib:parse_float(Float_str) of
{ok, F} ->
{ok,
{{literal_float, F},
Remaining,
tokenizer_ffi:code_unit_length(Float_str)}};
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 383).
-spec read_digits_loop(binary(), list(binary())) -> {binary(), binary()}.
read_digits_loop(Source, Acc) ->
case pop_codepoint(Source) of
{ok, {Char, Rest}} ->
case is_digit(Char) of
true ->
read_digits_loop(Rest, [Char | Acc]);
false ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end;
{error, nil} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 375).
-spec read_digits(binary(), binary()) -> {binary(), binary()}.
read_digits(Source, Prefix) ->
Initial = case Prefix of
<<""/utf8>> ->
[];
P ->
[P]
end,
read_digits_loop(Source, Initial).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 346).
-spec read_number(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
read_number(Source, Prefix) ->
{Digits, Remaining} = read_digits(Source, Prefix),
case pop_codepoint(Remaining) of
{ok, {<<"."/utf8>>, After_dot}} ->
case pop_codepoint(After_dot) of
{ok, {Next_char, _}} when Next_char =:= <<"."/utf8>> ->
parse_integer(Digits, Remaining);
{ok, {Next_char@1, _}} ->
case is_digit(Next_char@1) of
true ->
{Decimal_digits, Final_remaining} = read_digits(
After_dot,
<<""/utf8>>
),
Float_str = <<<<Digits/binary, "."/utf8>>/binary,
Decimal_digits/binary>>,
parse_float(Float_str, Final_remaining);
false ->
parse_integer(Digits, Remaining)
end;
{error, nil} ->
parse_integer(Digits, Remaining)
end;
_ ->
parse_integer(Digits, Remaining)
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 334).
-spec read_string(binary(), list(binary())) -> {ok, {binary(), binary()}} |
{error, nil}.
read_string(Source, Acc) ->
case pop_codepoint(Source) of
{ok, {<<"\""/utf8>>, Rest}} ->
{ok, {erlang:list_to_binary(lists:reverse(Acc)), Rest}};
{ok, {<<"\n"/utf8>>, _}} ->
{error, nil};
{ok, {Char, Rest@1}} ->
read_string(Rest@1, [Char | Acc]);
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 323).
-spec read_until_newline_loop(binary(), list(binary())) -> {binary(), binary()}.
read_until_newline_loop(Source, Acc) ->
case pop_codepoint(Source) of
{ok, {<<"\n"/utf8>>, _}} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source};
{ok, {Char, Rest}} ->
read_until_newline_loop(Rest, [Char | Acc]);
{error, nil} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 319).
-spec read_until_newline(binary()) -> {binary(), binary()}.
read_until_newline(Source) ->
read_until_newline_loop(Source, []).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 311).
-spec count_indentation(binary(), integer()) -> {integer(), binary()}.
count_indentation(Source, Count) ->
case pop_codepoint(Source) of
{ok, {<<" "/utf8>>, Rest}} ->
count_indentation(Rest, Count + 1);
{ok, {<<"\t"/utf8>>, Rest@1}} ->
count_indentation(Rest@1, Count + 2);
_ ->
{Count, Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 304).
-spec skip_empty_lines(binary(), integer()) -> {binary(), integer()}.
skip_empty_lines(Source, Count) ->
case pop_codepoint(Source) of
{ok, {<<"\n"/utf8>>, Rest}} ->
skip_empty_lines(Rest, Count + 1);
_ ->
{Source, Count}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 291).
-spec emit_token_n(
tokenizer_state(),
binary(),
integer(),
caffeine_lang@frontend@token:token(),
list(caffeine_lang@frontend@token:positioned_token())
) -> {ok, list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
emit_token_n(State, Rest, Len, Tok, Acc) ->
tokenize_loop(
advance(State, Rest, Len),
[{positioned_token,
Tok,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 282).
-spec emit_token(
tokenizer_state(),
binary(),
caffeine_lang@frontend@token:token(),
list(caffeine_lang@frontend@token:positioned_token())
) -> {ok, list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
emit_token(State, Rest, Tok, Acc) ->
emit_token_n(State, Rest, 1, Tok, Acc).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 57).
-spec tokenize_loop(
tokenizer_state(),
list(caffeine_lang@frontend@token:positioned_token())
) -> {ok, list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
tokenize_loop(State, Acc) ->
case pop_codepoint(erlang:element(2, State)) of
{error, nil} ->
{ok,
[{positioned_token,
e_o_f,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]};
{ok, {Char, Rest}} ->
case Char of
<<"\n"/utf8>> ->
{Remaining, Skipped} = skip_empty_lines(Rest, 0),
New_state = {tokenizer_state,
Remaining,
(erlang:element(3, State) + 1) + Skipped,
1,
true},
tokenize_loop(
New_state,
[{positioned_token,
whitespace_newline,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
<<"\r"/utf8>> ->
tokenize_loop(advance(State, Rest, 1), Acc);
<<" "/utf8>> when erlang:element(5, State) ->
{Indent_count, Remaining@1} = count_indentation(
erlang:element(2, State),
0
),
tokenize_loop(
advance(State, Remaining@1, Indent_count),
[{positioned_token,
{whitespace_indent, Indent_count},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
<<"\t"/utf8>> when erlang:element(5, State) ->
{Indent_count, Remaining@1} = count_indentation(
erlang:element(2, State),
0
),
tokenize_loop(
advance(State, Remaining@1, Indent_count),
[{positioned_token,
{whitespace_indent, Indent_count},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
<<" "/utf8>> ->
tokenize_loop(advance(State, Rest, 1), Acc);
<<"\t"/utf8>> ->
tokenize_loop(advance(State, Rest, 2), Acc);
<<"#"/utf8>> ->
case pop_codepoint(Rest) of
{ok, {<<"#"/utf8>>, After_hash}} ->
case pop_codepoint(After_hash) of
{ok, {<<"#"/utf8>>, After_third_hash}} ->
{Comment_text, Remaining@2} = read_until_newline(
After_third_hash
),
tokenize_loop(
advance(
State,
Remaining@2,
3 + tokenizer_ffi:code_unit_length(
Comment_text
)
),
[{positioned_token,
{comment_doc, Comment_text},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
_ ->
{Comment_text@1, Remaining@3} = read_until_newline(
After_hash
),
tokenize_loop(
advance(
State,
Remaining@3,
2 + tokenizer_ffi:code_unit_length(
Comment_text@1
)
),
[{positioned_token,
{comment_section,
Comment_text@1},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
)
end;
_ ->
{Comment_text@2, Remaining@4} = read_until_newline(
Rest
),
tokenize_loop(
advance(
State,
Remaining@4,
1 + tokenizer_ffi:code_unit_length(
Comment_text@2
)
),
[{positioned_token,
{comment_line, Comment_text@2},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
)
end;
<<"\""/utf8>> ->
case read_string(Rest, []) of
{ok, {Str_content, Remaining@5}} ->
tokenize_loop(
advance(
State,
Remaining@5,
2 + tokenizer_ffi:code_unit_length(
Str_content
)
),
[{positioned_token,
{literal_string, Str_content},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{unterminated_string,
erlang:element(3, State),
erlang:element(4, State)}}
end;
<<"{"/utf8>> ->
emit_token(State, Rest, symbol_left_brace, Acc);
<<"}"/utf8>> ->
emit_token(State, Rest, symbol_right_brace, Acc);
<<"("/utf8>> ->
emit_token(State, Rest, symbol_left_paren, Acc);
<<")"/utf8>> ->
emit_token(State, Rest, symbol_right_paren, Acc);
<<"["/utf8>> ->
emit_token(State, Rest, symbol_left_bracket, Acc);
<<"]"/utf8>> ->
emit_token(State, Rest, symbol_right_bracket, Acc);
<<":"/utf8>> ->
emit_token(State, Rest, symbol_colon, Acc);
<<","/utf8>> ->
emit_token(State, Rest, symbol_comma, Acc);
<<"*"/utf8>> ->
emit_token(State, Rest, symbol_star, Acc);
<<"|"/utf8>> ->
emit_token(State, Rest, symbol_pipe, Acc);
<<"="/utf8>> ->
emit_token(State, Rest, symbol_equals, Acc);
<<"."/utf8>> ->
case pop_codepoint(Rest) of
{ok, {<<"."/utf8>>, After_dot}} ->
emit_token_n(
State,
After_dot,
2,
symbol_dot_dot,
Acc
);
_ ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"-"/utf8>> ->
case read_number(Rest, <<"-"/utf8>>) of
{ok, {Tok, Remaining@6, Len}} ->
{Final_tok, Final_remaining, Final_len} = maybe_duration_or_percentage(
Tok,
Remaining@6,
Len
),
tokenize_loop(
advance(State, Final_remaining, Final_len),
[{positioned_token,
Final_tok,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
<<"-"/utf8>>}}
end;
<<"0"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"1"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"2"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"3"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"4"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"5"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"6"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"7"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"8"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"9"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@7, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_duration_or_percentage(
Tok@1,
Remaining@7,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
_ ->
case is_identifier_start(Char) of
true ->
{Word, Remaining@8} = read_identifier(
erlang:element(2, State)
),
tokenize_loop(
advance(
State,
Remaining@8,
tokenizer_ffi:code_unit_length(Word)
),
[{positioned_token,
keyword_or_identifier(Word),
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
false ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end
end
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 49).
?DOC(" Tokenizes source text into a list of positioned tokens.\n").
-spec tokenize(binary()) -> {ok,
list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
tokenize(Source) ->
State = {tokenizer_state, Source, 1, 1, true},
_pipe = tokenize_loop(State, []),
gleam@result:map(_pipe, fun lists:reverse/1).