Packages
caffeine_lang
4.9.0
6.3.1
6.3.0
6.2.2
6.2.1
6.2.0
6.1.2
6.1.1
6.1.0
6.0.0
5.6.0
5.5.0
5.4.4
5.4.3
5.4.2
5.4.1
5.4.0
5.3.0
5.2.0
5.1.1
5.1.0
5.0.12
5.0.11
5.0.10
5.0.8
5.0.7
5.0.6
5.0.5
5.0.4
5.0.1
5.0.0
4.10.0
4.9.0
4.8.3
4.8.2
4.8.1
4.8.0
4.7.9
4.7.8
4.7.7
4.7.6
4.7.5
4.6.7
4.6.6
4.6.5
4.6.4
4.6.3
4.6.2
4.6.0
4.5.1
4.5.0
4.4.4
4.4.3
4.4.1
4.4.0
4.3.7
4.3.6
3.0.6
3.0.5
3.0.4
3.0.3
3.0.2
3.0.1
3.0.0
2.0.5
2.0.4
2.0.3
2.0.2
2.0.1
2.0.0
1.0.2
1.0.1
0.1.0
0.0.24
0.0.23
0.0.22
0.0.21
0.0.20
0.0.19
0.0.18
0.0.17
0.0.16
0.0.15
0.0.14
0.0.13
0.0.12
0.0.11
0.0.10
0.0.9
0.0.8
0.0.7
0.0.6
0.0.5
0.0.4
0.0.2
0.0.1
A compiler for generating reliability artifacts from service expectation definitions.
Current section
Files
Jump to
Current section
Files
src/caffeine_lang@frontend@tokenizer.erl
-module(caffeine_lang@frontend@tokenizer).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/caffeine_lang/frontend/tokenizer.gleam").
-export([tokenize/1]).
-export_type([tokenizer_state/0]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
-type tokenizer_state() :: {tokenizer_state,
binary(),
integer(),
integer(),
boolean()}.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 210).
-spec advance(tokenizer_state(), binary(), integer()) -> tokenizer_state().
advance(State, Source, Len) ->
{tokenizer_state,
Source,
erlang:element(3, State),
erlang:element(4, State) + Len,
false}.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 241).
-spec skip_empty_lines(binary(), integer()) -> {binary(), integer()}.
skip_empty_lines(Source, Count) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<"\n"/utf8>>, Rest}} ->
skip_empty_lines(Rest, Count + 1);
_ ->
{Source, Count}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 248).
-spec count_indentation(binary(), integer()) -> {integer(), binary()}.
count_indentation(Source, Count) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<" "/utf8>>, Rest}} ->
count_indentation(Rest, Count + 1);
{ok, {<<"\t"/utf8>>, Rest@1}} ->
count_indentation(Rest@1, Count + 2);
_ ->
{Count, Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 260).
-spec read_until_newline_loop(binary(), list(binary())) -> {binary(), binary()}.
read_until_newline_loop(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<"\n"/utf8>>, _}} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source};
{ok, {Char, Rest}} ->
read_until_newline_loop(Rest, [Char | Acc]);
{error, nil} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 256).
-spec read_until_newline(binary()) -> {binary(), binary()}.
read_until_newline(Source) ->
read_until_newline_loop(Source, []).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 271).
-spec read_string(binary(), list(binary())) -> {ok, {binary(), binary()}} |
{error, nil}.
read_string(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<"\""/utf8>>, Rest}} ->
{ok, {erlang:list_to_binary(lists:reverse(Acc)), Rest}};
{ok, {<<"\n"/utf8>>, _}} ->
{error, nil};
{ok, {Char, Rest@1}} ->
read_string(Rest@1, [Char | Acc]);
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 332).
-spec parse_integer(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
parse_integer(Digits, Remaining) ->
case gleam_stdlib:parse_int(Digits) of
{ok, N} ->
{ok, {{literal_integer, N}, Remaining, string:length(Digits)}};
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 342).
-spec parse_float(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
parse_float(Float_str, Remaining) ->
case gleam_stdlib:parse_float(Float_str) of
{ok, F} ->
{ok, {{literal_float, F}, Remaining, string:length(Float_str)}};
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 353).
?DOC(" Checks if a number token is followed by `%`, converting it to LiteralPercentage.\n").
-spec maybe_percentage(
caffeine_lang@frontend@token:token(),
binary(),
integer()
) -> {caffeine_lang@frontend@token:token(), binary(), integer()}.
maybe_percentage(Tok, Remaining, Len) ->
case gleam_stdlib:string_pop_grapheme(Remaining) of
{ok, {<<"%"/utf8>>, After_percent}} ->
Float_val = case Tok of
{literal_integer, N} ->
erlang:float(N);
{literal_float, F} ->
F;
_ ->
+0.0
end,
{{literal_percentage, Float_val}, After_percent, Len + 1};
_ ->
{Tok, Remaining, Len}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 371).
-spec is_digit(binary()) -> boolean().
is_digit(Char) ->
case gleam@string:to_utf_codepoints(Char) of
[Cp] ->
Code = gleam_stdlib:identity(Cp),
(Code >= 48) andalso (Code =< 57);
_ ->
false
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 320).
-spec read_digits_loop(binary(), list(binary())) -> {binary(), binary()}.
read_digits_loop(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {Char, Rest}} ->
case is_digit(Char) of
true ->
read_digits_loop(Rest, [Char | Acc]);
false ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end;
{error, nil} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 312).
-spec read_digits(binary(), binary()) -> {binary(), binary()}.
read_digits(Source, Prefix) ->
Initial = case Prefix of
<<""/utf8>> ->
[];
P ->
[P]
end,
read_digits_loop(Source, Initial).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 283).
-spec read_number(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
read_number(Source, Prefix) ->
{Digits, Remaining} = read_digits(Source, Prefix),
case gleam_stdlib:string_pop_grapheme(Remaining) of
{ok, {<<"."/utf8>>, After_dot}} ->
case gleam_stdlib:string_pop_grapheme(After_dot) of
{ok, {Next_char, _}} when Next_char =:= <<"."/utf8>> ->
parse_integer(Digits, Remaining);
{ok, {Next_char@1, _}} ->
case is_digit(Next_char@1) of
true ->
{Decimal_digits, Final_remaining} = read_digits(
After_dot,
<<""/utf8>>
),
Float_str = <<<<Digits/binary, "."/utf8>>/binary,
Decimal_digits/binary>>,
parse_float(Float_str, Final_remaining);
false ->
parse_integer(Digits, Remaining)
end;
{error, nil} ->
parse_integer(Digits, Remaining)
end;
_ ->
parse_integer(Digits, Remaining)
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 389).
-spec is_letter(binary()) -> boolean().
is_letter(Char) ->
case gleam@string:to_utf_codepoints(Char) of
[Cp] ->
Code = gleam_stdlib:identity(Cp),
((Code >= 65) andalso (Code =< 90)) orelse ((Code >= 97) andalso (Code
=< 122));
_ ->
false
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 381).
-spec is_identifier_start(binary()) -> boolean().
is_identifier_start(Char) ->
is_letter(Char) orelse (Char =:= <<"_"/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 385).
-spec is_identifier_char(binary()) -> boolean().
is_identifier_char(Char) ->
(is_letter(Char) orelse is_digit(Char)) orelse (Char =:= <<"_"/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 403).
-spec read_identifier_loop(binary(), list(binary())) -> {binary(), binary()}.
read_identifier_loop(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {Char, Rest}} ->
case is_identifier_char(Char) of
true ->
read_identifier_loop(Rest, [Char | Acc]);
false ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end;
{error, nil} ->
{erlang:list_to_binary(lists:reverse(Acc)), Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 399).
-spec read_identifier(binary()) -> {binary(), binary()}.
read_identifier(Source) ->
read_identifier_loop(Source, []).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 418).
-spec keyword_or_identifier(binary()) -> caffeine_lang@frontend@token:token().
keyword_or_identifier(Word) ->
case Word of
<<"Blueprints"/utf8>> ->
keyword_blueprints;
<<"Expectations"/utf8>> ->
keyword_expectations;
<<"for"/utf8>> ->
keyword_for;
<<"extends"/utf8>> ->
keyword_extends;
<<"Requires"/utf8>> ->
keyword_requires;
<<"Provides"/utf8>> ->
keyword_provides;
<<"in"/utf8>> ->
keyword_in;
<<"x"/utf8>> ->
keyword_x;
<<"String"/utf8>> ->
keyword_string;
<<"Integer"/utf8>> ->
keyword_integer;
<<"Float"/utf8>> ->
keyword_float;
<<"Boolean"/utf8>> ->
keyword_boolean;
<<"List"/utf8>> ->
keyword_list;
<<"Dict"/utf8>> ->
keyword_dict;
<<"Optional"/utf8>> ->
keyword_optional;
<<"Defaulted"/utf8>> ->
keyword_defaulted;
<<"Type"/utf8>> ->
keyword_type;
<<"URL"/utf8>> ->
keyword_u_r_l;
<<"Percentage"/utf8>> ->
keyword_percentage;
<<"true"/utf8>> ->
literal_true;
<<"false"/utf8>> ->
literal_false;
_ ->
{identifier, Word}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 228).
-spec emit_token_n(
tokenizer_state(),
binary(),
integer(),
caffeine_lang@frontend@token:token(),
list(caffeine_lang@frontend@token:positioned_token())
) -> {ok, list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
emit_token_n(State, Rest, Len, Tok, Acc) ->
tokenize_loop(
advance(State, Rest, Len),
[{positioned_token,
Tok,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 21).
-spec tokenize_loop(
tokenizer_state(),
list(caffeine_lang@frontend@token:positioned_token())
) -> {ok, list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
tokenize_loop(State, Acc) ->
case gleam_stdlib:string_pop_grapheme(erlang:element(2, State)) of
{error, nil} ->
{ok,
[{positioned_token,
e_o_f,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]};
{ok, {Char, Rest}} ->
case Char of
<<"\n"/utf8>> ->
{Remaining, Skipped} = skip_empty_lines(Rest, 0),
New_state = {tokenizer_state,
Remaining,
(erlang:element(3, State) + 1) + Skipped,
1,
true},
tokenize_loop(
New_state,
[{positioned_token,
whitespace_newline,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
<<"\r"/utf8>> ->
tokenize_loop(advance(State, Rest, 1), Acc);
<<" "/utf8>> when erlang:element(5, State) ->
{Indent_count, Remaining@1} = count_indentation(
erlang:element(2, State),
0
),
tokenize_loop(
advance(State, Remaining@1, Indent_count),
[{positioned_token,
{whitespace_indent, Indent_count},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
<<"\t"/utf8>> when erlang:element(5, State) ->
{Indent_count, Remaining@1} = count_indentation(
erlang:element(2, State),
0
),
tokenize_loop(
advance(State, Remaining@1, Indent_count),
[{positioned_token,
{whitespace_indent, Indent_count},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
<<" "/utf8>> ->
tokenize_loop(advance(State, Rest, 1), Acc);
<<"\t"/utf8>> ->
tokenize_loop(advance(State, Rest, 2), Acc);
<<"#"/utf8>> ->
case gleam_stdlib:string_pop_grapheme(Rest) of
{ok, {<<"#"/utf8>>, After_hash}} ->
{Comment_text, Remaining@2} = read_until_newline(
After_hash
),
tokenize_loop(
advance(
State,
Remaining@2,
2 + string:length(Comment_text)
),
[{positioned_token,
{comment_section, Comment_text},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
_ ->
{Comment_text@1, Remaining@3} = read_until_newline(
Rest
),
tokenize_loop(
advance(
State,
Remaining@3,
1 + string:length(Comment_text@1)
),
[{positioned_token,
{comment_line, Comment_text@1},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
)
end;
<<"\""/utf8>> ->
case read_string(Rest, []) of
{ok, {Str_content, Remaining@4}} ->
tokenize_loop(
advance(
State,
Remaining@4,
2 + string:length(Str_content)
),
[{positioned_token,
{literal_string, Str_content},
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{unterminated_string,
erlang:element(3, State),
erlang:element(4, State)}}
end;
<<"{"/utf8>> ->
emit_token(State, Rest, symbol_left_brace, Acc);
<<"}"/utf8>> ->
emit_token(State, Rest, symbol_right_brace, Acc);
<<"("/utf8>> ->
emit_token(State, Rest, symbol_left_paren, Acc);
<<")"/utf8>> ->
emit_token(State, Rest, symbol_right_paren, Acc);
<<"["/utf8>> ->
emit_token(State, Rest, symbol_left_bracket, Acc);
<<"]"/utf8>> ->
emit_token(State, Rest, symbol_right_bracket, Acc);
<<":"/utf8>> ->
emit_token(State, Rest, symbol_colon, Acc);
<<","/utf8>> ->
emit_token(State, Rest, symbol_comma, Acc);
<<"*"/utf8>> ->
emit_token(State, Rest, symbol_star, Acc);
<<"+"/utf8>> ->
emit_token(State, Rest, symbol_plus, Acc);
<<"|"/utf8>> ->
emit_token(State, Rest, symbol_pipe, Acc);
<<"="/utf8>> ->
emit_token(State, Rest, symbol_equals, Acc);
<<"."/utf8>> ->
case gleam_stdlib:string_pop_grapheme(Rest) of
{ok, {<<"."/utf8>>, After_dot}} ->
emit_token_n(
State,
After_dot,
2,
symbol_dot_dot,
Acc
);
_ ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"-"/utf8>> ->
case read_number(Rest, <<"-"/utf8>>) of
{ok, {Tok, Remaining@5, Len}} ->
{Final_tok, Final_remaining, Final_len} = maybe_percentage(
Tok,
Remaining@5,
Len
),
tokenize_loop(
advance(State, Final_remaining, Final_len),
[{positioned_token,
Final_tok,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
<<"-"/utf8>>}}
end;
<<"0"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"1"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"2"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"3"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"4"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"5"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"6"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"7"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"8"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"9"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@6, Len@1}} ->
{Final_tok@1, Final_remaining@1, Final_len@1} = maybe_percentage(
Tok@1,
Remaining@6,
Len@1
),
tokenize_loop(
advance(State, Final_remaining@1, Final_len@1),
[{positioned_token,
Final_tok@1,
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
_ ->
case is_identifier_start(Char) of
true ->
{Word, Remaining@7} = read_identifier(
erlang:element(2, State)
),
tokenize_loop(
advance(State, Remaining@7, string:length(Word)),
[{positioned_token,
keyword_or_identifier(Word),
erlang:element(3, State),
erlang:element(4, State)} |
Acc]
);
false ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end
end
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 219).
-spec emit_token(
tokenizer_state(),
binary(),
caffeine_lang@frontend@token:token(),
list(caffeine_lang@frontend@token:positioned_token())
) -> {ok, list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
emit_token(State, Rest, Tok, Acc) ->
emit_token_n(State, Rest, 1, Tok, Acc).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 15).
?DOC(" Tokenizes source text into a list of positioned tokens.\n").
-spec tokenize(binary()) -> {ok,
list(caffeine_lang@frontend@token:positioned_token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
tokenize(Source) ->
State = {tokenizer_state, Source, 1, 1, true},
_pipe = tokenize_loop(State, []),
gleam@result:map(_pipe, fun lists:reverse/1).