Packages
caffeine_lang
3.0.0
6.3.1
6.3.0
6.2.2
6.2.1
6.2.0
6.1.2
6.1.1
6.1.0
6.0.0
5.6.0
5.5.0
5.4.4
5.4.3
5.4.2
5.4.1
5.4.0
5.3.0
5.2.0
5.1.1
5.1.0
5.0.12
5.0.11
5.0.10
5.0.8
5.0.7
5.0.6
5.0.5
5.0.4
5.0.1
5.0.0
4.10.0
4.9.0
4.8.3
4.8.2
4.8.1
4.8.0
4.7.9
4.7.8
4.7.7
4.7.6
4.7.5
4.6.7
4.6.6
4.6.5
4.6.4
4.6.3
4.6.2
4.6.0
4.5.1
4.5.0
4.4.4
4.4.3
4.4.1
4.4.0
4.3.7
4.3.6
3.0.6
3.0.5
3.0.4
3.0.3
3.0.2
3.0.1
3.0.0
2.0.5
2.0.4
2.0.3
2.0.2
2.0.1
2.0.0
1.0.2
1.0.1
0.1.0
0.0.24
0.0.23
0.0.22
0.0.21
0.0.20
0.0.19
0.0.18
0.0.17
0.0.16
0.0.15
0.0.14
0.0.13
0.0.12
0.0.11
0.0.10
0.0.9
0.0.8
0.0.7
0.0.6
0.0.5
0.0.4
0.0.2
0.0.1
A compiler for generating reliability artifacts from service expectation definitions.
Current section
Files
Jump to
Current section
Files
src/caffeine_lang@frontend@tokenizer.erl
-module(caffeine_lang@frontend@tokenizer).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/caffeine_lang/frontend/tokenizer.gleam").
-export([tokenize/1]).
-export_type([tokenizer_state/0]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
-type tokenizer_state() :: {tokenizer_state,
binary(),
integer(),
integer(),
boolean()}.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 160).
-spec advance(tokenizer_state(), binary(), integer()) -> tokenizer_state().
advance(State, Source, Len) ->
{tokenizer_state,
Source,
erlang:element(3, State),
erlang:element(4, State) + Len,
false}.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 188).
-spec skip_empty_lines(binary()) -> binary().
skip_empty_lines(Source) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<"\n"/utf8>>, Rest}} ->
skip_empty_lines(Rest);
_ ->
Source
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 195).
-spec count_indentation(binary(), integer()) -> {integer(), binary()}.
count_indentation(Source, Count) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<" "/utf8>>, Rest}} ->
count_indentation(Rest, Count + 1);
{ok, {<<"\t"/utf8>>, Rest@1}} ->
count_indentation(Rest@1, Count + 2);
_ ->
{Count, Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 207).
-spec read_until_newline_loop(binary(), binary()) -> {binary(), binary()}.
read_until_newline_loop(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<"\n"/utf8>>, _}} ->
{Acc, Source};
{ok, {Char, Rest}} ->
read_until_newline_loop(Rest, <<Acc/binary, Char/binary>>);
{error, nil} ->
{Acc, Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 203).
-spec read_until_newline(binary()) -> {binary(), binary()}.
read_until_newline(Source) ->
read_until_newline_loop(Source, <<""/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 215).
-spec read_string(binary(), binary()) -> {ok, {binary(), binary()}} |
{error, nil}.
read_string(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {<<"\""/utf8>>, Rest}} ->
{ok, {Acc, Rest}};
{ok, {<<"\n"/utf8>>, _}} ->
{error, nil};
{ok, {Char, Rest@1}} ->
read_string(Rest@1, <<Acc/binary, Char/binary>>);
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 265).
-spec parse_integer(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
parse_integer(Digits, Remaining) ->
case gleam_stdlib:parse_int(Digits) of
{ok, N} ->
{ok, {{literal_integer, N}, Remaining, string:length(Digits)}};
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 275).
-spec parse_float(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
parse_float(Float_str, Remaining) ->
case gleam_stdlib:parse_float(Float_str) of
{ok, F} ->
{ok, {{literal_float, F}, Remaining, string:length(Float_str)}};
{error, nil} ->
{error, nil}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 285).
-spec is_digit(binary()) -> boolean().
is_digit(Char) ->
case gleam@string:to_utf_codepoints(Char) of
[Cp] ->
Code = gleam_stdlib:identity(Cp),
(Code >= 48) andalso (Code =< 57);
_ ->
false
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 253).
-spec read_digits(binary(), binary()) -> {binary(), binary()}.
read_digits(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {Char, Rest}} ->
case is_digit(Char) of
true ->
read_digits(Rest, <<Acc/binary, Char/binary>>);
false ->
{Acc, Source}
end;
{error, nil} ->
{Acc, Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 224).
-spec read_number(binary(), binary()) -> {ok,
{caffeine_lang@frontend@token:token(), binary(), integer()}} |
{error, nil}.
read_number(Source, Prefix) ->
{Digits, Remaining} = read_digits(Source, Prefix),
case gleam_stdlib:string_pop_grapheme(Remaining) of
{ok, {<<"."/utf8>>, After_dot}} ->
case gleam_stdlib:string_pop_grapheme(After_dot) of
{ok, {Next_char, _}} when Next_char =:= <<"."/utf8>> ->
parse_integer(Digits, Remaining);
{ok, {Next_char@1, _}} ->
case is_digit(Next_char@1) of
true ->
{Decimal_digits, Final_remaining} = read_digits(
After_dot,
<<""/utf8>>
),
Float_str = <<<<Digits/binary, "."/utf8>>/binary,
Decimal_digits/binary>>,
parse_float(Float_str, Final_remaining);
false ->
parse_integer(Digits, Remaining)
end;
{error, nil} ->
parse_integer(Digits, Remaining)
end;
_ ->
parse_integer(Digits, Remaining)
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 303).
-spec is_letter(binary()) -> boolean().
is_letter(Char) ->
case gleam@string:to_utf_codepoints(Char) of
[Cp] ->
Code = gleam_stdlib:identity(Cp),
((Code >= 65) andalso (Code =< 90)) orelse ((Code >= 97) andalso (Code
=< 122));
_ ->
false
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 295).
-spec is_identifier_start(binary()) -> boolean().
is_identifier_start(Char) ->
is_letter(Char) orelse (Char =:= <<"_"/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 299).
-spec is_identifier_char(binary()) -> boolean().
is_identifier_char(Char) ->
(is_letter(Char) orelse is_digit(Char)) orelse (Char =:= <<"_"/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 317).
-spec read_identifier_loop(binary(), binary()) -> {binary(), binary()}.
read_identifier_loop(Source, Acc) ->
case gleam_stdlib:string_pop_grapheme(Source) of
{ok, {Char, Rest}} ->
case is_identifier_char(Char) of
true ->
read_identifier_loop(Rest, <<Acc/binary, Char/binary>>);
false ->
{Acc, Source}
end;
{error, nil} ->
{Acc, Source}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 313).
-spec read_identifier(binary()) -> {binary(), binary()}.
read_identifier(Source) ->
read_identifier_loop(Source, <<""/utf8>>).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 329).
-spec keyword_or_identifier(binary()) -> caffeine_lang@frontend@token:token().
keyword_or_identifier(Word) ->
case Word of
<<"Blueprints"/utf8>> ->
keyword_blueprints;
<<"Expects"/utf8>> ->
keyword_expects;
<<"for"/utf8>> ->
keyword_for;
<<"extends"/utf8>> ->
keyword_extends;
<<"Requires"/utf8>> ->
keyword_requires;
<<"Provides"/utf8>> ->
keyword_provides;
<<"in"/utf8>> ->
keyword_in;
<<"x"/utf8>> ->
keyword_x;
<<"String"/utf8>> ->
keyword_string;
<<"Integer"/utf8>> ->
keyword_integer;
<<"Float"/utf8>> ->
keyword_float;
<<"Boolean"/utf8>> ->
keyword_boolean;
<<"List"/utf8>> ->
keyword_list;
<<"Dict"/utf8>> ->
keyword_dict;
<<"Optional"/utf8>> ->
keyword_optional;
<<"Defaulted"/utf8>> ->
keyword_defaulted;
<<"true"/utf8>> ->
literal_true;
<<"false"/utf8>> ->
literal_false;
_ ->
{identifier, Word}
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 178).
-spec emit_token_n(
tokenizer_state(),
binary(),
integer(),
caffeine_lang@frontend@token:token(),
list(caffeine_lang@frontend@token:token())
) -> {ok, list(caffeine_lang@frontend@token:token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
emit_token_n(State, Rest, Len, Tok, Acc) ->
tokenize_loop(advance(State, Rest, Len), [Tok | Acc]).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 21).
-spec tokenize_loop(
tokenizer_state(),
list(caffeine_lang@frontend@token:token())
) -> {ok, list(caffeine_lang@frontend@token:token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
tokenize_loop(State, Acc) ->
case gleam_stdlib:string_pop_grapheme(erlang:element(2, State)) of
{error, nil} ->
{ok, [e_o_f | Acc]};
{ok, {Char, Rest}} ->
case Char of
<<"\n"/utf8>> ->
New_state = {tokenizer_state,
skip_empty_lines(Rest),
erlang:element(3, State) + 1,
1,
true},
tokenize_loop(New_state, [whitespace_newline | Acc]);
<<"\r"/utf8>> ->
tokenize_loop(advance(State, Rest, 1), Acc);
<<" "/utf8>> when erlang:element(5, State) ->
{Indent_count, Remaining} = count_indentation(
erlang:element(2, State),
0
),
tokenize_loop(
advance(State, Remaining, Indent_count),
[{whitespace_indent, Indent_count} | Acc]
);
<<"\t"/utf8>> when erlang:element(5, State) ->
{Indent_count, Remaining} = count_indentation(
erlang:element(2, State),
0
),
tokenize_loop(
advance(State, Remaining, Indent_count),
[{whitespace_indent, Indent_count} | Acc]
);
<<" "/utf8>> ->
tokenize_loop(advance(State, Rest, 1), Acc);
<<"\t"/utf8>> ->
tokenize_loop(advance(State, Rest, 2), Acc);
<<"#"/utf8>> ->
case gleam_stdlib:string_pop_grapheme(Rest) of
{ok, {<<"#"/utf8>>, After_hash}} ->
{Comment_text, Remaining@1} = read_until_newline(
After_hash
),
tokenize_loop(
advance(
State,
Remaining@1,
2 + string:length(Comment_text)
),
[{comment_section, Comment_text} | Acc]
);
_ ->
{Comment_text@1, Remaining@2} = read_until_newline(
Rest
),
tokenize_loop(
advance(
State,
Remaining@2,
1 + string:length(Comment_text@1)
),
[{comment_line, Comment_text@1} | Acc]
)
end;
<<"\""/utf8>> ->
case read_string(Rest, <<""/utf8>>) of
{ok, {Str_content, Remaining@3}} ->
tokenize_loop(
advance(
State,
Remaining@3,
2 + string:length(Str_content)
),
[{literal_string, Str_content} | Acc]
);
{error, nil} ->
{error,
{unterminated_string,
erlang:element(3, State),
erlang:element(4, State)}}
end;
<<"{"/utf8>> ->
emit_token(State, Rest, symbol_left_brace, Acc);
<<"}"/utf8>> ->
emit_token(State, Rest, symbol_right_brace, Acc);
<<"("/utf8>> ->
emit_token(State, Rest, symbol_left_paren, Acc);
<<")"/utf8>> ->
emit_token(State, Rest, symbol_right_paren, Acc);
<<"["/utf8>> ->
emit_token(State, Rest, symbol_left_bracket, Acc);
<<"]"/utf8>> ->
emit_token(State, Rest, symbol_right_bracket, Acc);
<<":"/utf8>> ->
emit_token(State, Rest, symbol_colon, Acc);
<<","/utf8>> ->
emit_token(State, Rest, symbol_comma, Acc);
<<"*"/utf8>> ->
emit_token(State, Rest, symbol_star, Acc);
<<"+"/utf8>> ->
emit_token(State, Rest, symbol_plus, Acc);
<<"|"/utf8>> ->
emit_token(State, Rest, symbol_pipe, Acc);
<<"="/utf8>> ->
emit_token(State, Rest, symbol_equals, Acc);
<<"."/utf8>> ->
case gleam_stdlib:string_pop_grapheme(Rest) of
{ok, {<<"."/utf8>>, After_dot}} ->
emit_token_n(
State,
After_dot,
2,
symbol_dot_dot,
Acc
);
_ ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"-"/utf8>> ->
case read_number(Rest, <<"-"/utf8>>) of
{ok, {Tok, Remaining@4, Len}} ->
tokenize_loop(
advance(State, Remaining@4, Len),
[Tok | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
<<"-"/utf8>>}}
end;
<<"0"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"1"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"2"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"3"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"4"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"5"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"6"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"7"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"8"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
<<"9"/utf8>> ->
case read_number(erlang:element(2, State), <<""/utf8>>) of
{ok, {Tok@1, Remaining@5, Len@1}} ->
tokenize_loop(
advance(State, Remaining@5, Len@1),
[Tok@1 | Acc]
);
{error, nil} ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end;
_ ->
case is_identifier_start(Char) of
true ->
{Word, Remaining@6} = read_identifier(
erlang:element(2, State)
),
tokenize_loop(
advance(State, Remaining@6, string:length(Word)),
[keyword_or_identifier(Word) | Acc]
);
false ->
{error,
{invalid_character,
erlang:element(3, State),
erlang:element(4, State),
Char}}
end
end
end.
-file("src/caffeine_lang/frontend/tokenizer.gleam", 15).
?DOC(" Tokenizes source text into a list of tokens.\n").
-spec tokenize(binary()) -> {ok, list(caffeine_lang@frontend@token:token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
tokenize(Source) ->
State = {tokenizer_state, Source, 1, 1, true},
_pipe = tokenize_loop(State, []),
gleam@result:map(_pipe, fun lists:reverse/1).
-file("src/caffeine_lang/frontend/tokenizer.gleam", 169).
-spec emit_token(
tokenizer_state(),
binary(),
caffeine_lang@frontend@token:token(),
list(caffeine_lang@frontend@token:token())
) -> {ok, list(caffeine_lang@frontend@token:token())} |
{error, caffeine_lang@frontend@tokenizer_error:tokenizer_error()}.
emit_token(State, Rest, Tok, Acc) ->
emit_token_n(State, Rest, 1, Tok, Acc).