sqlode

-module(sqlode@lexer).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/sqlode/lexer.gleam").
-export([tokenize/2, tokens_to_string/2]).
-export_type([token_render_options/0, token/0]).

-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.

-type token_render_options() :: {token_render_options,
        boolean(),
        boolean(),
        gleam@option:option(sqlode@model:engine())}.

-type token() :: {keyword, binary()} |
    {ident, binary()} |
    {quoted_ident, binary()} |
    {string_lit, binary()} |
    {number_lit, binary()} |
    {placeholder, binary()} |
    {operator, binary()} |
    l_paren |
    r_paren |
    comma |
    semicolon |
    dot |
    star.

-file("src/sqlode/lexer.gleam", 288).
-spec skip_line_comment(list(binary())) -> list(binary()).
skip_line_comment(Input) ->
    case Input of
        [] ->
            [];

        [<<"\n"/utf8>> | Rest] ->
            Rest;

        [_ | Rest@1] ->
            skip_line_comment(Rest@1)
    end.

-file("src/sqlode/lexer.gleam", 300).
-spec skip_block_comment_loop(list(binary()), integer()) -> list(binary()).
skip_block_comment_loop(Input, Depth) ->
    case Input of
        [] ->
            [];

        _ ->
            case Depth =< 0 of
                true ->
                    Input;

                false ->
                    case Input of
                        [<<"*"/utf8>>, <<"/"/utf8>> | Rest] ->
                            skip_block_comment_loop(Rest, Depth - 1);

                        [<<"/"/utf8>>, <<"*"/utf8>> | Rest@1] ->
                            skip_block_comment_loop(Rest@1, Depth + 1);

                        [_ | Rest@2] ->
                            skip_block_comment_loop(Rest@2, Depth);

                        [] ->
                            []
                    end
            end
    end.

-file("src/sqlode/lexer.gleam", 296).
-spec skip_block_comment(list(binary())) -> list(binary()).
skip_block_comment(Input) ->
    skip_block_comment_loop(Input, 1).

-file("src/sqlode/lexer.gleam", 319).
-spec read_single_quoted_string(list(binary()), list(binary())) -> {binary(),
    list(binary())}.
read_single_quoted_string(Input, Acc) ->
    case Input of
        [] ->
            {begin
                    _pipe = Acc,
                    _pipe@1 = lists:reverse(_pipe),
                    erlang:list_to_binary(_pipe@1)
                end,
                []};

        [<<"'"/utf8>>, <<"'"/utf8>> | Rest] ->
            read_single_quoted_string(Rest, [<<"'"/utf8>> | Acc]);

        [<<"'"/utf8>> | Rest@1] ->
            {begin
                    _pipe@2 = Acc,
                    _pipe@3 = lists:reverse(_pipe@2),
                    erlang:list_to_binary(_pipe@3)
                end,
                Rest@1};

        [G | Rest@2] ->
            read_single_quoted_string(Rest@2, [G | Acc])
    end.

-file("src/sqlode/lexer.gleam", 365).
-spec read_dollar_tag_chars_lex(list(binary()), list(binary())) -> {ok,
        {binary(), list(binary())}} |
    {error, nil}.
read_dollar_tag_chars_lex(Chars, Acc) ->
    case Chars of
        [] ->
            {error, nil};

        [<<"$"/utf8>> | Rest] ->
            Tag = begin
                _pipe = Acc,
                _pipe@1 = lists:reverse(_pipe),
                erlang:list_to_binary(_pipe@1)
            end,
            {ok, {Tag, Rest}};

        [C | Rest@1] ->
            case sqlode@char_utils:is_alnum_or_underscore(C) of
                true ->
                    read_dollar_tag_chars_lex(Rest@1, [C | Acc]);

                false ->
                    {error, nil}
            end
    end.

-file("src/sqlode/lexer.gleam", 351).
-spec try_dollar_tag_lex(list(binary())) -> {ok, {binary(), list(binary())}} |
    {error, nil}.
try_dollar_tag_lex(Chars) ->
    case Chars of
        [<<"$"/utf8>> | Rest] ->
            {ok, {<<""/utf8>>, Rest}};

        [C | Rest@1] ->
            case sqlode@char_utils:is_alpha_or_underscore(C) of
                true ->
                    read_dollar_tag_chars_lex(Rest@1, [C]);

                false ->
                    {error, nil}
            end;

        [] ->
            {error, nil}
    end.

-file("src/sqlode/lexer.gleam", 391).
-spec match_tag_then_dollar_lex(list(binary()), list(binary())) -> {ok,
        list(binary())} |
    {error, nil}.
match_tag_then_dollar_lex(Chars, Tag_chars) ->
    case Tag_chars of
        [] ->
            case Chars of
                [<<"$"/utf8>> | Rest] ->
                    {ok, Rest};

                _ ->
                    {error, nil}
            end;

        [Expected | Tag_rest] ->
            case Chars of
                [Actual | Chars_rest] when Actual =:= Expected ->
                    match_tag_then_dollar_lex(Chars_rest, Tag_rest);

                _ ->
                    {error, nil}
            end
    end.

-file("src/sqlode/lexer.gleam", 383).
-spec try_match_closing_dollar_tag_lex(list(binary()), binary()) -> {ok,
        list(binary())} |
    {error, nil}.
try_match_closing_dollar_tag_lex(Chars, Tag) ->
    Tag_chars = gleam@string:to_graphemes(Tag),
    match_tag_then_dollar_lex(Chars, Tag_chars).

-file("src/sqlode/lexer.gleam", 335).
-spec read_dollar_quoted_string(list(binary()), binary(), list(binary())) -> {binary(),
    list(binary())}.
read_dollar_quoted_string(Input, Tag, Acc) ->
    case Input of
        [] ->
            {begin
                    _pipe = Acc,
                    _pipe@1 = lists:reverse(_pipe),
                    erlang:list_to_binary(_pipe@1)
                end,
                []};

        [<<"$"/utf8>> | Rest] ->
            case try_match_closing_dollar_tag_lex(Rest, Tag) of
                {ok, Remaining} ->
                    {begin
                            _pipe@2 = Acc,
                            _pipe@3 = lists:reverse(_pipe@2),
                            erlang:list_to_binary(_pipe@3)
                        end,
                        Remaining};

                {error, _} ->
                    read_dollar_quoted_string(Rest, Tag, [<<"$"/utf8>> | Acc])
            end;

        [G | Rest@1] ->
            read_dollar_quoted_string(Rest@1, Tag, [G | Acc])
    end.

-file("src/sqlode/lexer.gleam", 410).
-spec read_quoted(list(binary()), binary(), list(binary())) -> {binary(),
    list(binary())}.
read_quoted(Input, Closer, Acc) ->
    case Input of
        [] ->
            {begin
                    _pipe = Acc,
                    _pipe@1 = lists:reverse(_pipe),
                    erlang:list_to_binary(_pipe@1)
                end,
                []};

        [G, G2 | Rest] when ((G =:= Closer) andalso (G2 =:= Closer)) andalso (Closer =/= <<"]"/utf8>>) ->
            read_quoted(Rest, Closer, [G | Acc]);

        [G@1 | Rest@1] ->
            case G@1 =:= Closer of
                true ->
                    {begin
                            _pipe@2 = Acc,
                            _pipe@3 = lists:reverse(_pipe@2),
                            erlang:list_to_binary(_pipe@3)
                        end,
                        Rest@1};

                false ->
                    read_quoted(Rest@1, Closer, [G@1 | Acc])
            end
    end.

-file("src/sqlode/lexer.gleam", 433).
?DOC(
    " Detect SQL prefixed string literals: E'..', U&'..', B'..', X'..', N'..'.\n"
    " Returns the tokens after the prefix and a tag describing the kind, so\n"
    " the caller can read the body via read_single_quoted_string. Case\n"
    " insensitive on the prefix character.\n"
).
-spec detect_string_literal_prefix(binary(), list(binary())) -> gleam@option:option({list(binary()),
    binary()}).
detect_string_literal_prefix(G, Rest) ->
    case {G, Rest} of
        {P, [<<"'"/utf8>> | After]} when (P =:= <<"E"/utf8>>) orelse (P =:= <<"e"/utf8>>) ->
            {some, {After, <<"escape"/utf8>>}};

        {P@1, [<<"'"/utf8>> | After@1]} when (P@1 =:= <<"B"/utf8>>) orelse (P@1 =:= <<"b"/utf8>>) ->
            {some, {After@1, <<"bit"/utf8>>}};

        {P@2, [<<"'"/utf8>> | After@2]} when (P@2 =:= <<"X"/utf8>>) orelse (P@2 =:= <<"x"/utf8>>) ->
            {some, {After@2, <<"hex"/utf8>>}};

        {P@3, [<<"'"/utf8>> | After@3]} when (P@3 =:= <<"N"/utf8>>) orelse (P@3 =:= <<"n"/utf8>>) ->
            {some, {After@3, <<"national"/utf8>>}};

        {P@4, [<<"&"/utf8>>, <<"'"/utf8>> | After@4]} when (P@4 =:= <<"U"/utf8>>) orelse (P@4 =:= <<"u"/utf8>>) ->
            {some, {After@4, <<"unicode"/utf8>>}};

        {_, _} ->
            none
    end.

-file("src/sqlode/lexer.gleam", 463).
-spec is_hex_digit(binary()) -> boolean().
is_hex_digit(G) ->
    case G of
        <<"0"/utf8>> ->
            true;

        <<"1"/utf8>> ->
            true;

        <<"2"/utf8>> ->
            true;

        <<"3"/utf8>> ->
            true;

        <<"4"/utf8>> ->
            true;

        <<"5"/utf8>> ->
            true;

        <<"6"/utf8>> ->
            true;

        <<"7"/utf8>> ->
            true;

        <<"8"/utf8>> ->
            true;

        <<"9"/utf8>> ->
            true;

        <<"a"/utf8>> ->
            true;

        <<"b"/utf8>> ->
            true;

        <<"c"/utf8>> ->
            true;

        <<"d"/utf8>> ->
            true;

        <<"e"/utf8>> ->
            true;

        <<"f"/utf8>> ->
            true;

        <<"A"/utf8>> ->
            true;

        <<"B"/utf8>> ->
            true;

        <<"C"/utf8>> ->
            true;

        <<"D"/utf8>> ->
            true;

        <<"E"/utf8>> ->
            true;

        <<"F"/utf8>> ->
            true;

        _ ->
            false
    end.

-file("src/sqlode/lexer.gleam", 449).
-spec read_hex_digits(list(binary()), list(binary())) -> {binary(),
    list(binary())}.
read_hex_digits(Input, Acc) ->
    case Input of
        [G | Rest] ->
            case is_hex_digit(G) of
                true ->
                    read_hex_digits(Rest, [G | Acc]);

                false ->
                    {begin
                            _pipe = Acc,
                            _pipe@1 = lists:reverse(_pipe),
                            erlang:list_to_binary(_pipe@1)
                        end,
                        Input}
            end;

        [] ->
            {begin
                    _pipe@2 = Acc,
                    _pipe@3 = lists:reverse(_pipe@2),
                    erlang:list_to_binary(_pipe@3)
                end,
                []}
    end.

-file("src/sqlode/lexer.gleam", 474).
-spec read_word(list(binary()), list(binary())) -> {binary(), list(binary())}.
read_word(Input, Acc) ->
    case Input of
        [G | Rest] ->
            case sqlode@char_utils:is_alnum_or_underscore(G) of
                true ->
                    read_word(Rest, [G | Acc]);

                false ->
                    {begin
                            _pipe = Acc,
                            _pipe@1 = lists:reverse(_pipe),
                            erlang:list_to_binary(_pipe@1)
                        end,
                        Input}
            end;

        [] ->
            {begin
                    _pipe@2 = Acc,
                    _pipe@3 = lists:reverse(_pipe@2),
                    erlang:list_to_binary(_pipe@3)
                end,
                []}
    end.

-file("src/sqlode/lexer.gleam", 497).
-spec is_sqlode_marker(binary()) -> boolean().
is_sqlode_marker(Word) ->
    (gleam_stdlib:string_starts_with(Word, <<"__sqlode_param_"/utf8>>) orelse gleam_stdlib:string_starts_with(
        Word,
        <<"__sqlode_slice_"/utf8>>
    ))
    andalso gleam_stdlib:string_ends_with(Word, <<"__"/utf8>>).

-file("src/sqlode/lexer.gleam", 505).
-spec is_sql_keyword(binary()) -> boolean().
is_sql_keyword(Word) ->
    case Word of
        <<"select"/utf8>> ->
            true;

        <<"from"/utf8>> ->
            true;

        <<"where"/utf8>> ->
            true;

        <<"and"/utf8>> ->
            true;

        <<"or"/utf8>> ->
            true;

        <<"not"/utf8>> ->
            true;

        <<"in"/utf8>> ->
            true;

        <<"is"/utf8>> ->
            true;

        <<"null"/utf8>> ->
            true;

        <<"as"/utf8>> ->
            true;

        <<"on"/utf8>> ->
            true;

        <<"join"/utf8>> ->
            true;

        <<"inner"/utf8>> ->
            true;

        <<"left"/utf8>> ->
            true;

        <<"right"/utf8>> ->
            true;

        <<"full"/utf8>> ->
            true;

        <<"outer"/utf8>> ->
            true;

        <<"cross"/utf8>> ->
            true;

        <<"natural"/utf8>> ->
            true;

        <<"using"/utf8>> ->
            true;

        <<"order"/utf8>> ->
            true;

        <<"by"/utf8>> ->
            true;

        <<"group"/utf8>> ->
            true;

        <<"rollup"/utf8>> ->
            true;

        <<"cube"/utf8>> ->
            true;

        <<"sets"/utf8>> ->
            true;

        <<"having"/utf8>> ->
            true;

        <<"limit"/utf8>> ->
            true;

        <<"offset"/utf8>> ->
            true;

        <<"union"/utf8>> ->
            true;

        <<"intersect"/utf8>> ->
            true;

        <<"except"/utf8>> ->
            true;

        <<"all"/utf8>> ->
            true;

        <<"distinct"/utf8>> ->
            true;

        <<"insert"/utf8>> ->
            true;

        <<"into"/utf8>> ->
            true;

        <<"values"/utf8>> ->
            true;

        <<"update"/utf8>> ->
            true;

        <<"set"/utf8>> ->
            true;

        <<"delete"/utf8>> ->
            true;

        <<"create"/utf8>> ->
            true;

        <<"table"/utf8>> ->
            true;

        <<"view"/utf8>> ->
            true;

        <<"type"/utf8>> ->
            true;

        <<"index"/utf8>> ->
            true;

        <<"alter"/utf8>> ->
            true;

        <<"add"/utf8>> ->
            true;

        <<"column"/utf8>> ->
            true;

        <<"drop"/utf8>> ->
            true;

        <<"if"/utf8>> ->
            true;

        <<"exists"/utf8>> ->
            true;

        <<"primary"/utf8>> ->
            true;

        <<"key"/utf8>> ->
            true;

        <<"foreign"/utf8>> ->
            true;

        <<"references"/utf8>> ->
            true;

        <<"unique"/utf8>> ->
            true;

        <<"check"/utf8>> ->
            true;

        <<"default"/utf8>> ->
            true;

        <<"constraint"/utf8>> ->
            true;

        <<"cascade"/utf8>> ->
            true;

        <<"restrict"/utf8>> ->
            true;

        <<"no"/utf8>> ->
            true;

        <<"action"/utf8>> ->
            true;

        <<"autoincrement"/utf8>> ->
            true;

        <<"serial"/utf8>> ->
            true;

        <<"bigserial"/utf8>> ->
            true;

        <<"smallserial"/utf8>> ->
            true;

        <<"returning"/utf8>> ->
            true;

        <<"case"/utf8>> ->
            true;

        <<"when"/utf8>> ->
            true;

        <<"then"/utf8>> ->
            true;

        <<"else"/utf8>> ->
            true;

        <<"end"/utf8>> ->
            true;

        <<"cast"/utf8>> ->
            true;

        <<"between"/utf8>> ->
            true;

        <<"like"/utf8>> ->
            true;

        <<"ilike"/utf8>> ->
            true;

        <<"with"/utf8>> ->
            true;

        <<"recursive"/utf8>> ->
            true;

        <<"temporary"/utf8>> ->
            true;

        <<"temp"/utf8>> ->
            true;

        <<"unlogged"/utf8>> ->
            true;

        <<"true"/utf8>> ->
            true;

        <<"false"/utf8>> ->
            true;

        <<"asc"/utf8>> ->
            true;

        <<"desc"/utf8>> ->
            true;

        <<"nulls"/utf8>> ->
            true;

        <<"first"/utf8>> ->
            true;

        <<"last"/utf8>> ->
            true;

        <<"over"/utf8>> ->
            true;

        <<"partition"/utf8>> ->
            true;

        <<"window"/utf8>> ->
            true;

        <<"row"/utf8>> ->
            true;

        <<"rows"/utf8>> ->
            true;

        <<"range"/utf8>> ->
            true;

        <<"preceding"/utf8>> ->
            true;

        <<"following"/utf8>> ->
            true;

        <<"current"/utf8>> ->
            true;

        <<"unbounded"/utf8>> ->
            true;

        <<"enum"/utf8>> ->
            true;

        <<"replace"/utf8>> ->
            true;

        <<"conflict"/utf8>> ->
            true;

        <<"do"/utf8>> ->
            true;

        <<"nothing"/utf8>> ->
            true;

        <<"begin"/utf8>> ->
            true;

        <<"commit"/utf8>> ->
            true;

        <<"rollback"/utf8>> ->
            true;

        <<"transaction"/utf8>> ->
            true;

        <<"for"/utf8>> ->
            true;

        <<"each"/utf8>> ->
            true;

        <<"trigger"/utf8>> ->
            true;

        <<"execute"/utf8>> ->
            true;

        <<"procedure"/utf8>> ->
            true;

        <<"function"/utf8>> ->
            true;

        <<"returns"/utf8>> ->
            true;

        <<"language"/utf8>> ->
            true;

        <<"volatile"/utf8>> ->
            true;

        <<"stable"/utf8>> ->
            true;

        <<"immutable"/utf8>> ->
            true;

        <<"security"/utf8>> ->
            true;

        <<"definer"/utf8>> ->
            true;

        <<"invoker"/utf8>> ->
            true;

        <<"grant"/utf8>> ->
            true;

        <<"revoke"/utf8>> ->
            true;

        <<"to"/utf8>> ->
            true;

        <<"schema"/utf8>> ->
            true;

        <<"database"/utf8>> ->
            true;

        <<"extension"/utf8>> ->
            true;

        <<"only"/utf8>> ->
            true;

        <<"lateral"/utf8>> ->
            true;

        <<"any"/utf8>> ->
            true;

        <<"some"/utf8>> ->
            true;

        <<"array"/utf8>> ->
            true;

        <<"of"/utf8>> ->
            true;

        <<"collate"/utf8>> ->
            true;

        _ ->
            false
    end.

-file("src/sqlode/lexer.gleam", 485).
-spec classify_word(binary()) -> token().
classify_word(Word) ->
    Lowered = string:lowercase(Word),
    case is_sqlode_marker(Word) of
        true ->
            {placeholder, Word};

        false ->
            case is_sql_keyword(Lowered) of
                true ->
                    {keyword, Lowered};

                false ->
                    {ident, Word}
            end
    end.

-file("src/sqlode/lexer.gleam", 648).
-spec read_number(list(binary()), list(binary())) -> {binary(), list(binary())}.
read_number(Input, Acc) ->
    case Input of
        [G | Rest] ->
            case sqlode@char_utils:is_digit(G) orelse (G =:= <<"."/utf8>>) of
                true ->
                    read_number(Rest, [G | Acc]);

                false ->
                    case (G =:= <<"e"/utf8>>) orelse (G =:= <<"E"/utf8>>) of
                        true ->
                            case Rest of
                                [<<"+"/utf8>> | _] ->
                                    read_number(
                                        gleam@list:drop(Rest, 1),
                                        [case Rest of
                                                [S | _] ->
                                                    S;

                                                _ ->
                                                    <<""/utf8>>
                                            end, G | Acc]
                                    );

                                [<<"-"/utf8>> | _] ->
                                    read_number(
                                        gleam@list:drop(Rest, 1),
                                        [case Rest of
                                                [S | _] ->
                                                    S;

                                                _ ->
                                                    <<""/utf8>>
                                            end, G | Acc]
                                    );

                                _ ->
                                    read_number(Rest, [G | Acc])
                            end;

                        false ->
                            {begin
                                    _pipe = Acc,
                                    _pipe@1 = lists:reverse(_pipe),
                                    erlang:list_to_binary(_pipe@1)
                                end,
                                Input}
                    end
            end;

        [] ->
            {begin
                    _pipe@2 = Acc,
                    _pipe@3 = lists:reverse(_pipe@2),
                    erlang:list_to_binary(_pipe@3)
                end,
                []}
    end.

-file("src/sqlode/lexer.gleam", 680).
-spec read_placeholder_or_ident(list(binary())) -> {token(), list(binary())}.
read_placeholder_or_ident(Input) ->
    case Input of
        [<<"$"/utf8>> | Rest] ->
            {Word, Remaining} = read_word(Rest, [<<"$"/utf8>>]),
            {{placeholder, Word}, Remaining};

        _ ->
            {{operator, <<"$"/utf8>>}, Input}
    end.

-file("src/sqlode/lexer.gleam", 690).
-spec read_placeholder_question(list(binary()), list(binary())) -> {binary(),
    list(binary())}.
read_placeholder_question(Input, Acc) ->
    case Input of
        [G | Rest] ->
            case sqlode@char_utils:is_digit(G) of
                true ->
                    read_placeholder_question(Rest, [G | Acc]);

                false ->
                    {begin
                            _pipe = Acc,
                            _pipe@1 = lists:reverse(_pipe),
                            erlang:list_to_binary(_pipe@1)
                        end,
                        Input}
            end;

        [] ->
            {begin
                    _pipe@2 = Acc,
                    _pipe@3 = lists:reverse(_pipe@2),
                    erlang:list_to_binary(_pipe@3)
                end,
                []}
    end.

-file("src/sqlode/lexer.gleam", 706).
-spec read_operator(list(binary())) -> {binary(), list(binary())}.
read_operator(Input) ->
    case Input of
        [<<"<"/utf8>>, <<">"/utf8>> | Rest] ->
            {<<"<>"/utf8>>, Rest};

        [<<"<"/utf8>>, <<"="/utf8>> | Rest@1] ->
            {<<"<="/utf8>>, Rest@1};

        [<<"<"/utf8>>, <<"@"/utf8>> | Rest@2] ->
            {<<"<@"/utf8>>, Rest@2};

        [<<">"/utf8>>, <<"="/utf8>> | Rest@3] ->
            {<<">="/utf8>>, Rest@3};

        [<<"!"/utf8>>, <<"="/utf8>> | Rest@4] ->
            {<<"!="/utf8>>, Rest@4};

        [<<"|"/utf8>>, <<"|"/utf8>> | Rest@5] ->
            {<<"||"/utf8>>, Rest@5};

        [<<"&"/utf8>>, <<"&"/utf8>> | Rest@6] ->
            {<<"&&"/utf8>>, Rest@6};

        [<<"-"/utf8>>, <<">"/utf8>>, <<">"/utf8>> | Rest@7] ->
            {<<"->>"/utf8>>, Rest@7};

        [<<"-"/utf8>>, <<">"/utf8>> | Rest@8] ->
            {<<"->"/utf8>>, Rest@8};

        [G | Rest@9] ->
            {G, Rest@9};

        [] ->
            {<<""/utf8>>, []}
    end.

-file("src/sqlode/lexer.gleam", 56).
-spec do_tokenize(list(binary()), sqlode@model:engine(), list(token())) -> list(token()).
do_tokenize(Input, Engine, Acc) ->
    case Input of
        [] ->
            Acc;

        [G | Rest] ->
            case G of
                <<" "/utf8>> ->
                    do_tokenize(Rest, Engine, Acc);

                <<"\t"/utf8>> ->
                    do_tokenize(Rest, Engine, Acc);

                <<"\n"/utf8>> ->
                    do_tokenize(Rest, Engine, Acc);

                <<"\r"/utf8>> ->
                    do_tokenize(Rest, Engine, Acc);

                <<"-"/utf8>> ->
                    case Rest of
                        [<<"-"/utf8>> | After_dash] ->
                            Remaining = skip_line_comment(After_dash),
                            do_tokenize(Remaining, Engine, Acc);

                        _ ->
                            {Op, Remaining@1} = read_operator([G | Rest]),
                            do_tokenize(
                                Remaining@1,
                                Engine,
                                [{operator, Op} | Acc]
                            )
                    end;

                <<"/"/utf8>> ->
                    case Rest of
                        [<<"*"/utf8>> | After_star] ->
                            Remaining@2 = skip_block_comment(After_star),
                            do_tokenize(Remaining@2, Engine, Acc);

                        _ ->
                            do_tokenize(
                                Rest,
                                Engine,
                                [{operator, <<"/"/utf8>>} | Acc]
                            )
                    end;

                <<"#"/utf8>> ->
                    case Engine of
                        my_s_q_l ->
                            Remaining@3 = skip_line_comment(Rest),
                            do_tokenize(Remaining@3, Engine, Acc);

                        _ ->
                            case Rest of
                                [<<">"/utf8>>, <<">"/utf8>> | After] ->
                                    do_tokenize(
                                        After,
                                        Engine,
                                        [{operator, <<"#>>"/utf8>>} | Acc]
                                    );

                                [<<">"/utf8>> | After@1] ->
                                    do_tokenize(
                                        After@1,
                                        Engine,
                                        [{operator, <<"#>"/utf8>>} | Acc]
                                    );

                                _ ->
                                    {Op@1, Remaining@4} = read_operator(
                                        [G | Rest]
                                    ),
                                    do_tokenize(
                                        Remaining@4,
                                        Engine,
                                        [{operator, Op@1} | Acc]
                                    )
                            end
                    end;

                <<"'"/utf8>> ->
                    {Content, Remaining@5} = read_single_quoted_string(Rest, []),
                    do_tokenize(
                        Remaining@5,
                        Engine,
                        [{string_lit, Content} | Acc]
                    );

                <<"$"/utf8>> ->
                    case Engine of
                        postgre_s_q_l ->
                            case try_dollar_tag_lex(Rest) of
                                {ok, {Tag, After_tag}} ->
                                    {Content@1, Remaining@6} = read_dollar_quoted_string(
                                        After_tag,
                                        Tag,
                                        []
                                    ),
                                    do_tokenize(
                                        Remaining@6,
                                        Engine,
                                        [{string_lit, Content@1} | Acc]
                                    );

                                {error, _} ->
                                    {Token, Remaining@7} = read_placeholder_or_ident(
                                        [<<"$"/utf8>> | Rest]
                                    ),
                                    do_tokenize(
                                        Remaining@7,
                                        Engine,
                                        [Token | Acc]
                                    )
                            end;

                        _ ->
                            {Token@1, Remaining@8} = read_placeholder_or_ident(
                                [<<"$"/utf8>> | Rest]
                            ),
                            do_tokenize(Remaining@8, Engine, [Token@1 | Acc])
                    end;

                <<"\""/utf8>> ->
                    case Engine of
                        my_s_q_l ->
                            {Content@2, Remaining@9} = read_quoted(
                                Rest,
                                <<"\""/utf8>>,
                                []
                            ),
                            do_tokenize(
                                Remaining@9,
                                Engine,
                                [{string_lit, Content@2} | Acc]
                            );

                        _ ->
                            {Content@3, Remaining@10} = read_quoted(
                                Rest,
                                <<"\""/utf8>>,
                                []
                            ),
                            do_tokenize(
                                Remaining@10,
                                Engine,
                                [{quoted_ident, Content@3} | Acc]
                            )
                    end;

                <<"`"/utf8>> ->
                    {Content@4, Remaining@11} = read_quoted(
                        Rest,
                        <<"`"/utf8>>,
                        []
                    ),
                    do_tokenize(
                        Remaining@11,
                        Engine,
                        [{quoted_ident, Content@4} | Acc]
                    );

                <<"["/utf8>> ->
                    case Engine of
                        s_q_lite ->
                            {Content@5, Remaining@12} = read_quoted(
                                Rest,
                                <<"]"/utf8>>,
                                []
                            ),
                            do_tokenize(
                                Remaining@12,
                                Engine,
                                [{quoted_ident, Content@5} | Acc]
                            );

                        _ ->
                            {Op@2, Remaining@13} = read_operator([G | Rest]),
                            do_tokenize(
                                Remaining@13,
                                Engine,
                                [{operator, Op@2} | Acc]
                            )
                    end;

                <<"("/utf8>> ->
                    do_tokenize(Rest, Engine, [l_paren | Acc]);

                <<")"/utf8>> ->
                    do_tokenize(Rest, Engine, [r_paren | Acc]);

                <<","/utf8>> ->
                    do_tokenize(Rest, Engine, [comma | Acc]);

                <<";"/utf8>> ->
                    do_tokenize(Rest, Engine, [semicolon | Acc]);

                <<"*"/utf8>> ->
                    do_tokenize(Rest, Engine, [star | Acc]);

                <<"."/utf8>> ->
                    case Rest of
                        [Next | _] ->
                            case sqlode@char_utils:is_digit(Next) of
                                true ->
                                    {Num, Remaining@14} = read_number(Rest, [G]),
                                    do_tokenize(
                                        Remaining@14,
                                        Engine,
                                        [{number_lit, Num} | Acc]
                                    );

                                false ->
                                    do_tokenize(Rest, Engine, [dot | Acc])
                            end;

                        _ ->
                            do_tokenize(Rest, Engine, [dot | Acc])
                    end;

                <<"?"/utf8>> ->
                    case Rest of
                        [<<"|"/utf8>> | After@2] ->
                            do_tokenize(
                                After@2,
                                Engine,
                                [{operator, <<"?|"/utf8>>} | Acc]
                            );

                        [<<"&"/utf8>> | After@3] ->
                            do_tokenize(
                                After@3,
                                Engine,
                                [{operator, <<"?&"/utf8>>} | Acc]
                            );

                        _ ->
                            {Ph, Remaining@15} = read_placeholder_question(
                                Rest,
                                [G]
                            ),
                            do_tokenize(
                                Remaining@15,
                                Engine,
                                [{placeholder, Ph} | Acc]
                            )
                    end;

                <<":"/utf8>> ->
                    case Rest of
                        [<<":"/utf8>> | After_colon] ->
                            do_tokenize(
                                After_colon,
                                Engine,
                                [{operator, <<"::"/utf8>>} | Acc]
                            );

                        [Next@1 | _] ->
                            case sqlode@char_utils:is_alpha_or_underscore(
                                Next@1
                            ) of
                                true ->
                                    {Ph@1, Remaining@16} = read_word(Rest, [G]),
                                    do_tokenize(
                                        Remaining@16,
                                        Engine,
                                        [{placeholder, Ph@1} | Acc]
                                    );

                                false ->
                                    do_tokenize(
                                        Rest,
                                        Engine,
                                        [{operator, <<":"/utf8>>} | Acc]
                                    )
                            end;

                        _ ->
                            do_tokenize(
                                Rest,
                                Engine,
                                [{operator, <<":"/utf8>>} | Acc]
                            )
                    end;

                <<"@"/utf8>> ->
                    case Rest of
                        [<<">"/utf8>> | After@4] ->
                            do_tokenize(
                                After@4,
                                Engine,
                                [{operator, <<"@>"/utf8>>} | Acc]
                            );

                        [Next@2 | _] ->
                            case sqlode@char_utils:is_alpha_or_underscore(
                                Next@2
                            ) of
                                true ->
                                    {Ph@2, Remaining@17} = read_word(Rest, [G]),
                                    do_tokenize(
                                        Remaining@17,
                                        Engine,
                                        [{placeholder, Ph@2} | Acc]
                                    );

                                false ->
                                    do_tokenize(
                                        Rest,
                                        Engine,
                                        [{operator, <<"@"/utf8>>} | Acc]
                                    )
                            end;

                        _ ->
                            do_tokenize(
                                Rest,
                                Engine,
                                [{operator, <<"@"/utf8>>} | Acc]
                            )
                    end;

                _ ->
                    case sqlode@char_utils:is_digit(G) of
                        true ->
                            case {G, Rest} of
                                {<<"0"/utf8>>, [X | Hex_rest]} when (X =:= <<"x"/utf8>>) orelse (X =:= <<"X"/utf8>>) ->
                                    {Digits, Remaining@18} = read_hex_digits(
                                        Hex_rest,
                                        []
                                    ),
                                    do_tokenize(
                                        Remaining@18,
                                        Engine,
                                        [{number_lit,
                                                <<<<"0"/utf8, X/binary>>/binary,
                                                    Digits/binary>>} |
                                            Acc]
                                    );

                                {_, _} ->
                                    {Num@1, Remaining@19} = read_number(
                                        Rest,
                                        [G]
                                    ),
                                    do_tokenize(
                                        Remaining@19,
                                        Engine,
                                        [{number_lit, Num@1} | Acc]
                                    )
                            end;

                        false ->
                            case sqlode@char_utils:is_alpha_or_underscore(G) of
                                true ->
                                    case detect_string_literal_prefix(G, Rest) of
                                        {some, {After_prefix, _}} ->
                                            {Content@6, Remaining@20} = read_single_quoted_string(
                                                After_prefix,
                                                []
                                            ),
                                            do_tokenize(
                                                Remaining@20,
                                                Engine,
                                                [{string_lit, Content@6} | Acc]
                                            );

                                        none ->
                                            {Word, Remaining@21} = read_word(
                                                Rest,
                                                [G]
                                            ),
                                            Token@2 = classify_word(Word),
                                            do_tokenize(
                                                Remaining@21,
                                                Engine,
                                                [Token@2 | Acc]
                                            )
                                    end;

                                false ->
                                    {Op@3, Remaining@22} = read_operator(
                                        [G | Rest]
                                    ),
                                    do_tokenize(
                                        Remaining@22,
                                        Engine,
                                        [{operator, Op@3} | Acc]
                                    )
                            end
                    end
            end
    end.

-file("src/sqlode/lexer.gleam", 50).
?DOC(
    " Tokenize a SQL string into a list of tokens.\n"
    " Comments are stripped. String literals and quoted identifiers are preserved\n"
    " as single tokens with their content.\n"
).
-spec tokenize(binary(), sqlode@model:engine()) -> list(token()).
tokenize(Sql, Engine) ->
    Graphemes = gleam@string:to_graphemes(Sql),
    _pipe = do_tokenize(Graphemes, Engine, []),
    lists:reverse(_pipe).

-file("src/sqlode/lexer.gleam", 759).
-spec token_to_string(token(), token_render_options()) -> binary().
token_to_string(Token, Options) ->
    case Token of
        {keyword, K} ->
            case erlang:element(2, Options) of
                true ->
                    string:uppercase(K);

                false ->
                    K
            end;

        {ident, Name} ->
            Name;

        {quoted_ident, Name@1} ->
            case erlang:element(3, Options) of
                true ->
                    case erlang:element(4, Options) of
                        {some, my_s_q_l} ->
                            <<<<"`"/utf8, Name@1/binary>>/binary, "`"/utf8>>;

                        _ ->
                            <<<<"\""/utf8, Name@1/binary>>/binary, "\""/utf8>>
                    end;

                false ->
                    Name@1
            end;

        {string_lit, Value} ->
            case erlang:element(3, Options) of
                true ->
                    <<<<"'"/utf8,
                            (gleam@string:replace(
                                Value,
                                <<"'"/utf8>>,
                                <<"''"/utf8>>
                            ))/binary>>/binary,
                        "'"/utf8>>;

                false ->
                    <<<<"'"/utf8, Value/binary>>/binary, "'"/utf8>>
            end;

        {number_lit, N} ->
            N;

        {placeholder, P} ->
            P;

        {operator, Op} ->
            Op;

        l_paren ->
            <<"("/utf8>>;

        r_paren ->
            <<")"/utf8>>;

        comma ->
            <<","/utf8>>;

        semicolon ->
            <<";"/utf8>>;

        dot ->
            <<"."/utf8>>;

        star ->
            <<"*"/utf8>>
    end.

-file("src/sqlode/lexer.gleam", 734).
-spec tokens_to_string_loop(
    list(token()),
    list(binary()),
    token_render_options()
) -> list(binary()).
tokens_to_string_loop(Tokens, Acc, Options) ->
    case Tokens of
        [] ->
            Acc;

        [Token | Rest] ->
            S = token_to_string(Token, Options),
            With_space = case {Acc, Token} of
                {_, comma} ->
                    [S | Acc];

                {_, semicolon} ->
                    [S | Acc];

                {_, l_paren} ->
                    [S | Acc];

                {_, r_paren} ->
                    [S | Acc];

                {_, dot} ->
                    [S | Acc];

                {_, star} ->
                    [S | Acc];

                {[<<"("/utf8>> | _], _} ->
                    [S | Acc];

                {[<<"."/utf8>> | _], _} ->
                    [S | Acc];

                {_, {operator, <<"["/utf8>>}} ->
                    [S | Acc];

                {[<<"]"/utf8>> | _], _} ->
                    [S | Acc];

                {[<<"["/utf8>> | _], _} ->
                    [S | Acc];

                {[], _} ->
                    [S];

                {_, _} ->
                    [S, <<" "/utf8>> | Acc]
            end,
            tokens_to_string_loop(Rest, With_space, Options)
    end.

-file("src/sqlode/lexer.gleam", 725).
?DOC(" Render a list of tokens back to a SQL string with smart spacing.\n").
-spec tokens_to_string(list(token()), token_render_options()) -> binary().
tokens_to_string(Tokens, Options) ->
    _pipe = tokens_to_string_loop(Tokens, [], Options),
    _pipe@1 = lists:reverse(_pipe),
    erlang:list_to_binary(_pipe@1).