starfuzz

-module(starfuzz@tokenize).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/starfuzz/tokenize.gleam").
-export([characters/1, words/1, character_ngrams/2, word_ngrams/2]).
-export_type([tokenize_error/0]).

-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.

?MODULEDOC(
    " A module providing tokenization capabilities.\n"
    " Tokenization splits raw text into smaller components such as characters,\n"
    " words, or n-grams before similarity metrics or vector transformations.\n"
).

-type tokenize_error() :: {invalid_n, integer()}.

-file("src/starfuzz/tokenize.gleam", 16).
?DOC(" Splits the string into its constituent Unicode grapheme characters.\n").
-spec characters(binary()) -> list(binary()).
characters(Text) ->
    gleam@string:to_graphemes(Text).

-file("src/starfuzz/tokenize.gleam", 42).
-spec is_whitespace(binary()) -> boolean().
is_whitespace(Char) ->
    case Char of
        <<" "/utf8>> ->
            true;

        <<"\t"/utf8>> ->
            true;

        <<"\n"/utf8>> ->
            true;

        <<"\r"/utf8>> ->
            true;

        _ ->
            false
    end.

-file("src/starfuzz/tokenize.gleam", 21).
?DOC(" Splits the string into words, using whitespace characters as boundaries.\n").
-spec words(binary()) -> list(binary()).
words(Text) ->
    Chars = gleam@string:to_graphemes(Text),
    {Acc, Current_word} = gleam@list:fold(
        Chars,
        {[], []},
        fun(State, G) ->
            {Word_list, Current} = State,
            case is_whitespace(G) of
                true ->
                    case Current of
                        [] ->
                            {Word_list, []};

                        _ ->
                            {[gleam@string:join(
                                        lists:reverse(Current),
                                        <<""/utf8>>
                                    ) |
                                    Word_list],
                                []}
                    end;

                false ->
                    {Word_list, [G | Current]}
            end
        end
    ),
    Final_list = case Current_word of
        [] ->
            Acc;

        _ ->
            [gleam@string:join(lists:reverse(Current_word), <<""/utf8>>) | Acc]
    end,
    lists:reverse(Final_list).

-file("src/starfuzz/tokenize.gleam", 50).
?DOC(" Generates character n-grams of size `n` from the input text.\n").
-spec character_ngrams(binary(), integer()) -> {ok, list(binary())} |
    {error, tokenize_error()}.
character_ngrams(Text, N) ->
    Chars = characters(Text),
    case starfuzz@internal@ngram:ngrams(Chars, N) of
        {ok, Windows} ->
            {ok,
                gleam@list:map(
                    Windows,
                    fun(W) -> gleam@string:join(W, <<""/utf8>>) end
                )};

        {error, nil} ->
            {error, {invalid_n, N}}
    end.

-file("src/starfuzz/tokenize.gleam", 59).
?DOC(" Generates word n-grams of size `n` from the input text.\n").
-spec word_ngrams(binary(), integer()) -> {ok, list(list(binary()))} |
    {error, tokenize_error()}.
word_ngrams(Text, N) ->
    Wrds = words(Text),
    case starfuzz@internal@ngram:ngrams(Wrds, N) of
        {ok, Windows} ->
            {ok, Windows};

        {error, nil} ->
            {error, {invalid_n, N}}
    end.