glisp

-module(glisp@tokenizer).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).

-export([tokenize/1]).

-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.

-file("src/glisp/tokenizer.gleam", 14).
?DOC(
    " Tokenize a string into a list of tokens.\n"
    "\n"
    " This function splits the input string into tokens, ensuring that parentheses\n"
    " are treated as separate tokens.\n"
    "\n"
    " ## Examples\n"
    "\n"
    " ```gleam\n"
    " tokenize(\"(add 1 2)\") // -> [\"(\", \"add\", \"1\", \"2\", \")\"]\n"
    " ```\n"
).
-spec tokenize(binary()) -> list(binary()).
tokenize(Source) ->
    Spaced = gleam@string:replace(Source, <<"("/utf8>>, <<" ( "/utf8>>),
    Spaced@1 = gleam@string:replace(Spaced, <<")"/utf8>>, <<" ) "/utf8>>),
    _pipe = gleam@string:split(Spaced@1, <<" "/utf8>>),
    gleam@list:filter(_pipe, fun(S) -> S /= <<""/utf8>> end).