Current section
Files
Jump to
Current section
Files
src/glugify@unicode.erl
-module(glugify@unicode).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/glugify/unicode.gleam").
-export([transliterate_text_with/3, transliterate_text/1, validate_ascii_only/1, validate_ascii_or_unicode/3]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
-file("src/glugify/unicode.gleam", 165).
-spec lookup_with_locale(binary(), glugify@locale:locale()) -> {ok, binary()} |
{error, nil}.
lookup_with_locale(Grapheme, Locale) ->
case glugify@internal@char_maps:lookup_locale(Grapheme, Locale) of
{ok, Replacement} ->
{ok, Replacement};
{error, nil} ->
glugify@internal@char_maps:lookup(Grapheme)
end.
-file("src/glugify/unicode.gleam", 203).
?DOC(
" Combining marks (U+0300-U+036F), Hebrew cantillation and niqqud\n"
" (U+0591-U+05C7), Arabic tashkeel (U+064B-U+0652), zero-width\n"
" joiners/non-joiners and bidirectional marks (U+200C-U+200F), and\n"
" variation selectors (U+FE00-U+FE0F) carry no slug content of their own.\n"
).
-spec is_ignorable_codepoint(integer()) -> boolean().
is_ignorable_codepoint(Code) ->
(((((Code >= 16#0300) andalso (Code =< 16#036F)) orelse ((Code >= 16#0591)
andalso (Code =< 16#05C7)))
orelse ((Code >= 16#064B) andalso (Code =< 16#0652)))
orelse ((Code >= 16#200C) andalso (Code =< 16#200F)))
orelse ((Code >= 16#FE00) andalso (Code =< 16#FE0F)).
-file("src/glugify/unicode.gleam", 176).
?DOC(
" Fallback for graphemes that have no direct mapping: decompose into\n"
" codepoints, drop combining marks and invisible joiners, and map each\n"
" remaining codepoint individually. Handles NFD (decomposed) input such\n"
" as \"e\" followed by U+0301. Codepoints with no mapping are stripped.\n"
).
-spec transliterate_codepoints(binary(), glugify@locale:locale()) -> binary().
transliterate_codepoints(Grapheme, Locale) ->
_pipe = Grapheme,
_pipe@1 = gleam@string:to_utf_codepoints(_pipe),
_pipe@2 = gleam@list:filter_map(
_pipe@1,
fun(Codepoint) ->
Code = gleam_stdlib:identity(Codepoint),
case is_ignorable_codepoint(Code) of
true ->
{error, nil};
false ->
Char = gleam_stdlib:utf_codepoint_list_to_string(
[Codepoint]
),
case lookup_with_locale(Char, Locale) of
{ok, Replacement} ->
{ok, Replacement};
{error, _} ->
case (Code >= 32) andalso (Code =< 126) of
true ->
{ok, Char};
false ->
{error, nil}
end
end
end
end
),
gleam@string:join(_pipe@2, <<""/utf8>>).
-file("src/glugify/unicode.gleam", 211).
-spec is_ascii_safe(binary()) -> boolean().
is_ascii_safe(Char) ->
case string:length(Char) of
1 ->
case gleam_stdlib:string_pop_grapheme(Char) of
{ok, {Grapheme, <<""/utf8>>}} ->
case gleam@string:to_utf_codepoints(Grapheme) of
[Codepoint] ->
Code = gleam_stdlib:identity(Codepoint),
(Code >= 32) andalso (Code =< 126);
_ ->
false
end;
_ ->
false
end;
_ ->
false
end.
-file("src/glugify/unicode.gleam", 130).
-spec transliterate_graphemes(
list(binary()),
glugify@locale:locale(),
list(binary()),
list(binary())
) -> {ok, list(binary())} | {error, glugify@errors:slugify_error()}.
transliterate_graphemes(Graphemes, Locale, Ignore, Acc) ->
case Graphemes of
[] ->
{ok, lists:reverse(Acc)};
[Grapheme | Rest] ->
case gleam@list:contains(Ignore, Grapheme) of
true ->
transliterate_graphemes(
Rest,
Locale,
Ignore,
[Grapheme | Acc]
);
false ->
case lookup_with_locale(Grapheme, Locale) of
{ok, Replacement} ->
transliterate_graphemes(
Rest,
Locale,
Ignore,
[Replacement | Acc]
);
{error, _} ->
case is_ascii_safe(Grapheme) of
true ->
transliterate_graphemes(
Rest,
Locale,
Ignore,
[Grapheme | Acc]
);
false ->
transliterate_graphemes(
Rest,
Locale,
Ignore,
[transliterate_codepoints(
Grapheme,
Locale
) |
Acc]
)
end
end
end
end.
-file("src/glugify/unicode.gleam", 49).
?DOC(
" Transliterates Unicode text to ASCII using locale-specific rules,\n"
" keeping the graphemes in `ignore` verbatim.\n"
"\n"
" Locale rules take precedence over the general tables, so with\n"
" `locale.German` the text \"Über\" becomes \"Ueber\" rather than \"Uber\".\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" import glugify/locale\n"
"\n"
" transliterate_text_with(\"Über München\", locale.German, [])\n"
" // -> Ok(\"Ueber Muenchen\")\n"
"\n"
" transliterate_text_with(\"嗨 hello\", locale.Default, [\"嗨\"])\n"
" // -> Ok(\"嗨 hello\")\n"
" ```\n"
).
-spec transliterate_text_with(binary(), glugify@locale:locale(), list(binary())) -> {ok,
binary()} |
{error, glugify@errors:slugify_error()}.
transliterate_text_with(Text, Locale, Ignore) ->
_pipe = Text,
_pipe@1 = gleam@string:to_graphemes(_pipe),
_pipe@2 = transliterate_graphemes(_pipe@1, Locale, Ignore, []),
gleam@result:map(
_pipe@2,
fun(_capture) -> gleam@string:join(_capture, <<""/utf8>>) end
).
-file("src/glugify/unicode.gleam", 28).
?DOC(
" Transliterates Unicode text to ASCII equivalents.\n"
"\n"
" This function converts accented characters, Cyrillic, Greek, typographic\n"
" punctuation and common symbols to their ASCII equivalents. Decomposed\n"
" (NFD) characters are handled by stripping combining marks and mapping the\n"
" base character. Characters with no known mapping (such as emoji) are\n"
" stripped from the output.\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" transliterate_text(\"café\")\n"
" // -> Ok(\"cafe\")\n"
"\n"
" transliterate_text(\"naïve\")\n"
" // -> Ok(\"naive\")\n"
"\n"
" transliterate_text(\"Résumé 🚀\")\n"
" // -> Ok(\"Resume \")\n"
" ```\n"
).
-spec transliterate_text(binary()) -> {ok, binary()} |
{error, glugify@errors:slugify_error()}.
transliterate_text(Text) ->
transliterate_text_with(Text, default, []).
-file("src/glugify/unicode.gleam", 115).
-spec validate_ascii_graphemes(list(binary()), list(binary())) -> {ok, nil} |
{error, glugify@errors:slugify_error()}.
validate_ascii_graphemes(Graphemes, Ignore) ->
case Graphemes of
[] ->
{ok, nil};
[Grapheme | Rest] ->
case is_ascii_safe(Grapheme) orelse gleam@list:contains(
Ignore,
Grapheme
) of
true ->
validate_ascii_graphemes(Rest, Ignore);
false ->
{error, {transliteration_failed, Grapheme}}
end
end.
-file("src/glugify/unicode.gleam", 75).
?DOC(
" Validates that text contains only ASCII characters.\n"
" \n"
" This function checks that all characters in the input are within\n"
" the printable ASCII range (32-126). Non-ASCII characters will cause\n"
" an error to be returned.\n"
" \n"
" ## Examples\n"
" \n"
" ```gleam\n"
" validate_ascii_only(\"hello world\")\n"
" // -> Ok(\"hello world\")\n"
" \n"
" validate_ascii_only(\"café\")\n"
" // -> Error(TransliterationFailed(\"é\"))\n"
" ```\n"
).
-spec validate_ascii_only(binary()) -> {ok, binary()} |
{error, glugify@errors:slugify_error()}.
validate_ascii_only(Text) ->
_pipe = Text,
_pipe@1 = gleam@string:to_graphemes(_pipe),
_pipe@2 = validate_ascii_graphemes(_pipe@1, []),
gleam@result:map(_pipe@2, fun(_) -> Text end).
-file("src/glugify/unicode.gleam", 100).
?DOC(
" Validates text based on Unicode allowance settings.\n"
"\n"
" If `allow_unicode` is `True`, all text is accepted.\n"
" If `allow_unicode` is `False`, only ASCII text and graphemes listed\n"
" in `ignore` are accepted.\n"
"\n"
" ## Examples\n"
"\n"
" ```gleam\n"
" validate_ascii_or_unicode(\"café\", True, [])\n"
" // -> Ok(\"café\")\n"
"\n"
" validate_ascii_or_unicode(\"café\", False, [])\n"
" // -> Error(TransliterationFailed(\"é\"))\n"
"\n"
" validate_ascii_or_unicode(\"café\", False, [\"é\"])\n"
" // -> Ok(\"café\")\n"
" ```\n"
).
-spec validate_ascii_or_unicode(binary(), boolean(), list(binary())) -> {ok,
binary()} |
{error, glugify@errors:slugify_error()}.
validate_ascii_or_unicode(Text, Allow_unicode, Ignore) ->
case Allow_unicode of
true ->
{ok, Text};
false ->
_pipe = Text,
_pipe@1 = gleam@string:to_graphemes(_pipe),
_pipe@2 = validate_ascii_graphemes(_pipe@1, Ignore),
gleam@result:map(_pipe@2, fun(_) -> Text end)
end.