Current section

Files

Jump to
localize lib localize collation normalizer.ex
Raw

lib/localize/collation/normalizer.ex

defmodule Localize.Collation.Normalizer do
@moduledoc """
Unicode NFD normalization for collation.
Delegates to Erlang's `:unicode` module.
"""
@doc """
Normalize a string to NFD (Canonical Decomposition) form.
Uses Erlang's `:unicode.characters_to_nfd_binary/1` followed by a canonical
reordering pass using the `unicode` package's CCC data to correct ordering
for newer Unicode codepoints.
### Arguments
* `string` - a UTF-8 binary string.
### Returns
The NFD-normalized string as a UTF-8 binary.
### Examples
iex> "café" |> Localize.Collation.Normalizer.nfd() |> String.to_charlist() |> length()
5
iex> Localize.Collation.Normalizer.nfd("e\u0301")
"e\u0301"
"""
@spec nfd(String.t()) :: String.t()
def nfd(string) when is_binary(string) do
string
|> :unicode.characters_to_nfd_binary()
|> String.to_charlist()
|> canonical_reorder()
|> List.to_string()
end
@doc """
Convert a string to a list of integer codepoints.
### Arguments
* `string` - a UTF-8 binary string.
### Returns
A list of integer codepoints.
### Examples
iex> Localize.Collation.Normalizer.to_codepoints("abc")
[97, 98, 99]
"""
@spec to_codepoints(String.t()) :: [non_neg_integer(), ...]
def to_codepoints(string) when is_binary(string) do
String.to_charlist(string)
end
@doc """
Optionally normalize a string and convert it to a list of integer codepoints.
### Arguments
* `string` - a UTF-8 binary string.
* `normalize?` - whether to apply NFD normalization first (default: `false`).
### Returns
A list of integer codepoints, optionally NFD-normalized.
### Examples
iex> Localize.Collation.Normalizer.normalize_to_codepoints("abc")
[97, 98, 99]
iex> Localize.Collation.Normalizer.normalize_to_codepoints("café", true)
[99, 97, 102, 101, 769]
"""
@spec normalize_to_codepoints(String.t(), boolean()) :: [non_neg_integer()]
def normalize_to_codepoints(string, normalize? \\ false) do
string
|> then(fn s -> if normalize?, do: nfd(s), else: s end)
|> to_codepoints()
end
defp canonical_reorder(codepoints) do
case reorder_pass(codepoints, false) do
{result, true} -> canonical_reorder(result)
{result, false} -> result
end
end
defp reorder_pass([a, b | rest], swapped) do
ccc_a = combining_class(a)
ccc_b = combining_class(b)
if ccc_a > ccc_b and ccc_b > 0 do
{tail, s} = reorder_pass([a | rest], true)
{[b | tail], s}
else
{tail, s} = reorder_pass([b | rest], swapped)
{[a | tail], s}
end
end
defp reorder_pass([cp], swapped), do: {[cp], swapped}
defp reorder_pass([], swapped), do: {[], swapped}
defp combining_class(cp) do
Localize.Collation.Unicode.combining_class(cp)
end
end