Current section
Files
Jump to
Current section
Files
lib/cldr/collation/table.ex
defmodule Cldr.Collation.Table do
@moduledoc """
Persistent-term-backed collation element table.
Stores the collation table parsed from `FractionalUCA.txt` for fast
concurrent lookups using `:persistent_term`, which provides zero-copy
reads for data that is written once and never modified.
Handles both single codepoint mappings and contractions
(multi-codepoint sequences).
"""
use GenServer
alias Cldr.Collation.Element
alias Cldr.Collation.Table.Parser
@table_name :collation_table
@contractions_table :collation_contractions
@fractional_keys "FractionalUCA.txt"
@doc """
Ensure the collation table is loaded.
Loads the `FractionalUCA.txt` data file on first call. Subsequent calls
are no-ops.
### Returns
* `:ok` - the table is loaded and ready for lookups.
### Examples
iex> Cldr.Collation.Table.ensure_loaded()
:ok
"""
@spec ensure_loaded() :: :ok
def ensure_loaded do
case :persistent_term.get(@table_name, nil) do
nil -> GenServer.call(__MODULE__, :load, :infinity)
_map -> :ok
end
end
@doc """
Look up collation elements for a codepoint or codepoint sequence.
### Arguments
* `codepoint` - a single integer codepoint, or a list of integer.
codepoints (contraction). Lists are internally converted to the
compact key format (integer for single, tuple for multi).
### Returns
* `{:ok, [%Cldr.Collation.Element{}]}` - the collation elements for.
the entry.
* `:unmapped` - no entry found in the table.
### Examples
iex> Cldr.Collation.Table.ensure_loaded()
iex> {:ok, elements} = Cldr.Collation.Table.lookup(0x0041)
iex> Cldr.Collation.Element.primary(hd(elements)) > 0
true
iex> Cldr.Collation.Table.ensure_loaded()
iex> Cldr.Collation.Table.lookup(0x10FFFF)
:unmapped
"""
@spec lookup(non_neg_integer() | [non_neg_integer()]) :: {:ok, [Element.t()]} | :unmapped
def lookup(codepoint) when is_integer(codepoint) do
table = :persistent_term.get(@table_name)
case Map.get(table, codepoint) do
nil -> :unmapped
elements -> {:ok, elements}
end
end
def lookup(codepoints) when is_list(codepoints) do
table = :persistent_term.get(@table_name)
key = Parser.codepoints_to_key(codepoints)
case Map.get(table, key) do
nil -> :unmapped
elements -> {:ok, elements}
end
end
@doc """
Check if a codepoint begins any multi-codepoint contraction.
### Arguments
* `codepoint` - an integer codepoint to check.
### Returns
* A list of contraction lengths that start with this codepoint, or `[]` if.
this codepoint does not begin any contractions.
### Examples
iex> Cldr.Collation.Table.ensure_loaded()
iex> lengths = Cldr.Collation.Table.contraction_starters(0x006C)
iex> is_list(lengths)
true
"""
@spec contraction_starters(non_neg_integer()) :: [pos_integer()]
def contraction_starters(codepoint) do
contractions = :persistent_term.get(@contractions_table)
Map.get(contractions, codepoint, [])
end
@doc """
Find the longest matching entry for the given codepoint sequence.
Tries contractions from longest to shortest, falling back to a single
codepoint lookup.
### Arguments
* `codepoints` - a list of integer codepoints to match against.
### Returns
* `{matched_cps, elements, remaining_cps}` - a successful match with the.
matched codepoints, their collation elements, and the remaining unprocessed tail.
* `{:unmapped, codepoint, remaining_cps}` - the first codepoint has no table entry.
* `:done` - the input list is empty.
### Examples
iex> Cldr.Collation.Table.ensure_loaded()
iex> {matched, _elements, rest} = Cldr.Collation.Table.longest_match([0x0041, 0x0042])
iex> matched
[65]
iex> rest
[66]
"""
@spec longest_match([non_neg_integer()]) ::
{[non_neg_integer()], [Element.t()], [non_neg_integer()]}
| {:unmapped, non_neg_integer(), [non_neg_integer()]}
| :done
def longest_match([cp | rest] = _codepoints) do
# Check if this codepoint starts any contractions
lengths = contraction_starters(cp)
if lengths == [] do
# No contractions, just look up single codepoint
case lookup(cp) do
{:ok, elements} -> {[cp], elements, rest}
:unmapped -> {:unmapped, cp, rest}
end
else
# Try contractions from longest to shortest
max_len = Enum.max(lengths)
available = [cp | Enum.take(rest, max_len - 1)]
result =
max_len..2//-1
|> Enum.reduce_while(nil, fn len, _acc ->
if len <= length(available) do
candidate = Enum.take(available, len)
case lookup(candidate) do
{:ok, elements} ->
remaining = Enum.drop([cp | rest], len)
{:halt, {candidate, elements, remaining}}
:unmapped ->
{:cont, nil}
end
else
{:cont, nil}
end
end)
case result do
nil ->
# No contraction matched, try single codepoint
case lookup(cp) do
{:ok, elements} -> {[cp], elements, rest}
:unmapped -> {:unmapped, cp, rest}
end
match ->
match
end
end
end
# Note: longest_match returns matched codepoints as lists for downstream compatibility.
# The lookup functions handle the list-to-key conversion internally.
def longest_match([]), do: :done
@doc """
Look up collation elements with a tailoring overlay checked first.
### Arguments
* `codepoints` - a single integer codepoint, or a list of integer codepoints.
* `overlay` - a map of `%{key => [%Cldr.Collation.Element{}]}` tailoring.
entries, where keys are integers (single CP) or tuples (contractions).
### Returns
* Same as `lookup/1`, but checks the overlay map before falling back.
to the root table.
### Examples
iex> Cldr.Collation.Table.ensure_loaded()
iex> overlay = %{0x0041 => [{0xFFFF, 0x0020, 0x0008, false}]}
iex> {:ok, [elem]} = Cldr.Collation.Table.lookup_with_overlay(0x0041, overlay)
iex> Cldr.Collation.Element.primary(elem)
0xFFFF
"""
@spec lookup_with_overlay(non_neg_integer() | [non_neg_integer()], map() | nil) ::
{:ok, [Element.t()]} | :unmapped
def lookup_with_overlay(codepoint, overlay) when is_integer(codepoint) do
lookup_with_overlay_key(codepoint, overlay)
end
def lookup_with_overlay(codepoints, overlay) when is_list(codepoints) do
key = Parser.codepoints_to_key(codepoints)
lookup_with_overlay_key(key, overlay)
end
defp lookup_with_overlay_key(key, nil) do
table = :persistent_term.get(@table_name)
case Map.get(table, key) do
nil -> :unmapped
elements -> {:ok, elements}
end
end
defp lookup_with_overlay_key(key, overlay) when is_map(overlay) do
case Map.get(overlay, key) do
nil ->
table = :persistent_term.get(@table_name)
case Map.get(table, key) do
nil -> :unmapped
elements -> {:ok, elements}
end
elements ->
{:ok, elements}
end
end
@doc """
Find the longest matching entry, checking a tailoring overlay first.
### Arguments
* `codepoints` - a list of integer codepoints to match.
* `overlay` - a tailoring overlay map, or `nil` for root-only lookups.
### Returns
Same as `longest_match/1`.
### Examples
iex> Cldr.Collation.Table.ensure_loaded()
iex> {matched, _elems, rest} = Cldr.Collation.Table.longest_match_with_overlay([0x0041, 0x0042], nil)
iex> matched
[65]
iex> rest
[66]
"""
@spec longest_match_with_overlay([non_neg_integer()], map() | nil) ::
{[non_neg_integer()], [Element.t()], [non_neg_integer()]}
| {:unmapped, non_neg_integer(), [non_neg_integer()]}
| :done
def longest_match_with_overlay(codepoints, nil), do: longest_match(codepoints)
def longest_match_with_overlay([cp | rest] = _codepoints, overlay) when is_map(overlay) do
# Check overlay contractions first (try longest possible)
overlay_max_len = overlay_max_contraction_length(cp, overlay)
overlay_result =
if overlay_max_len > 0 do
available = [cp | Enum.take(rest, overlay_max_len - 1)]
overlay_max_len..1//-1
|> Enum.reduce_while(nil, fn len, _acc ->
if len <= length(available) do
candidate = Enum.take(available, len)
key = Parser.codepoints_to_key(candidate)
case Map.get(overlay, key) do
nil ->
{:cont, nil}
elements ->
remaining = Enum.drop([cp | rest], len)
{:halt, {candidate, elements, remaining}}
end
else
{:cont, nil}
end
end)
else
nil
end
case overlay_result do
nil ->
# No overlay match, fall back to root table
longest_match([cp | rest])
match ->
match
end
end
def longest_match_with_overlay([], _overlay), do: :done
# Find the maximum contraction length in the overlay starting with cp.
# Overlay keys are integers (single CP) or tuples (contractions).
defp overlay_max_contraction_length(cp, overlay) do
overlay
|> Map.keys()
|> Enum.reduce(0, fn
^cp, acc -> max(acc, 1)
key, acc when is_tuple(key) and elem(key, 0) == cp -> max(acc, tuple_size(key))
_, acc -> acc
end)
end
# GenServer
def start_link(options \\ []) do
GenServer.start_link(__MODULE__, options, name: __MODULE__)
end
@impl true
def init(_options) do
if nif_enabled?() do
{:ok, %{loaded: false}}
else
{:ok, %{loaded: false}, {:continue, :load}}
end
end
@impl true
def handle_continue(:load, state) do
load_table()
{:noreply, %{state | loaded: true}}
end
@impl true
def handle_call(:load, _from, %{loaded: true} = state) do
{:reply, :ok, state}
end
def handle_call(:load, _from, %{loaded: false} = state) do
load_table()
{:reply, :ok, %{state | loaded: true}}
end
defp nif_enabled? do
System.get_env("CLDR_COLLATION_NIF") == "true" or
Application.get_env(:ex_cldr_collation, :nif, false) == true
end
defp load_table do
fractional_path = data_path(@fractional_keys)
%{entries: all_entries} = Parser.parse(fractional_path)
# Build contraction starters map.
# Keys are now integers (single CP) or tuples (contractions).
contractions =
Enum.reduce(all_entries, %{}, fn {key, _elements}, acc ->
case key do
cp when is_integer(cp) ->
acc
tuple when is_tuple(tuple) ->
first = elem(tuple, 0)
len = tuple_size(tuple)
existing = Map.get(acc, first, MapSet.new())
Map.put(acc, first, MapSet.put(existing, len))
end
end)
contractions =
Map.new(contractions, fn {cp, lengths} -> {cp, MapSet.to_list(lengths)} end)
# Store atomically in persistent_term
:persistent_term.put(@table_name, all_entries)
:persistent_term.put(@contractions_table, contractions)
# Build fast Latin lookup table
Cldr.Collation.FastLatin.build()
end
defp data_path(filename) do
case :code.priv_dir(:ex_cldr_collation) do
{:error, :bad_name} ->
Path.join([File.cwd!(), "priv", filename])
priv_dir ->
Path.join(priv_dir, filename)
end
end
end