Packages
localize
0.3.0
1.0.0-rc.4
1.0.0-rc.3
1.0.0-rc.2
1.0.0-rc.1
1.0.0-rc.0
0.50.0
0.49.0
0.48.0
0.47.0
0.46.0
0.45.0
0.44.0
0.41.3
0.41.2
0.41.1
0.41.0
0.40.0
0.39.0
0.38.0
0.37.0
0.36.0
0.35.0
0.34.0
0.33.0
0.32.0
0.31.0
0.30.1
0.30.0
retired
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.0
0.23.0
0.22.0
0.21.0
0.20.0
0.19.0
0.18.0
0.16.0
0.15.0
0.14.0
0.13.0
0.12.0
0.11.0
0.10.0
0.9.0
0.8.0
0.7.0
0.6.0
0.5.0
0.4.0
0.3.0
0.2.0
0.1.0
0.1.0-alpha.1
Localization (parsing, formatting) of numbers, dates/time/calendar, units of measure, messages and lists. Includes localized collation.
Current section
Files
Jump to
Current section
Files
lib/localize/collation/implicit_weights.ex
defmodule Localize.Collation.ImplicitWeights do
@moduledoc """
Computes implicit collation elements for codepoints not in the DUCET/CLDR allkeys table.
The UCA defines an algorithm for computing implicit weights for:
* CJK Unified Ideographs (Han characters).
* Hangul syllables (decomposed algorithmically).
* Unassigned codepoints.
See UTS #10 Section 10.1 for the implicit weight computation.
"""
import Bitwise
alias Localize.Collation.Element
@cjk_unified_start 0x4E00
@cjk_unified_end 0x9FFF
@cjk_compat_start 0xF900
@cjk_compat_end 0xFAFF
@cjk_ext_a_start 0x3400
@cjk_ext_a_end 0x4DBF
@cjk_ext_b_start 0x20000
@cjk_ext_b_end 0x2A6DF
@cjk_ext_c_start 0x2A700
@cjk_ext_c_end 0x2B81D
@cjk_ext_d_start 0x2B820
@cjk_ext_d_end 0x2CEAD
@cjk_ext_e_start 0x2CEB0
@cjk_ext_e_end 0x2EBE0
@cjk_ext_f_start 0x2EBF0
@cjk_ext_f_end 0x2EE5D
@cjk_ext_g_start 0x30000
@cjk_ext_g_end 0x3134A
@cjk_ext_h_start 0x31350
@cjk_ext_h_end 0x33479
@hangul_start 0xAC00
@hangul_end 0xD7A3
@sbase 0xAC00
@lbase 0x1100
@vbase 0x1161
@tbase 0x11A7
@vcount 21
@tcount 28
@ncount @vcount * @tcount
@han_base 0xFB40
@han_ext_base 0xFB80
@unassigned_base 0xFBC0
@doc """
Check if a codepoint is a CJK Unified Ideograph.
### Arguments
* `cp` - an integer codepoint.
### Returns
* `true` if the codepoint is a CJK Unified Ideograph.
* `false` otherwise.
### Examples
iex> Localize.Collation.ImplicitWeights.unified_ideograph?(0x4E00)
true
iex> Localize.Collation.ImplicitWeights.unified_ideograph?(0x0041)
false
"""
@spec unified_ideograph?(non_neg_integer()) :: boolean()
def unified_ideograph?(cp) do
(cp >= @cjk_unified_start and cp <= @cjk_unified_end) or
(cp >= @cjk_ext_a_start and cp <= @cjk_ext_a_end) or
(cp >= @cjk_ext_b_start and cp <= @cjk_ext_b_end) or
(cp >= @cjk_ext_c_start and cp <= @cjk_ext_c_end) or
(cp >= @cjk_ext_d_start and cp <= @cjk_ext_d_end) or
(cp >= @cjk_ext_e_start and cp <= @cjk_ext_e_end) or
(cp >= @cjk_ext_f_start and cp <= @cjk_ext_f_end) or
(cp >= @cjk_ext_g_start and cp <= @cjk_ext_g_end) or
(cp >= @cjk_ext_h_start and cp <= @cjk_ext_h_end) or
(cp >= @cjk_compat_start and cp <= @cjk_compat_end) or
cp in [
0xFA0E,
0xFA0F,
0xFA11,
0xFA13,
0xFA14,
0xFA1F,
0xFA21,
0xFA23,
0xFA24,
0xFA27,
0xFA28,
0xFA29
]
end
@doc """
Check if a codepoint is a Hangul syllable.
### Arguments
* `cp` - an integer codepoint.
### Returns
* `true` if the codepoint is a Hangul syllable.
* `false` otherwise.
### Examples
iex> Localize.Collation.ImplicitWeights.hangul_syllable?(0xAC00)
true
iex> Localize.Collation.ImplicitWeights.hangul_syllable?(0x0041)
false
"""
@spec hangul_syllable?(non_neg_integer()) :: boolean()
def hangul_syllable?(cp), do: cp >= @hangul_start and cp <= @hangul_end
@doc """
Compute implicit collation elements for a codepoint not in the allkeys table.
### Arguments
* `cp` - an integer codepoint.
### Returns
* `{:hangul_decompose, jamo}` - for Hangul syllables.
* `[element, element]` - two implicit CEs for CJK or unassigned codepoints.
### Examples
iex> [ce1, ce2] = Localize.Collation.ImplicitWeights.compute(0x4E00)
iex> Localize.Collation.Element.primary(ce1) >= 0xFB40
true
iex> Localize.Collation.Element.secondary(ce2)
0
"""
@spec compute(non_neg_integer()) :: {:hangul_decompose, [non_neg_integer()]} | [Element.t()]
def compute(cp) do
cond do
hangul_syllable?(cp) ->
decompose_hangul(cp)
unified_ideograph?(cp) ->
compute_han_implicit(cp)
true ->
compute_unassigned(cp)
end
end
@doc """
Decompose a Hangul syllable into its constituent jamo codepoints.
### Arguments
* `cp` - an integer codepoint for a Hangul syllable (U+AC00..U+D7A3).
### Returns
A list of 2 or 3 jamo codepoints: `[lead, vowel]` or `[lead, vowel, trail]`.
### Examples
iex> Localize.Collation.ImplicitWeights.decompose_hangul_to_jamo(0xAC00)
[0x1100, 0x1161]
iex> Localize.Collation.ImplicitWeights.decompose_hangul_to_jamo(0xAC01)
[0x1100, 0x1161, 0x11A8]
"""
@spec decompose_hangul_to_jamo(non_neg_integer()) :: [non_neg_integer()]
def decompose_hangul_to_jamo(cp) do
sindex = cp - @sbase
lindex = div(sindex, @ncount)
vindex = div(rem(sindex, @ncount), @tcount)
tindex = rem(sindex, @tcount)
l = @lbase + lindex
v = @vbase + vindex
if tindex > 0 do
t = @tbase + tindex
[l, v, t]
else
[l, v]
end
end
defp decompose_hangul(cp) do
jamo = decompose_hangul_to_jamo(cp)
{:hangul_decompose, jamo}
end
defp compute_han_implicit(cp) do
base =
if (cp >= @cjk_unified_start and cp <= @cjk_unified_end) or
(cp >= @cjk_ext_a_start and cp <= @cjk_ext_a_end) do
@han_base
else
@han_ext_base
end
compute_implicit_pair(cp, base)
end
defp compute_unassigned(cp) do
compute_implicit_pair(cp, @unassigned_base)
end
defp compute_implicit_pair(cp, base) do
aaaa = base + (cp >>> 15)
bbbb = (cp &&& 0x7FFF) ||| 0x8000
[
Element.new(aaaa, 0x0020, 0x0002),
Element.new(bbbb, 0x0000, 0x0000)
]
end
end