Packages
unicode_idna
0.1.0
Pure-Elixir implementation of UTS #46 (IDNA 2008 Compatibility Processing) with Punycode (RFC 3492), bidi (RFC 5893) and CONTEXTJ joiner rules.
Current section
Files
Jump to
Current section
Files
lib/unicode/idna/utils.ex
defmodule Unicode.IDNA.Utils do
@moduledoc false
@doc """
Parses the UTS #46 IDNA mapping table.
Returns a list of `{first_codepoint, last_codepoint, status,
mapping}` tuples where:
* `status` is one of `:valid`, `:ignored`, `:mapped`,
`:deviation` or `:disallowed`.
* `mapping` is a list of target code points (possibly empty)
for `:mapped` and `:deviation`, and `nil` for the other
statuses.
The list is sorted by `first_codepoint`.
"""
@idna_mapping_path Path.join(Unicode.IDNA.data_dir(), "idna_mapping_table.txt")
@external_resource @idna_mapping_path
def idna_mapping do
@idna_mapping_path
|> File.stream!()
|> Stream.map(&strip_comment/1)
|> Stream.reject(&blank?/1)
|> Stream.map(&parse_mapping_row/1)
|> Enum.to_list()
|> Enum.sort_by(fn {first, _, _, _} -> first end)
end
@doc """
Parses a Unicode `extracted/Derived*.txt`-style file.
Returns a list of `{first_codepoint, last_codepoint, value}`
tuples where `value` is the second semicolon-separated field as
an atom (downcased, whitespace removed).
"""
def derived_property(path) do
path
|> File.stream!()
|> Stream.map(&strip_comment/1)
|> Stream.reject(&blank?/1)
|> Stream.map(&parse_derived_row/1)
|> Enum.to_list()
|> Enum.sort_by(fn {first, _, _} -> first end)
end
defp strip_comment(line) do
line
|> String.split("#", parts: 2)
|> hd()
|> String.trim()
end
defp blank?(""), do: true
defp blank?(_), do: false
defp parse_mapping_row(line) do
fields = line |> String.split(";") |> Enum.map(&String.trim/1)
{first, last} = parse_range(Enum.at(fields, 0))
status_atom = fields |> Enum.at(1) |> String.to_atom()
target_field = Enum.at(fields, 2, "")
mapping =
case {status_atom, target_field} do
{status, _} when status in [:mapped, :deviation] ->
parse_codepoint_list(target_field)
_ ->
nil
end
{first, last, status_atom, mapping}
end
defp parse_derived_row(line) do
[range_field, value | _] = line |> String.split(";") |> Enum.map(&String.trim/1)
{first, last} = parse_range(range_field)
{first, last, value |> String.downcase() |> String.to_atom()}
end
defp parse_range(field) do
case String.split(field, "..") do
[single] ->
cp = String.to_integer(single, 16)
{cp, cp}
[first, last] ->
{String.to_integer(first, 16), String.to_integer(last, 16)}
end
end
defp parse_codepoint_list(""), do: []
defp parse_codepoint_list(field) do
field
|> String.split(" ", trim: true)
|> Enum.map(&String.to_integer(&1, 16))
end
end