Packages

Library to check for use of Unicode codepoints outside subsets specified in RFC 9839.

Current section

Files

Jump to
rfc9839 lib rfc9839.ex
Raw

lib/rfc9839.ex

defmodule RFC9839 do
@moduledoc """
Module implementing checks for whether a string, charlist or codepoint fits
within the Unicode subsets specified by [RFC 9839][rfc9839]. These subsets are
designed to exclude "problematic" codepoints, for varying definitions of
"problematic".
[rfc9839]: https://www.rfc-editor.org/rfc/rfc9839
"""
# Codepoint ranges are checked in guards. That is significantly faster than
# iterating over a list of ranges and checking each one.
@doc """
Check if a codepoint, charlist, or a string is within the unicode subset of
"Unicode Scalars" as defined by RFC 9839.
Unicode Scalars includes all valid unicode codepoints except surrogates.
Note that Elixir strings are binaries containing UTF-8 and are not allowed to
contain surrogates or invalid codepoints. Calling this function with a binary
that does will raise an exception instead of returning `false`, so for strings
this function is mostly useless.
## Examples
iex> RFC9839.is_unicode_scalar?(?a)
true
iex> RFC9839.is_unicode_scalar?(0xD800) # Surrogate
false
iex> RFC9839.is_unicode_scalar?(~c"Hello World")
true
iex> RFC9839.is_unicode_scalar?([0xD800])
false
iex> RFC9839.is_unicode_scalar?("Hello World")
true
iex> RFC9839.is_unicode_scalar?(<<0xED, 0xA0, 0x80>>)
** (UnicodeConversionError) invalid encoding starting at <<237, 160, 128>>
"""
def is_unicode_scalar?(string_or_charlist_or_codepoint)
def is_unicode_scalar?(cp) when is_integer(cp) and cp in 0x0..0xD7FF, do: true
def is_unicode_scalar?(cp) when is_integer(cp) and cp in 0xE000..0x10FFFF, do: true
def is_unicode_scalar?(cp) when is_integer(cp), do: false
def is_unicode_scalar?(list) when is_list(list) do
Enum.all?(list, &is_unicode_scalar?/1)
end
def is_unicode_scalar?(string) when is_binary(string) do
is_unicode_scalar?(String.to_charlist(string))
end
@doc """
Check if a codepoint, charlist, or a string is within the unicode subset of
"XML Characters" as defined by RFC 9839.
XML Characters includes all unicode codepoints except surrogates, legacy "C0"
control codes, and the noncharacters U+FFFE and U+FFFF.
## Examples
iex> RFC9839.is_xml_character?(?a)
true
iex> RFC9839.is_xml_character?(?\\b)
false
iex> RFC9839.is_xml_character?(~c"Hello World")
true
iex> RFC9839.is_xml_character?(~c"Hello \\b World")
false
iex> RFC9839.is_xml_character?("Hello World")
true
iex> RFC9839.is_xml_character?("Hello \\b World")
false
"""
def is_xml_character?(string_or_charlist_or_codepoint)
# useful controls
def is_xml_character?(cp) when is_integer(cp) and cp in 0x9..0x9, do: true
def is_xml_character?(cp) when is_integer(cp) and cp in 0xA..0xA, do: true
def is_xml_character?(cp) when is_integer(cp) and cp in 0xD..0xD, do: true
# exclude surrogates
def is_xml_character?(cp) when is_integer(cp) and cp in 0x20..0xD7FF, do: true
# exclude FFFE and FFFF nonchars
def is_xml_character?(cp) when is_integer(cp) and cp in 0xE000..0xFFFD, do: true
def is_xml_character?(cp) when is_integer(cp) and cp in 0x10000..0x10FFFF, do: true
def is_xml_character?(cp) when is_integer(cp), do: false
def is_xml_character?(list) when is_list(list) do
Enum.all?(list, &is_xml_character?/1)
end
def is_xml_character?(string) when is_binary(string) do
is_xml_character?(String.to_charlist(string))
end
@doc """
Check if a codepoint, charlist, or a string is within the unicode subset of
"Unicode Assignables" as defined by RFC 9839.
Unicode Assignables includes all codepoints that are currently assigned or
that might be assigned in the future, except surrogates, legacy control codes
and noncharacters.
## Examples
iex> RFC9839.is_unicode_assignable?(?a)
true
iex> RFC9839.is_unicode_assignable?(0x0080) # C1 control
false
iex> RFC9839.is_unicode_assignable?(~c"Hello World")
true
iex> RFC9839.is_unicode_assignable?(~c"Hello \\u0080 World")
false
iex> RFC9839.is_unicode_assignable?("Hello World")
true
iex> RFC9839.is_unicode_assignable?("Hello \\u0080 World")
false
"""
def is_unicode_assignable?(string_or_charlist_or_codepoint)
# useful controls
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x9..0x9, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xA..0xA, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xD..0xD, do: true
# exclude C1 controls and DEL
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x20..0x7E, do: true
# exclude surrogates
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xA0..0xD7FF, do: true
# exclude FDD0 nonchars
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xE000..0xFDCF, do: true
# exclude FFFE and FFFF nonchars
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xFDF0..0xFFFD, do: true
# (repeat per plane)
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x10000..0x1FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x20000..0x2FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x30000..0x3FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x40000..0x4FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x50000..0x5FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x60000..0x6FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x70000..0x7FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x80000..0x8FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x90000..0x9FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xA0000..0xAFFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xB0000..0xBFFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xC0000..0xCFFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xD0000..0xDFFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xE0000..0xEFFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0xF0000..0xFFFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp) and cp in 0x100000..0x10FFFD, do: true
def is_unicode_assignable?(cp) when is_integer(cp), do: false
def is_unicode_assignable?(list) when is_list(list) do
Enum.all?(list, &is_unicode_assignable?/1)
end
def is_unicode_assignable?(string) when is_binary(string) do
is_unicode_assignable?(String.to_charlist(string))
end
end