Current section

Files

Jump to
unicode_data lib script.ex
Raw

lib/script.ex

defmodule UnicodeData.Script do
@moduledoc false
@external_resource scripts_path = Path.join([__DIR__, "Scripts.txt"])
lines = File.stream!(scripts_path, [], :line)
|> Stream.filter(&String.match?(&1, ~r/^[0-9A-F]+/))
for line <- lines do
[r, s, _] = String.split(line, ~r/[\#;]/)
r = String.trim(r)
|> String.split("..")
|> Enum.map(&Integer.parse(&1, 16))
|> Enum.map(fn {x, _} -> x end)
s = String.trim(s)
case r do
[x] ->
def script_from_codepoint(unquote(x)) do
unquote(s)
end
[a, b] ->
def script_from_codepoint(n) when n in unquote(a)..unquote(b) do
unquote(s)
end
end
end
def script_to_tag(script) do
# This maps the Unicode Script property to an OpenType script tag
# Data from http://www.microsoft.com/typography/otspec/scripttags.htm
# and http://www.unicode.org/Public/UNIDATA/PropertyValueAliases.txt.
#
# Updated through to Unicode 10.0
unicode_scripts = %{
"Ahom" => "ahom",
"Anatolian_Hieroglyphs" => "hluw",
"Caucasian_Albanian" => "aghb",
"Arabic" => "arab",
"Imperial_Aramaic"=> "armi",
"Armenian"=> "armn",
"Avestan"=> "avst",
"Balinese"=> "bali",
"Bamum"=> "bamu",
"Bassa_Vah"=> "bass",
"Batak"=> "batk",
"Bengali"=> ["bng2", "beng"],
"Bopomofo"=> "bopo",
"Brahmi"=> "brah",
"Braille"=> "brai",
"Buginese"=> "bugi",
"Buhid"=> "buhd",
"Byzantine_Music"=> "byzm",
"Chakma"=> "cakm",
#"Canadian_Syllabics"=> "cans",
"Canadian_Aboriginal"=> "cans",
"Carian"=> "cari",
"Cham"=> "cham",
"Cherokee"=> "cher",
"Coptic"=> "copt",
"Cypriot"=> "cprt",
"Cyrillic"=> "cyrl",
"Devanagari"=> ["dev2", "deva"],
"Deseret"=> "dsrt",
"Duployan"=> "dupl",
"Egyptian_Hieroglyphs"=> "egyp",
"Elbasan"=> "elba",
"Ethiopic"=> "ethi",
"Georgian"=> "geor",
"Glagolitic"=> "glag",
"Gothic"=> "goth",
"Grantha"=> "gran",
"Greek"=> "grek",
"Gujarati"=> ["gjr2", "gujr"],
"Gurmukhi"=> ["gur2", "guru"],
"Hangul"=> "hang",
"Han"=> "hani",
"Hanunoo"=> "hano",
"Hebrew"=> "hebr",
"Hiragana"=> "hira",
"Pahawh_Hmong"=> "hmng",
"Katakana_Or_Hiragana"=> "hrkt",
"Old_Italic"=> "ital",
"Javanese"=> "java",
"Kayah_Li"=> "kali",
"Katakana"=> "kana",
"Kharoshthi"=> "khar",
"Khmer"=> "khmr",
"Khojki"=> "khoj",
"Kannada"=> ["knd2", "knda"],
"Kaithi"=> "kthi",
"Tai_Tham"=> "lana",
"Lao"=> "lao ",
"Latin"=> "latn",
"Lepcha"=> "lepc",
"Limbu"=> "limb",
"Linear_A"=> "lina",
"Linear_B"=> "linb",
"Lisu"=> "lisu",
"Lycian"=> "lyci",
"Lydian"=> "lydi",
"Mahajani"=> "mahj",
"Mandaic"=> "mand",
"Manichaean"=> "mani",
"Mende_Kikakui"=> "mend",
"Meroitic_Cursive"=> "merc",
"Meroitic_Hieroglyphs"=> "mero",
"Malayalam"=> ["mlm2", "mlym"],
"Modi"=> "modi",
"Mongolian"=> "mong",
"Mro"=> "mroo",
"Meetei_Mayek"=> "mtei",
"Myanmar"=> ["mym2", "mymr"],
"Old_North_Arabian"=> "narb",
"Nabataean"=> "nbat",
"Nko"=> "nko ",
"Ogham"=> "ogam",
"Ol_Chiki"=> "olck",
"Old_Turkic"=> "orkh",
"Odia"=> ["ory2", "orya"],
"Osmanya"=> "osma",
"Palmyrene"=> "palm",
"Pau_Cin_Hau"=> "pauc",
"Old_Permic"=> "perm",
"Phags_Pa"=> "phag",
"Inscriptional_Pahlavi"=> "phli",
"Psalter_Pahlavi"=> "phlp",
"Phoenician"=> "phnx",
"Miao"=> "plrd",
"Inscriptional_Parthian"=> "prti",
"Rejang"=> "rjng",
"Runic"=> "runr",
"Samaritan"=> "samr",
"Old_South_Arabian"=> "sarb",
"Saurashtra"=> "saur",
"Shavian"=> "shaw",
"Sharada"=> "shrd",
"Siddham"=> "sidd",
"Khudawadi"=> "sind",
"Sinhala"=> "sinh",
"Sora_Sompeng"=> "sora",
"Sundanese"=> "sund",
"Syloti_Nagri"=> "sylo",
"Syriac"=> "syrc",
"Tagbanwa"=> "tagb",
"Takri"=> "takr",
"Tai_Le"=> "tale",
"New_Tai_Lue"=> "talu",
"Tamil"=> ["tml2", "taml"],
"Tai_Viet"=> "tavt",
"Telugu"=> ["tel2", "telu"],
"Tifinagh"=> "tfng",
"Tagalog"=> "tglg",
"Thaana"=> "thaa",
"Thai"=> "thai",
"Tibetan"=> "tibt",
"Tirhuta"=> "tirh",
"Ugaritic"=> "ugar",
"Vai"=> "vai ",
"Warang_Citi"=> "wara",
"Old_Persian"=> "xpeo",
"Cuneiform"=> "xsux",
"Yi"=> "yi ",
# Unicode 8.0
"Hatran" => "hatr",
"Multani" => "mult",
"Old_Hungarian" => "hung",
"SignWriting" => "sgnw",
# Unicode 9.0
"Adlam" => "adlm",
"Bhaisuki" => "bhks",
"Marchen" => "marc",
"Newa" => "newa",
"Osage" => "osge",
"Tangut" => "tang",
# Unicode 10.0
"Masaram_Gondi" => "gonm",
"Nushu" => "nshu",
"Soyombo" => "soyo",
"Zanabazar_Square" => "zanb",
# always at the bottom as they are special
"Inherited"=> "zinh",
"Common"=> "zyyy",
"Unknown"=> "zzzz"
}
Map.get(unicode_scripts, script, "zzzz")
end
# this list comes from ISO 15924
# There's a handy sortable table on
# https://en.wikipedia.org/wiki/ISO_15924
#
# Updated through to Unicode 10.0
def right_to_left?(script) do
rtl_scripts = [
# unicode 7.0
"arab", "hebr", "syrc", "thaa", "cprt",
"khar", "phnx", "nko ", "lydi", "avst",
"armi", "phli", "prti", "sarb", "orkh",
"samr", "mand", "merc", "mero", "mani",
"mend", "nbat", "narb", "palm", "phlp",
# unicode 8.0
"hatr", "hung",
# unicode 9.0
"adlm"
]
script in rtl_scripts
end
def uses_joining_type?(script) do
joining_scripts = ["arab", "mong", "syrc", "nko ",
"phag", "mand", "mani", "phlp"]
script in joining_scripts
end
def detect_script(text) do
x = String.codepoints(text)
|> Stream.map(fn <<x::utf8>> -> x end)
|> Stream.map(&script_from_codepoint(&1))
|> Stream.filter(fn x -> !(x in ["Common", "Inherited", "Unknown"]) end)
|> Enum.to_list
script = if x == [], do: "Unknown", else: hd(x)
script_to_tag(script)
end
# used for cursive scripts -- Arabic, Syriac, N'Ko, Mongolian, etc
@external_resource shaping_path = Path.join([__DIR__, "ArabicShaping.txt"])
lines = File.stream!(shaping_path, [], :line)
|> Stream.filter(&String.match?(&1, ~r/^[0-9A-F]+/))
for line <- lines do
[cp, _, t, join_group] = String.split(line, ~r/[\#;]/)
cp = cp
|> String.trim
|> Integer.parse(16)
|> elem(0)
join_group = String.trim(join_group)
# use the join type unless in special-cased join group
jt = if join_group in ["ALAPH", "DALATH RISH"] do
join_group
else
String.trim(t)
end
def arabic_shaping_from_codepoint(unquote(cp)) do
unquote(jt)
end
def jointype_from_codepoint(unquote(cp)) do
unquote(String.trim(t))
end
def joingroup_from_codepoint(unquote(cp)) do
unquote(join_group)
end
end
def arabic_shaping_from_codepoint(_cp) do
"U"
end
def jointype_from_codepoint(_cp) do
"U"
end
def joingroup_from_codepoint(_cp) do
"No_Joining_Group"
end
# Cursive scripts have the following join types:
# R Right_Joining
# L Left_Joining
# D Dual_Joining
# C Join_Causing
# U Non_Joining
# T Transparent
#
# special joining rules for:
# ALAPH
# DALATH RISH
#
def arabic_shaping([], _prev, output), do: Enum.reverse(output)
# start
def arabic_shaping([type | types], nil, []) do
curr = if type == "U", do: nil, else: "isol"
arabic_shaping(types, type, [curr])
end
# case A: previous is U
def arabic_shaping([type | types], "U", [prev | output]) do
curr = if type == "U", do: nil, else: "isol"
arabic_shaping(types, type, [curr, prev | output])
end
# case B: previous is DALATH RISH
def arabic_shaping([type | types], "DALATH RISH", [prev | output]) do
curr = case type do
"U" -> nil
"ALAPH" -> "fin3"
_ -> "isol"
end
arabic_shaping(types, type, [curr, prev | output])
end
# case C: previous is R
def arabic_shaping([type | types], "R", [prev | output]) do
curr = case type do
"U" -> nil
"ALAPH" -> "fin2"
_ -> "isol"
end
arabic_shaping(types, type, [curr, prev | output])
end
# case C.2: previous is ALAPH in isol form
def arabic_shaping([type | types], "ALAPH", ["isol" | output]) do
curr = case type do
"U" -> nil
"ALAPH" -> "fin2"
_ -> "isol"
end
arabic_shaping(types, type, [curr, "isol" | output])
end
# case D: previous is ALAPH in fina form
def arabic_shaping([type | types], "ALAPH", ["fina" | output]) do
{prev, curr} = case type do
"U" -> {"fina", nil}
"L" -> {"fina", "isol"}
"ALAPH" -> {"med2", "fin2"}
_ -> {"med2", "isol"}
end
arabic_shaping(types, type, [curr, prev | output])
end
# case E: previous is D or C in fina form
def arabic_shaping([type | types], prev_type, ["fina" | output]) when prev_type in ["D", "C"] do
{prev, curr} = case type do
"U" -> {"fina", nil}
"L" -> {"fina", "isol"}
_ -> {"medi", "fina"}
end
arabic_shaping(types, type, [curr, prev | output])
end
# case F: previous is D, C, or L in isol form
def arabic_shaping([type | types], prev_type, ["isol" | output]) when prev_type in ["D", "C", "L"] do
{prev, curr} = case type do
"U" -> {"isol", nil}
"L" -> {"isol", "isol"}
_ -> {"init", "fina"}
end
arabic_shaping(types, type, [curr, prev | output])
end
# case G: previous is ALAPH in fin2/fin3 form
def arabic_shaping([type | types], "ALAPH", [prev | output]) when prev in ["fin2", "fin3"] do
{prev, curr} = case type do
"U" -> {prev, nil}
"L" -> {prev, "isol"}
"ALAPH" -> {"isol", "fin2"}
_ -> {"isol", "isol"}
end
arabic_shaping(types, type, [curr, prev | output])
end
# transparent -- should be ignored by joining rules
# combining marks are an example as they are zero-width
def arabic_shaping(["T" | types], prev_type, [prev | output]) do
arabic_shaping(types, prev_type, [nil, prev | output])
end
# fallback -- do no shaping for current glyph
def arabic_shaping([type | types], _prev_type, [prev | output]) do
arabic_shaping(types, type, [nil, prev | output])
end
# shape arabic and other cursive scripts
# arabic, mongolian, syriac, n'ko, phags pa,
# mandiac, manichaean, psalter pahlavi
def shape_glyphs(script, text) when script in ["arab", "mong", "syrc", "nko ", "phag", "mand", "mani", "phlp"] do
features = ["isol", "medi", "init", "fina", "med2", "fin2", "fin3"]
# look up shaping types
x = text
|> String.codepoints
|> Stream.map(fn <<x::utf8>> -> x end)
|> Stream.map(&arabic_shaping_from_codepoint(&1))
|> Enum.to_list
# record the locations of transparent (do not affect joining)
glyphs_T = x
|> Enum.with_index
|> Enum.filter(fn {n, _} -> n == "T" end)
|> Enum.map(fn {"T", i} -> i end)
# remove the transparent glyphs
trimmed = x
|> Enum.filter(fn n -> n != "T" end)
# do the shaping
shaped = arabic_shaping(trimmed, nil, [])
# transparent glyphs have no shape class, insert nil at appropriate locations
shaped = Enum.reduce(glyphs_T, shaped, fn i, acc -> List.insert_at(acc, i, nil) end)
{features, shaped}
end
# default shaper does nothing
def shape_glyphs(_script, _glyphs) do
{[], []}
end
end