Packages

Text analysis and processing for Elixir including ngram, language detection and more.

Current section

Files

Jump to
text lib hyphenation parser.ex
Raw

lib/hyphenation/parser.ex

defmodule Text.Hyphenation.Parser do
@moduledoc false
# Parses a TeX-style hyphenation pattern file (hyph-utf8 / Knuth format)
# into the lookup structures used by Text.Hyphenation.
#
# The output is a `{patterns, exceptions}` tuple:
#
# * `patterns` is a map from the *letters-only* pattern string to a
# list of priorities, one entry between each adjacent letter pair
# plus one entry at each end. So `.ach4` parses to
# `{".ach", [0, 0, 0, 0, 4]}`. The leading "." marks a word-start
# anchor; a trailing "." marks word-end.
#
# * `exceptions` is a map from a fully-lowercased exception word (no
# hyphens) to the list of zero-based hyphenation point indices.
# `as-so-ciate` parses to `{"associate", [2, 5]}`.
@doc "Parses a TeX hyphenation pattern file into `{patterns, exceptions}`."
def parse_tex(content) when is_binary(content) do
cleaned = strip_comments(content)
patterns =
cleaned
|> extract_block("\\patterns{")
|> tokens()
|> Map.new(&parse_pattern/1)
exceptions =
cleaned
|> extract_block("\\hyphenation{")
|> tokens()
|> Map.new(&parse_exception/1)
{patterns, exceptions}
end
defp strip_comments(content) do
content
|> String.split("\n")
|> Enum.map(&String.replace(&1, ~r/%.*$/, ""))
|> Enum.join("\n")
end
defp extract_block(content, opener) do
case String.split(content, opener, parts: 2) do
[_before, after_open] ->
case String.split(after_open, "}", parts: 2) do
[block, _rest] -> block
[block] -> block
end
[_only] ->
""
end
end
defp tokens(block) do
block
|> String.split(~r/\s+/, trim: true)
|> Enum.reject(&(&1 == ""))
end
@doc false
def parse_pattern(token) do
chars = String.to_charlist(token)
do_split(chars, [], [0])
end
# Walk the chars; build `letters` (reversed) and `priorities` (reversed).
# Invariant: priorities has one more element than letters.
defp do_split([], letters, priorities) do
{letters |> Enum.reverse() |> List.to_string(), Enum.reverse(priorities)}
end
defp do_split([c | rest], letters, [_last | priors_tail]) when c >= ?0 and c <= ?9 do
do_split(rest, letters, [c - ?0 | priors_tail])
end
defp do_split([c | rest], letters, priorities) do
do_split(rest, [c | letters], [0 | priorities])
end
@doc false
def parse_exception(token) do
parts = String.split(token, "-")
word = Enum.join(parts)
points =
parts
|> Enum.drop(-1)
|> Enum.scan(0, fn part, acc -> acc + String.length(part) end)
{word, points}
end
end