Packages
gettext
0.20.0
1.0.2
1.0.1
retired
1.0.0
0.26.2
0.26.1
0.26.0
retired
0.25.0
0.24.0
0.23.1
0.23.0
0.22.3
0.22.2
0.22.1
0.22.0
0.21.0
0.20.0
0.19.1
0.19.0
0.18.2
0.18.1
0.18.0
0.17.4
0.17.3
0.17.2
0.17.1
0.17.0
0.16.1
0.16.0
0.15.0
0.14.1
0.14.0
0.13.1
0.13.0
0.12.2
0.12.1
0.12.0
0.11.0
0.10.0
0.9.0
0.8.0
0.7.0
0.6.1
0.6.0
0.5.0
Internationalization and localization through gettext
Current section
Files
Jump to
Current section
Files
lib/gettext/po/parser.ex
defmodule Gettext.PO.Parser do
@moduledoc false
alias Gettext.PO.Translations
alias Gettext.PO.Translation
alias Gettext.PO.PluralTranslation
@doc """
Parses a list of tokens into lists of comments, headers, and translations.
"""
@spec parse([Gettext.PO.Tokenizer.token()]) ::
{:ok, [binary], [binary], [Gettext.PO.translation()]} | Gettext.PO.parse_error()
def parse(tokens) when is_list(tokens) do
case :gettext_po_parser.parse(tokens) do
{:ok, translations} ->
parse_yecc_result(translations)
{:error, _reason} = error ->
parse_error(error)
end
end
defp parse_yecc_result(translations) do
translations = Enum.map(translations, &to_struct/1)
with :ok <- check_for_duplicates(translations) do
{top_comments, headers, translations} = extract_top_comments_and_headers(translations)
{:ok, top_comments, headers, translations}
end
end
defp to_struct({:translation, translation}) do
struct(Translation, translation)
|> extract_references()
|> extract_extracted_comments()
|> extract_flags()
end
defp to_struct({:plural_translation, translation}) do
struct(PluralTranslation, translation)
|> extract_references()
|> extract_extracted_comments()
|> extract_flags()
end
defp parse_error({:error, {line, _module, reason}}) do
{:error, line, parse_error_reason(reason)}
end
defp extract_references(%{__struct__: _, comments: comments} = translation) do
{reference_comments, other_comments} = Enum.split_with(comments, &match?("#:" <> _, &1))
references =
reference_comments
|> Enum.reject(fn "#:" <> comm -> String.trim(comm) == "" end)
|> Enum.flat_map(&parse_references/1)
%{translation | references: references, comments: other_comments}
end
defp parse_references("#:" <> comment) do
regex = ~r/
([\w\.:][\w\s\.:\/]*?)
:
(\d+)
/x
for [_file_and_line, file, line] <- Regex.scan(regex, comment) do
{file, String.to_integer(line)}
end
end
defp extract_extracted_comments(%{__struct__: _, comments: comments} = translation) do
{extracted_comments, other_comments} = Enum.split_with(comments, &match?("#." <> _, &1))
extracted_comments =
Enum.reject(extracted_comments, fn "#." <> comm -> String.trim(comm) == "" end)
%{translation | extracted_comments: extracted_comments, comments: other_comments}
end
defp extract_flags(%{__struct__: _, comments: comments} = translation) do
{flag_comments, other_comments} = Enum.split_with(comments, &match?("#," <> _, &1))
%{translation | flags: parse_flags(flag_comments), comments: other_comments}
end
defp parse_flags(flag_comments) do
flag_comments
|> Stream.map(fn "#," <> content -> content end)
|> Stream.flat_map(&String.split(&1, ","))
|> Stream.map(&String.trim/1)
|> Stream.reject(&(&1 == ""))
|> MapSet.new()
end
# If the first translation has an empty msgid, it's assumed to represent
# headers. Headers will be in the msgstr of this "fake" translation, one on
# each line. For now, we'll just separate those lines in order to get a list
# of headers.
defp extract_top_comments_and_headers([%Translation{msgid: id, msgstr: headers} = t | rest])
when id == "" or id == [""] do
{t.comments, headers, rest}
end
defp extract_top_comments_and_headers(translations) do
{[], [], translations}
end
defp check_for_duplicates(translations) do
check_for_duplicates(translations, %{})
end
defp check_for_duplicates([t | translations], existing) do
key = Translations.key(t)
case Map.fetch(existing, key) do
{:ok, old_line} ->
build_duplicated_error(t, old_line)
:error ->
check_for_duplicates(translations, Map.put(existing, key, t.po_source_line))
end
end
defp check_for_duplicates([], _existing) do
:ok
end
defp build_duplicated_error(%Translation{} = t, old_line) do
id = IO.iodata_to_binary(t.msgid)
{:error, t.po_source_line, "found duplicate on line #{old_line} for msgid: '#{id}'"}
end
defp build_duplicated_error(%PluralTranslation{} = t, old_line) do
id = IO.iodata_to_binary(t.msgid)
idp = IO.iodata_to_binary(t.msgid_plural)
msg = "found duplicate on line #{old_line} for msgid: '#{id}' and msgid_plural: '#{idp}'"
{:error, t.po_source_line, msg}
end
# We need to explicitly parse the error reason that yecc spits out because a
# `{type, line, token}` token is printed as the Erlang term in the error (by
# yecc). So, for example, if a token has a binary value then yecc will return
# something like:
#
# syntax error before: <<"my token">>
#
# which is not what we want, as we want the term to be printed as an Elixir
# term. While this is ugly, it's necessary (as yecc is not very extensible)
# and is what Elixir itself does
# (https://github.com/elixir-lang/elixir/blob/b80651/lib/elixir/src/elixir_errors.erl#L51-L103).
defp parse_error_reason([error, token]) do
IO.chardata_to_string(parse_error_reason(error, to_string(token)))
end
defp parse_error_reason('syntax error before: ' = prefix, "<<" <> rest),
do: [prefix, binary_part(rest, 0, byte_size(rest) - 2)]
defp parse_error_reason(error, token), do: [error, token]
end