Current section

Files

Jump to
gettext_mapper lib mix tasks gettext_mapper.extract.ex
Raw

lib/mix/tasks/gettext_mapper.extract.ex

defmodule Mix.Tasks.GettextMapper.Extract do
@shortdoc "Extract translations from static gettext_mapper maps to populate .po/.pot files"
@moduledoc """
Extracts both msgids and translations from static gettext_mapper maps and populates .po/.pot files.
This task scans your codebase for `gettext_mapper(%{...})` calls with static maps and:
1. Extracts the msgid (custom msgid if specified, otherwise default locale message)
2. Creates/updates .pot template files with all msgids (empty msgstr)
3. Extracts all translations and populates corresponding .po files with msgstr entries
## Examples
# Extract and populate all .po files
mix gettext_mapper.extract
# Extract from specific files
mix gettext_mapper.extract lib/my_app/models.ex
# Dry run (show what would be extracted without modifying files)
mix gettext_mapper.extract --dry-run
# Use specific backend and priv directory
mix gettext_mapper.extract --backend MyApp.Gettext --priv priv/gettext
## How it works
1. Finds all `gettext_mapper(%{...})` calls with static translation maps
2. Extracts the msgid (custom msgid option takes priority over default locale message)
3. For each locale in the map, updates the corresponding .po file with the translation
4. Creates new msgid/msgstr entries or updates existing ones
## Custom Message IDs
The task supports the `msgid` option for stable translation keys:
gettext_mapper(%{"en" => "Hello", "de" => "Hallo"}, msgid: "greeting.hello")
This creates .po entries with the custom msgid:
# de/LC_MESSAGES/default.po
msgid "greeting.hello"
msgstr "Hallo"
This allows you to use stable keys that don't change when the source text changes.
## Example
From code:
```elixir
def greeting do
gettext_mapper(%{"de" => "Hello World", "en" => "Hello World", "uk" => "Hello World"})
end
```
Creates in de/LC_MESSAGES/default.po:
```
msgid "Hello World"
msgstr "Hallo Welt"
```
Creates in uk/LC_MESSAGES/default.po:
```
msgid "Hello World"
msgstr "Привіт Світ"
```
"""
use Mix.Task
@impl Mix.Task
def run(args) do
Mix.Task.run("compile")
Mix.Task.run("app.start")
{opts, paths, _} =
OptionParser.parse(args,
switches: [dry_run: :boolean, backend: :string, priv: :string],
aliases: [d: :dry_run, b: :backend, p: :priv]
)
dry_run = Keyword.get(opts, :dry_run, false)
backend = get_backend(opts)
priv_dir = Keyword.get(opts, :priv, "priv/gettext")
files_to_process =
if Enum.empty?(paths) do
find_elixir_files()
else
paths
end
if dry_run do
Mix.shell().info("Running in dry-run mode. No .po files will be modified.")
end
# Collect all translation maps from files
all_translations = collect_translation_maps(files_to_process)
if Enum.empty?(all_translations) do
Mix.shell().info("No static translation maps found in specified files.")
else
# Group by msgid and populate .po files
populate_po_files(all_translations, backend, priv_dir, dry_run)
count = length(all_translations)
Mix.shell().info("Processed #{count} translation maps and updated .po files.")
end
end
defp get_backend(opts) do
case Keyword.get(opts, :backend) do
nil ->
try do
GettextMapper.GettextAPI.gettext_module()
rescue
_error ->
if File.exists?("test/test_helper.exs") do
Code.require_file("test/test_helper.exs")
GettextMapper.GettextAPI.gettext_module()
else
reraise "No gettext backend configured. Use --backend YourApp.Gettext",
__STACKTRACE__
end
end
backend_string ->
String.to_existing_atom(backend_string)
end
end
defp find_elixir_files do
"lib/**/*.ex"
|> Path.wildcard()
|> Enum.filter(&File.exists?/1)
end
defp collect_translation_maps(files) do
Enum.flat_map(files, &extract_maps_from_file/1)
end
defp extract_maps_from_file(file_path) do
try do
content = File.read!(file_path)
extract_maps_from_content(content, file_path)
rescue
error ->
Mix.shell().error("Error reading #{file_path}: #{Exception.message(error)}")
[]
end
end
defp extract_maps_from_content(content, file_path) do
# First extract module-level domain if present
module_domain = extract_module_domain(content)
# Regex to find gettext_mapper calls with static maps, optionally with domain and/or msgid
# Captures: 1=map_content, 2=first_opt_key, 3=first_opt_value, 4=second_opt_key, 5=second_opt_value
regex =
~r/gettext_mapper\(\s*%\{([^}]+)\}\s*(?:,\s*(domain|msgid):\s*"([^"]+)")?(?:,\s*(domain|msgid):\s*"([^"]+)")?\s*\)/s
Regex.scan(regex, content, capture: :all)
|> Enum.with_index()
|> Enum.flat_map(fn {captures, index} ->
full_match = Enum.at(captures, 0, "")
map_content = Enum.at(captures, 1, "")
opt1_key = Enum.at(captures, 2, "")
opt1_val = Enum.at(captures, 3, "")
opt2_key = Enum.at(captures, 4, "")
opt2_val = Enum.at(captures, 5, "")
# Check if this match is in a comment
if comment_text?(content, full_match) do
# Skip matches that are in comments
[]
else
# Parse options from captured groups
opts = parse_options(opt1_key, opt1_val, opt2_key, opt2_val)
# Determine domain priority: call-level > module-level > default
domain =
cond do
Map.has_key?(opts, "domain") -> Map.get(opts, "domain")
module_domain != nil -> module_domain
true -> GettextMapper.GettextAPI.default_domain()
end
custom_msgid = Map.get(opts, "msgid")
case parse_translation_map(map_content) do
{:ok, translations} ->
[{translations, domain, custom_msgid, "#{file_path}:#{index + 1}"}]
:error ->
Mix.shell().warn(
"Could not parse translation map in #{file_path} at position #{index + 1}"
)
[]
end
end
end)
end
defp parse_options(key1, val1, key2, val2) do
opts = %{}
opts =
if key1 != "" and val1 != "" do
Map.put(opts, key1, val1)
else
opts
end
if key2 != "" and val2 != "" do
Map.put(opts, key2, val2)
else
opts
end
end
defp comment_text?(content, match_text) do
# Find the position of this match in the content
case :binary.match(content, match_text) do
{pos, _len} ->
comment_by_position?(content, pos)
:nomatch ->
false
end
end
defp comment_by_position?(content, pos) do
# Check if this position is in a line comment, @moduledoc, or @doc section
line_comment?(content, pos) or doc_section?(content, pos)
end
defp line_comment?(content, pos) do
# Check if there's a # character before this position in the same line
content_before_match = String.slice(content, 0, pos)
# Find the last newline before the match
last_newline_pos =
content_before_match
|> String.reverse()
|> :binary.match("\n")
|> case do
{reverse_pos, _} -> pos - reverse_pos - 1
:nomatch -> 0
end
line_before_match =
String.slice(content_before_match, last_newline_pos, pos - last_newline_pos)
# Check if there's a # in the line before the match
String.contains?(line_before_match, "#")
end
defp doc_section?(content, pos) do
# Check if the match is inside @moduledoc """ or @doc """ blocks
content_before_match = String.slice(content, 0, pos)
# Find all doc block starts (either @doc """ or @moduledoc """)
doc_starts =
(Regex.scan(~r/@doc\s+"""/, content_before_match, return: :index) ++
Regex.scan(~r/@moduledoc\s+"""/, content_before_match, return: :index))
|> Enum.map(fn [{start, _}] -> start end)
|> Enum.sort()
# Find all standalone """ that are not part of @doc or @moduledoc
all_triple_quotes =
Regex.scan(~r/"""/, content_before_match, return: :index)
|> Enum.map(fn [{start, _}] -> start end)
|> Enum.sort()
# Filter out the """ that are part of @doc/@moduledoc declarations
doc_ends =
Enum.reject(all_triple_quotes, fn quote_pos ->
# Check if this """ is preceded by @doc or @moduledoc on the same or previous lines
Enum.any?(doc_starts, fn doc_start ->
# Rough heuristic for same declaration
abs(quote_pos - doc_start) < 50
end)
end)
# Count how many doc blocks are currently open
open_blocks = length(doc_starts) - length(doc_ends)
# If we have more opening doc blocks than closing """, we're inside a doc block
open_blocks > 0
end
defp parse_translation_map(map_string) do
try do
# Clean up the map string
cleaned =
map_string
|> String.replace(~r/\s+/, " ")
|> String.trim()
# Parse key-value pairs
pairs = Regex.scan(~r/"([^"]+)"\s*=>\s*"([^"]*)"/, cleaned)
translations =
Enum.into(pairs, %{}, fn [_, key, value] ->
{key, value}
end)
if map_size(translations) > 0 do
{:ok, translations}
else
:error
end
rescue
_ -> :error
end
end
defp populate_po_files(translation_maps, backend, priv_dir, dry_run) do
# Get default locale to use as msgid source
default_locale = get_default_locale(backend)
# Group by msgid and domain
grouped_by_msgid_and_domain =
Enum.reduce(translation_maps, %{}, fn {translations, domain, custom_msgid, source}, acc ->
# Use custom msgid if provided, otherwise use default locale message
msgid = custom_msgid || Map.get(translations, default_locale)
case msgid do
nil ->
acc
msgid ->
key = {msgid, domain}
existing = Map.get(acc, key, [])
Map.put(acc, key, [{translations, source} | existing])
end
end)
# Group msgids by domain for .pot file generation
msgids_by_domain =
Enum.reduce(grouped_by_msgid_and_domain, %{}, fn {{msgid, domain}, _}, acc ->
existing = Map.get(acc, domain, [])
Map.put(acc, domain, [msgid | existing])
end)
# Create/update .pot files for each domain
Enum.each(msgids_by_domain, fn {domain, msgids} ->
update_pot_file(priv_dir, domain, Enum.uniq(msgids), dry_run)
end)
# For each msgid/domain combination, update all relevant .po files
Enum.each(grouped_by_msgid_and_domain, fn {{msgid, domain}, translation_groups} ->
# Merge all translations for this msgid (in case of duplicates)
merged_translations = merge_translation_groups(translation_groups)
# Check if this is a custom msgid (msgid differs from default locale text)
default_locale_text = Map.get(merged_translations, default_locale)
has_custom_msgid = default_locale_text != nil and default_locale_text != msgid
# Update .po files for each locale
# Skip default locale only if msgid equals the default locale text
Enum.each(merged_translations, fn {locale, msgstr} ->
if locale != default_locale or has_custom_msgid do
update_po_file(priv_dir, locale, domain, msgid, msgstr, dry_run)
end
end)
end)
end
defp update_pot_file(priv_dir, domain, msgids, dry_run) do
default_domain = GettextMapper.GettextAPI.default_domain()
pot_filename = if domain == default_domain, do: "#{default_domain}.pot", else: "#{domain}.pot"
pot_file_path = Path.join(priv_dir, pot_filename)
if dry_run do
Mix.shell().info("Would update #{pot_file_path} with #{length(msgids)} msgid(s)")
else
ensure_pot_file_exists(pot_file_path)
Enum.each(msgids, fn msgid ->
add_msgid_to_pot(pot_file_path, msgid)
end)
end
end
defp ensure_pot_file_exists(pot_file_path) do
unless File.exists?(pot_file_path) do
File.mkdir_p!(Path.dirname(pot_file_path))
initial_content = """
## POT (Portable Object Template) file
## Extracted from gettext_mapper static translation maps.
msgid ""
msgstr ""
"Content-Type: text/plain; charset=UTF-8\\n"
"""
File.write!(pot_file_path, initial_content)
end
end
defp add_msgid_to_pot(pot_file_path, msgid) do
content = File.read!(pot_file_path)
# Check if msgid already exists
escaped_msgid = Regex.escape(msgid)
msgid_pattern = ~r/msgid "#{escaped_msgid}"\s*\nmsgstr ""/s
unless Regex.match?(msgid_pattern, content) do
# Add new msgid with empty msgstr
new_entry = "\nmsgid \"#{escape_string(msgid)}\"\nmsgstr \"\"\n"
File.write!(pot_file_path, content <> new_entry)
end
end
defp merge_translation_groups(translation_groups) do
Enum.reduce(translation_groups, %{}, fn {translations, _source}, acc ->
Map.merge(acc, translations)
end)
end
defp update_po_file(priv_dir, locale, domain, msgid, msgstr, dry_run) do
default_domain = GettextMapper.GettextAPI.default_domain()
po_filename = if domain == default_domain, do: "#{default_domain}.po", else: "#{domain}.po"
po_file_path = Path.join([priv_dir, locale, "LC_MESSAGES", po_filename])
if dry_run do
Mix.shell().info("Would update #{po_file_path}: msgid \"#{msgid}\" -> msgstr \"#{msgstr}\"")
else
ensure_po_file_exists(po_file_path, locale)
add_or_update_translation(po_file_path, msgid, msgstr)
end
end
defp ensure_po_file_exists(po_file_path, locale) do
unless File.exists?(po_file_path) do
File.mkdir_p!(Path.dirname(po_file_path))
initial_content = """
## "Language: #{locale}" translations
msgid ""
msgstr ""
"Language: #{locale}\\n"
"Content-Type: text/plain; charset=UTF-8\\n"
"""
File.write!(po_file_path, initial_content)
end
end
defp add_or_update_translation(po_file_path, msgid, msgstr) do
content = File.read!(po_file_path)
# Check if msgid already exists
msgid_pattern = ~r/msgid "#{Regex.escape(msgid)}"\s*\nmsgstr "[^"]*"/s
if Regex.match?(msgid_pattern, content) do
# Update existing translation
updated_content =
Regex.replace(
msgid_pattern,
content,
"msgid \"#{msgid}\"\nmsgstr \"#{escape_string(msgstr)}\""
)
File.write!(po_file_path, updated_content)
else
# Add new translation
new_entry = "\nmsgid \"#{msgid}\"\nmsgstr \"#{escape_string(msgstr)}\"\n"
File.write!(po_file_path, content <> new_entry)
end
end
defp escape_string(str) do
str
|> String.replace("\\", "\\\\")
|> String.replace("\"", "\\\"")
end
defp extract_module_domain(content) do
# Look for `use GettextMapper, domain: "domain_name"`
case Regex.run(~r/use\s+GettextMapper\s*,\s*domain:\s*"([^"]+)"/, content) do
[_, domain] -> domain
_ -> nil
end
end
defp get_default_locale(backend) do
try do
backend.__gettext__(:default_locale)
rescue
_ -> "en"
end
end
end