Current section
Files
Jump to
Current section
Files
lib/codepagex.ex
defmodule Codepagex do
# unfortunately exdoc doesnt support ``` fenced blocks
@moduledoc (
File.read!("README.md")
|> String.split("\n")
|> Enum.reject(&(String.match?(&1, ~r/#.Codepagex/)))
|> Enum.reject(
&(String.match?(&1, ~r/```|Build Status|Documentation Status/))
)
|> Enum.join("\n")
)
require Codepagex.Mappings
alias Codepagex.Mappings
@type to_s_missing_inner ::
((binary, term) -> {:ok, String.t, binary, term} | {:error, term})
@type to_s_missing_outer ::
((String.t) -> {:ok, to_s_missing_inner} | {:error, term})
@type from_s_missing_inner ::
((String.t, term) -> {:ok, binary, String.t, term} | {:error, term})
@type from_s_missing_outer ::
((String.t) -> {:ok, from_s_missing_inner} | {:error, term})
@type encoding :: atom | String.t
@on_load :load_atoms
def load_atoms do
Code.ensure_loaded? Codepagex.Mappings
:ok
end
format_helper = fn {a, m} ->
" | #{inspect(a) |> String.ljust(15)} | #{m} |"
end
@aliases_markdown (
Mappings.aliases(:all)
|> Enum.map(format_helper)
|> Enum.join("\n")
)
@doc """
Returns a list of shorthand aliases that may be used instead of the full name
of the encoding.
The available aliases are:
| Alias | Full name |
|------:|:----------|
#{@aliases_markdown}
Some of these may not be available depending on mix configuration. If the
`selection` parameter is `:all` then all possible aliases are listed,
otherwise, only the available aliases are listed
For a full list of encodings, see `encoding_list/1`
"""
@spec aliases(atom) :: list(atom)
def aliases(selection \\ nil)
defdelegate aliases(selection), to: Mappings
@encodings_markdown (
# format as table with 3 columns
Mappings.encoding_list(:all)
|> Enum.map(&(String.ljust(&1, 30)))
|> Enum.chunk(3, 3, ["", ""])
|> Enum.map(&(Enum.join(&1, " | ")))
|> Enum.map(&("| #{&1} |"))
|> Enum.join("\n")
)
@encodings_atom (
Mappings.encoding_list(:all)
|> Enum.map(&String.to_atom/1)
)
@doc """
Returns a list of the supported encodings. These are extracted from
http://unicode.org/ and the names correspond to a encoding file on that page
`encoding_list/1` is normally called without any parameters to list the
encodings that are currently configured during compilation. To see all
available options, even those unavailable, use `encoding_list(:all)`
The available encodings are:
#{@encodings_markdown}
For more information about configuring encodings, refer to `Codepagex`.
For a list of shorthand names, see `aliases/1`
"""
@spec encoding_list(atom) :: list(String.t)
def encoding_list(selection \\ nil)
defdelegate encoding_list(selection), to: Mappings
@doc false
def encoding_atoms, do: @encodings_atom
# This is the default missing_fun
defp error_on_missing do
fn _ ->
{:ok, fn _, _ ->
{:error, "Invalid bytes for encoding", nil}
end}
end
end
@doc """
This function may be used as a parameter to `to_string/4` or `to_string!/4`
such that any bytes in the input binary that don't have a proper encoding are
replaced with a special unicode character and the function will not
fail.
If this function is used, `to_string/4` will never return an error.
The accumulator input `acc` of `to_string/4` is incremented by the number of
replacements made.
## Examples
iex> iso = "Hello æøå!" |> from_string!(:iso_8859_1)
iex> to_string!(iso, :ascii, use_utf_replacement())
"Hello ���!"
iex> iso = "Hello æøå!" |> from_string!(:iso_8859_1)
iex> to_string(iso, :ascii, use_utf_replacement())
{:ok, "Hello ���!", 3}
"""
@spec use_utf_replacement :: to_s_missing_outer
def use_utf_replacement do
fn _encoding ->
{:ok, &use_utf_replacement_inner/2}
end
end
defp use_utf_replacement_inner(<<_, rest::binary>>, acc) do
# � replacement character used to replace an unknown or
# unrepresentable character
new_acc = if is_integer(acc), do: acc + 1, else: 1
{:ok, <<0xFFFD :: utf8>>, rest, new_acc}
end
@doc """
This function may be used in conjunction with to `from_string/4` or
`from_string!/4`. If there are utf-8 codepoints in the source string that are
not possible to represent in the target encoding, they are replaced with a
String.
When using this function, `from_string/4` will never return an error if
`replace_with` converts to the target encoding without errors.
The accumulator input `acc` of `from_string/4` is incremented on each
replacement done.
## Examples
iex> from_string!("Hello æøå!", :ascii, replace_nonexistent("_"))
"Hello ___!"
iex> from_string("Hello æøå!", :ascii, replace_nonexistent("_"), 100)
{:ok, "Hello ___!", 103}
"""
@spec replace_nonexistent(String.t) :: from_s_missing_outer
def replace_nonexistent(replace_with) do
fn encoding ->
case from_string(replace_with, encoding) do
{:ok, encoded_replace_with} ->
inner =
fn <<_ :: utf8, rest :: binary>>, acc ->
new_acc = if is_integer(acc), do: acc + 1, else: 1
{:ok, encoded_replace_with, rest, new_acc}
end
{:ok, inner}
err ->
err
end
end
end
@spec strip_acc({atom, term, integer}) :: {atom, term}
defp strip_acc({code, return_value, _acc}), do: {code, return_value}
@doc """
Converts a binary in a specified encoding to an Elixir string in utf-8
encoding.
The encoding parameter should be in `encoding_list/0` (passed as atoms or
strings), or in `aliases/0`.
## Examples
iex> to_string(<<72, 201, 166, 166, 211>>, :iso_8859_1)
{:ok, "Hɦ¦Ó"}
iex> to_string(<<128>>, "ETSI/GSM0338")
{:error, "Invalid bytes for encoding"}
"""
@spec to_string(binary, encoding) :: {:ok, String.t} | {:error, term}
def to_string(binary, encoding) do
to_string(binary, encoding, error_on_missing())
|> strip_acc
end
@doc """
Convert a binary in a specified encoding into an Elixir string in utf-8
encoding
Compared to `to_string/2`, you may pass a `missing_fun` function parameter to
handle encoding errors in the `binary`. The function `use_utf_replacement/0`
may be used as a default error handling machanism.
## Implementing missing_fun
The `missing_fun` must be an anonymous function that returns a second
function. The outer function will receive the encoding used by `to_string/4`,
and must then return `{:ok, inner_function}` or `{:error, reason}`. Returning
`:error` will cause `to_string/4` to fail.
The returned inner function must receive two arguments.
- a binary containing the remainder of the `binary` parameter that is still
unprocessed.
- the accumulator `acc`
The return value must be
- `{:ok, replacement, new_rest, new_acc}` to continue processing
- `{:error, reason, new_acc}` to cause `to_string/4` to fail
The `acc` parameter from `to_string/4` is passed between every invocation of
the inner function then returned by `to_string/4`. In many use cases, `acc`
may be ignored.
## Examples
Using the `use_utf_replacement/0` function to handle invalid bytes:
iex> iso = "Hello æøå!" |> from_string!(:iso_8859_1)
iex> to_string(iso, :ascii, use_utf_replacement())
{:ok, "Hello ���!", 3}
iex> iso = "Hello æøå!" |> from_string!(:iso_8859_1)
iex> missing_fun =
...> fn encoding ->
...> case to_string("#", encoding) do
...> {:ok, replacement} ->
...> inner_fun =
...> fn <<_, rest :: binary>>, acc ->
...> {:ok, replacement, rest, acc + 1}
...> end
...> {:ok, inner_fun}
...> err ->
...> err
...> end
...> end
iex> to_string(iso, :ascii, missing_fun, 0)
{:ok, "Hello ###!", 3}
The previous code was included for completeness. If you know your replacement
is valid in the target encoding, you might as well do:
iex> iso = "Hello æøå!" |> from_string!(:iso_8859_1)
iex> missing_fun =
...> fn _encoding ->
...> inner_fun =
...> fn <<_, rest :: binary>>, acc ->
...> {:ok, "#", rest, acc + 1}
...> end
...> {:ok, inner_fun}
...> end
iex> to_string(iso, :ascii, missing_fun, 10)
{:ok, "Hello ###!", 13}
"""
@spec to_string(binary, encoding, to_s_missing_outer, term) ::
{:ok, String.t, integer} | {:error, term, integer}
def to_string(binary, encoding, missing_fun, acc \\ nil)
# create a forwarding to_string implementation for each alias
for {aliaz, encoding} <- Mappings.aliases do
def to_string(binary, unquote(aliaz), missing_fun, acc) do
to_string(binary, unquote(encoding |> String.to_atom), missing_fun, acc)
end
end
def to_string(binary, encoding, missing_fun, acc) when is_atom(encoding) do
case missing_fun.(encoding) do
{:ok, inner_fun} ->
Mappings.to_string(binary, encoding, inner_fun, acc)
err ->
err
end
end
def to_string(binary, encoding, missing_fun, acc) when is_binary(encoding) do
try do
to_string(binary, String.to_existing_atom(encoding), missing_fun, acc)
rescue
ArgumentError ->
{:error, "Unknown encoding #{inspect encoding}", acc}
end
end
@doc """
Like `to_string/2` but raises exceptions on errors.
## Examples
iex> to_string!(<<72, 201, 166, 166, 211>>, :iso_8859_1)
"Hɦ¦Ó"
iex> to_string!(<<128>>, "ETSI/GSM0338")
** (Codepagex.Error) Invalid bytes for encoding
"""
@spec to_string!(binary, encoding) :: String.t | no_return
def to_string!(binary, encoding) do
to_string!(binary, encoding, error_on_missing(), nil)
end
@doc """
Like `to_string/4` but raises exceptions on errors.
## Examples
iex> iso = "Hello æøå!" |> from_string!(:iso_8859_1)
iex> to_string!(iso, :ascii, use_utf_replacement())
"Hello ���!"
"""
@spec to_string!(binary, encoding, to_s_missing_outer, term) ::
String.t | no_return
def to_string!(binary, encoding, missing_fun, acc \\ nil) do
case to_string(binary, encoding, missing_fun, acc) do
{:ok, result, _} ->
result
{:error, reason, _} ->
raise Codepagex.Error, reason
end
end
@doc """
Converts an Elixir string in utf-8 encoding to a binary in another encoding.
The `encoding` parameter should be in `encoding_list/0` as an atom or String,
or in `aliases/0`.
## Examples
iex> from_string("Hɦ¦Ó", :iso_8859_1)
{:ok, <<72, 201, 166, 166, 211>>}
iex> from_string("Hɦ¦Ó", :"ISO8859/8859-1") # without alias
{:ok, <<72, 201, 166, 166, 211>>}
iex> from_string("ʒ", :iso_8859_1)
{:error, "Invalid bytes for encoding"}
"""
@spec from_string(String.t, encoding) :: {:ok, binary} | {:error, term}
def from_string(string, encoding) do
from_string(string, encoding, error_on_missing(), nil)
|> strip_acc
end
@doc """
Convert an Elixir String in utf-8 to a binary in a specified encoding. A
function parameter specifies how to deal with codepoints that are not
representable in the target encoding.
Compared to `from_string/2`, you may pass a `missing_fun` function parameter
to handle encoding errors in `string`. The function `replace_nonexistent/1`
may be used as a default error handling machanism.
The `encoding` parameter should be in `encoding_list/0` as an atom or String,
or in `aliases/0`.
## Implementing missing_fun
The `missing_fun` must be an anonymous function that returns a second
function. The outer function will receive the encoding used by
`from_string/4`, and must then return `{:ok, inner_function}` or `{:error,
reason}`. Returning `:error` will cause `from_string/4` to fail.
The returned inner function must receive two arguments.
- a String containing the remainder of the `string` parameter that is still
unprocessed.
- the accumulator `acc`
The return value must be
- `{:ok, replacement, new_rest, new_acc}` to continue processing
- `{:error, reason, new_acc}` to cause `from_string/4` to fail
The `acc` parameter from `from_string/4` is passed between every invocation
of the inner function then returned by `to_string/4`. In many use cases,
`acc` may be ignored.
## Examples
Using the `replace_nonexistent/1` function to handle invalid bytes:
iex> from_string("Hello æøå!", :ascii, replace_nonexistent("_"))
{:ok, "Hello ___!", 3}
Defining a custom `missing_fun`:
iex> missing_fun =
...> fn encoding ->
...> case from_string("#", encoding) do
...> {:ok, replacement} ->
...> inner_fun =
...> fn <<_ :: utf8, rest :: binary>>, acc ->
...> {:ok, replacement, rest, acc + 1}
...> end
...> {:ok, inner_fun}
...> err ->
...> err
...> end
...> end
iex> from_string("Hello æøå!", :ascii, missing_fun, 0)
{:ok, "Hello ###!", 3}
The previous code was included for completeness. If you know your replacement
is valid in the target encoding, you might as well do:
iex> missing_fun = fn _encoding ->
...> inner_fun =
...> fn <<_ :: utf8, rest :: binary>>, acc ->
...> {:ok, "#", rest, acc + 1}
...> end
...> {:ok, inner_fun}
...> end
iex> from_string("Hello æøå!", :ascii, missing_fun, 10)
{:ok, "Hello ###!", 13}
"""
@spec from_string(binary, encoding, from_s_missing_outer, term) ::
{:ok, String.t, integer} | {:error, term, integer}
def from_string(string, encoding, missing_fun, acc \\ nil)
# aliases are forwarded to proper name
for {aliaz, encoding} <- Mappings.aliases do
def from_string(string, unquote(aliaz), missing_fun, acc) do
from_string(
string, unquote(encoding |> String.to_atom), missing_fun, acc)
end
end
def from_string(string, encoding, missing_fun, acc) when is_atom(encoding) do
case missing_fun.(encoding) do
{:ok, inner_fun} ->
Mappings.from_string(string, encoding, inner_fun, acc)
err ->
err
end
end
def from_string(
string, encoding, missing_fun, acc
) when is_binary(encoding) do
try do
from_string(string, String.to_existing_atom(encoding), missing_fun, acc)
rescue
ArgumentError ->
{:error, "Unknown encoding #{inspect encoding}", acc}
end
end
@doc """
Like `from_string/2` but raising exceptions on errors.
## Examples
iex> from_string!("Hɦ¦Ó", :iso_8859_1)
<<72, 201, 166, 166, 211>>
iex> from_string!("ʒ", :iso_8859_1)
** (Codepagex.Error) Invalid bytes for encoding
"""
@spec from_string!(String.t, encoding) :: binary | no_return
def from_string!(binary, encoding) do
from_string! binary, encoding, error_on_missing(), nil
end
@doc """
Like `from_string/4` but raising exceptions on errors.
## Examples
iex> missing_fun = replace_nonexistent("_")
iex> from_string!("Hello æøå!", :ascii, missing_fun)
"Hello ___!"
"""
@spec from_string!(String.t, encoding, from_s_missing_outer, term) ::
binary | no_return
def from_string!(string, encoding, missing_fun, acc \\ nil) do
case from_string(string, encoding, missing_fun, acc) do
{:ok, result, _} ->
result
{:error, reason, _} ->
raise Codepagex.Error, reason
end
end
@doc """
Convert a binary in one encoding to a binary in another encoding. The string
is converted to utf-8 internally in the process.
The encoding parameters should be in `encoding_list/0` or `aliases/0`. It may
be passed as an atom, or a string for full encoding names.
## Examples
iex> translate(<<174>>, :iso_8859_1, :iso_8859_15)
{:ok, <<174>>}
iex> translate(<<174>>, :iso_8859_1, :iso_8859_2)
{:error, "Invalid bytes for encoding"}
"""
@spec translate(binary, encoding, encoding) :: {:ok, binary} | {:error, term}
def translate(binary, encoding_from, encoding_to) do
case to_string(binary, encoding_from) do
{:ok, string} ->
from_string(string, encoding_to)
err ->
err
end
end
@doc """
Like `translate/3` but raises exceptions on errors
## Examples
iex> translate!(<<174>>, :iso_8859_1, :iso_8859_15)
<<174>>
iex> translate!(<<174>>, :iso_8859_1,:iso_8859_2)
** (Codepagex.Error) Invalid bytes for encoding
"""
@spec translate!(binary, encoding, encoding) :: binary
def translate!(binary, encoding_from, encoding_to) do
binary
|> to_string!(encoding_from)
|> from_string!(encoding_to)
end
end