Packages

Elixir library for encoding and decoding binaries in Base1 - Binary encoding inspired by unary numbers.

Current section

Files

Jump to
base1 lib base1.ex
Raw

lib/base1.ex

defmodule Base1 do
@moduledoc """
This module encodes and decodes Base1.
## Overview
According to the author of Base1:
```
Base1 encodes arbitrary binary data as a string of repeated "A" characters.
With analogy to the unary numeral system, the binary data is encoded in the length of the string.
```
Base1 is designed to be relatively inefficient, but accurate.
There are three main ways to encode and decode data as Base1 in this module:
* Using `encode/1` and `decode/1` or `decode!/1` to produce a sequence of repeated "A" characters. This does not scale.
* Using `encode_length/1` and `decode_length!/1` or `decode_length/1`. This scales better, though is still relatively inefficient. It is much more compact, however and supports larger inputs.
* Using `encode_length_bin/1` and `decode_length_bin!/1` or `decode_length_bin/1`. This scales better, but not as well as pure length encoding due to the string conversion. This exists mostly to provide a convenience for interfaces that need Base1 and expect Elixir binary input and output.
## Resources
* [Base 1 Canonical Implementation](https://github.com/qntm/base1)
"""
@typedoc """
A Base1 encoded binary.
"""
@type base1_binary() :: <<_::8>>
@max_bin_size 4294967295
@doc """
Encodes a binary as a Base1 string.
## Examples
iex> Base1.encode!(<<1>>)
"AA"
iex> Base1.encode!("b")
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
iex> Base1.encode!(<<>>)
<<>>
"""
@spec encode!(binary()) :: base1_binary()
def encode!(data) when is_binary(data) do
do_encode(data)
end
def encode!(_data) do
raise ArgumentError, "data must be a valid binary"
end
@doc """
Encodes a binary as a Base1 string.
## Examples
iex> Base1.encode(<<1>>)
{:ok, "AA"}
iex> Base1.encode("b")
{:ok, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
iex> Base1.encode(<<>>)
{:ok, <<>>}
"""
@spec encode(binary()) :: {:ok, base1_binary()} | :error
def encode(data) do
{:ok, do_encode(data)}
rescue
ArgumentError -> :error
end
@doc """
Encodes a binary as a Base1 length.
## Examples
Base1.encode_length(<<0>>
1
Base1.encode_length(<<1>>)
2
Base1.encode_length("hi")
26986
Base1.encode_length(<<0x03, 0xC0>>)
1217
"""
@spec encode_length(binary()) :: non_neg_integer()
def encode_length(data) when is_binary(data) do
do_encode_length(data)
end
@doc """
Encodes a binary as a Base1 length string.
## Examples
Base1.encode_length_bin(<<0>>
"1"
Base1.encode_length_bin(<<1>>)
"2"
Base1.encode_length_bin("hi")
"26986"
Base1.encode_length_bin(<<0x03, 0xC0>>)
"1217"
"""
@spec encode_length_bin(binary()) :: binary()
def encode_length_bin(data) do
encode_length(data) |> Integer.to_string()
end
@doc """
Decodes a Base1 string as binary.
Raises an ArgumentError if the input string is not a Base1 binary.
## Examples
iex> Base1.decode!("A")
<<0>>
iex> Base1.decode!("AA")
<<1>>
iex> Base1.decode!("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
"b"
"""
@spec decode!(base1_binary()) :: binary()
def decode!(string) when is_binary(string) do
do_decode(string)
end
@doc """
Decodes a Base1 string as binary.
Returns `:error` if the input string is not a Base1 binary.
## Examples
iex> Base1.decode("A")
{:ok, <<0>>}
iex> Base1.decode("AA")
{:ok, <<1>>}
iex> Base1.decode("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
{:ok, "b"}
iex> Base1.decode("AAAAAAHHH BLACK BEAST OF")
:error
"""
@spec decode(base1_binary()) :: {:ok, binary()} | :error
def decode(string) when is_binary(string) do
{:ok, decode!(string)}
rescue
ArgumentError -> :error
end
@doc """
Decodes a Base1 string length.
Raises an ArgumentError if the length is not a non-negative integer.
## Examples
iex> Base1.decode_length!(1)
<<0>>
iex> Base1.decode_length!(2)
<<1>>
iex> Base1.decode_length!(381115146191751804757868696628784997)
"Hello Cleveland"
iex> Base1.decode_length!(9479543125109159158650175321189934146175361938985120077868441490465756706997907027887945065)
"It is probably better to encode length"
"""
@spec decode_length!(non_neg_integer()) :: binary()
def decode_length!(number) when is_integer(number) and number >= 0 do
decode_blocks(number)
end
def decode_length!(_number) do
raise ArgumentError, "Base1 lengths are non-negative integers."
end
@doc """
Decodes a Base1 string length.
Returns `:error` if the length is not a non-negative integer.
## Examples
iex> Base1.decode_length(1)
{:ok, <<0>>}
iex> Base1.decode_length(2)
{:ok, <<1>>}
iex> Base1.decode_length(381115146191751804757868696628784997)
{:ok, "Hello Cleveland"}
iex> Base1.decode_length(9479543125109159158650175321189934146175361938985120077868441490465756706997907027887945065)
{:ok, "It is probably better to encode length"}
iex> Base1.decode_length(-256)
:error
"""
@spec decode_length(non_neg_integer()) :: {:ok, binary()} | :error
def decode_length(number) do
{:ok, decode_length!(number)}
rescue
ArgumentError -> :error
end
@doc """
Decodes a Base1 string length encoded as a string.
Raises an ArgumentError if the length is not a non-negative integer string.
## Examples
iex> Base1.decode_length_bin!("1")
<<0>>
iex> Base1.decode_length_bin!("2")
<<1>>
iex> Base1.decode_length_bin!("381115146191751804757868696628784997")
"Hello Cleveland"
iex> Base1.decode_length_bin!("9479543125109159158650175321189934146175361938985120077868441490465756706997907027887945065")
"It is probably better to encode length"
"""
@spec decode_length_bin!(binary()) :: binary()
def decode_length_bin!(data) do
String.to_integer(data) |> decode_length!()
end
@doc """
Decodes a Base1 string length encoded as a string.
`:error` is returned if the length is not a non-negative integer string.
## Examples
iex> Base1.decode_length_bin("1")
{:ok, <<0>>}
iex> Base1.decode_length_bin("2")
{:ok, <<1>>}
iex> Base1.decode_length_bin("381115146191751804757868696628784997")
{:ok, "Hello Cleveland"}
iex> Base1.decode_length_bin("9479543125109159158650175321189934146175361938985120077868441490465756706997907027887945065")
{:ok, "It is probably better to encode length"}
iex> Base1.decode_length_bin("cheddar cheese please")
:error
"""
@spec decode_length_bin(binary()) :: binary()
def decode_length_bin(data) do
{:ok, decode_length_bin!(data)}
rescue
ArgumentError -> :error
end
#===============================================================================
# Privacy
#===============================================================================
# this meager power function doesn't convert from integer or overflow as easily
# as the built-in :math.pow(), so we use it here
defp pow(n, k) do
pow(n, k, 1)
end
defp pow(_n, 0, acc) do
acc
end
defp pow(n, k, acc) do
pow(n, k - 1, n * acc)
end
defp do_encode(<<>> = data) do
data
end
defp do_encode(data) do
data_length = do_encode_length(data)
#TODO: resolve limited binary size issue
# There is a hard limit on the maximum size of a binary
# in 64-bit VMs it is 2,305,843,009,213,693,951 bytes
# in 32-bit VMs it is 536,870,911 bytes
# further, when trying to invoke binary copy, a large size can easily cause malloc issues depending on the config
# this can bring down the entire VM due to the NIF call
# Here we try to rescue things in advance
# One option is to construct several binaries and return a series perhaps, but we will still probably risk exceeding memory limits
# this is a sloppy patch for now, probably there is better system_info to use or perhaps it's better to check some info about the heap
# it is probably best to never use base1, but here we are
# max_bin_size = case :erlang.system_info(:wordsize) do
# 4 -> 536_870_911
# _ -> 2_305_843_009_213_693_951
# end
if data_length > @max_bin_size do
raise ArgumentError, "Data is too large to binary encode as Base1 #{inspect data_length} bytes are required. Use encode_length/1 and decode_length/1 instead."
end
:binary.copy("A", data_length)
end
defp do_encode_length(<<>>) do
0
end
defp do_encode_length(data) do
blocks = byte_size(data) - 1
do_encode_length(blocks, 0) + :binary.decode_unsigned(data)
end
defp do_encode_length(0, acc) do
acc + 1
end
defp do_encode_length(remaining_blocks, acc) do
val = pow(256, remaining_blocks)
do_encode_length(remaining_blocks - 1, acc + val)
end
defp do_decode(<<>>) do
<<>>
end
defp do_decode(string) do
# verify we have only As to ensure we are running very slow, but accurate
# using byte_size(data) would not ensure we have valid Base1 data and be faster, which we don't want
bin_length = calculate_length(string, 0)
decode_blocks(bin_length, 1, 0)
end
defp calculate_length(<<>>, acc) do
acc
end
defp calculate_length(<<"A", rest::binary>>, acc) do
calculate_length(rest, acc + 1)
end
defp calculate_length(_string, _acc) do
raise ArgumentError, "Input string is not a valid Base1 string."
end
defp decode_blocks(0) do
<<>>
end
defp decode_blocks(bin_length) do
decode_blocks(bin_length, 1, 0)
end
defp decode_blocks(bin_length, block_size, acc) when bin_length >= block_size do
decode_blocks(bin_length - block_size, block_size * 256, acc + 1)
end
defp decode_blocks(bin_length, _block_size, acc) do
encode_buffer(bin_length, acc - 1, [])
end
defp encode_buffer(bin_length, byte_number, encoded_chars) when byte_number >= 0 do
# likely we can do some tricks like using encoding unsigned to make this faster and skip some pre-steps,
# but we want to adhere to the inherent sluggishness of this algorithm
#TODO: something better for unicode - we follow the JS algorithm, so we need to check if it's acceptable to do this another way
char = rem(bin_length, 256)
quotient = (bin_length - char) |> div(256)
encode_buffer(quotient, byte_number - 1, [char | encoded_chars])
end
defp encode_buffer(_bin_length, _byte_number, encoded_chars) do
# we take an ugly approach from the JS so we work with unicode unsafe here.
encoded_chars |> IO.iodata_to_binary()
end
end