Packages
avrora
0.5.0
0.30.2
0.30.1
0.30.0
0.29.2
0.29.1
0.29.0
0.28.0
0.27.0
0.26.0
0.25.0
0.24.2
0.24.1
0.24.0
0.23.0
0.22.0
0.21.1
0.21.0
0.20.0
0.19.0
0.18.1
0.18.0
0.17.0
0.16.0
0.15.0
0.14.1
0.14.0
0.13.0
0.12.0
0.11.0
0.10.0
0.9.1
0.9.0
0.8.1
0.8.0
0.7.1
0.7.0
0.6.0
0.5.1
0.5.0
0.4.0
0.3.1
0.3.0
0.2.2
0.2.1
0.2.0
0.1.1-beta
0.1.0-beta
An Elixir library for working with Avro messages conveniently. It supports local schema files and Confluent® schema registry.
Current section
Files
Jump to
Current section
Files
lib/avrora/encoder.ex
defmodule Avrora.Encoder do
@moduledoc """
Encodes and decodes avro messages created with or without extra Schema Registry
version.
"""
require Logger
alias Avrora.{Mapper, Name, Resolver}
@registry_magic_bytes <<0::size(8)>>
@object_container_magic_bytes <<"Obj", 1>>
@doc """
Decodes given message with a schema eather loaded from the Object Container Files
or from the configured schema registry.
## Examples
...> payload = <<0, 0, 0, 0, 8, 72, 48, 48, 48, 48, 48, 48, 48, 48, 45, 48,
48, 48, 48, 45, 48, 48, 48, 48, 45, 48, 48, 48, 48, 45, 48, 48, 48, 48, 48,
48, 48, 48, 48, 48, 48, 48, 123, 20, 174, 71, 225, 250, 47, 64>>
...> Avrora.Encoder.decode(payload)
{:ok, %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99}}
"""
@spec decode(binary()) :: {:ok, map()} | {:error, term()}
def decode(payload) when is_binary(payload) do
case payload do
<<@registry_magic_bytes, <<id::size(32)>>, body::binary>> ->
with {:ok, avro} <- Resolver.resolve(id), do: do_decode(avro.schema, body)
<<@object_container_magic_bytes, _::binary>> ->
do_decode(payload)
_ ->
{:error, :undecodable}
end
end
@doc """
Decodes given message with a schema eather loaded from the local file or from
the configured schema registry.
## Examples
...> payload = <<72, 48, 48, 48, 48, 48, 48, 48, 48, 45, 48, 48, 48, 48, 45,
48, 48, 48, 48, 45, 48, 48, 48, 48, 45, 48, 48, 48, 48, 48, 48, 48, 48, 48,
48, 48, 48, 123, 20, 174, 71, 225, 250, 47, 64>>
...> Avrora.Encoder.decode(payload, schema_name: "io.confluent.Payment")
{:ok, %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99}}
"""
@spec decode(binary(), keyword(String.t())) :: {:ok, map()} | {:error, term()}
def decode(payload, schema_name: schema_name) when is_binary(payload) do
with {:ok, schema_name} <- Name.parse(schema_name) do
unless is_nil(schema_name.version) do
Logger.warn(
"decoding message with schema version is not supported, `#{schema_name.name}` used instead"
)
end
{schema_id, body} =
case payload do
<<@registry_magic_bytes, <<id::size(32)>>, body::binary>> ->
{id, body}
<<@object_container_magic_bytes, _::binary>> ->
Logger.warn("message contains embeded schema, given schema name will be ignored")
{:embeded, payload}
<<body::binary>> ->
{schema_name.name, body}
end
case schema_id do
:embeded ->
do_decode(payload)
_ ->
with {:ok, avro} <- Resolver.resolve_any([schema_id, schema_name.name]),
do: do_decode(avro.schema, body)
end
end
end
@doc """
Encodes given message with a schema eather loaded from the local file or from
the configured schema registry.
You can control formatting with a `:format` option, possible variants are:
* :guess - behaves like :ocf if can't behave like :registry (default)
* :ocf - embeds schema with Object Container Files format
* :registry - embeds Confluent Schema Registry magic version
* :plain - gives nothing, but only encoded message
## Examples
...> payload = %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99}
...> Avrora.Encoder.encode(payload, schema_name: "io.confluent.Payment", format: :plain)
{:ok, <<72, 48, 48, 48, 48, 48, 48, 48, 48, 45, 48, 48, 48, 48, 45,
48, 48, 48, 48, 45, 48, 48, 48, 48, 45, 48, 48, 48, 48, 48, 48, 48, 48, 48,
48, 48, 48, 123, 20, 174, 71, 225, 250, 47, 64>>}
"""
@spec encode(map(), keyword(String.t())) :: {:ok, binary()} | {:error, term()}
def encode(payload, schema_name: schema_name) when is_map(payload),
do: encode(payload, schema_name: schema_name, format: :guess)
def encode(payload, schema_name: schema_name, format: format) when is_map(payload) do
with {:ok, schema_name} <- Name.parse(schema_name),
{:ok, avro} <- Resolver.resolve(schema_name.name),
{:ok, body} <- do_encode(avro.schema, payload) do
unless is_nil(schema_name.version) do
Logger.warn(
"encoding message with schema version is not supported yet, `#{schema_name.name}` used instead"
)
end
case format do
:guess ->
if is_nil(avro.version),
do: do_embed_schema(avro.schema, body),
else: do_embed_id(avro.id, body)
:registry ->
if is_nil(avro.version),
do: {:error, :invalid_schema_version},
else: do_embed_id(avro.id, body)
:ocf ->
do_embed_schema(avro.schema, body)
:plain ->
{:ok, body}
_ ->
{:error, :unknown_format}
end
end
end
defp do_decode(payload) do
{_, _, decoded} = :avro_ocf.decode_binary(payload)
{:ok, Mapper.to_map(decoded)}
rescue
error -> {:error, error}
end
defp do_decode(schema, payload) do
decoded =
payload
|> :avro_binary_decoder.decode(:undefined, fn _ -> schema end)
|> Mapper.to_map()
{:ok, decoded}
rescue
error -> {:error, error}
end
defp do_encode(schema, payload) do
encoded =
schema
|> :avro_record.new(payload)
|> :avro_binary_encoder.encode_value()
|> :erlang.list_to_binary()
{:ok, encoded}
rescue
error -> {:error, error}
end
defp do_embed_id(id, payload) do
encoded = <<@registry_magic_bytes, <<id::size(32)>>, payload::binary>>
{:ok, encoded}
end
defp do_embed_schema(schema, payload) do
encoded =
schema
|> :avro_ocf.make_header()
|> :avro_ocf.make_ocf(List.wrap(payload))
|> :erlang.list_to_binary()
{:ok, encoded}
rescue
error -> {:error, error}
end
end