Current section
Files
Jump to
Current section
Files
lib/eyeon/binary_decoder/v10.ex
defmodule Eyeon.BinaryDecoder.V10 do
@moduledoc """
Decodes Ion 1.0 binary format to Elixir values.
This module expects data WITHOUT the Ion Version Marker (BVM) prefix.
The BVM is stripped by `Eyeon.BinaryDecoder` (the router) before dispatching here.
"""
alias Eyeon.{Binary, Catalog, LocalSymbolTable}
alias Eyeon.BinaryDecoder.SymbolTable
alias Eyeon.BinaryDecoder.Timestamp, as: BinTimestamp
alias Eyeon.Timestamp
import Bitwise
@bvm <<0xE0, 0x01, 0x00, 0xEA>>
@spec decode_all(binary(), Catalog.t(), map()) :: {:ok, [any()]} | {:error, any()}
def decode_all(data, catalog \\ Catalog.new([]), options \\ %{}) do
state = %{symbol_table: LocalSymbolTable.new([], []), catalog: catalog, options: options}
{:ok, decode_top_level(data, state, [])}
rescue
e -> {:error, Exception.message(e)}
end
@spec decode(binary(), Catalog.t(), map()) :: {:ok, any()} | {:error, any()}
def decode(data, catalog \\ Catalog.new([]), options \\ %{}) do
state = %{symbol_table: LocalSymbolTable.new([], []), catalog: catalog, options: options}
values = decode_top_level(data, state, [])
case values do
[] -> {:ok, []}
[single] -> {:ok, single}
multiple -> {:ok, multiple}
end
rescue
e -> {:error, Exception.message(e)}
end
# --- Top-level value stream ---
defp decode_top_level(<<>>, _state, acc), do: Enum.reverse(acc)
defp decode_top_level(data, state, acc) do
case read_type_descriptor(data) do
{:bvm, rest} ->
new_state = %{state | symbol_table: LocalSymbolTable.new([], [])}
decode_top_level(rest, new_state, acc)
{:nop, _len, rest} ->
decode_top_level(rest, state, acc)
{type_id, len, representation, rest} ->
case maybe_annotation_wrapper(type_id, len, representation, rest, state) do
{:symtab, new_state, rest2} ->
decode_top_level(rest2, new_state, acc)
{:value, value, rest2, new_state} ->
decode_top_level(rest2, new_state, [value | acc])
end
end
end
# --- Type descriptor reading ---
defp read_type_descriptor(<<@bvm, rest::binary>>) do
{:bvm, rest}
end
defp read_type_descriptor(<<type_id::4, 0x0F::4, rest::binary>>) when type_id != 0x0F do
{type_id, :null, <<>>, rest}
end
defp read_type_descriptor(<<0x0F::8, _rest::binary>>) do
raise "reserved type descriptor 0x0F encountered"
end
defp read_type_descriptor(<<0x00, rest::binary>>) do
{:nop, 0, rest}
end
defp read_type_descriptor(<<0x00::4, 0x0E::4, rest::binary>>) do
{len, rest2} = Binary.read_varuint(rest)
<<_pad::binary-size(len), rest3::binary>> = rest2
{:nop, len, rest3}
end
defp read_type_descriptor(<<0x00::4, len::4, rest::binary>>) when len < 14 do
<<_pad::binary-size(len), rest2::binary>> = rest
{:nop, len, rest2}
end
# Bool with L=14 (VarUInt length) is always invalid
defp read_type_descriptor(<<0x01::4, 0x0E::4, _rest::binary>>) do
raise "invalid bool type descriptor: L=14 (bools cannot use VarUInt length)"
end
defp read_type_descriptor(<<type_id::4, 0x0E::4, rest::binary>>) do
{len, rest2} = Binary.read_varuint(rest)
<<representation::binary-size(len), rest3::binary>> = rest2
{type_id, len, representation, rest3}
end
# Bool: L encodes the value (0=false, 1=true), not a byte length
defp read_type_descriptor(<<0x01::4, len::4, rest::binary>>) when len in [0, 1] do
{0x01, len, <<>>, rest}
end
defp read_type_descriptor(<<0x01::4, len::4, _rest::binary>>) when len > 1 and len < 15 do
raise "invalid bool type descriptor: L=#{len} (only 0, 1, 15 are valid)"
end
# Struct with L=1: sorted struct, actual length is VarUInt that follows
defp read_type_descriptor(<<0x0D::4, 0x01::4, rest::binary>>) do
{len, rest2} = Binary.read_varuint(rest)
if len == 0, do: raise("ordered struct must not be empty (use D0)")
<<representation::binary-size(len), rest3::binary>> = rest2
{0x0D, len, representation, rest3}
end
defp read_type_descriptor(<<type_id::4, len::4, rest::binary>>) do
<<representation::binary-size(len), rest2::binary>> = rest
{type_id, len, representation, rest2}
end
# --- Annotation wrapper handling ---
defp maybe_annotation_wrapper(0x0E, _len, representation, rest, state) do
{annot_length, inner} = Binary.read_varuint(representation)
if annot_length == 0 do
raise "annotation wrapper must have at least one annotation"
end
<<annot_bytes::binary-size(annot_length), value_bytes::binary>> = inner
if byte_size(value_bytes) == 0 do
raise "annotation wrapper must contain a value"
end
annotations = read_annotation_sids(annot_bytes, state, [])
# Validate that the wrapped value is not another annotation wrapper or NOP
validate_annotation_wrapped_value(value_bytes)
if symtab_annotation?(annotations) do
new_state = decode_and_process_symtab(value_bytes, state)
{:symtab, new_state, rest}
else
{inner_value, remaining} = decode_value_from_bytes(value_bytes, state)
if byte_size(remaining) > 0 do
raise "annotation wrapper contains extra bytes after value"
end
annotated = wrap_annotations(annotations, inner_value)
{:value, annotated, rest, state}
end
end
defp maybe_annotation_wrapper(type_id, len, representation, rest, state) do
value = decode_typed_value(type_id, len, representation, state)
{:value, value, rest, state}
end
defp validate_annotation_wrapped_value(<<0x0E::4, _::4, _rest::binary>>) do
raise "annotation wrappers cannot be nested"
end
defp validate_annotation_wrapped_value(<<0x00::4, len::4, _rest::binary>>) when len != 0x0F do
raise "NOP pad cannot be annotated"
end
defp validate_annotation_wrapped_value(_), do: :ok
defp decode_and_process_symtab(value_bytes, state) do
case read_type_descriptor(value_bytes) do
{0x0D, _len, rep, _rest} ->
fields = decode_struct_fields_as_list(rep, state, [])
SymbolTable.process_symbol_table({:raw_symtab_fields, fields}, state)
_ ->
{inner_value, _} = decode_value_from_bytes(value_bytes, state)
SymbolTable.process_symbol_table(inner_value, state)
end
end
defp decode_struct_fields_as_list(<<>>, _state, acc), do: Enum.reverse(acc)
defp decode_struct_fields_as_list(data, state, acc) do
{field_sid, rest} = Binary.read_varuint(data)
field_name = SymbolTable.resolve_symbol_text(field_sid, state)
case read_type_descriptor(rest) do
{:nop, _len, rest2} ->
decode_struct_fields_as_list(rest2, state, acc)
{type_id, len, representation, rest2} ->
value = decode_typed_value(type_id, len, representation, state)
decode_struct_fields_as_list(rest2, state, [{field_name, value} | acc])
end
end
defp symtab_annotation?([{:symbol, "$ion_symbol_table"} | _]), do: true
defp symtab_annotation?(_), do: false
defp wrap_annotations([], value), do: value
defp wrap_annotations([ann], value), do: {:annotated, ann, value}
defp wrap_annotations([ann | rest], value) do
{:annotated, ann, wrap_annotations(rest, value)}
end
defp read_annotation_sids(<<>>, _state, acc), do: Enum.reverse(acc)
defp read_annotation_sids(data, state, acc) do
{sid, rest} = Binary.read_varuint(data)
name = SymbolTable.resolve_symbol(sid, state)
read_annotation_sids(rest, state, [name | acc])
end
# --- Value decoding from a bytes chunk (used inside containers) ---
defp decode_value_from_bytes(<<>>, _state), do: {nil, <<>>}
defp decode_value_from_bytes(data, state) do
case read_type_descriptor(data) do
{:bvm, _rest} ->
raise "IVM cannot appear inside an annotation wrapper"
{:nop, _len, rest} ->
decode_value_from_bytes(rest, state)
{0x0E, _len, representation, rest} ->
{annot_length, inner} = Binary.read_varuint(representation)
<<annot_bytes::binary-size(annot_length), value_bytes::binary>> = inner
annotations = read_annotation_sids(annot_bytes, state, [])
{inner_value, _} = decode_value_from_bytes(value_bytes, state)
{wrap_annotations(annotations, inner_value), rest}
{type_id, len, representation, rest} ->
value = decode_typed_value(type_id, len, representation, state)
{value, rest}
end
end
# --- Decode container children ---
defp decode_container_values(<<>>, _state, acc), do: Enum.reverse(acc)
defp decode_container_values(data, state, acc) do
case read_type_descriptor(data) do
{:bvm, _rest} ->
raise "IVM cannot appear inside a container"
{:nop, _len, rest} ->
decode_container_values(rest, state, acc)
{0x0E, _len, representation, rest} ->
{annot_length, inner} = Binary.read_varuint(representation)
<<annot_bytes::binary-size(annot_length), value_bytes::binary>> = inner
annotations = read_annotation_sids(annot_bytes, state, [])
{inner_value, _} = decode_value_from_bytes(value_bytes, state)
value = wrap_annotations(annotations, inner_value)
decode_container_values(rest, state, [value | acc])
{type_id, len, representation, rest} ->
value = decode_typed_value(type_id, len, representation, state)
decode_container_values(rest, state, [value | acc])
end
end
# --- Typed value decoders (all clauses grouped together) ---
# T=0: null
defp decode_typed_value(0x00, :null, _, _state), do: nil
defp decode_typed_value(0x00, _len, _rep, _state), do: nil
# T=1: bool
defp decode_typed_value(0x01, :null, _, _state), do: {:null, :bool}
defp decode_typed_value(0x01, 0, _, _state), do: false
defp decode_typed_value(0x01, 1, _, _state), do: true
# T=2: positive integer
defp decode_typed_value(0x02, :null, _, _state), do: {:null, :int}
defp decode_typed_value(0x02, 0, _, _state), do: 0
defp decode_typed_value(0x02, len, rep, _state) do
{value, _} = Binary.read_uint(rep, len)
value
end
# T=3: negative integer
defp decode_typed_value(0x03, :null, _, _state), do: {:null, :int}
defp decode_typed_value(0x03, 0, _, _state) do
raise "negative zero integer is not valid in Ion"
end
defp decode_typed_value(0x03, len, rep, _state) do
{value, _} = Binary.read_uint(rep, len)
if value == 0, do: raise("negative zero integer is not valid in Ion")
-value
end
# T=4: float
defp decode_typed_value(0x04, :null, _, _state), do: {:null, :float}
defp decode_typed_value(0x04, 0, _, _state), do: 0.0
defp decode_typed_value(0x04, 4, rep, _state) do
case rep do
<<0::1, 0xFF::8, fraction::23>> when fraction != 0 -> :nan
<<1::1, 0xFF::8, fraction::23>> when fraction != 0 -> :nan
<<0x7F800000::32>> -> :infinity
<<0xFF800000::32>> -> :neg_infinity
<<value::float-32>> -> value * 1.0
end
end
defp decode_typed_value(0x04, 8, rep, _state) do
case rep do
<<0::1, 0x7FF::11, fraction::52>> when fraction != 0 -> :nan
<<1::1, 0x7FF::11, fraction::52>> when fraction != 0 -> :nan
<<0x7FF0000000000000::64>> -> :infinity
<<0xFFF0000000000000::64>> -> :neg_infinity
<<value::float-64>> -> value
end
end
# T=5: decimal
defp decode_typed_value(0x05, :null, _, _state), do: {:null, :decimal}
defp decode_typed_value(0x05, 0, _, _state), do: Decimal.new("0")
defp decode_typed_value(0x05, _len, rep, _state) do
{exponent, coef_bytes} = Binary.read_varint(rep)
coef_len = byte_size(coef_bytes)
{coefficient, _} =
if coef_len == 0 do
{0, <<>>}
else
Binary.read_int(coef_bytes, coef_len)
end
{sign, abs_coef} =
cond do
coefficient < 0 -> {-1, -coefficient}
coef_len > 0 and coefficient == 0 -> check_negative_zero(coef_bytes)
true -> {1, coefficient}
end
%Decimal{sign: sign, coef: abs_coef, exp: exponent}
end
# T=6: timestamp
defp decode_typed_value(0x06, :null, _, _state), do: {:null, :timestamp}
defp decode_typed_value(0x06, _len, rep, state) do
{offset, rest} = Binary.read_varint(rep)
{year, rest2} = Binary.read_varuint(rest)
{month, day, hour, minute, second, frac_str, _rest_final} =
BinTimestamp.decode_timestamp_fields(rest2)
BinTimestamp.validate_timestamp_date(year, month, day)
if Map.get(state.options, :timestamp, :native) == :raw do
{:timestamp,
Timestamp.from_components(year, month, day, hour, minute, second, frac_str, offset)}
else
Timestamp.components_to_native(year, month, day, hour, minute, second, frac_str, offset)
end
end
# T=7: symbol
defp decode_typed_value(0x07, :null, _, _state), do: {:null, :symbol}
defp decode_typed_value(0x07, 0, _, state), do: SymbolTable.resolve_symbol(0, state)
defp decode_typed_value(0x07, len, rep, state) do
{sid, _} = Binary.read_uint(rep, len)
SymbolTable.resolve_symbol(sid, state)
end
# T=8: string
defp decode_typed_value(0x08, :null, _, _state), do: {:null, :string}
defp decode_typed_value(0x08, 0, _, _state), do: ""
defp decode_typed_value(0x08, _len, rep, _state) do
if String.valid?(rep) do
rep
else
raise "string contains invalid UTF-8 encoding"
end
end
# T=9: clob
defp decode_typed_value(0x09, :null, _, _state), do: {:null, :clob}
defp decode_typed_value(0x09, _len, rep, _state), do: {:clob, rep}
# T=0xA: blob
defp decode_typed_value(0x0A, :null, _, _state), do: {:null, :blob}
defp decode_typed_value(0x0A, _len, rep, _state), do: {:blob, rep}
# T=0xB: list
defp decode_typed_value(0x0B, :null, _, _state), do: {:null, :list}
defp decode_typed_value(0x0B, _len, rep, state) do
decode_container_values(rep, state, [])
end
# T=0xC: sexp
defp decode_typed_value(0x0C, :null, _, _state), do: {:null, :sexp}
defp decode_typed_value(0x0C, _len, rep, state) do
{:sexp, decode_container_values(rep, state, [])}
end
# T=0xD: struct
defp decode_typed_value(0x0D, :null, _, _state), do: {:null, :struct}
defp decode_typed_value(0x0D, 0, _, _state), do: %{}
defp decode_typed_value(0x0D, _len, rep, state) do
decode_struct_fields(rep, state, [])
|> Map.new()
end
# T=0x0F: reserved
defp decode_typed_value(0x0F, _, _, _state) do
raise "reserved type code 0x0F"
end
# Catch-all for typed nulls not explicitly listed
defp decode_typed_value(type_id, :null, _, _state) do
{:null, type_id_to_atom(type_id)}
end
# --- Struct field decoding ---
defp decode_struct_fields(<<>>, _state, acc), do: Enum.reverse(acc)
defp decode_struct_fields(data, state, acc) do
{field_sid, rest} = Binary.read_varuint(data)
field_name = SymbolTable.resolve_symbol_text(field_sid, state)
case read_type_descriptor(rest) do
{:nop, _len, rest2} ->
decode_struct_fields(rest2, state, acc)
{0x0E, _len, representation, rest2} ->
{annot_length, inner} = Binary.read_varuint(representation)
<<annot_bytes::binary-size(annot_length), value_bytes::binary>> = inner
annotations = read_annotation_sids(annot_bytes, state, [])
{inner_value, _} = decode_value_from_bytes(value_bytes, state)
value = wrap_annotations(annotations, inner_value)
decode_struct_fields(rest2, state, [{field_name, value} | acc])
{type_id, len, representation, rest2} ->
value = decode_typed_value(type_id, len, representation, state)
decode_struct_fields(rest2, state, [{field_name, value} | acc])
end
end
# --- Helpers ---
defp check_negative_zero(<<first_byte, _rest::binary>>) do
if band(first_byte, 0x80) != 0 do
{-1, 0}
else
{1, 0}
end
end
defp check_negative_zero(<<>>), do: {1, 0}
defp type_id_to_atom(0x01), do: :bool
defp type_id_to_atom(0x02), do: :int
defp type_id_to_atom(0x03), do: :int
defp type_id_to_atom(0x04), do: :float
defp type_id_to_atom(0x05), do: :decimal
defp type_id_to_atom(0x06), do: :timestamp
defp type_id_to_atom(0x07), do: :symbol
defp type_id_to_atom(0x08), do: :string
defp type_id_to_atom(0x09), do: :clob
defp type_id_to_atom(0x0A), do: :blob
defp type_id_to_atom(0x0B), do: :list
defp type_id_to_atom(0x0C), do: :sexp
defp type_id_to_atom(0x0D), do: :struct
defp type_id_to_atom(_), do: :null
end