Packages

Elixir client for the S2 durable stream API

Current section

Files

Jump to
s2_client lib s2 patterns serialization.ex
Raw

lib/s2/patterns/serialization.ex

defmodule S2.Patterns.Serialization do
@moduledoc """
High-level serialize/deserialize pipeline for S2 records.
Composes chunking, framing, and deduplication into two operations:
- `prepare/3` — serialize a term into an `AppendInput` ready to send
- `decode/3` — decode sequenced records back into terms, with dedup + reassembly
Accepts a serializer map with `:serialize` and `:deserialize` functions:
serializer = %{
serialize: &Jason.encode!/1,
deserialize: &Jason.decode!/1
}
writer = Serialization.writer()
{input, writer} = Serialization.prepare(writer, %{"event" => "click"}, serializer)
# input is an %S2.V1.AppendInput{} ready to append
reader = Serialization.reader()
{messages, reader} = Serialization.decode(reader, sequenced_records, serializer)
"""
alias S2.Patterns.{RecordFraming, Dedupe}
@type serializer :: S2.Store.serializer()
@type writer :: Dedupe.Writer.t()
@type reader :: %{filter: Dedupe.Filter.t(), assembler: RecordFraming.Assembler.t()}
@doc "Create a new writer state for stamping records with dedup headers."
@spec writer() :: writer()
def writer, do: Dedupe.Writer.new()
@doc "Create a new reader state for dedup filtering and chunk reassembly."
@spec reader() :: reader()
def reader, do: %{filter: Dedupe.Filter.new(), assembler: RecordFraming.Assembler.new()}
@doc """
Serialize a term into an `%S2.V1.AppendInput{}` ready to send.
Applies serialization, chunking, framing, and dedup stamping. Returns
the input and updated writer state (for monotonic dedup sequence).
"""
@spec prepare(writer(), term(), serializer()) :: {S2.V1.AppendInput.t(), writer()}
def prepare(writer, message, serializer) do
bytes = serializer.serialize.(message)
records = RecordFraming.frame(bytes)
{stamped, writer} = Dedupe.Writer.stamp_records(writer, records)
{%S2.V1.AppendInput{records: stamped}, writer}
end
@doc """
Decode sequenced records back into terms with dedup + reassembly.
Returns a list of results and the updated reader. Each result is either
a successfully deserialized term or `{:error, {:deserialization_error, exception}}`
if the deserializer raised. Assembler errors produce
`{:error, {:assembly_error, reason}}`.
Callers should pattern match on results to handle errors:
{results, reader} = Serialization.decode(reader, records, serializer)
Enum.each(results, fn
{:error, reason} -> Logger.warning("decode error: \#{inspect(reason)}")
message -> process(message)
end)
"""
@spec decode(reader(), [S2.V1.SequencedRecord.t()], serializer()) :: {[term()], reader()}
def decode(reader, records, serializer) do
{reversed_messages, reader} =
Enum.reduce(records, {[], reader}, fn record, {msgs, reader} ->
case Dedupe.Filter.check(reader.filter, record) do
:duplicate ->
{msgs, reader}
{:ok, filter} ->
reader = %{reader | filter: filter}
case RecordFraming.Assembler.add(reader.assembler, record) do
{:ok, data, assembler} ->
reader = %{reader | assembler: assembler}
case safe_deserialize(serializer, data) do
{:ok, message} -> {[message | msgs], reader}
{:error, _} = err -> {[err | msgs], reader}
end
{:incomplete, assembler} ->
{msgs, %{reader | assembler: assembler}}
{:error, reason} ->
{[{:error, {:assembly_error, reason}} | msgs], reader}
end
end
end)
{Enum.reverse(reversed_messages), reader}
end
defp safe_deserialize(serializer, data) do
{:ok, serializer.deserialize.(data)}
rescue
e -> {:error, {:deserialization_error, e}}
end
end