Packages

Elixir client for the S2 durable stream API

Current section

Files

Jump to
s2_client lib s2 store.ex
Raw

lib/s2/store.ex

defmodule S2.Store do
@moduledoc """
High-level store module, similar to `Ecto.Repo`.
Define a store in your application:
defmodule MyApp.S2 do
use S2.Store,
otp_app: :my_app,
basin: "my-basin"
end
Add it to your supervision tree:
# lib/my_app/application.ex
children = [MyApp.S2]
Configure in config:
# config/config.exs
config :my_app, MyApp.S2,
base_url: "https://aws.s2.dev",
token: System.get_env("S2_TOKEN")
Then use it:
MyApp.S2.append("my-stream", %{event: "click"})
MyApp.S2.listen("my-stream", fn msg -> IO.inspect(msg) end)
Each stream gets its own process with a persistent HTTP/2 connection
and open `AppendSession`. Workers are started lazily on first use.
## Options
* `:max_retries` — Maximum reconnection attempts before giving up (default: `:infinity`).
* `:base_delay` — Base delay in ms for exponential backoff between retries (default: 500).
* `:max_queue_size` — Maximum pending appends per stream worker before returning `{:error, :overloaded}` (default: 1000).
* `:recv_timeout` — Timeout in milliseconds for individual S2S data plane operations (default: 5000).
* `:compression` — Compression for S2S frames: `:none`, `:gzip`, or `:zstd` (default: `:none`). Zstd requires the optional `:ezstd` dependency.
* `:call_timeout` — Timeout in milliseconds for `append/2` and `append_batch/2` calls (default: 5000). Set higher if appends may take longer due to large payloads or high latency.
* `:serializer` — A map with `:serialize` and `:deserialize` functions (default: JSON via Jason).
## Listener options
`listen/3` accepts these additional options:
* `:from` — Where to start reading: an integer sequence number (default: `0`) or `:tail` to start from the end of the stream.
* `:serializer` — Override the store's default serializer for this listener.
## Callbacks
When you `use S2.Store`, the following functions are generated:
* `start_link/1` — Start the store supervisor.
* `child_spec/1` — Supervisor child specification.
* `config/0` — Returns the compile-time store configuration.
* `append/2` — Append a message to a stream.
* `append/3` — Append a message with a custom serializer.
* `append_batch/2` — Append multiple messages to a stream.
* `append_batch/3` — Append multiple messages with a custom serializer.
* `create_stream/1` — Create a stream via the control plane.
* `delete_stream/1` — Delete a stream via the control plane.
* `listen/2` — Start a listener on a stream.
* `listen/3` — Start a listener with options.
* `stop_listener/1` — Stop a running listener by PID.
"""
@typedoc "A serializer map with `:serialize` and `:deserialize` functions."
@type serializer :: %{
serialize: (term() -> binary()),
deserialize: (binary() -> term())
}
defmacro __using__(opts) do
quote bind_quoted: [opts: opts] do
@otp_app Keyword.fetch!(opts, :otp_app)
@basin Keyword.fetch!(opts, :basin)
@serializer Keyword.get(opts, :serializer, %{
serialize: &Jason.encode!/1,
deserialize: &Jason.decode!/1
})
@doc false
@spec child_spec(keyword()) :: Supervisor.child_spec()
def child_spec(opts) do
%{
id: __MODULE__,
start: {__MODULE__, :start_link, [opts]},
type: :supervisor
}
end
@doc "Start the store and its supervision tree."
@spec start_link(keyword()) :: Supervisor.on_start()
def start_link(_opts \\ []) do
app_config = Application.get_env(@otp_app, __MODULE__, [])
config = %{
store: __MODULE__,
basin: @basin,
serializer: @serializer,
base_url: Keyword.get(app_config, :base_url, "http://localhost:4243"),
token: Keyword.get(app_config, :token),
max_retries: Keyword.get(app_config, :max_retries, :infinity),
base_delay: Keyword.get(app_config, :base_delay, 500),
max_queue_size: Keyword.get(app_config, :max_queue_size, 1000),
recv_timeout: Keyword.get(app_config, :recv_timeout, 5_000),
compression: Keyword.get(app_config, :compression, :none),
call_timeout: Keyword.get(app_config, :call_timeout, 5_000)
}
unless config.compression in [:none, :gzip, :zstd] do
raise ArgumentError,
"invalid compression: #{inspect(config.compression)}, must be :none, :gzip, or :zstd"
end
S2.Store.Supervisor.start_link(config)
end
@doc "Returns compile-time store configuration (basin, serializer, otp_app)."
@spec config() :: map()
def config do
%{basin: @basin, serializer: @serializer, otp_app: @otp_app}
end
@doc """
Append a message to a stream.
The message is serialized, framed, and deduplicated before being sent.
Returns `{:ok, ack}` with an `S2.V1.AppendAck` on success.
"""
@spec append(String.t(), term(), S2.Store.serializer()) ::
{:ok, S2.V1.AppendAck.t()} | {:error, term()}
def append(stream, message, serializer \\ @serializer) do
with {:ok, _pid} <- S2.Store.Supervisor.ensure_worker(__MODULE__, stream) do
timeout = S2.Store.Supervisor.get_config(__MODULE__).call_timeout
S2.Store.StreamWorker.append(__MODULE__, stream, message, serializer, timeout)
end
end
@doc """
Append multiple messages to a stream in a single batch.
All messages are serialized and sent as one `AppendInput`. Returns
`{:ok, ack}` with an `S2.V1.AppendAck` on success.
"""
@spec append_batch(String.t(), [term()], S2.Store.serializer()) ::
{:ok, S2.V1.AppendAck.t()} | {:error, term()}
def append_batch(stream, messages, serializer \\ @serializer) when is_list(messages) do
with {:ok, _pid} <- S2.Store.Supervisor.ensure_worker(__MODULE__, stream) do
timeout = S2.Store.Supervisor.get_config(__MODULE__).call_timeout
S2.Store.StreamWorker.append_batch(__MODULE__, stream, messages, serializer, timeout)
end
end
@doc "Create a stream via the control plane."
@spec create_stream(String.t()) :: {:ok, S2.StreamInfo.t()} | {:error, S2.Error.t()}
def create_stream(stream) do
S2.Store.Supervisor.create_stream(__MODULE__, stream)
end
@doc "Delete a stream via the control plane."
@spec delete_stream(String.t()) :: :ok | {:error, S2.Error.t()}
def delete_stream(stream) do
S2.Store.Supervisor.delete_stream(__MODULE__, stream)
end
@doc """
Start a listener that calls `callback` for each message on the stream.
The listener runs in a supervised task. Returns `{:ok, pid}` where the PID
can be passed to `stop_listener/1`.
## Options
* `:from` — Starting position: integer sequence number (default: `0`) or `:tail`.
* `:serializer` — Override the store's default serializer.
"""
@spec listen(String.t(), (term() -> any()), keyword()) :: {:ok, pid()}
def listen(stream, callback, opts \\ []) do
S2.Store.Supervisor.listen(__MODULE__, stream, callback, opts)
end
@doc "Stop a running listener by PID."
@spec stop_listener(pid()) :: :ok | {:error, :not_found}
def stop_listener(pid) do
S2.Store.Supervisor.stop_listener(__MODULE__, pid)
end
# Allow `use MyApp.S2, serializer: ...` in downstream modules
# to bind stream functions to this store with a specific serializer.
defmacro __using__(stream_opts) do
store = __MODULE__
serializer_ast =
case Keyword.get(stream_opts, :serializer) do
nil -> Macro.escape(@serializer)
ast -> ast
end
quote do
@__store__ unquote(store)
@doc false
def __serializer__, do: unquote(serializer_ast)
@spec append(String.t(), term()) :: {:ok, S2.V1.AppendAck.t()} | {:error, term()}
def append(stream, message) do
@__store__.append(stream, message, __serializer__())
end
@spec append_batch(String.t(), [term()]) ::
{:ok, S2.V1.AppendAck.t()} | {:error, term()}
def append_batch(stream, messages) do
@__store__.append_batch(stream, messages, __serializer__())
end
@spec listen(String.t(), (term() -> any()), keyword()) :: {:ok, pid()}
def listen(stream, callback, opts \\ []) do
opts = Keyword.put_new(opts, :serializer, __serializer__())
@__store__.listen(stream, callback, opts)
end
@spec create_stream(String.t()) :: {:ok, S2.StreamInfo.t()} | {:error, S2.Error.t()}
def create_stream(stream) do
@__store__.create_stream(stream)
end
@spec delete_stream(String.t()) :: :ok | {:error, S2.Error.t()}
def delete_stream(stream) do
@__store__.delete_stream(stream)
end
@spec stop_listener(pid()) :: :ok | {:error, :not_found}
def stop_listener(pid) do
@__store__.stop_listener(pid)
end
end
end
defoverridable child_spec: 1
end
end
end