Packages
An in-memory caching library using tuples as the core data type, with support for structs, key/value pairs, queues, and stacks. Includes experimental distributed caching.
Current section
Files
Jump to
Current section
Files
lib/distributed/struct.ex
defmodule SuperCache.Distributed.Struct do
@moduledoc """
Cluster-aware struct store.
Writes (`add`, `remove`, `remove_all`, `init`) are routed to the primary
node for the partition that owns the struct type. Reads (`get`, `get_all`)
default to the local node but can be forwarded to the primary or resolved
via quorum when stronger consistency is needed.
Write replication mode is controlled by the cluster-wide `:replication_mode`
setting (see `SuperCache.Cluster.Bootstrap`):
| Mode | Guarantee | Extra latency |
|-----------|------------------------|------------------|
| `:async` | Eventual (default) | None |
| `:sync` | At-least-once delivery | +1 RTT per write |
| `:strong` | Three-phase commit | +3 RTTs per write|
## Read modes
| Mode | Consistency | Latency |
|------------|-------------------------|-----------------------|
| `:local` | Eventual (default) | Zero extra latency |
| `:primary` | Strong (per key) | +1 RTT if non-primary |
| `:quorum` | Majority vote | +1 RTT (parallel) |
## Example
alias SuperCache.Distributed.Struct, as: DS
DS.init(%Order{}, :id)
DS.add(%Order{id: "o-1", customer: "Alice", total: 59.99, status: :pending})
DS.get(%Order{id: "o-1"})
DS.get(%Order{id: "o-1"}, read_mode: :primary)
DS.get(%Order{id: "o-1"}, read_mode: :quorum)
{:ok, all} = DS.get_all(%Order{})
DS.remove(%Order{id: "o-1"})
DS.remove_all(%Order{})
"""
alias SuperCache.{Storage, Partition}
alias SuperCache.Cluster.{Manager, Replicator, ThreePhaseCommit}
require Logger
## ── Public API ──────────────────────────────────────────────────────────────
@spec init(map, atom) :: true | {:error, any}
def init(%{__struct__: _} = struct, key \\ :id) when is_atom(key) do
with true <- Map.has_key?(struct, key),
{:error, :key_not_found} <- get_key_field(struct) do
route_write(struct, :local_init, [struct, key])
else
false -> {:error, "key does not exist on struct"}
{:ok, _} -> {:error, "struct already initialised"}
end
end
@spec add(map) :: {:ok, map} | {:error, any}
def add(%{__struct__: _} = struct) do
with {:ok, _key} <- get_key_field(struct) do
route_write(struct, :local_add, [struct])
end
end
@spec get(map, keyword) :: {:ok, map} | {:error, :not_found | any}
def get(%{__struct__: _} = struct, opts \\ []) do
with {:ok, _key} <- get_key_field(struct) do
route_read(struct, :local_get, [struct], Keyword.get(opts, :read_mode, :local))
end
end
@spec get_all(map, keyword) :: {:ok, list} | {:error, any}
def get_all(%{__struct__: _} = struct, opts \\ []) do
with {:ok, _key} <- get_key_field(struct) do
route_read(struct, :local_get_all, [struct], Keyword.get(opts, :read_mode, :local))
end
end
@spec remove(map) :: {:ok, map} | {:error, any}
def remove(%{__struct__: _} = struct) do
with {:ok, _key} <- get_key_field(struct) do
case get(struct) do
{:error, :not_found} = err ->
err
{:ok, _existing} = return ->
case route_write(struct, :local_remove, [struct]) do
:ok -> return
{:ok, _} -> return
other -> other
end
end
end
end
@spec remove_all(map) :: {:ok, :removed} | {:error, any}
def remove_all(%{__struct__: struct_name} = struct) do
with {:ok, _key} <- get_key_field(struct) do
Logger.debug(fn -> "super_cache, dist.struct, remove_all #{inspect(struct_name)}" end)
case route_write(struct, :local_remove_all, [struct]) do
:ok -> {:ok, :removed}
{:ok, _} -> {:ok, :removed}
other -> other
end
end
end
## ── Remote entry points — writes (via :erpc, do NOT call directly) ──────────
@doc false
def local_init(%{__struct__: struct_name} = struct, key) do
ns = namespace(struct)
partition = Partition.get_partition(ns)
idx = Partition.get_partition_order(ns)
apply_write(idx, partition, [{:put, {{:struct_storage, :key, struct_name}, key}}])
true
end
@doc false
def local_add(%{__struct__: struct_name} = struct) do
with {:ok, key} <- get_key_field(struct) do
ns = namespace(struct)
partition = Partition.get_partition(ns)
idx = Partition.get_partition_order(ns)
key_data = Map.get(struct, key)
ets_key = {{:struct_storage, :struct, struct_name}, key_data}
Logger.debug(fn ->
"super_cache, dist.struct, add #{inspect(struct_name)} key=#{inspect(key_data)}"
end)
apply_write(idx, partition, [{:delete, ets_key}, {:put, {ets_key, struct}}])
{:ok, struct}
end
end
@doc false
def local_remove(%{__struct__: struct_name} = struct) do
with {:ok, key} <- get_key_field(struct) do
ns = namespace(struct)
partition = Partition.get_partition(ns)
idx = Partition.get_partition_order(ns)
ets_key = {{:struct_storage, :struct, struct_name}, Map.get(struct, key)}
apply_write(idx, partition, [{:delete, ets_key}])
:ok
end
end
@doc false
def local_remove_all(%{__struct__: struct_name} = struct) do
ns = namespace(struct)
partition = Partition.get_partition(ns)
idx = Partition.get_partition_order(ns)
pattern = {{{:struct_storage, :struct, struct_name}, :_}, :_}
apply_write(idx, partition, [{:delete_match, pattern}])
{:ok, :removed}
end
## ── Remote entry points — reads (via :erpc, do NOT call directly) ───────────
@doc false
def local_get(%{__struct__: struct_name} = struct) do
with {:ok, key} <- get_key_field(struct) do
ns = namespace(struct)
partition = Partition.get_partition(ns)
ets_key = {{:struct_storage, :struct, struct_name}, Map.get(struct, key)}
case Storage.get(ets_key, partition) do
[] -> {:error, :not_found}
[{_, result}] -> {:ok, result}
end
end
end
@doc false
def local_get_all(%{__struct__: struct_name} = struct) do
with {:ok, _key} <- get_key_field(struct) do
ns = namespace(struct)
partition = Partition.get_partition(ns)
results =
Storage.get_by_match_object(
{{{:struct_storage, :struct, struct_name}, :_}, :_},
partition
)
|> Enum.map(fn {_, value} -> value end)
{:ok, results}
end
end
@doc false
def local_get_key_field(struct_name) do
ns = {:struct_storage, struct_name}
partition = Partition.get_partition(ns)
case Storage.get({:struct_storage, :key, struct_name}, partition) do
[{_, key}] -> {:ok, key}
[] -> {:error, :key_not_found}
end
end
## ── Private ──────────────────────────────────────────────────────────────────
defp namespace(%{__struct__: struct_name}), do: {:struct_storage, struct_name}
defp get_key_field(%{__struct__: struct_name} = struct) do
ns = namespace(struct)
partition = Partition.get_partition(ns)
case Storage.get({:struct_storage, :key, struct_name}, partition) do
[{_, key}] ->
{:ok, key}
[] ->
# Not found locally — node may not hold this partition. Fall back to primary.
fetch_key_field_from_primary(struct, struct_name)
end
end
defp fetch_key_field_from_primary(struct, struct_name) do
idx = Partition.get_partition_order(namespace(struct))
{primary, _replicas} = Manager.get_replicas(idx)
if primary == node() do
# We are the primary yet got a miss — struct was never init'd.
{:error, :key_not_found}
else
Logger.debug(fn ->
"super_cache, dist.struct, get_key_field fallback → #{inspect(primary)}"
end)
:erpc.call(primary, __MODULE__, :local_get_key_field, [struct_name], 5_000)
end
end
# ── Write routing ─────────────────────────────────────────────────────────────
defp route_write(struct, fun, args) do
idx = Partition.get_partition_order(namespace(struct))
{primary, _replicas} = Manager.get_replicas(idx)
if primary == node() do
apply(__MODULE__, fun, args)
else
Logger.debug(fn -> "super_cache, dist.struct, fwd #{fun} → #{inspect(primary)}" end)
:erpc.call(primary, __MODULE__, fun, args, 5_000)
end
end
# In :strong mode ThreePhaseCommit.commit/2 handles both local Storage apply
# and replica propagation atomically (primary applies last after replicas ACK).
# In other modes Storage is written locally first then replicated async.
defp apply_write(idx, partition, ops) do
case Manager.replication_mode() do
:strong ->
case ThreePhaseCommit.commit(idx, ops) do
:ok ->
:ok
{:error, reason} ->
Logger.error("super_cache, dist.struct, 3pc failed: #{inspect(reason)}")
{:error, reason}
end
_ ->
Enum.each(ops, fn
{:put, record} ->
Storage.put(record, partition)
Replicator.replicate(idx, :put, record)
{:delete, key} ->
Storage.delete(key, partition)
Replicator.replicate(idx, :delete, key)
{:delete_match, pattern} ->
Storage.delete_match(pattern, partition)
Replicator.replicate(idx, :delete_match, pattern)
{:delete_all, _} ->
Storage.delete_all(partition)
Replicator.replicate(idx, :delete_all, nil)
end)
:ok
end
end
# ── Read routing ──────────────────────────────────────────────────────────────
defp route_read(_struct, fun, args, :local) do
apply(__MODULE__, fun, args)
end
defp route_read(struct, fun, args, :primary) do
idx = Partition.get_partition_order(namespace(struct))
{primary, _replicas} = Manager.get_replicas(idx)
if primary == node() do
apply(__MODULE__, fun, args)
else
Logger.debug(fn ->
"super_cache, dist.struct, read_primary #{fun} → #{inspect(primary)}"
end)
:erpc.call(primary, __MODULE__, fun, args, 5_000)
end
end
defp route_read(struct, fun, args, :quorum) do
idx = Partition.get_partition_order(namespace(struct))
{primary, replicas} = Manager.get_replicas(idx)
results =
[primary | replicas]
|> Task.async_stream(
fn
n when n == node() -> apply(__MODULE__, fun, args)
n -> :erpc.call(n, __MODULE__, fun, args, 5_000)
end,
timeout: 5_000,
on_timeout: :kill_task
)
|> Enum.flat_map(fn
{:ok, result} -> [result]
_ -> []
end)
majority = div(length(results), 2) + 1
case Enum.find(Enum.frequencies(results), fn {_, c} -> c >= majority end) do
{result, _} ->
result
nil ->
if primary == node(),
do: apply(__MODULE__, fun, args),
else: :erpc.call(primary, __MODULE__, fun, args, 5_000)
end
end
end