Current section
Files
Jump to
Current section
Files
lib/cache/hash_ring.ex
defmodule Cache.HashRing do
@moduledoc """
Consistent hash ring strategy adapter using `libring`.
This strategy distributes cache keys across Erlang cluster nodes using a
consistent hash ring. When a key is hashed to the local node, the operation
is executed directly. When it hashes to a remote node, the operation is
forwarded via a configurable RPC module (defaults to `:erpc`).
The ring automatically tracks Erlang node membership using
`HashRing.Managed` with `monitor_nodes: true`, so nodes joining or leaving
the cluster are reflected in the ring automatically.
## Usage
```elixir
defmodule MyApp.DistributedCache do
use Cache,
adapter: {Cache.HashRing, Cache.ETS},
name: :distributed_cache,
opts: [read_concurrency: true]
end
```
## Options
#{NimbleOptions.docs([
ring_opts: [
type: :keyword_list,
doc: "Options passed to `HashRing.Worker`, such as `node_blacklist` and `node_whitelist`.",
default: []
],
node_weight: [
type: :pos_integer,
doc: "Number of virtual nodes (shards) per node on the ring. Higher values give more even distribution.",
default: 128
],
rpc_module: [
type: :atom,
doc: "Module used for remote calls. Must implement `call/4` with the same signature as `:erpc.call/4`.",
default: :erpc
]
])}
## How It Works
Each node in the cluster starts the same underlying adapter locally. When a
cache operation is performed:
1. The key is hashed to determine which node owns it via the consistent ring.
2. If the owning node is `Node.self()`, the operation is executed locally.
3. If the owning node is a remote node, the operation is forwarded via the
configured `rpc_module` (default `:erpc`).
This ensures that each key is always stored on the same node (with the same
ring configuration), enabling efficient distributed caching without a
centralised store.
## Read-Repair
When the ring topology changes (node up/down), some keys will hash to a
different node. `Cache.HashRing.RingMonitor` snapshots the ring before each
change, keeping up to `ring_history_size` previous rings.
On a `get` miss, the previous rings are consulted in order (newest first).
For each previous ring, if the key hashed to a different (live) node, a
`get` is attempted there. On a hit:
1. The value is returned immediately.
2. It is written to the current owning node (migration).
3. It is deleted from the old node asynchronously.
This lazily migrates hot keys after rebalancing without scanning the ring.
> **Note**: When `sandbox?: true`, the ring is bypassed and all operations
> are executed locally against the sandbox adapter.
"""
@behaviour Cache.Strategy
@strategy_keys [:ring_opts, :node_weight, :rpc_module, :ring_history_size, :__cache_module__]
@opts_definition [
ring_opts: [
type: :keyword_list,
doc: "Options passed to HashRing.Worker.",
default: []
],
node_weight: [
type: :pos_integer,
doc: "Number of virtual nodes (shards) per node on the ring.",
default: 128
],
rpc_module: [
type: :atom,
doc: "Module used for remote calls (must implement call/4).",
default: :erpc
],
ring_history_size: [
type: :pos_integer,
doc: "Number of previous ring snapshots to keep for read-repair fallback.",
default: 3
]
]
@impl Cache.Strategy
def opts_definition, do: @opts_definition
@impl Cache.Strategy
def child_spec({cache_name, underlying_adapter, adapter_opts}) do
ring_name = ring_name(cache_name)
user_ring_opts = adapter_opts[:ring_opts] || []
ring_opts = Keyword.merge([monitor_nodes: true], user_ring_opts)
underlying_opts = validate_underlying_opts(underlying_adapter, Keyword.drop(adapter_opts, @strategy_keys))
managed_ring_spec = %{
id: ring_name,
type: :worker,
start: {HashRing.Worker, :start_link, [[{:name, ring_name} | ring_opts]]}
}
ring_monitor_spec = %{
id: :"#{cache_name}_ring_monitor",
start:
{Cache.HashRing.RingMonitor, :start_link,
[
[
cache_name: cache_name,
ring_name: ring_name,
history_size: adapter_opts[:ring_history_size] || 3,
node_blacklist: user_ring_opts[:node_blacklist] || [~r/^remsh.*$/, ~r/^rem-.*$/],
node_whitelist: user_ring_opts[:node_whitelist] || []
]
]}
}
%{
id: :"#{cache_name}_hash_ring_supervisor",
type: :supervisor,
start:
{Supervisor, :start_link,
[
[
underlying_adapter.child_spec({cache_name, underlying_opts}),
managed_ring_spec,
ring_monitor_spec
],
[strategy: :one_for_one]
]}
}
end
@impl Cache.Strategy
def get(cache_name, key, underlying_adapter, adapter_opts) do
target_node = key_to_node(cache_name, key)
rpc = adapter_opts[:rpc_module] || :erpc
underlying_opts = validate_underlying_opts(underlying_adapter, Keyword.drop(adapter_opts, @strategy_keys))
result =
if target_node === Node.self() do
underlying_adapter.get(cache_name, key, underlying_opts)
else
rpc.call(target_node, underlying_adapter, :get, [cache_name, key, underlying_opts])
end
case result do
{:ok, nil} ->
read_repair(cache_name, key, target_node, underlying_adapter, underlying_opts, rpc)
{:ok, encoded} ->
{:ok, Cache.TermEncoder.decode(encoded)}
{:error, _} = error ->
error
end
end
@impl Cache.Strategy
def put(cache_name, key, ttl, value, underlying_adapter, adapter_opts) do
target_node = key_to_node(cache_name, key)
rpc = adapter_opts[:rpc_module] || :erpc
underlying_opts = validate_underlying_opts(underlying_adapter, Keyword.drop(adapter_opts, @strategy_keys))
encoded = Cache.TermEncoder.encode(value, underlying_opts[:compression_level])
if target_node === Node.self() do
underlying_adapter.put(cache_name, key, ttl, encoded, underlying_opts)
else
rpc.call(target_node, underlying_adapter, :put, [cache_name, key, ttl, encoded, underlying_opts])
end
end
@impl Cache.Strategy
def delete(cache_name, key, underlying_adapter, adapter_opts) do
target_node = key_to_node(cache_name, key)
rpc = adapter_opts[:rpc_module] || :erpc
underlying_opts = validate_underlying_opts(underlying_adapter, Keyword.drop(adapter_opts, @strategy_keys))
if target_node === Node.self() do
underlying_adapter.delete(cache_name, key, underlying_opts)
else
rpc.call(target_node, underlying_adapter, :delete, [cache_name, key, underlying_opts])
end
end
defp read_repair(cache_name, key, current_node, underlying_adapter, underlying_opts, rpc) do
previous_rings = Cache.HashRing.RingMonitor.previous_rings(cache_name)
result =
Enum.reduce_while(previous_rings, {:not_found, MapSet.new()}, fn ring, {:not_found, tried} ->
old_node = HashRing.key_to_node(ring, key)
cond do
old_node === current_node ->
{:cont, {:not_found, tried}}
MapSet.member?(tried, old_node) ->
{:cont, {:not_found, tried}}
true ->
case rpc_get(rpc, old_node, underlying_adapter, cache_name, key, underlying_opts) do
{:ok, nil} ->
{:cont, {:not_found, MapSet.put(tried, old_node)}}
{:ok, encoded} ->
{:halt, {:found, encoded, old_node}}
:unavailable ->
{:cont, {:not_found, MapSet.put(tried, old_node)}}
end
end
end)
case result do
{:found, encoded, old_node} ->
value = Cache.TermEncoder.decode(encoded)
migrate_value(cache_name, key, value, current_node, old_node, underlying_adapter, underlying_opts, rpc)
{:ok, value}
{:not_found, _tried} ->
{:ok, nil}
end
end
defp migrate_value(cache_name, key, value, current_node, old_node, underlying_adapter, underlying_opts, rpc) do
encoded = Cache.TermEncoder.encode(value, underlying_opts[:compression_level])
if current_node === Node.self() do
underlying_adapter.put(cache_name, key, nil, encoded, underlying_opts)
else
rpc.call(current_node, underlying_adapter, :put, [cache_name, key, nil, encoded, underlying_opts])
end
Task.start(fn ->
rpc.call(old_node, underlying_adapter, :delete, [cache_name, key, underlying_opts])
end)
end
defp rpc_get(rpc, node, adapter, cache_name, key, opts) do
case rpc.call(node, adapter, :get, [cache_name, key, opts]) do
{:ok, _} = result -> result
{:error, _} -> :unavailable
{:badrpc, _} -> :unavailable
end
rescue
_ -> :unavailable
catch
:exit, _ -> :unavailable
end
defp key_to_node(cache_name, key) do
ring = ring_name(cache_name)
case HashRing.Managed.key_to_node(ring, key) do
{:error, {:invalid_ring, :no_nodes}} -> Node.self()
{:error, :no_such_ring} -> Node.self()
node -> node
end
end
defp validate_underlying_opts(adapter, opts) do
if Code.ensure_loaded?(adapter) and function_exported?(adapter, :opts_definition, 0) do
NimbleOptions.validate!(opts, adapter.opts_definition())
else
opts
end
end
defp ring_name(cache_name), do: :"#{cache_name}_hash_ring"
end