Current section
Files
Jump to
Current section
Files
lib/aerospike/cluster/supervisor.ex
defmodule Aerospike.Cluster.Supervisor do
@moduledoc """
Top-level supervisor for one named Aerospike cluster.
Starts the table owner, per-node pool supervisor, partition-map writer,
and tend-cycle worker under `rest_for_one` so the blast radius of a crash
matches which process owns what state.
Crash semantics under `rest_for_one`:
* Tend-cycle crash → only the tend-cycle worker restarts. ETS tables
survive, the pool supervisor and writer survive, and the restarted
worker rehydrates its `:ready` flag from the `:meta` table.
* Writer crash → the tend-cycle worker also restarts (it is after the
writer). The
next tend-cycle write would target a dead writer anyway; taking
the worker with it keeps the cycle consistent.
* Pool-supervisor crash → writer and tend-cycle worker also restart.
* Table-owner crash → the whole subtree restarts with fresh tables.
This module only supervises. It does not start pools, does not run
tend cycles, and does not own any application state.
"""
alias Aerospike.Cluster.NodeSupervisor
alias Aerospike.Cluster.PartitionMapWriter
alias Aerospike.Cluster.TableOwner
alias Aerospike.Cluster.Tender
alias Aerospike.Policy
@typedoc "Authentication mode used during node login."
@type auth_mode :: :internal | :external | :pki
@typedoc "Cluster tend scheduling mode."
@type tend_trigger :: :timer | :manual
@typedoc """
Transport connection option accepted inside startup `:connect_opts`.
TCP transports use the timeout and socket tuning keys. TLS transports
also use the `:tls_*` keys. Auth credentials are configured as top-level
startup options; the Tender forwards derived auth/session values to
transports internally. Other atom keys are forwarded to custom
transports and ignored by the built-in transports.
"""
@type connect_option ::
{:connect_timeout_ms, pos_integer()}
| {:info_timeout, pos_integer()}
| {:login_timeout_ms, pos_integer()}
| {:tcp_nodelay, boolean()}
| {:tcp_keepalive, boolean()}
| {:tcp_sndbuf, pos_integer() | nil}
| {:tcp_rcvbuf, pos_integer() | nil}
| {:tls_name, String.t() | nil}
| {:tls_cacertfile, Path.t() | nil}
| {:tls_certfile, Path.t() | nil}
| {:tls_keyfile, Path.t() | nil}
| {:tls_verify, :verify_peer | :verify_none}
| {:tls_opts, keyword()}
| {atom(), term()}
@typedoc "Keyword list of transport connection options."
@type connect_options :: [connect_option()]
@typedoc """
Startup option accepted by `Aerospike.start_link/1` and
`Aerospike.child_spec/1`.
* `:name` — atom used as the cluster identity (required). Becomes
the cluster process name, table prefix, and pool-supervisor name.
* `:transport` — module implementing `Aerospike.Cluster.NodeTransport`
(required).
* `:hosts` — list of `"host:port"` or `"host"` seed strings
(required, non-empty).
* `:namespaces` — list of namespace strings the cluster must serve
before `Aerospike.Cluster.ready?/1` returns `true` (required,
non-empty).
Pool-level knobs live at the top level of the keyword list because
the pool supervisor, not the transport, applies them. TCP and TLS tuning
knobs live inside `:connect_opts` because the transport owns socket setup.
Pool and lifecycle keys:
* `:pool_size` — workers per node. Positive integer.
* `:min_connections_per_node` — warm connection target per node.
Non-negative integer.
* `:idle_timeout_ms` — idle worker eviction timeout in milliseconds.
Positive integer.
* `:max_idle_pings` — maximum idle workers dropped per verification
cycle. Positive integer.
* `:tend_interval_ms` — automatic tend period in milliseconds.
Positive integer.
* `:tend_trigger` — `:timer` or `:manual`.
Breaker and command-default keys:
* `:failure_threshold` — consecutive tend failures before node
demotion. Non-negative integer.
* `:circuit_open_threshold` — consecutive node failures before the
breaker refuses commands. Non-negative integer.
* `:max_concurrent_ops_per_node` — per-node in-flight plus queued
command cap. Positive integer.
* `:max_retries`, `:sleep_between_retries_ms`, `:replica_policy` —
default retry policy. See `t:Aerospike.RetryPolicy.option/0`.
* `:use_compression` — cluster-wide request compression opt-in.
Discovery and identity keys:
* `:use_services_alternate` — use alternate peer endpoints during
discovery.
* `:seed_only_cluster` — skip peer discovery and use only configured
seeds.
* `:cluster_name` — expected server cluster name.
* `:application_id` — client application identity sent to supporting
servers.
`:user` and `:password` are cluster-wide session-login credentials. Both
must be present together; neither present disables auth. PKI auth uses the
TLS client certificate and must omit both.
"""
@type option ::
{:name, atom()}
| {:transport, module()}
| {:hosts, [String.t(), ...]}
| {:namespaces, [String.t(), ...]}
| {:connect_opts, connect_options()}
| {:pool_size, pos_integer()}
| {:min_connections_per_node, non_neg_integer()}
| {:idle_timeout_ms, pos_integer()}
| {:max_idle_pings, pos_integer()}
| {:tend_interval_ms, pos_integer()}
| {:tend_trigger, tend_trigger()}
| {:failure_threshold, non_neg_integer()}
| {:circuit_open_threshold, non_neg_integer()}
| {:max_concurrent_ops_per_node, pos_integer()}
| {:max_retries, non_neg_integer()}
| {:sleep_between_retries_ms, non_neg_integer()}
| {:replica_policy, Aerospike.RetryPolicy.replica_policy()}
| {:use_compression, boolean()}
| {:use_services_alternate, boolean()}
| {:seed_only_cluster, boolean()}
| {:cluster_name, String.t()}
| {:application_id, String.t()}
| {:auth_mode, auth_mode()}
| {:user, String.t()}
| {:password, String.t()}
| {:login_timeout_ms, pos_integer()}
@typedoc "Keyword list of startup options."
@type options :: [option()]
@doc """
Returns the OTP child specification for one named Aerospike cluster.
Accepted options are documented by `t:option/0`.
"""
@spec child_spec(options()) :: Supervisor.child_spec()
def child_spec(opts) when is_list(opts) do
name = Keyword.fetch!(opts, :name)
%{
id: {__MODULE__, name},
start: {__MODULE__, :start_link, [opts]},
type: :supervisor,
restart: :permanent,
shutdown: :infinity
}
end
@doc """
Starts the top-level supervisor for one named cluster.
Returns the `Supervisor` pid on success. The supervisor registers
itself under `sup_name/1` so callers can reach it by name.
Accepted options are documented by `t:option/0`. `:name`,
`:transport`, `:hosts`, and `:namespaces` are required.
"""
@spec start_link(options()) :: Supervisor.on_start()
def start_link(opts) when is_list(opts) do
validated = validate!(opts)
name = Keyword.fetch!(validated, :name)
hosts = Keyword.fetch!(validated, :hosts)
tender_opts =
validated
|> Keyword.drop([:tables, :node_supervisor, :writer, :hosts])
|> Keyword.put(:seeds, parse_hosts!(hosts))
|> Keyword.put(:node_supervisor, NodeSupervisor.sup_name(name))
|> Keyword.put(:writer, PartitionMapWriter.via(name))
children = [
{TableOwner, name: name},
{NodeSupervisor, name: name},
%{
id: {PartitionMapWriter, name},
start: {__MODULE__, :start_writer, [name]},
type: :worker,
restart: :permanent,
shutdown: 5_000
},
%{
id: {Tender, name},
start: {__MODULE__, :start_tender, [name, tender_opts]},
type: :worker,
restart: :permanent,
shutdown: 5_000
}
]
Supervisor.start_link(children, strategy: :rest_for_one, name: sup_name(name))
end
@doc false
# Resolves TableOwner tables at supervisor init time. TableOwner is already
# up at this point, so `tables/1` is a synchronous call on a live process.
@spec start_writer(atom()) :: GenServer.on_start()
def start_writer(name) do
tables = TableOwner.tables(TableOwner.via(name))
PartitionMapWriter.start_link(name: name, tables: tables)
end
@doc false
# Resolves TableOwner tables at supervisor init time. TableOwner is already
# up at this point, so `tables/1` is a synchronous call on a live process.
@spec start_tender(atom(), keyword()) :: GenServer.on_start()
def start_tender(name, tender_opts) do
tables = TableOwner.tables(TableOwner.via(name))
tender_opts
|> Keyword.put(:tables, tables)
|> Tender.start_link()
end
@doc """
Returns the registered name atom used by `start_link/1` for `name`.
"""
@spec sup_name(atom()) :: atom()
def sup_name(name) when is_atom(name), do: :"#{name}_sup"
## Validation
@required_keys [:name, :transport, :hosts, :namespaces]
defp validate!(opts) do
Enum.each(@required_keys, fn key ->
Keyword.has_key?(opts, key) or
raise ArgumentError,
"Aerospike.Cluster.Supervisor: missing required option #{inspect(key)}"
end)
name = Keyword.fetch!(opts, :name)
is_atom(name) or
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :name must be an atom, got #{inspect(name)}"
transport = Keyword.fetch!(opts, :transport)
is_atom(transport) or
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :transport must be a module, got #{inspect(transport)}"
hosts = Keyword.fetch!(opts, :hosts)
(is_list(hosts) and hosts != []) or
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :hosts must be a non-empty list of \"host:port\" strings"
_ = parse_hosts!(hosts)
namespaces = Keyword.fetch!(opts, :namespaces)
(is_list(namespaces) and namespaces != []) or
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :namespaces must be a non-empty list of strings"
validate_namespaces!(namespaces)
validate_pos_integer!(opts, :pool_size)
validate_non_neg_integer!(opts, :min_connections_per_node)
validate_min_connections!(opts)
validate_pos_integer!(opts, :idle_timeout_ms)
validate_pos_integer!(opts, :max_idle_pings)
validate_pos_integer!(opts, :tend_interval_ms)
validate_pos_integer!(opts, :login_timeout_ms)
validate_non_neg_integer!(opts, :failure_threshold)
validate_non_neg_integer!(opts, :circuit_open_threshold)
validate_pos_integer!(opts, :max_concurrent_ops_per_node)
validate_auth_opts!(opts)
validate_auth_mode!(opts)
validate_non_empty_string!(opts, :cluster_name)
validate_non_empty_string!(opts, :application_id)
validate_top_level_bool!(opts, :use_compression)
validate_top_level_bool!(opts, :use_services_alternate)
validate_top_level_bool!(opts, :seed_only_cluster)
validate_tend_trigger!(opts)
validate_cluster_policy!(opts)
validate_connect_opts!(opts)
opts
end
# `:user` / `:password` are cluster-wide credentials. Reject a partial
# pair at `start_link/1` rather than letting a half-configured cluster
# silently disable auth on one socket and require it on the next.
defp validate_auth_opts!(opts) do
user = Keyword.get(opts, :user)
password = Keyword.get(opts, :password)
case {user, password} do
{nil, nil} ->
:ok
{u, p} when is_binary(u) and is_binary(p) ->
:ok
_ ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :user and :password must both be strings or both be absent"
end
end
defp validate_auth_mode!(opts) do
auth_mode = Keyword.get(opts, :auth_mode, :internal)
auth_mode in [:internal, :external, :pki] or
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :auth_mode must be :internal, :external, or :pki, " <>
"got #{inspect(auth_mode)}"
validate_auth_mode_transport!(opts, auth_mode)
validate_auth_mode_credentials!(opts, auth_mode)
end
defp validate_auth_mode_transport!(opts, auth_mode) when auth_mode in [:external, :pki] do
case Keyword.fetch!(opts, :transport) do
Aerospike.Transport.Tls ->
:ok
transport ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: #{inspect(auth_mode)} auth requires Aerospike.Transport.Tls, " <>
"got #{inspect(transport)}"
end
end
defp validate_auth_mode_transport!(_opts, _auth_mode), do: :ok
defp validate_auth_mode_credentials!(opts, :external) do
if is_binary(Keyword.get(opts, :user)) and is_binary(Keyword.get(opts, :password)) do
:ok
else
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :external auth requires :user and :password strings"
end
end
defp validate_auth_mode_credentials!(opts, :pki) do
case {Keyword.get(opts, :user), Keyword.get(opts, :password)} do
{nil, nil} ->
:ok
_ ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :pki auth uses the TLS client certificate; " <>
":user and :password must be absent"
end
end
defp validate_auth_mode_credentials!(_opts, _auth_mode), do: :ok
defp validate_min_connections!(opts) do
min_connections = Keyword.get(opts, :min_connections_per_node)
pool_size = Keyword.get(opts, :pool_size)
cond do
is_nil(min_connections) or is_nil(pool_size) ->
:ok
min_connections <= pool_size ->
:ok
true ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :min_connections_per_node must be less than or equal to :pool_size"
end
end
defp validate_pos_integer!(opts, key) do
case Keyword.fetch(opts, key) do
:error -> :ok
{:ok, value} when is_integer(value) and value > 0 -> :ok
{:ok, value} -> raise_pos_integer!(key, value)
end
end
defp validate_non_neg_integer!(opts, key) do
case Keyword.fetch(opts, key) do
:error ->
:ok
{:ok, value} when is_integer(value) and value >= 0 ->
:ok
{:ok, value} ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: #{inspect(key)} must be a non-negative integer, " <>
"got #{inspect(value)}"
end
end
defp raise_pos_integer!(key, value) do
raise ArgumentError,
"Aerospike.Cluster.Supervisor: #{inspect(key)} must be a positive integer, " <>
"got #{inspect(value)}"
end
defp parse_hosts!(hosts) do
Enum.map(hosts, &parse_host!/1)
end
defp parse_host!(host_port) when is_binary(host_port) and host_port != "" do
case String.split(host_port, ":", parts: 2) do
[host, port] when host != "" ->
{host, parse_host_port!(host_port, port)}
[host] when host != "" ->
{host, 3000}
_ ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: each host must be a non-empty string in " <>
"\"host:port\" or \"host\" form, got #{inspect(host_port)}"
end
end
defp parse_host!(host_port) do
raise ArgumentError,
"Aerospike.Cluster.Supervisor: each host must be a non-empty string in " <>
"\"host:port\" or \"host\" form, got #{inspect(host_port)}"
end
defp parse_host_port!(host_port, port) do
case Integer.parse(port) do
{value, ""} when value in 1..65_535 ->
value
_ ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: invalid host port in #{inspect(host_port)}"
end
end
defp validate_namespaces!(namespaces) do
Enum.each(namespaces, fn
namespace when is_binary(namespace) and namespace != "" ->
:ok
namespace ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: each namespace must be a non-empty string, " <>
"got #{inspect(namespace)}"
end)
end
defp validate_top_level_bool!(opts, key) do
case Keyword.fetch(opts, key) do
:error ->
:ok
{:ok, value} when is_boolean(value) ->
:ok
{:ok, value} ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: #{inspect(key)} must be a boolean, got #{inspect(value)}"
end
end
defp validate_non_empty_string!(opts, key) do
case Keyword.fetch(opts, key) do
:error ->
:ok
{:ok, value} when is_binary(value) and value != "" ->
:ok
{:ok, value} ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: #{inspect(key)} must be a non-empty string, " <>
"got #{inspect(value)}"
end
end
defp validate_tend_trigger!(opts) do
case Keyword.fetch(opts, :tend_trigger) do
:error ->
:ok
{:ok, value} when value in [:timer, :manual] ->
:ok
{:ok, value} ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :tend_trigger must be :timer or :manual, " <>
"got #{inspect(value)}"
end
end
defp validate_cluster_policy!(opts) do
_ = Policy.cluster_defaults(opts)
:ok
end
# Rejects the two fat-fingers an operator is most likely to hit inside
# `:connect_opts`: a non-boolean boolean or a non-positive buffer size.
# `Transport.Tcp.connect/3` also raises on bad values — doing it here
# too means `start_link/1` fails synchronously instead of later when
# the first pool worker tries to connect.
defp validate_connect_opts!(opts) do
case Keyword.fetch(opts, :connect_opts) do
:error ->
:ok
{:ok, connect_opts} when is_list(connect_opts) ->
validate_bool!(connect_opts, :tcp_nodelay)
validate_bool!(connect_opts, :tcp_keepalive)
validate_optional_pos_integer!(connect_opts, :tcp_sndbuf)
validate_optional_pos_integer!(connect_opts, :tcp_rcvbuf)
validate_pos_integer!(connect_opts, :connect_timeout_ms)
validate_pos_integer!(connect_opts, :info_timeout)
validate_pos_integer!(connect_opts, :login_timeout_ms)
validate_tls_opts!(connect_opts)
:ok
{:ok, value} ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: :connect_opts must be a keyword list, got #{inspect(value)}"
end
end
# TLS-specific opt shape checks. The transport itself also raises on
# bad values, but catching them here gives `start_link/1` a synchronous
# failure point — operators see a typo in their config before the first
# pool worker tries to connect.
defp validate_tls_opts!(connect_opts) do
validate_string_or_nil!(connect_opts, :tls_name)
validate_path_or_nil!(connect_opts, :tls_cacertfile)
validate_path_or_nil!(connect_opts, :tls_certfile)
validate_path_or_nil!(connect_opts, :tls_keyfile)
validate_tls_verify!(connect_opts)
validate_tls_cert_key_pair!(connect_opts)
:ok
end
defp validate_string_or_nil!(opts, key) do
case Keyword.fetch(opts, key) do
:error -> :ok
{:ok, nil} -> :ok
{:ok, value} when is_binary(value) -> :ok
{:ok, value} -> raise_string!(key, value)
end
end
defp validate_path_or_nil!(opts, key) do
case Keyword.fetch(opts, key) do
:error -> :ok
{:ok, nil} -> :ok
{:ok, value} when is_binary(value) -> :ok
{:ok, value} -> raise_string!(key, value)
end
end
defp validate_tls_verify!(opts) do
case Keyword.fetch(opts, :tls_verify) do
:error -> :ok
{:ok, :verify_peer} -> :ok
{:ok, :verify_none} -> :ok
{:ok, value} -> raise_tls_verify!(value)
end
end
# `:tls_certfile` and `:tls_keyfile` ship together or not at all —
# mTLS requires both; standard TLS sets neither. A half-configured
# pair is a common copy-paste mistake worth rejecting synchronously.
defp validate_tls_cert_key_pair!(opts) do
cert = Keyword.get(opts, :tls_certfile)
key = Keyword.get(opts, :tls_keyfile)
case {cert, key} do
{nil, nil} ->
:ok
{c, k} when is_binary(c) and is_binary(k) ->
:ok
_ ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: connect_opts :tls_certfile and :tls_keyfile " <>
"must both be strings or both be absent, " <>
"got certfile=#{inspect(cert)} keyfile=#{inspect(key)}"
end
end
defp raise_string!(key, value) do
raise ArgumentError,
"Aerospike.Cluster.Supervisor: connect_opts #{inspect(key)} must be a string or nil, " <>
"got #{inspect(value)}"
end
defp raise_tls_verify!(value) do
raise ArgumentError,
"Aerospike.Cluster.Supervisor: connect_opts :tls_verify must be :verify_peer or " <>
":verify_none, got #{inspect(value)}"
end
defp validate_bool!(opts, key) do
case Keyword.fetch(opts, key) do
:error ->
:ok
{:ok, value} when is_boolean(value) ->
:ok
{:ok, value} ->
raise ArgumentError,
"Aerospike.Cluster.Supervisor: connect_opts #{inspect(key)} must be a boolean, " <>
"got #{inspect(value)}"
end
end
defp validate_optional_pos_integer!(opts, key) do
case Keyword.fetch(opts, key) do
:error -> :ok
{:ok, nil} -> :ok
{:ok, value} when is_integer(value) and value > 0 -> :ok
{:ok, value} -> raise_pos_integer!(key, value)
end
end
end