Packages
phoenix_gen_api
2.19.0
2.22.0
2.21.1
2.21.0
2.20.1
2.20.0
2.19.0
2.18.1
2.18.0
2.17.1
2.17.0
2.16.0
2.15.0
2.14.0
2.13.0
2.12.0
2.11.0
2.10.0
2.9.1
2.9.0
2.8.0
2.7.0
2.6.1
2.6.0
2.5.0
2.4.0
2.3.0
2.2.0
2.1.1
2.1.0
2.0.1
2.0.0
1.2.0
1.1.3
1.1.2
1.1.1
1.1.0
1.0.1
1.0.0
0.2.1
0.2.0
0.1.1
0.1.0
0.0.16
0.0.15
0.0.14
0.0.13
0.0.12
0.0.11
0.0.10
0.0.9
0.0.8
0.0.7
0.0.6
0.0.5
0.0.4
0.0.3
0.0.2
0.0.1
A library for fast develop APIs in Elixir cluster, using Phoenix Channels for transport data, auto pull api configs from service nodes.
Current section
Files
Jump to
Current section
Files
lib/phoenix_gen_api/node_selector.ex
defmodule PhoenixGenApi.NodeSelector do
@moduledoc """
Provides node selection strategies for distributed request execution.
This module implements various strategies for selecting target nodes from a list of
available nodes. The selection strategy determines how requests are distributed across
nodes in a cluster.
## Supported Selection Strategies
### :random
Selects a random node from the available nodes list. This provides simple load balancing
with no guarantees about distribution fairness.
### :hash
Uses consistent hashing based on the request ID to select a node. The same request ID
will always map to the same node, which is useful for caching and stateful operations.
### {:hash, hash_key}
Uses consistent hashing based on a specific field from the request. The `hash_key` can
reference a field in `request.args` or a field directly on the request struct (like
`user_id` or `device_id`). This ensures requests with the same hash_key value always
go to the same node.
### :round_robin
Distributes requests evenly across all nodes in a circular fashion. Uses an atomic
counter for true global round-robin distribution across all processes.
### {:sticky, hash_key}
Selects a node on the first request and "sticks" it to the given hash_key value
(e.g., `user_id`). Subsequent requests with the same value go to the same node.
The sticky mapping expires after 1 hour (configurable via `@sticky_ttl_ms`),
after which a new node is randomly selected. If the node is removed from the
available nodes list, a new node is selected immediately.
## Dynamic Node Resolution
Instead of a static list of nodes, you can provide a module-function-args tuple that
will be called at runtime to get the current list of nodes:
nodes: {MyApp.NodeRegistry, :get_active_nodes, []}
This allows for dynamic node discovery and automatic adaptation to cluster changes.
## Node Selection for Retry/Fallback
The `get_nodes/2` function returns a list of nodes suitable for the retry strategy:
- `:random` and `:hash` return a single-node list (the selected node)
- `:round_robin` returns all nodes (for fallback to other nodes on failure)
## Fault Tolerance
- Nodes are validated before selection
- Empty node lists return `{:error, :no_nodes_available}`
- Invalid node formats are filtered out
- Failed node resolution is logged with details
## Examples
# Random selection
config = %FunConfig{
nodes: ["node1@host", "node2@host", "node3@host"],
choose_node_mode: :random
}
{:ok, node} = NodeSelector.get_node(config, request)
# Hash by request ID (consistent)
config = %FunConfig{
nodes: ["node1@host", "node2@host"],
choose_node_mode: :hash
}
{:ok, node} = NodeSelector.get_node(config, request)
# Hash by custom field
request = %Request{
request_id: "req_123",
user_id: "user_456",
args: %{"session_id" => "sess_789"}
}
config = %FunConfig{
nodes: ["node1@host", "node2@host"],
choose_node_mode: {:hash, "user_id"}
}
{:ok, node} = NodeSelector.get_node(config, request)
# Round-robin (global, atomic counter)
config = %FunConfig{
nodes: ["node1@host", "node2@host", "node3@host"],
choose_node_mode: :round_robin
}
{:ok, node1} = NodeSelector.get_node(config, request)
{:ok, node2} = NodeSelector.get_node(config, request)
{:ok, node3} = NodeSelector.get_node(config, request)
{:ok, node1_again} = NodeSelector.get_node(config, request) # wraps around
# Sticky node affinity
config = %FunConfig{
nodes: ["node1@host", "node2@host"],
choose_node_mode: {:sticky, "user_id"}
}
# First request from user_123 -> randomly selects node1
# Subsequent requests from user_123 -> always node1 (for up to 1 hour)
{:ok, node} = NodeSelector.get_node(config, request)
## Notes
- Round-robin uses an atomic counter for true global distribution (no process dictionary)
- Hash functions use `:erlang.phash2/2` for deterministic hashing
- Dynamic node resolution happens on every call, allowing real-time cluster updates
- If a hash_key is not found in the request, falls back to random selection
- Sticky mappings are stored in ETS and cleaned up periodically by `ConfigPuller` (every 1 hour)
- Use `NodeSelector.cleanup_sticky_table/0` for manual cleanup
- Returns `{:ok, node}` on success or `{:error, reason}` on failure
"""
alias PhoenixGenApi.Structs.{FunConfig, Request}
alias PhoenixGenApi.Helpers.Shared
require Logger
# Atomic counter reference for round-robin. Uses :atomics for lock-free increments.
@round_robin_counter_name :phoenix_gen_api_round_robin_counter
# ETS table for sticky node affinity.
# Stores {sticky_key, node, timestamp} tuples.
# sticky_key is derived from the hash_key value (e.g., user_id).
@sticky_table_name :phoenix_gen_api_sticky_nodes
# 1 hour TTL for sticky mappings
@sticky_ttl_ms 3_600_000
@doc """
Selects a single target node based on the configuration and request.
This function examines the `choose_node_mode` in the configuration and applies
the appropriate node selection strategy. If the `nodes` field is a tuple, it
will be called as a function to dynamically resolve the node list.
## Parameters
- `config` - A `FunConfig` struct containing:
- `nodes` - Either a list of node names or a `{module, function, args}` tuple
- `choose_node_mode` - The selection strategy (`:random`, `:hash`, `{:hash, key}`, `:round_robin`)
- `request` - A `Request` struct containing the request details
## Returns
- `{:ok, node}` - The selected node
- `{:error, reason}` - Selection failed
## Examples
config = %FunConfig{
nodes: ["node1@host", "node2@host"],
choose_node_mode: :random
}
request = %Request{
request_id: "req_123",
request_type: "get_user",
user_id: "user_456",
args: %{"user_id" => "user_456"}
}
{:ok, node} = NodeSelector.get_node(config, request)
"""
@spec get_node(FunConfig.t(), Request.t()) ::
{:ok, node :: atom() | String.t()} | {:error, term()}
def get_node(config = %FunConfig{}, request = %Request{}) do
with {:ok, resolved_config} <- resolve_nodes(config),
{:ok, nodes} <- validate_and_get_nodes(resolved_config) do
select_node(nodes, resolved_config.choose_node_mode, request)
end
end
@doc """
Selects a list of target nodes based on the configuration and request.
The returned list is ordered by preference for fallback/retry purposes:
- `:random` - Returns a shuffled list of all nodes (selected node first)
- `:hash` / `{:hash, key}` - Returns all nodes with the hashed node first
- `:round_robin` - Returns all nodes starting from the round-robin position
This is useful for the executor's fallback mechanism where if the primary
node fails, it tries the remaining nodes in order.
## Parameters
- `config` - A `FunConfig` struct
- `request` - A `Request` struct
## Returns
- `{:ok, [node, ...]}` - Ordered list of nodes (primary first)
- `{:error, reason}` - Selection failed
## Examples
config = %FunConfig{
nodes: ["node1@host", "node2@host", "node3@host"],
choose_node_mode: :random
}
{:ok, [primary | fallbacks]} = NodeSelector.get_nodes(config, request)
"""
@spec get_nodes(FunConfig.t(), Request.t()) :: {:ok, [atom() | String.t()]} | {:error, term()}
def get_nodes(config = %FunConfig{}, request = %Request{}) do
with {:ok, resolved_config} <- resolve_nodes(config),
{:ok, nodes} <- validate_and_get_nodes(resolved_config) do
select_nodes_ordered(nodes, resolved_config.choose_node_mode, request)
end
end
@doc """
Resolves dynamic node configuration to a concrete node list.
If `nodes` is an MFA tuple `{module, function, args}`, calls the function
to get the node list at runtime. If `nodes` is already a list, returns
the config unchanged. If `nodes` is `:local`, returns the config unchanged.
## Parameters
- `config` - A `FunConfig` struct
## Returns
- `{:ok, %FunConfig{}}` - Config with resolved nodes
- `{:error, reason}` - Resolution failed
"""
@spec resolve_nodes(FunConfig.t()) :: {:ok, FunConfig.t()} | {:error, term()}
def resolve_nodes(config = %FunConfig{nodes: :local}), do: {:ok, config}
def resolve_nodes(config = %FunConfig{nodes: nodes}) when is_list(nodes), do: {:ok, config}
def resolve_nodes(config = %FunConfig{nodes: {m, f, a}}) do
case resolve_dynamic_nodes(m, f, a) do
{:ok, nodes} ->
{:ok, %{config | nodes: nodes}}
{:error, reason} ->
Logger.error(
"[NodeSelector] resolve_nodes: failed to resolve dynamic nodes via MFA, reason: #{inspect(reason)}"
)
{:error, {:dynamic_node_resolution_failed, reason}}
end
end
def resolve_nodes(%FunConfig{nodes: other}) do
Logger.error("[NodeSelector] resolve_nodes: invalid nodes configuration: #{inspect(other)}")
{:error, {:invalid_nodes_configuration, other}}
end
@doc """
Resolves nodes and returns the raw node list regardless of configuration type.
Unlike `resolve_nodes/1` which returns the full config, this returns just
the list of nodes. Useful for getting all available nodes for retry strategies.
## Parameters
- `config` - A `FunConfig` struct
## Returns
- `{:ok, [node]}` - List of resolved nodes
- `{:error, reason}` - Resolution failed
"""
@spec resolve_nodes_list(FunConfig.t()) :: {:ok, [atom() | String.t()]} | {:error, term()}
def resolve_nodes_list(%FunConfig{nodes: :local}) do
{:ok, [node()]}
end
def resolve_nodes_list(config = %FunConfig{}) do
case resolve_nodes(config) do
{:ok, resolved} -> {:ok, Shared.validate_nodes(resolved.nodes)}
{:error, reason} -> {:error, reason}
end
end
@doc """
Validates that the `choose_node_mode` is a recognized strategy.
## Returns
- `true` if the mode is valid
- `false` otherwise
"""
@spec choose_node_valid?(FunConfig.t()) :: boolean()
def choose_node_valid?(%FunConfig{choose_node_mode: mode}) do
case mode do
:random -> true
:hash -> true
{:hash, _} -> true
:round_robin -> true
{:sticky, _} -> true
_ -> false
end
end
@doc """
Calculates a retry backoff delay based on the attempt number.
Uses exponential backoff with jitter to prevent thundering herd problems.
## Parameters
- `attempt` - The current attempt number (1-based)
- `opts` - Options keyword list:
- `:base_ms` - Base delay in milliseconds (default: 100)
- `:max_ms` - Maximum delay in milliseconds (default: 5000)
- `:jitter` - Whether to add random jitter (default: true)
## Returns
- Delay in milliseconds before the next retry
## Examples
iex> NodeSelector.calculate_backoff(1)
# Returns ~100ms (with jitter)
iex> NodeSelector.calculate_backoff(3)
# Returns ~400ms (with jitter)
iex> NodeSelector.calculate_backoff(5, max_ms: 2000)
# Returns capped at ~2000ms (with jitter)
"""
@spec calculate_backoff(pos_integer(), keyword()) :: non_neg_integer()
def calculate_backoff(attempt, opts \\ []) when is_integer(attempt) and attempt > 0 do
base_ms = Keyword.get(opts, :base_ms, 100)
max_ms = Keyword.get(opts, :max_ms, 5_000)
jitter? = Keyword.get(opts, :jitter, true)
# Exponential backoff: base * 2^(attempt-1)
delay = (base_ms * :math.pow(2, attempt - 1)) |> trunc()
delay = min(delay, max_ms)
if jitter? do
# Add random jitter: 0.5x to 1.5x the delay
jitter_factor = 0.5 + :rand.uniform()
trunc(delay * jitter_factor)
else
delay
end
end
@doc """
Resets the round-robin counter.
This is primarily useful for testing. In production, the counter
should be allowed to increment naturally.
"""
@spec reset_round_robin() :: :ok
def reset_round_robin do
case :ets.whereis(@round_robin_counter_name) do
:undefined ->
:ok
_table ->
:ets.insert(@round_robin_counter_name, {:counter, 0})
:ok
end
end
# --- Private Functions ---
defp resolve_dynamic_nodes(m, f, a) when is_atom(m) and is_atom(f) and is_list(a) do
try do
case apply(m, f, a) do
nodes when is_list(nodes) ->
{:ok, nodes}
other ->
{:error, {:invalid_return_type, other}}
end
rescue
error ->
{:error, {:exception, Exception.message(error)}}
catch
kind, value ->
{:error, {kind, value}}
end
end
defp resolve_dynamic_nodes(_, _, _) do
{:error, :invalid_mfa_format}
end
defp validate_and_get_nodes(%FunConfig{nodes: :local}) do
{:ok, [node()]}
end
defp validate_and_get_nodes(%FunConfig{nodes: nodes}) do
validated = Shared.validate_nodes(nodes)
if validated == [] do
Logger.error("[NodeSelector] no valid nodes available after filtering")
{:error, :no_nodes_available}
else
{:ok, validated}
end
end
# Select a single node (backward compatible with get_node/2)
defp select_node(nodes, mode, request) do
case mode do
:random ->
{:ok, Enum.random(nodes)}
:hash ->
{:ok, hash_node(request, nodes)}
{:hash, hash_key} ->
hash_node_with_fallback(request, nodes, hash_key)
:round_robin ->
{:ok, round_robin_node(nodes)}
{:sticky, hash_key} ->
sticky_node(request, nodes, hash_key)
_ ->
Logger.error("[NodeSelector] invalid choose_node_mode: #{inspect(mode)}")
{:error, {:invalid_choose_node_mode, mode}}
end
end
# Select an ordered list of nodes (primary first, then fallbacks)
defp select_nodes_ordered(nodes, mode, request) do
case mode do
:random ->
# Shuffle nodes, putting a random one first
primary = Enum.random(nodes)
fallbacks = List.delete(nodes, primary) |> Enum.shuffle()
{:ok, [primary | fallbacks]}
:hash ->
primary = hash_node(request, nodes)
fallbacks = List.delete(nodes, primary)
{:ok, [primary | fallbacks]}
{:hash, hash_key} ->
# hash_node_with_fallback always returns {:ok, node} (falling back to random on miss)
{:ok, primary} = hash_node_with_fallback(request, nodes, hash_key)
fallbacks = List.delete(nodes, primary)
{:ok, [primary | fallbacks]}
:round_robin ->
# Return all nodes starting from the round-robin position
primary_idx = get_round_robin_index(length(nodes))
{before, after_nodes} = Enum.split(nodes, primary_idx)
{:ok, after_nodes ++ before}
{:sticky, hash_key} ->
case sticky_node(request, nodes, hash_key) do
{:ok, primary} ->
fallbacks = List.delete(nodes, primary)
{:ok, [primary | fallbacks]}
error ->
error
end
_ ->
Logger.error("[NodeSelector] invalid choose_node_mode: #{inspect(mode)}")
{:error, {:invalid_choose_node_mode, mode}}
end
end
defp hash_node(request, nodes) do
hash_order = :erlang.phash2(request.request_id, length(nodes))
Enum.at(nodes, hash_order)
end
defp hash_node_with_fallback(request, nodes, hash_key) do
value =
Map.get(request.args, hash_key) ||
case request do
%{^hash_key => v} when not is_nil(v) -> v
_ -> nil
end
case value do
nil ->
Logger.warning(
"[NodeSelector] hash key #{inspect(hash_key)} not found in request args, falling back to random selection"
)
{:ok, Enum.random(nodes)}
val ->
hash_order = :erlang.phash2(val, length(nodes))
{:ok, Enum.at(nodes, hash_order)}
end
end
# Round-robin using an atomic counter stored in ETS.
# This provides true global round-robin across all processes,
# unlike the previous process dictionary approach.
defp round_robin_node(nodes) do
nodes_length = length(nodes)
idx = get_round_robin_index(nodes_length)
Enum.at(nodes, idx)
end
defp get_round_robin_index(nodes_length) when nodes_length <= 1, do: 0
defp get_round_robin_index(nodes_length) do
ensure_round_robin_counter()
# Use :ets.update_counter for atomic lock-free increment.
# :ets.update_counter/4 uses the same underlying mechanism as :atomics
# (lock-free atomic operations on ETS entries) and scales linearly with cores.
# This avoids the single-writer bottleneck of a GenServer-based counter.
counter =
:ets.update_counter(
@round_robin_counter_name,
:counter,
{2, 1},
{@round_robin_counter_name, 0}
)
# counter is the value AFTER the increment, so use (counter - 1) for 0-based indexing
rem(counter - 1, nodes_length)
end
# Idempotent initialization of the round-robin counter ETS table.
# Uses :ets.update_counter for lock-free atomic increments (same underlying
# mechanism as :atomics, but without the ETS table ownership issue).
defp ensure_round_robin_counter do
case :ets.whereis(@round_robin_counter_name) do
:undefined ->
try do
:ets.new(@round_robin_counter_name, [
:named_table,
:public,
:set,
write_concurrency: true,
read_concurrency: true
])
:ets.insert(@round_robin_counter_name, {:counter, 0})
rescue
ArgumentError ->
# Table was created by another process concurrently; that's fine
:ok
end
_ ->
:ok
end
end
# --- Sticky Node Affinity ---
# Gets or creates a sticky node assignment for the given hash_key value.
# Returns {:ok, node} or {:error, reason}.
defp sticky_node(request, nodes, hash_key) do
value = get_sticky_value(request, hash_key)
if is_nil(value) do
Logger.warning(
"[NodeSelector] sticky hash key #{inspect(hash_key)} not found in request, falling back to random selection"
)
select_and_store_sticky(nodes, hash_key)
else
ensure_sticky_table()
handle_sticky_lookup(nodes, hash_key)
end
end
defp handle_sticky_lookup(nodes, sticky_key) do
case :ets.lookup(@sticky_table_name, sticky_key) do
[{^sticky_key, node, _ts}] ->
handle_existing_sticky(nodes, node, sticky_key)
_ ->
# No sticky assignment or node no longer in list, select new one
select_and_store_sticky(nodes, sticky_key)
end
end
defp handle_existing_sticky(nodes, node, sticky_key) do
if Enum.member?(nodes, node) do
if sticky_valid?(sticky_key) do
{:ok, node}
else
# TTL expired, re-select
select_and_store_sticky(nodes, sticky_key)
end
else
# Node no longer in list, select new one
select_and_store_sticky(nodes, sticky_key)
end
end
defp get_sticky_value(request, hash_key) do
Map.get(request.args, hash_key) ||
case request do
%{^hash_key => v} when not is_nil(v) -> v
_ -> nil
end
end
defp ensure_sticky_table do
case :ets.whereis(@sticky_table_name) do
:undefined ->
try do
:ets.new(@sticky_table_name, [
:named_table,
:public,
:set,
write_concurrency: true
])
rescue
ArgumentError ->
# Table was created by another process concurrently
:ok
end
_ ->
:ok
end
end
defp sticky_valid?(sticky_key) do
case :ets.lookup(@sticky_table_name, sticky_key) do
[{^sticky_key, _node, ts}] ->
now_ms = System.system_time(:millisecond)
now_ms - ts < @sticky_ttl_ms
_ ->
false
end
end
defp select_and_store_sticky(nodes, sticky_key) do
ensure_sticky_table()
node = Enum.random(nodes)
ts = System.system_time(:millisecond)
:ets.insert(@sticky_table_name, {sticky_key, node, ts})
{:ok, node}
end
@doc """
Cleans up expired sticky mappings.
This function should be called periodically (e.g., every hour) to remove
stale sticky node assignments that have exceeded their TTL.
## Returns
- `:ok` - Cleanup completed
"""
def cleanup_sticky_table do
now_ms = System.system_time(:millisecond)
:ets.foldl(@sticky_table_name, [], fn {key, node, ts}, acc ->
if ts < now_ms - @sticky_ttl_ms do
[{key, node, ts} | acc]
else
acc
end
end)
|> Enum.each(fn {key, node, ts} ->
:ets.delete_object(@sticky_table_name, {key, node, ts})
end)
:ok
end
end