Current section
Files
Jump to
Current section
Files
lib/asm/extensions/routing/health_tracker.ex
defmodule ASM.Extensions.Routing.HealthTracker do
@moduledoc """
Provider health state machine with temporary exclusion windows.
"""
alias ASM.Error
@typedoc "Unique provider candidate identifier used by the router."
@type provider_id :: term()
@typedoc "Per-provider health status."
@type health_status :: %{
status: :healthy | :degraded | :unhealthy,
failures: non_neg_integer(),
last_error: term() | nil,
excluded_until_ms: integer() | nil,
excluded?: boolean()
}
@typep tracker_state :: %{
required(:status) => :healthy | :degraded | :unhealthy,
required(:failures) => non_neg_integer(),
required(:last_error) => term() | nil,
required(:excluded_until_ms) => integer() | nil
}
@type t :: %__MODULE__{
by_provider: %{provider_id() => tracker_state()},
failure_cooldown_ms: pos_integer()
}
@enforce_keys [:by_provider, :failure_cooldown_ms]
defstruct [:by_provider, :failure_cooldown_ms]
@spec new([provider_id()], keyword()) :: {:ok, t()} | {:error, Error.t()}
def new(provider_ids, opts \\ []) when is_list(provider_ids) and is_list(opts) do
cooldown = Keyword.get(opts, :failure_cooldown_ms, 30_000)
cond do
provider_ids == [] ->
{:error, config_error("routing requires at least one provider candidate")}
not is_integer(cooldown) or cooldown <= 0 ->
{:error, config_error(":failure_cooldown_ms must be a positive integer")}
true ->
by_provider =
provider_ids
|> Enum.uniq()
|> Enum.reduce(%{}, fn provider_id, acc ->
Map.put(acc, provider_id, fresh_state())
end)
{:ok, %__MODULE__{by_provider: by_provider, failure_cooldown_ms: cooldown}}
end
end
@spec refresh(t(), integer()) :: t()
def refresh(%__MODULE__{} = tracker, now_ms) when is_integer(now_ms) do
by_provider =
Enum.into(tracker.by_provider, %{}, fn {provider_id, state} ->
{provider_id, maybe_recover_to_degraded(state, now_ms)}
end)
%{tracker | by_provider: by_provider}
end
@spec mark_success(t(), provider_id(), integer()) :: {:ok, t()} | {:error, Error.t()}
def mark_success(%__MODULE__{} = tracker, provider_id, now_ms) when is_integer(now_ms) do
with {:ok, state} <- fetch_state(tracker, provider_id) do
recovered = maybe_recover_to_degraded(state, now_ms)
next_state = %{
recovered
| status: :healthy,
failures: 0,
last_error: nil,
excluded_until_ms: nil
}
{:ok, put_state(tracker, provider_id, next_state)}
end
end
@spec mark_failure(t(), provider_id(), term(), integer()) :: {:ok, t()} | {:error, Error.t()}
def mark_failure(%__MODULE__{} = tracker, provider_id, reason, now_ms)
when is_integer(now_ms) do
with {:ok, state} <- fetch_state(tracker, provider_id) do
current = maybe_recover_to_degraded(state, now_ms)
next_state = %{
current
| status: :unhealthy,
failures: current.failures + 1,
last_error: reason,
excluded_until_ms: now_ms + tracker.failure_cooldown_ms
}
{:ok, put_state(tracker, provider_id, next_state)}
end
end
@spec status(t(), provider_id(), integer()) :: {:ok, health_status()} | {:error, Error.t()}
def status(%__MODULE__{} = tracker, provider_id, now_ms) when is_integer(now_ms) do
with {:ok, state} <- fetch_state(tracker, provider_id) do
state = maybe_recover_to_degraded(state, now_ms)
{:ok, to_health_status(state, now_ms)}
end
end
@spec snapshot(t(), integer()) :: %{provider_id() => health_status()}
def snapshot(%__MODULE__{} = tracker, now_ms) when is_integer(now_ms) do
Enum.into(tracker.by_provider, %{}, fn {provider_id, state} ->
{provider_id, to_health_status(maybe_recover_to_degraded(state, now_ms), now_ms)}
end)
end
@spec available_provider_ids(t(), integer()) :: [provider_id()]
def available_provider_ids(%__MODULE__{} = tracker, now_ms) when is_integer(now_ms) do
tracker
|> snapshot(now_ms)
|> Enum.filter(fn {_provider_id, status} -> not status.excluded? end)
|> Enum.map(fn {provider_id, _status} -> provider_id end)
end
defp fetch_state(%__MODULE__{} = tracker, provider_id) do
case Map.fetch(tracker.by_provider, provider_id) do
{:ok, state} ->
{:ok, state}
:error ->
{:error,
Error.new(
:config_invalid,
:config,
"unknown routing provider candidate: #{inspect(provider_id)}"
)}
end
end
defp put_state(%__MODULE__{} = tracker, provider_id, state) do
%{tracker | by_provider: Map.put(tracker.by_provider, provider_id, state)}
end
defp fresh_state do
%{status: :healthy, failures: 0, last_error: nil, excluded_until_ms: nil}
end
defp maybe_recover_to_degraded(
%{status: :unhealthy, excluded_until_ms: until_ms} = state,
now_ms
)
when is_integer(until_ms) and now_ms >= until_ms do
%{state | status: :degraded, excluded_until_ms: nil}
end
defp maybe_recover_to_degraded(state, _now_ms), do: state
defp to_health_status(state, now_ms) do
excluded? =
case state.excluded_until_ms do
until_ms when is_integer(until_ms) -> now_ms < until_ms
_other -> false
end
%{
status: state.status,
failures: state.failures,
last_error: state.last_error,
excluded_until_ms: state.excluded_until_ms,
excluded?: excluded?
}
end
defp config_error(message) do
Error.new(:config_invalid, :config, message)
end
end