Packages
snakepit
0.8.8
0.13.0
0.12.0
0.11.1
0.11.0
0.10.1
0.10.0
0.9.1
0.9.0
0.8.9
0.8.8
0.8.7
0.8.6
0.8.5
0.8.4
0.8.3
0.8.2
0.8.1
0.8.0
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.11
0.6.10
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.2
0.6.1
0.6.0
0.5.1
0.5.0
0.4.3
0.4.2
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.1
0.2.0
0.1.2
0.1.1
0.1.0
High-performance pooler and session manager for external language integrations. Supports Python, Node.js, Ruby, and more with gRPC streaming, session management, and production-ready process cleanup.
Current section
Files
Jump to
Current section
Files
lib/snakepit/health_monitor.ex
defmodule Snakepit.HealthMonitor do
@moduledoc """
Monitors worker health and crash patterns.
Tracks crashes within a rolling window and determines overall pool health.
Can be used to trigger circuit breaker actions or alerting.
## Usage
{:ok, hm} = HealthMonitor.start_link(
name: :my_pool_health,
pool: :default,
max_crashes: 10,
crash_window_ms: 60_000
)
HealthMonitor.record_crash(hm, "worker_1", %{reason: :segfault})
if HealthMonitor.healthy?(hm) do
# Pool is healthy
else
# Too many crashes, consider action
end
"""
use GenServer
alias Snakepit.Defaults
require Logger
@type worker_stats :: %{
crash_count: non_neg_integer(),
last_crash_time: integer() | nil,
crash_reasons: [term()]
}
@type t :: %{
pool: atom(),
workers: %{String.t() => worker_stats()},
crash_window_ms: pos_integer(),
max_crashes: pos_integer(),
total_crashes: non_neg_integer(),
check_interval_ms: pos_integer(),
check_timer: reference() | nil
}
# Client API
@doc """
Starts a health monitor.
## Options
- `:name` - GenServer name (required)
- `:pool` - Pool name to monitor (required)
- `:check_interval_ms` - Health check interval (default: 30000)
- `:crash_window_ms` - Rolling window for crash counting (default: 60000)
- `:max_crashes` - Max crashes in window before unhealthy (default: 10)
"""
@spec start_link(keyword()) :: GenServer.on_start()
def start_link(opts) do
name = Keyword.fetch!(opts, :name)
GenServer.start_link(__MODULE__, opts, name: name)
end
@doc """
Records a worker crash.
"""
@spec record_crash(GenServer.server(), String.t(), map()) :: :ok
def record_crash(server, worker_id, info \\ %{}) do
GenServer.cast(server, {:record_crash, worker_id, info})
end
@doc """
Returns whether the pool is considered healthy.
"""
@spec healthy?(GenServer.server()) :: boolean()
def healthy?(server) do
GenServer.call(server, :healthy?)
end
@doc """
Returns health status for a specific worker.
"""
@spec worker_health(GenServer.server(), String.t()) :: map()
def worker_health(server, worker_id) do
GenServer.call(server, {:worker_health, worker_id})
end
@doc """
Returns comprehensive health statistics.
"""
@spec stats(GenServer.server()) :: map()
def stats(server) do
GenServer.call(server, :get_stats)
end
# GenServer callbacks
@impl true
def init(opts) do
pool = Keyword.fetch!(opts, :pool)
state = %{
pool: pool,
workers: %{},
crash_window_ms:
Keyword.get(opts, :crash_window_ms, Defaults.health_monitor_crash_window_ms()),
max_crashes: Keyword.get(opts, :max_crashes, Defaults.health_monitor_max_crashes()),
total_crashes: 0,
check_interval_ms:
Keyword.get(opts, :check_interval_ms, Defaults.health_monitor_check_interval()),
check_timer: nil
}
# Schedule periodic cleanup
timer = schedule_cleanup(state.check_interval_ms)
{:ok, %{state | check_timer: timer}}
end
@impl true
def handle_call(:healthy?, _from, state) do
crashes_in_window = count_crashes_in_window(state)
healthy = crashes_in_window < state.max_crashes
{:reply, healthy, state}
end
def handle_call({:worker_health, worker_id}, _from, state) do
worker_stats = Map.get(state.workers, worker_id, default_worker_stats())
health = %{
healthy: worker_stats.crash_count < 3,
crash_count: worker_stats.crash_count,
last_crash_time: worker_stats.last_crash_time
}
{:reply, health, state}
end
def handle_call(:get_stats, _from, state) do
crashes_in_window = count_crashes_in_window(state)
stats = %{
pool: state.pool,
total_crashes: state.total_crashes,
crashes_in_window: crashes_in_window,
workers_with_crashes: map_size(state.workers),
max_crashes: state.max_crashes,
crash_window_ms: state.crash_window_ms,
is_healthy: crashes_in_window < state.max_crashes
}
{:reply, stats, state}
end
@impl true
def handle_cast({:record_crash, worker_id, info}, state) do
now = System.monotonic_time(:millisecond)
worker_stats =
state.workers
|> Map.get(worker_id, default_worker_stats())
|> update_worker_crash(info, now)
workers = Map.put(state.workers, worker_id, worker_stats)
state = %{
state
| workers: workers,
total_crashes: state.total_crashes + 1
}
# Emit telemetry
:telemetry.execute(
[:snakepit, :worker, :crash],
%{},
%{
pool: state.pool,
worker_id: worker_id,
reason: Map.get(info, :reason)
}
)
{:noreply, state}
end
@impl true
def handle_info(:cleanup, state) do
# Remove old crash data
state = cleanup_old_crashes(state)
# Reschedule
timer = schedule_cleanup(state.check_interval_ms)
{:noreply, %{state | check_timer: timer}}
end
def handle_info(_msg, state) do
{:noreply, state}
end
# Private functions
defp schedule_cleanup(interval_ms) do
Process.send_after(self(), :cleanup, interval_ms)
end
defp default_worker_stats do
%{
crash_count: 0,
last_crash_time: nil,
crash_reasons: [],
crash_times: []
}
end
defp update_worker_crash(stats, info, now) do
reason = Map.get(info, :reason)
%{
stats
| crash_count: stats.crash_count + 1,
last_crash_time: now,
crash_reasons: [reason | Enum.take(stats.crash_reasons, 9)],
crash_times: [now | Enum.take(Map.get(stats, :crash_times, []), 99)]
}
end
defp count_crashes_in_window(state) do
now = System.monotonic_time(:millisecond)
window_start = now - state.crash_window_ms
state.workers
|> Map.values()
|> Enum.flat_map(fn stats -> Map.get(stats, :crash_times, []) end)
|> Enum.count(fn time -> time >= window_start end)
end
defp cleanup_old_crashes(state) do
now = System.monotonic_time(:millisecond)
window_start = now - state.crash_window_ms
workers =
state.workers
|> Enum.map(fn {id, stats} ->
crash_times =
stats
|> Map.get(:crash_times, [])
|> Enum.filter(fn time -> time >= window_start end)
{id, Map.put(stats, :crash_times, crash_times)}
end)
|> Enum.filter(fn {_id, stats} ->
# Keep workers with recent crashes
Map.get(stats, :crash_times, []) != []
end)
|> Map.new()
%{state | workers: workers}
end
end