Packages
Monitors status changes and fires alerts when a monitor transitions to a configured state.
Current section
Files
Jump to
Current section
Files
lib/integrations/status_change.ex
defmodule Handlers.StatusChange do
@moduledoc """
Fires when a monitor's status transitions to a configured set of states.
Tracks each monitor independently. A notification fires only when the status
changes AND has been confirmed by `confirm_after` consecutive checks.
Recovery (`:up`) fires immediately without confirmation. Re-notification
while a monitor stays down is supported via `resend_interval_minutes`.
## Params
* `:monitor_id` — only react to this monitor id (optional; nil means all).
* `:on_statuses` — comma-separated or list of statuses that trigger a fire.
Defaults to `"down,up"` (alert on down, recover on up).
* `:confirm_after` — consecutive checks required before firing. Defaults to `2`.
* `:resend_interval_minutes` — re-fire if monitor stays in a trigger state for this many
minutes. `0` disables re-notification (default).
* `:source_scope` — which reporting instances to react to in a clustered
deployment: `"local"` (default, this instance only),
`"all"`, or a comma-separated list of instance ids.
## Examples
Alert when a monitor goes down (2 confirmations), recover when it comes back:
%{on_statuses: "down,up", confirm_after: 2}
Alert immediately on first failure, no recovery notification:
%{on_statuses: "down", confirm_after: 1}
Alert on down, re-notify every hour while still down:
%{on_statuses: "down,up", confirm_after: 2, resend_interval_minutes: 60}
"""
use CodeNameRaven.Handler
alias CodeNameRaven.Channel.Notification
@impl true
def display_name, do: "Status Change"
@impl true
def params_template do
%{
monitor_id: "",
on_statuses: "down,up",
confirm_after: "2",
resend_interval_minutes: "0",
source_scope: "local"
}
end
@impl true
def params_schema do
[
monitor_id: [type: :string],
on_statuses: [type: :string, default: "down,up"],
confirm_after: [type: :string, default: "2"],
resend_interval_minutes: [type: :string, default: "0"],
source_scope: [type: :string, default: "local"]
]
end
@impl true
def handler_init(params) do
{:ok,
%{
monitor_id: nilify(params[:monitor_id] || params["monitor_id"]),
on_statuses: parse_statuses(params[:on_statuses] || params["on_statuses"] || "down,up"),
confirm_after: max(parse_int(params[:confirm_after] || params["confirm_after"], 2), 1),
resend_interval_minutes: parse_int(params[:resend_interval_minutes] || params["resend_interval_minutes"], 0),
source_scope: str(params[:source_scope] || params["source_scope"]) || "local",
last_statuses: %{},
# %{id => {status, count}} — awaiting confirmation
pending: %{},
# %{id => status} — currently firing
in_alert: %{},
# %{id => DateTime.t()} — for resend tracking
last_fired_at: %{},
# %{id => DateTime.t()} — when the *current* alert first fired, preserved
# across resends and the eventual recovery notification
alert_since: %{}
}}
end
@impl true
def on_status(%{monitor_id: id, status: new_status, checked_at: checked_at} = msg, params, state) do
instance_id = Map.get(msg, :instance_id)
monitor_name = Map.get(msg, :monitor_name)
scope = str(params[:source_scope] || params["source_scope"]) || state.source_scope
if matches_filter?(id, state.monitor_id) and matches_scope?(instance_id, scope) do
prev_status = Map.get(state.last_statuses, id)
state = %{state | last_statuses: Map.put(state.last_statuses, id, new_status)}
do_transition(id, new_status, prev_status, checked_at, monitor_name, instance_id, state)
else
{:ok, state}
end
end
def on_status(_msg, _params, state), do: {:ok, state}
# ---------------------------------------------------------------------------
# Source-scope filtering — which reporting instances this handler reacts to
# ---------------------------------------------------------------------------
defp matches_scope?(_instance_id, "all"), do: true
defp matches_scope?(instance_id, "local") do
instance_id in [nil, ""] or instance_id == CodeNameRaven.Runtime.instance_id()
end
defp matches_scope?(instance_id, scope) when is_binary(scope) do
scope
|> String.split(",")
|> Enum.map(&String.trim/1)
|> Enum.member?(instance_id)
end
defp matches_scope?(_instance_id, _scope), do: true
# ---------------------------------------------------------------------------
# Transition logic
# ---------------------------------------------------------------------------
# Recovery: fires immediately without confirmation if we were in alert.
defp do_transition(id, :up, _prev, checked_at, monitor_name, instance_id, state) do
alert_status = Map.get(state.in_alert, id)
alert_started_at = Map.get(Map.get(state, :alert_since, %{}), id, checked_at)
state = clear_pending(state, id) |> clear_alert(id)
if alert_status != nil and :up in state.on_statuses do
{:fire, notification(id, :up, alert_status, checked_at, monitor_name, instance_id, alert_started_at), state}
else
{:ok, state}
end
end
defp do_transition(id, new_status, prev_status, checked_at, monitor_name, instance_id, state) do
cond do
new_status not in state.on_statuses ->
{:ok, clear_pending(state, id)}
Map.get(state.in_alert, id) == new_status ->
maybe_resend(id, new_status, checked_at, monitor_name, instance_id, state)
true ->
confirm(id, new_status, prev_status, checked_at, monitor_name, instance_id, state)
end
end
defp confirm(id, new_status, prev_status, checked_at, monitor_name, instance_id, state) do
{pending_status, count} = Map.get(state.pending, id, {new_status, 0})
count = if pending_status == new_status, do: count + 1, else: 1
if count >= state.confirm_after do
state =
state
|> clear_pending(id)
|> put_in_alert(id, new_status)
|> put_last_fired_at(id, checked_at)
|> put_alert_since(id, checked_at)
{:fire, notification(id, new_status, prev_status, checked_at, monitor_name, instance_id, checked_at), state}
else
{:ok, %{state | pending: Map.put(state.pending, id, {new_status, count})}}
end
end
defp maybe_resend(_id, _status, _checked_at, _monitor_name, _instance_id, %{resend_interval_minutes: 0} = state) do
{:ok, state}
end
defp maybe_resend(id, new_status, checked_at, monitor_name, instance_id, state) do
last = Map.get(state.last_fired_at, id)
interval_s = state.resend_interval_minutes * 60
if last && DateTime.diff(checked_at, last, :second) >= interval_s do
alert_started_at = Map.get(Map.get(state, :alert_since, %{}), id, checked_at)
{:fire, notification(id, new_status, new_status, checked_at, monitor_name, instance_id, alert_started_at),
put_last_fired_at(state, id, checked_at)}
else
{:ok, state}
end
end
# ---------------------------------------------------------------------------
# State helpers
# ---------------------------------------------------------------------------
defp clear_pending(state, id), do: %{state | pending: Map.delete(state.pending, id)}
defp clear_alert(state, id) do
since = Map.get(state, :alert_since, %{})
%{state | in_alert: Map.delete(state.in_alert, id), last_fired_at: Map.delete(state.last_fired_at, id)}
|> Map.put(:alert_since, Map.delete(since, id))
end
defp put_in_alert(state, id, status), do: %{state | in_alert: Map.put(state.in_alert, id, status)}
defp put_last_fired_at(state, id, dt), do: %{state | last_fired_at: Map.put(state.last_fired_at, id, dt)}
defp put_alert_since(state, id, dt) do
since = Map.get(state, :alert_since, %{})
Map.put(state, :alert_since, Map.put(since, id, dt))
end
defp notification(id, status, prev_status, occurred_at, monitor_name, instance_id, alert_started_at) do
%Notification{
monitor_id: id,
monitor_name: monitor_name,
status: status,
previous_status: prev_status,
occurred_at: occurred_at,
instance_id: instance_id,
alert_started_at: alert_started_at
}
end
defp matches_filter?(_id, nil), do: true
defp matches_filter?(id, filter), do: id == filter
defp parse_statuses(list) when is_list(list) do
Enum.flat_map(list, fn
a when is_atom(a) -> [a]
s when is_binary(s) -> to_status_atom(s)
end)
end
defp parse_statuses(str) when is_binary(str) do
str |> String.split(",") |> Enum.flat_map(&to_status_atom(String.trim(&1)))
end
defp parse_statuses(_), do: [:down, :up]
defp to_status_atom("up"), do: [:up]
defp to_status_atom("degraded"), do: [:degraded]
defp to_status_atom("down"), do: [:down]
defp to_status_atom("unknown"), do: [:unknown]
defp to_status_atom(_), do: []
defp parse_int(nil, default), do: default
defp parse_int(v, _) when is_integer(v), do: v
defp parse_int(v, default) when is_binary(v) do
case Integer.parse(v) do
{n, _} -> n
:error -> default
end
end
defp parse_int(_, default), do: default
defp nilify(""), do: nil
defp nilify(v), do: v
defp str(nil), do: nil
defp str(""), do: nil
defp str(v), do: v
end