raven_integration_watchdog

defmodule Handlers.Watchdog do
  @moduledoc """
  Dead-man's-switch handler — fires a `:down` alert when a monitor stops
  reporting entirely, not just when it reports unhealthy.

  Every ordinary handler reacts to status *events*; a monitor that crashes,
  gets unscheduled, or loses network connectivity to the DataBus simply
  stops producing events and never fires an alert. Watchdog instead tracks
  the last time each monitor was *seen at all* and, on a periodic tick,
  fires `:down` for any monitor silent longer than `:stale_after_minutes`.
  Seeing the monitor report again fires an `:up` recovery and clears the
  alert.

  ## Params

    * `:stale_after_minutes`     — how long a monitor may go silent before
                                   it's considered stale. Defaults to `5`.
    * `:monitor_id`              — only watch this monitor id (optional; nil means all).
    * `:on_statuses`             — comma-separated or list of statuses this handler
                                   participates in. Defaults to `"down,up"` — `:up`
                                   must be included for the recovery notification to fire.
    * `:resend_interval_minutes` — re-fire while a monitor stays stale, every this many
                                   minutes. `0` disables re-notification (default).
    * `:source_scope`            — which reporting instances to track in a clustered
                                   deployment: `"local"` (default, this instance only)
                                   or `"all"`.
  """

  use CodeNameRaven.Handler

  alias CodeNameRaven.Channel.Notification

  @impl true
  def display_name, do: "Watchdog"

  @impl true
  def params_template do
    %{
      stale_after_minutes: "5",
      monitor_id: "",
      on_statuses: "down,up",
      resend_interval_minutes: "0",
      source_scope: "local"
    }
  end

  @impl true
  def params_schema do
    [
      stale_after_minutes: [type: :string, default: "5"],
      monitor_id: [type: :string],
      on_statuses: [type: :string, default: "down,up"],
      resend_interval_minutes: [type: :string, default: "0"],
      source_scope: [type: {:in, ["local", "all"]}, default: "local"]
    ]
  end

  @impl true
  def handler_init(params) do
    {:ok,
     %{
       stale_after_minutes: parse_int(params[:stale_after_minutes] || params["stale_after_minutes"], 5),
       monitor_id: nilify(params[:monitor_id] || params["monitor_id"]),
       on_statuses: parse_statuses(params[:on_statuses] || params["on_statuses"] || "down,up"),
       resend_interval_minutes: parse_int(params[:resend_interval_minutes] || params["resend_interval_minutes"], 0),
       source_scope: parse_scope(params[:source_scope] || params["source_scope"]),
       # %{id => %{checked_at, monitor_name, node, instance_id}} — last time
       # each monitor was seen at all, regardless of its reported status
       last_seen: %{},
       # %{id => DateTime.t()} — when the monitor was first found stale
       in_alert: %{},
       # %{id => DateTime.t()} — for resend tracking
       last_fired_at: %{}
     }}
  end

  @impl true
  def on_status(%{monitor_id: id, checked_at: checked_at} = msg, _params, state) do
    monitor_name = Map.get(msg, :monitor_name)
    node = Map.get(msg, :node)
    instance_id = Map.get(msg, :instance_id)

    if matches_filter?(id, state.monitor_id) and matches_scope?(instance_id, state.source_scope) do
      state = %{
        state
        | last_seen:
            Map.put(state.last_seen, id, %{
              checked_at: checked_at,
              monitor_name: monitor_name,
              node: node,
              instance_id: instance_id
            })
      }

      maybe_recover(id, state)
    else
      {:ok, state}
    end
  end

  def on_status(_msg, _params, state), do: {:ok, state}

  @impl true
  def on_tick(_params, state) do
    now = DateTime.utc_now()
    stale_window_s = state.stale_after_minutes * 60

    {notifications, state} =
      Enum.reduce(state.last_seen, {[], state}, fn {id, entry}, {acc, state} ->
        if DateTime.diff(now, entry.checked_at, :second) >= stale_window_s do
          check_stale(id, entry, now, state, acc)
        else
          {acc, state}
        end
      end)

    case notifications do
      [] -> {:ok, state}
      _ -> {:fire, Enum.reverse(notifications), state}
    end
  end

  # ---------------------------------------------------------------------------
  # Staleness / recovery logic
  # ---------------------------------------------------------------------------

  defp maybe_recover(id, state) do
    case Map.get(state.in_alert, id) do
      nil ->
        {:ok, state}

      _ ->
        entry = Map.get(state.last_seen, id)
        state = clear_alert(state, id)

        if :up in state.on_statuses do
          {:fire, notification(id, :up, :down, entry.checked_at, entry.monitor_name, entry.instance_id), state}
        else
          {:ok, state}
        end
    end
  end

  defp check_stale(id, entry, now, state, acc) do
    if Map.get(state.in_alert, id) do
      maybe_resend_stale(id, entry, now, state, acc)
    else
      state = state |> put_in_alert(id, now) |> put_last_fired_at(id, now)
      n = notification(id, :down, :up, now, entry.monitor_name, entry.instance_id)
      {[n | acc], state}
    end
  end

  defp maybe_resend_stale(_id, _entry, _now, %{resend_interval_minutes: 0} = state, acc), do: {acc, state}

  defp maybe_resend_stale(id, entry, now, state, acc) do
    last = Map.get(state.last_fired_at, id)
    interval_s = state.resend_interval_minutes * 60

    if last && DateTime.diff(now, last, :second) >= interval_s do
      state = put_last_fired_at(state, id, now)
      n = notification(id, :down, :down, now, entry.monitor_name, entry.instance_id)
      {[n | acc], state}
    else
      {acc, state}
    end
  end

  # ---------------------------------------------------------------------------
  # Source-scope filtering
  # ---------------------------------------------------------------------------

  defp parse_scope("all"), do: :all
  defp parse_scope(_), do: :local

  defp matches_scope?(_instance_id, :all), do: true

  defp matches_scope?(instance_id, :local) do
    instance_id in [nil, ""] or instance_id == CodeNameRaven.Runtime.instance_id()
  end

  # ---------------------------------------------------------------------------
  # State helpers
  # ---------------------------------------------------------------------------

  defp clear_alert(state, id) do
    %{state | in_alert: Map.delete(state.in_alert, id), last_fired_at: Map.delete(state.last_fired_at, id)}
  end

  defp put_in_alert(state, id, dt), do: %{state | in_alert: Map.put(state.in_alert, id, dt)}
  defp put_last_fired_at(state, id, dt), do: %{state | last_fired_at: Map.put(state.last_fired_at, id, dt)}

  defp notification(id, status, prev_status, occurred_at, monitor_name, instance_id) do
    %Notification{
      monitor_id: id,
      monitor_name: monitor_name,
      status: status,
      previous_status: prev_status,
      occurred_at: occurred_at,
      instance_id: instance_id
    }
  end

  defp matches_filter?(_id, nil), do: true
  defp matches_filter?(id, filter), do: id == filter

  defp parse_statuses(list) when is_list(list) do
    Enum.flat_map(list, fn
      a when is_atom(a) -> [a]
      s when is_binary(s) -> to_status_atom(s)
    end)
  end

  defp parse_statuses(str) when is_binary(str) do
    str |> String.split(",") |> Enum.flat_map(&to_status_atom(String.trim(&1)))
  end

  defp parse_statuses(_), do: [:down, :up]

  defp to_status_atom("up"), do: [:up]
  defp to_status_atom("degraded"), do: [:degraded]
  defp to_status_atom("down"), do: [:down]
  defp to_status_atom("unknown"), do: [:unknown]
  defp to_status_atom(_), do: []

  defp parse_int(nil, default), do: default
  defp parse_int(v, _) when is_integer(v), do: v

  defp parse_int(v, default) when is_binary(v) do
    case Integer.parse(v) do
      {n, _} -> n
      :error -> default
    end
  end

  defp parse_int(_, default), do: default

  defp nilify(""), do: nil
  defp nilify(v), do: v
end