Packages
ex_esdb
0.7.7
0.11.0
0.10.0
0.9.0
0.8.0
0.7.8
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.1
0.6.0
0.5.1
0.5.0
0.4.8
0.4.7
0.4.6
0.4.5
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.3
0.3.2
0.3.1
0.3.0
0.2.5
0.2.4
0.2.3
0.2.2
0.2.1
0.2.0
0.1.7
0.1.6
0.1.5
0.1.4
0.1.3
0.1.2
0.1.1
0.1.0
0.0.20
0.0.19
0.0.18
0.0.17
0.0.16
0.0.15
0.0.14-alpha
0.0.13-alpha
0.0.12-alpha
0.0.11-alpha
0.0.10-alpha
0.0.9-alpha
0.0.8-alpha
0.0.6-alpha
0.0.5-alpha
0.0.4-alpha
0.0.3-alpha
0.0.2-alfa
0.0.1-alfa
ExESDB is a reincarnation of rabbitmq/khepri, specialized for use as a BEAM-native event store.
Current section
Files
Jump to
Current section
Files
lib/ex_esdb/subscription_health_tracker.ex
defmodule ExESDB.SubscriptionHealthTracker do
@moduledoc """
Centralized tracker for subscription health events received via :ex_esdb_system PubSub.
This module:
- Subscribes to subscription health events from subscription proxies
- Maintains current health status for all subscriptions in a store
- Provides APIs to query subscription health
- Publishes aggregated health summaries
- Integrates with the broader ExESDB monitoring system
"""
use GenServer
alias ExESDB.StoreNaming
alias ExESDB.Themes, as: Themes
require Logger
@health_table_prefix :subscription_health_
@health_summary_interval :timer.seconds(30)
# Health status types
@type health_status :: :healthy | :degraded | :failed | :registering | :unknown
@type health_event :: %{
store_id: atom(),
subscription_name: String.t(),
event_type: atom(),
metadata: map()
}
@type health_data :: %{
subscription_name: String.t(),
current_status: health_status(),
last_seen: integer(),
event_count: non_neg_integer(),
last_event: health_event(),
error_count: non_neg_integer(),
last_error: term() | nil
}
## Public API
@doc """
Gets the current health status for all subscriptions in a store.
"""
@spec get_store_health_summary(atom()) :: {:ok, map()} | {:error, term()}
def get_store_health_summary(store_id) do
name = StoreNaming.genserver_name(__MODULE__, store_id)
GenServer.call(name, :get_health_summary)
end
@doc """
Gets detailed health information for a specific subscription.
"""
@spec get_subscription_health(atom(), String.t()) :: {:ok, health_data()} | {:error, :not_found}
def get_subscription_health(store_id, subscription_name) do
name = StoreNaming.genserver_name(__MODULE__, store_id)
GenServer.call(name, {:get_subscription_health, subscription_name})
end
@doc """
Lists all subscriptions currently being tracked.
"""
@spec list_tracked_subscriptions(atom()) :: {:ok, [String.t()]}
def list_tracked_subscriptions(store_id) do
name = StoreNaming.genserver_name(__MODULE__, store_id)
GenServer.call(name, :list_subscriptions)
end
## GenServer Implementation
def start_link(opts) do
store_id = StoreNaming.extract_store_id(opts)
name = StoreNaming.genserver_name(__MODULE__, store_id)
GenServer.start_link(__MODULE__, opts, name: name)
end
def child_spec(opts) do
store_id = StoreNaming.extract_store_id(opts)
%{
id: StoreNaming.child_spec_id(__MODULE__, store_id),
start: {__MODULE__, :start_link, [opts]},
type: :worker,
restart: :permanent,
shutdown: 5000
}
end
@impl GenServer
def init(opts) do
store_id = StoreNaming.extract_store_id(opts)
# Create ETS table for fast health lookups
table_name = String.to_atom("#{@health_table_prefix}#{store_id}")
:ets.new(table_name, [:named_table, :public, :set])
# Subscribe to all subscription health events for this store using dedicated health PubSub
store_topic = "store_health:#{store_id}"
subscription_topic_pattern = "subscription_health:#{store_id}:*"
:ok = Phoenix.PubSub.subscribe(:ex_esdb_health, store_topic)
# Note: PubSub doesn't support wildcard subscriptions, so we'll need to handle this differently
# We'll subscribe to events as subscriptions are discovered
# Schedule periodic health summary publishing
Process.send_after(self(), :publish_health_summary, @health_summary_interval)
state = %{
store_id: store_id,
health_table: table_name,
subscribed_topics: MapSet.new([store_topic]),
last_summary_published: System.system_time(:millisecond)
}
Logger.info("#{Themes.subscription_health_tracker(self(), "Started for store: #{store_id}")}")
{:ok, state}
end
@impl GenServer
def handle_info({:subscription_health, health_event}, state) do
updated_state = process_health_event(state, health_event)
{:noreply, updated_state}
end
@impl GenServer
def handle_info(:publish_health_summary, state) do
publish_health_summary(state)
# Schedule next summary
Process.send_after(self(), :publish_health_summary, @health_summary_interval)
updated_state = %{state | last_summary_published: System.system_time(:millisecond)}
{:noreply, updated_state}
end
@impl GenServer
def handle_call(:get_health_summary, _from, state) do
summary = build_health_summary(state)
{:reply, {:ok, summary}, state}
end
@impl GenServer
def handle_call({:get_subscription_health, subscription_name}, _from, state) do
case :ets.lookup(state.health_table, subscription_name) do
[{^subscription_name, health_data}] -> {:reply, {:ok, health_data}, state}
[] -> {:reply, {:error, :not_found}, state}
end
end
@impl GenServer
def handle_call(:list_subscriptions, _from, state) do
subscriptions =
:ets.tab2list(state.health_table)
|> Enum.map(fn {subscription_name, _} -> subscription_name end)
{:reply, {:ok, subscriptions}, state}
end
## Private Functions
defp process_health_event(state, health_event) do
%{
store_id: store_id,
subscription_name: subscription_name,
event_type: event_type,
metadata: metadata
} = health_event
# Ensure we're subscribed to this subscription's health events
updated_state = ensure_subscription_topic(state, store_id, subscription_name)
# Update health data
now = System.system_time(:millisecond)
current_health = get_current_health_data(updated_state.health_table, subscription_name)
updated_health = update_health_data(current_health, event_type, health_event, now)
# Store updated health data
:ets.insert(updated_state.health_table, {subscription_name, updated_health})
# Log significant health changes
log_health_change(event_type, subscription_name, metadata)
updated_state
end
defp ensure_subscription_topic(state, store_id, subscription_name) do
topic = "subscription_health:#{store_id}:#{subscription_name}"
if MapSet.member?(state.subscribed_topics, topic) do
state
else
:ok = Phoenix.PubSub.subscribe(:ex_esdb_health, topic)
%{state | subscribed_topics: MapSet.put(state.subscribed_topics, topic)}
end
end
defp get_current_health_data(table, subscription_name) do
case :ets.lookup(table, subscription_name) do
[{^subscription_name, health_data}] -> health_data
[] -> %{
subscription_name: subscription_name,
current_status: :unknown,
last_seen: 0,
event_count: 0,
last_event: nil,
error_count: 0,
last_error: nil
}
end
end
defp update_health_data(current_health, event_type, health_event, now) do
new_status = determine_health_status(event_type, health_event)
is_error = is_error_event?(event_type)
%{
current_health |
current_status: new_status,
last_seen: now,
event_count: current_health.event_count + 1,
last_event: health_event,
error_count: if(is_error, do: current_health.error_count + 1, else: current_health.error_count),
last_error: if(is_error, do: health_event, else: current_health.last_error)
}
end
defp determine_health_status(:registration_started, _), do: :registering
defp determine_health_status(:registration_success, _), do: :healthy
defp determine_health_status(:registration_failed, _), do: :failed
defp determine_health_status(:proxy_started, _), do: :healthy
defp determine_health_status(:proxy_stopped, _), do: :failed
defp determine_health_status(:proxy_crashed, _), do: :failed
defp determine_health_status(:circuit_breaker_opened, _), do: :degraded
defp determine_health_status(:circuit_breaker_closed, _), do: :healthy
defp determine_health_status(:event_delivery_success, _), do: :healthy
defp determine_health_status(:event_delivery_failed, _), do: :degraded
defp determine_health_status(:periodic_heartbeat, _), do: :healthy
defp determine_health_status(_, _), do: :unknown
defp is_error_event?(event_type) do
event_type in [:registration_failed, :proxy_crashed, :circuit_breaker_opened, :event_delivery_failed]
end
defp log_health_change(:registration_failed, subscription_name, metadata) do
error = Map.get(metadata, :error, "unknown")
Logger.warning("Subscription #{subscription_name} registration failed: #{inspect(error)}")
end
defp log_health_change(:proxy_crashed, subscription_name, metadata) do
reason = Map.get(metadata, :crash_reason, "unknown")
Logger.error("Subscription #{subscription_name} proxy crashed: #{inspect(reason)}")
end
defp log_health_change(:circuit_breaker_opened, subscription_name, metadata) do
reason = Map.get(metadata, :reason, "unknown")
Logger.warning("Circuit breaker opened for subscription #{subscription_name}: #{inspect(reason)}")
end
defp log_health_change(_, _, _), do: :ok
defp build_health_summary(state) do
all_health_data = :ets.tab2list(state.health_table)
summary = Enum.reduce(all_health_data, %{
total_subscriptions: 0,
healthy: 0,
degraded: 0,
failed: 0,
registering: 0,
unknown: 0
}, fn {_name, health_data}, acc ->
status = health_data.current_status
%{
acc |
total_subscriptions: acc.total_subscriptions + 1,
healthy: acc.healthy + (if status == :healthy, do: 1, else: 0),
degraded: acc.degraded + (if status == :degraded, do: 1, else: 0),
failed: acc.failed + (if status == :failed, do: 1, else: 0),
registering: acc.registering + (if status == :registering, do: 1, else: 0),
unknown: acc.unknown + (if status == :unknown, do: 1, else: 0)
}
end)
Map.put(summary, :store_id, state.store_id)
end
defp publish_health_summary(state) do
summary = build_health_summary(state)
topic = "health_summary:#{state.store_id}"
Phoenix.PubSub.broadcast(:ex_esdb_health, topic, {:health_summary, summary})
Logger.debug("Published health summary for store #{state.store_id}: #{inspect(summary)}")
end
end