Current section
Files
Jump to
Current section
Files
lib/auto_scaler/auto_scaler.ex
defmodule HareMq.AutoScaler do
use GenServer
require Logger
alias HareMq.AutoScalerConfiguration
@moduledoc """
A GenServer that automatically scales the number of consumers based on the
number of messages in the RabbitMQ queue.
"""
# How long to wait for a worker's :get_config call before trying the next one.
# Configurable via `config :hare_mq, :auto_scaler, worker_timeout_ms: 5_000`.
defp worker_call_timeout do
(Application.get_env(:hare_mq, :auto_scaler) || [])[:worker_timeout_ms] || 5_000
end
def start_link(%HareMq.AutoScalerConfiguration{module_name: module_name} = opts) do
GenServer.start_link(__MODULE__, opts, name: {:global, :"#{module_name}.AutoScaler"})
end
@impl true
def init(%AutoScalerConfiguration{} = config) do
schedule_check(config.check_interval_ms)
{:ok, %{config: config, current_consumer_count: config.initial_consumer_count}}
end
@impl true
def handle_info(:check_queue, state) do
queue_length = get_queue_length(state.config, state.current_consumer_count)
state = adjust_consumers(queue_length, state)
schedule_check(state.config.check_interval_ms)
{:noreply, state}
end
defp schedule_check(interval) do
Process.send_after(self(), :check_queue, interval)
end
# Walk workers W1..W{count} and return the queue depth from the first one
# that is alive and responsive. Falls back to 0 only if none can be reached,
# preventing a single dead worker from causing a spurious scale-down.
defp get_queue_length(%AutoScalerConfiguration{} = config, current_count) do
find_worker_queue_length(config.module_name, 1, max(current_count, 1))
end
defp find_worker_queue_length(_module_name, index, max) when index > max, do: 0
defp find_worker_queue_length(module_name, index, max) do
case :global.whereis_name("#{module_name}.W#{index}") do
pid when is_pid(pid) ->
try do
queue_config = GenServer.call(pid, :get_config, worker_call_timeout())
# Check if channel is available before attempting to get message count
if queue_config.channel do
AMQP.Queue.message_count(queue_config.channel, queue_config.queue_name)
else
# Worker hasn't finished connecting yet, try next worker
find_worker_queue_length(module_name, index + 1, max)
end
catch
:exit, _ -> find_worker_queue_length(module_name, index + 1, max)
end
_ ->
find_worker_queue_length(module_name, index + 1, max)
end
end
defp adjust_consumers(
queue_length,
%{config: config, current_consumer_count: current_count} = state
) do
target_consumer_count = calculate_target_consumer_count(queue_length, config)
Logger.debug(
"[#{config.module_name}] Queue check: #{queue_length} messages, current consumers: #{current_count}, target: #{target_consumer_count}"
)
cond do
target_consumer_count > current_count ->
supervisor = HareMq.DynamicSupervisor.supervisor_name(config.module_name)
scale_up_count = target_consumer_count - current_count
Logger.debug(
"[#{config.module_name}] Scaling UP from #{current_count} to #{target_consumer_count} consumers (adding #{scale_up_count})"
)
Enum.each(1..scale_up_count, fn index ->
HareMq.DynamicSupervisor.add_consumer(
supervisor,
worker: config.consumer_worker,
name: generate_consumer_name(config.module_name, current_count + index),
opts: config.consumer_opts
)
end)
target_consumer_count < current_count ->
supervisor = HareMq.DynamicSupervisor.supervisor_name(config.module_name)
scale_down_count = current_count - target_consumer_count
Logger.debug(
"[#{config.module_name}] Scaling DOWN from #{current_count} to #{target_consumer_count} consumers (removing #{scale_down_count})"
)
Enum.each((target_consumer_count + 1)..current_count, fn index ->
HareMq.DynamicSupervisor.remove_consumer(
supervisor,
generate_consumer_name(config.module_name, index)
)
end)
true ->
:ok
end
%{state | current_consumer_count: target_consumer_count}
end
def calculate_target_consumer_count(queue_length, config) do
# one consumer per N messages in the queue.
div(queue_length + config.messages_per_consumer - 1, config.messages_per_consumer)
|> max(config.min_consumers)
|> min(config.max_consumers)
end
defp generate_consumer_name(module_name, count) do
"#{module_name}.W#{count}"
end
end