Current section
Files
Jump to
Current section
Files
lib/gen_consumer/adapter/redis_consumer.ex
defmodule GenConsumer.RedisConsumer do
@moduledoc """
Consumer based on Redis streams
"""
use GenConsumer.Adapter
require Logger
defmodule State do
@moduledoc false
defstruct [:topics, :group, :consumer, :callback, :conn, :last_ids, :max_batch_size]
end
@impl true
def init(opts) do
topics = Keyword.fetch!(opts, :topics)
group = Keyword.fetch!(opts, :group)
consumer = Keyword.fetch!(opts, :consumer)
callback = Keyword.fetch!(opts, :callback)
max_batch_size = Keyword.get(opts, :max_batch_size, 100)
host = Keyword.get(opts, :host, "localhost")
port = Keyword.get(opts, :port, 6379)
{:ok, conn} = Redix.start_link(host: host, port: port)
last_ids = for _topic <- topics, do: nil
state =
%State{
max_batch_size: max_batch_size,
topics: topics,
group: group,
consumer: consumer,
callback: callback,
conn: conn,
last_ids: last_ids
}
create_group_if_not_existant(state)
Process.send_after self(), :block, 1000
{:ok, state}
end
def handle_info(:block, state) do
streams = state.topics
command =
if state.group do
ids = for _stream <- streams, do: ">"
(["XREADGROUP", "GROUP", state.group, state.consumer, "BLOCK", 1000, "COUNT", state.max_batch_size, "STREAMS"] ++ streams ++ ids)
else
(["XREAD", "BLOCK", 1000, "COUNT", state.max_batch_size, "STREAMS"] ++ streams ++ state.last_ids)
end
case Redix.command(state.conn, command, timeout: 60_000) do
{:ok, matched_streams} when is_list(matched_streams) ->
last_ids =
for [_stream, events] <- matched_streams do
events =
for [id, ["value", value]] <- events do
{id, value}
end
for {_id, event} <- events, do: state.callback.(event)
{last_id, _} = List.last(events)
last_id
end
Process.send_after self(), :block, 1000
state = %{state | last_ids: last_ids}
{:noreply, state, 60_000}
{:error, error} ->
Logger.error("Redix command failed with exception: #{inspect error}")
Process.send_after self(), :block, 1000
{:noreply, state, 60_000}
_ ->
Process.send_after self(), :block, 1000
{:noreply, state, 60_000}
end
rescue
exception ->
Logger.error(inspect(exception))
{:stop, :shutdown, state}
end
defp create_group_if_not_existant(state) do
# Logger.info "Ensuring group '#{state.group}' exists..."
if state.group do
for stream <- state.topics do
# try do
# command = ["XINFO", "GROUPS", stream]
# exists? =
# Redix.command!(state.conn, command)
# |> Enum.any?(fn [_, existing, _, _, _, _, _, _] -> existing == state.group end)
# |> IO.inspect
# if !exists? do
# Logger.info "Group '#{state.group}' does not exist. Creating group..."
command = ["XGROUP", "CREATE", stream, state.group, "$", "MKSTREAM"]
result =
try do
Redix.command(state.conn, command)
rescue
exception ->
{:error, exception}
end
case result do
{:error, %Redix.Error{message: "BUSYGROUP Consumer Group name already exists"}} ->
:ok
_ ->
:ok
end
# case Redix.command(state.conn, command) do
# {:error, %Redix.Error{message: "ERR no such key"}} ->
# Logger.info "Stream '#{stream}' does not exist. Creating stream...."
# {:ok, _} = Redix.command(state.conn, ["XADD", stream, "*", "", ""])
# :ok
# {:error, %Redix.Error{message: "BUSYGROUP Consumer Group name already exists"}} ->
# :ok
# {:error, error} ->
# raise error
# {:ok, _} ->
# :ok
# # end
# end
# rescue
# _exception ->
# nil
# end
end
end
end
end