Packages

Elixir messaging library using RabbitMQ, providing easy-to-use modules for message publishing, consuming, and retry handling

Current section

Files

Jump to
hare_mq lib dedup_cache dedup_cache.ex
Raw

lib/dedup_cache/dedup_cache.ex

defmodule HareMq.DedupCache do
@moduledoc """
ETS-backed deduplication cache for RabbitMQ message processing.
## Storage
Each cache instance owns a named ETS table (`:named_table, :public, :set`).
The table name is derived from the registered server name so lookups can go
directly to ETS without passing through the GenServer message queue.
## Writes vs reads
- `add/3` (and `add/4`) — synchronous `GenServer.call`. This guarantees that
a subsequent `is_dup?` call on any process will see the entry.
- `is_dup?/2` (and `is_dup?/3`) — reads ETS directly, completely bypassing
the GenServer. Concurrent read throughput is not limited by a single process.
## Expiry
A 1-second timer runs `handle_info(:clear_cache)` which calls
`:ets.select_delete/2` to remove all rows whose `expired_at` timestamp is in
the past. This is a C-level operation with no Elixir-side allocation,
keeping the GenServer responsive regardless of cache size.
## Named instances
Pass `name:` to `start_link/1` to run isolated caches side-by-side:
{HareMq.DedupCache, name: {:global, :dedup_cache_tenant_a}}
Then pass the same name as the third/fourth argument to `is_dup?/3` and
`add/4`. Publishers accept a `dedup_cache_name:` option.
"""
use GenServer
def start_link(opts \\ []) do
server_name = Keyword.get(opts, :name, {:global, __MODULE__})
table_name = ets_table_name(server_name)
GenServer.start_link(__MODULE__, table_name, name: server_name)
|> HareMq.CodeFlow.successful_start()
end
def init(table_name) do
:ets.new(table_name, [:named_table, :public, :set, {:read_concurrency, true}])
schedule_clear()
{:ok, table_name}
end
def is_dup?(message, deduplication_keys \\ []) do
ets_is_dup?(__MODULE__, message, deduplication_keys)
end
def is_dup?(message, deduplication_keys, cache_name) do
ets_is_dup?(ets_table_name(cache_name), message, deduplication_keys)
end
def add(message, deduplication_ttl, deduplication_keys \\ []) do
add(message, deduplication_ttl, deduplication_keys, {:global, __MODULE__})
end
def add(message, deduplication_ttl, deduplication_keys, cache_name) do
GenServer.call(cache_name, {:add, message, deduplication_ttl, deduplication_keys})
end
def handle_info(:clear_cache, table) do
now = :os.system_time(:millisecond)
:ets.select_delete(table, [{{"$1", :"$2", :"$3"}, [{:"=<", :"$3", now}], [true]}])
schedule_clear()
{:noreply, table}
end
def handle_call({:add, message, deduplication_ttl, deduplication_keys}, _from, table) do
hash = generate_hash(message, deduplication_keys)
# Store :infinity as the atom directly. Erlang term ordering places atoms
# above all numbers, so :infinity > any_integer is always true. This means
# ets_is_dup?'s '>' guard always passes, and clear_cache's '=<' guard
# never deletes these entries — achieving a true infinite TTL without
# relying on a large magic number that would eventually expire.
expired_at =
case deduplication_ttl do
:infinity -> :infinity
ms -> :os.system_time(:millisecond) + ms
end
:ets.insert(table, {hash, message, expired_at})
{:reply, :ok, table}
end
defp ets_is_dup?(table, message, deduplication_keys) do
hash = generate_hash(message, deduplication_keys)
now = :os.system_time(:millisecond)
# Use select so the expiry guard and key match are evaluated atomically
# in a single C-level ETS operation, with no window between lookup and check.
match_spec = [{{hash, :"$1", :"$2"}, [{:>, :"$2", now}], [:"$1"]}]
case :ets.select(table, match_spec) do
[cached_message] -> check_keys(message, cached_message, deduplication_keys)
_ -> false
end
rescue
# Table doesn't exist (cache not started)
ArgumentError -> false
end
defp check_keys(message, cached_message, deduplication_keys) when is_map(message) do
Enum.all?(deduplication_keys, fn key ->
message[key] === cached_message[key]
end)
end
defp check_keys(_message, _cached_message, _deduplication_keys), do: true
defp schedule_clear do
Process.send_after(self(), :clear_cache, 1_000)
end
defp ets_table_name({:global, name}), do: name
defp ets_table_name(name) when is_atom(name), do: name
defp ets_table_name(_), do: __MODULE__
defp generate_hash(message, _deduplication_keys) when is_binary(message) do
:crypto.hash(:md5, message) |> Base.encode16()
end
defp generate_hash(message, deduplication_keys) when is_map(message) do
message =
case deduplication_keys do
[_ | _] -> Map.take(message, deduplication_keys)
_ -> message
end
encoded_message = Jason.encode!(message)
:crypto.hash(:md5, encoded_message) |> Base.encode16()
end
end