Packages
reckon_db
5.2.2
5.11.0
5.10.4
5.10.3
5.10.1
5.10.0
5.9.1
5.9.0
5.8.3
5.8.2
5.8.1
5.8.0
5.7.0
5.6.1
5.6.0
5.5.5
5.5.4
5.5.3
5.5.2
5.5.1
5.5.0
5.4.0
5.2.2
5.2.1
5.2.0
5.1.0
5.0.0
4.0.0
3.1.2
3.1.1
3.0.0
2.3.7
2.3.6
2.3.5
2.3.4
2.3.3
2.3.2
2.3.1
2.3.0
2.2.2
2.2.0
2.1.4
2.1.3
2.1.2
2.1.1
2.1.0
2.0.0
1.7.7
1.7.6
1.7.5
1.7.4
1.7.3
1.7.2
1.7.1
1.7.0
1.6.3
1.6.2
1.6.1
1.6.0
1.5.1
1.5.0
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.3
1.3.2
1.3.1
1.3.0
1.2.7
1.2.6
1.2.5
1.2.4
1.2.3
1.2.2
1.2.1
1.2.0
1.1.1
1.1.0
1.0.3
1.0.2
1.0.1
1.0.0
BEAM-native Event Store built on Khepri/Ra with Raft consensus. Event sourcing, persistent subscriptions, snapshots, and automatic cluster formation via UDP multicast discovery. Ships embedded Rust NIFs for 3-15x acceleration of crypto, hashing, compression, aggregation, filter matching, and grap...
Current section
Files
Jump to
Current section
Files
src/reckon_db_discovery.erl
%% @doc Cluster discovery for reckon-db
%%
%% Handles node discovery via UDP multicast (LAN) or Kubernetes DNS.
%% Ported from LibCluster's gossip strategy.
%%
%% Protocol (v2, since 5.1.0):
%% 1. Broadcast {gossip_v2, NodeBin, Timestamp, Hmac} every
%% BROADCAST_INTERVAL, where Hmac = HMAC-SHA256 over the node name
%% and timestamp keyed with the cluster secret. The secret itself
%% never goes on the wire (v1 broadcast it in cleartext).
%% 2. On receive: safe-decode, constant-time HMAC check, freshness
%% window, THEN net_kernel:connect_node/1. The node name travels
%% as a binary and is only atomized after authentication, so
%% unauthenticated LAN datagrams can neither grow the atom table
%% nor trigger term decoding of attacker-shaped structures.
%% 3. On node up: trigger Khepri cluster join via StoreCoordinator
%%
%% Discovery requires an explicitly configured cluster secret: the
%% RECKON_DB_CLUSTER_SECRET env var, or the cluster_secret application
%% environment key. Without one, cluster-mode discovery stays passive:
%% no broadcasts, inbound gossip ignored. v1 shipped a hardcoded
%% default secret, which made every unconfigured deployment trust any
%% LAN host.
%%
%% @author rgfaber
-module(reckon_db_discovery).
-behaviour(gen_server).
-include("reckon_db.hrl").
-include("reckon_db_telemetry.hrl").
%% API
-export([start_link/1]).
-export([trigger_discovery/1]).
-export([get_discovered_nodes/1]).
%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]).
-ifdef(TEST).
-export([encode_gossip_message/2, decode_gossip/2, gossip_mac/3]).
-endif.
-define(DEFAULT_PORT, 45892).
-define(MULTICAST_ADDR, {239, 255, 0, 1}).
-define(BROADCAST_INTERVAL_MS, 5000).
-define(MULTICAST_TTL, 1).
%% Reject gossip whose timestamp is further than this from local time.
%% Bounds replay of captured datagrams; generous enough for LAN clock
%% skew. Replaying a fresh packet only re-announces the legitimate
%% node, which is harmless (dist cookie still gates the connection).
-define(GOSSIP_FRESHNESS_MS, 60_000).
%% Erlang node names are practically bounded; anything bigger is junk.
-define(MAX_NODE_NAME_BYTES, 255).
-record(state, {
store_id :: atom(),
config :: store_config(),
socket :: gen_udp:socket() | undefined,
port :: non_neg_integer() | undefined,
multicast_addr :: inet:ip4_address() | undefined,
cluster_secret :: binary() | undefined,
broadcast_interval :: non_neg_integer() | undefined,
discovered_nodes :: [node()]
}).
%%====================================================================
%% API
%%====================================================================
-spec start_link(store_config()) -> {ok, pid()} | {error, term()}.
start_link(#store_config{store_id = StoreId} = Config) ->
Name = reckon_db_naming:discovery_name(StoreId),
gen_server:start_link({local, Name}, ?MODULE, Config, []).
%% @doc Trigger immediate discovery broadcast
-spec trigger_discovery(atom()) -> ok.
trigger_discovery(StoreId) ->
Name = reckon_db_naming:discovery_name(StoreId),
gen_server:cast(Name, trigger_discovery).
%% @doc Get list of discovered nodes
-spec get_discovered_nodes(atom()) -> [node()].
get_discovered_nodes(StoreId) ->
Name = reckon_db_naming:discovery_name(StoreId),
gen_server:call(Name, get_discovered_nodes).
%%====================================================================
%% gen_server callbacks
%%====================================================================
init(#store_config{store_id = StoreId, mode = Mode} = Config) ->
case Mode of
cluster ->
init_cluster_mode(StoreId, Config);
single ->
%% In single mode, discovery is a no-op
logger:info("Discovery disabled in single-node mode (store: ~p)", [StoreId]),
{ok, #state{
store_id = StoreId,
config = Config,
socket = undefined,
discovered_nodes = []
}}
end.
init_cluster_mode(StoreId, Config) ->
case get_cluster_secret() of
{ok, ClusterSecret} ->
init_cluster_discovery(StoreId, Config, ClusterSecret);
error ->
%% Fail closed: without an explicit shared secret, anyone
%% on the LAN multicast segment could have us dial them.
%% Manual/static cluster joins keep working.
logger:error(
"Discovery disabled (store: ~p): no cluster secret configured. "
"Set RECKON_DB_CLUSTER_SECRET or {reckon_db, cluster_secret} "
"to enable multicast discovery.", [StoreId]),
{ok, #state{
store_id = StoreId,
config = Config,
socket = undefined,
discovered_nodes = []
}}
end.
init_cluster_discovery(StoreId, Config, ClusterSecret) ->
Port = get_config_value(discovery_port, ?DEFAULT_PORT),
MulticastAddr = get_config_value(multicast_addr, ?MULTICAST_ADDR),
BroadcastInterval = get_config_value(broadcast_interval, ?BROADCAST_INTERVAL_MS),
State = #state{
store_id = StoreId,
config = Config,
port = Port,
multicast_addr = MulticastAddr,
cluster_secret = ClusterSecret,
broadcast_interval = BroadcastInterval,
discovered_nodes = []
},
case open_multicast_socket(Port, MulticastAddr) of
{ok, Socket} ->
logger:info("Discovery started on port ~p (store: ~p)", [Port, StoreId]),
%% Schedule first broadcast
schedule_broadcast(BroadcastInterval),
{ok, State#state{socket = Socket}};
{error, Reason} ->
logger:warning("Failed to open multicast socket: ~p, running in passive mode", [Reason]),
%% Continue without socket - will use manual discovery
{ok, State#state{socket = undefined}}
end.
handle_call(get_discovered_nodes, _From, #state{discovered_nodes = Nodes} = State) ->
{reply, Nodes, State};
handle_call(_Request, _From, State) ->
{reply, {error, unknown_request}, State}.
handle_cast(trigger_discovery, #state{socket = undefined} = State) ->
%% No socket, can't broadcast
{noreply, State};
handle_cast(trigger_discovery, State) ->
broadcast_presence(State),
{noreply, State};
handle_cast(_Msg, State) ->
{noreply, State}.
handle_info(broadcast, #state{socket = undefined} = State) ->
%% No socket, schedule next attempt
schedule_broadcast(State#state.broadcast_interval),
{noreply, State};
handle_info(broadcast, #state{broadcast_interval = Interval} = State) ->
broadcast_presence(State),
schedule_broadcast(Interval),
{noreply, State};
handle_info({udp, _Socket, _IP, _Port, Data}, State) ->
NewState = handle_gossip_message(Data, State),
{noreply, NewState};
handle_info(_Info, State) ->
{noreply, State}.
terminate(_Reason, #state{socket = undefined}) ->
ok;
terminate(_Reason, #state{socket = Socket}) ->
gen_udp:close(Socket),
ok.
%%====================================================================
%% Internal functions
%%====================================================================
%% @private Open UDP multicast socket
-spec open_multicast_socket(non_neg_integer(), inet:ip4_address()) ->
{ok, gen_udp:socket()} | {error, term()}.
open_multicast_socket(Port, MulticastAddr) ->
Options = [
binary,
{active, true},
{reuseaddr, true},
{multicast_ttl, ?MULTICAST_TTL},
{multicast_loop, false},
{add_membership, {MulticastAddr, {0, 0, 0, 0}}}
],
gen_udp:open(Port, Options).
%% @private Broadcast presence to multicast group
-spec broadcast_presence(#state{}) -> ok.
broadcast_presence(#state{socket = Socket, port = Port, multicast_addr = Addr,
cluster_secret = Secret, store_id = StoreId}) ->
Message = encode_gossip_message(node(), Secret),
case gen_udp:send(Socket, Addr, Port, Message) of
ok ->
ok;
{error, Reason} ->
logger:warning("Failed to broadcast discovery message (store: ~p): ~p",
[StoreId, Reason])
end.
%% @private Encode gossip message (v2: authenticated, atom-free,
%% secret never on the wire).
-spec encode_gossip_message(node(), binary()) -> binary().
encode_gossip_message(Node, Secret) ->
Timestamp = erlang:system_time(millisecond),
NodeBin = atom_to_binary(Node, utf8),
Mac = gossip_mac(NodeBin, Timestamp, Secret),
term_to_binary({gossip_v2, NodeBin, Timestamp, Mac}).
-spec gossip_mac(binary(), integer(), binary()) -> binary().
gossip_mac(NodeBin, Timestamp, Secret) ->
crypto:mac(hmac, sha256, Secret,
<<Timestamp:64/signed-big, NodeBin/binary>>).
%% @private Handle incoming gossip message.
-spec handle_gossip_message(binary(), #state{}) -> #state{}.
handle_gossip_message(_Data, #state{cluster_secret = undefined} = State) ->
State;
handle_gossip_message(Data, #state{cluster_secret = Secret,
store_id = StoreId,
discovered_nodes = KnownNodes} = State) ->
case decode_gossip(Data, Secret) of
{ok, NodeBin} ->
%% Atomization is safe here: only holders of the cluster
%% secret reach this point, so atom creation is bounded
%% to authenticated peers.
Node = binary_to_atom(NodeBin, utf8),
case Node =:= node() of
true -> State;
false -> handle_discovered_node(Node, StoreId, KnownNodes, State)
end;
reject ->
State
end.
%% @private Decode + authenticate one gossip datagram.
%%
%% The datagram is untrusted LAN input. Decode with [safe] (a plain
%% binary_to_term here allowed remote atom-table exhaustion and
%% arbitrary-term allocation before any authentication), pattern-match
%% the exact v2 shape, verify the HMAC in constant time, then check
%% freshness. The try is required: [safe] raises badarg on unknown
%% atoms / garbage and a hostile packet must degrade to a no-op, not
%% crash discovery.
-spec decode_gossip(binary(), binary()) -> {ok, binary()} | reject.
decode_gossip(Data, Secret) ->
Decoded = try
{term, binary_to_term(Data, [safe])}
catch
error:badarg -> invalid
end,
authenticate_gossip_term(Decoded, Secret).
authenticate_gossip_term({term, {gossip_v2, NodeBin, Timestamp, Mac}}, Secret)
when is_binary(NodeBin), byte_size(NodeBin) > 0,
byte_size(NodeBin) =< ?MAX_NODE_NAME_BYTES,
is_integer(Timestamp),
is_binary(Mac), byte_size(Mac) =:= 32 ->
Expected = gossip_mac(NodeBin, Timestamp, Secret),
case crypto:hash_equals(Expected, Mac) andalso is_fresh(Timestamp) of
true ->
{ok, NodeBin};
false ->
logger:debug("Ignoring gossip with invalid MAC or stale timestamp"),
reject
end;
authenticate_gossip_term(_Invalid, _Secret) ->
reject.
is_fresh(Timestamp) ->
abs(erlang:system_time(millisecond) - Timestamp) =< ?GOSSIP_FRESHNESS_MS.
%% @private Handle a newly discovered node
-spec handle_discovered_node(node(), atom(), [node()], #state{}) -> #state{}.
handle_discovered_node(Node, StoreId, KnownNodes, State) ->
case lists:member(Node, KnownNodes) of
true ->
State;
false ->
logger:info("Discovered new node: ~p (store: ~p)", [Node, StoreId]),
connect_discovered_node(Node, StoreId, KnownNodes, State)
end.
connect_discovered_node(Node, StoreId, KnownNodes, State) ->
case net_kernel:connect_node(Node) of
true ->
logger:info("Connected to discovered node: ~p", [Node]),
telemetry:execute(
?CLUSTER_NODE_UP,
#{system_time => erlang:system_time(millisecond)},
#{store_id => StoreId, node => Node,
member_count => length(nodes()) + 1}
),
trigger_cluster_join(StoreId),
State#state{discovered_nodes = [Node | KnownNodes]};
false ->
logger:warning("Failed to connect to discovered node: ~p", [Node]),
State
end.
%% @private Trigger cluster join via store coordinator
-spec trigger_cluster_join(atom()) -> ok.
trigger_cluster_join(StoreId) ->
%% Use spawn to avoid blocking discovery
spawn(fun() ->
try
reckon_db_store_coordinator:join_cluster(StoreId)
catch
_:Reason ->
logger:warning("Failed to trigger cluster join: ~p", [Reason])
end
end),
ok.
%% @private Schedule next broadcast
-spec schedule_broadcast(non_neg_integer()) -> reference().
schedule_broadcast(Interval) ->
erlang:send_after(Interval, self(), broadcast).
%% @private Get cluster secret from environment or config.
%% No default: a hardcoded fallback secret means every unconfigured
%% deployment trusts any LAN host that knows the public source code.
-spec get_cluster_secret() -> {ok, binary()} | error.
get_cluster_secret() ->
case os:getenv("RECKON_DB_CLUSTER_SECRET") of
false ->
get_cluster_secret_from_env();
Secret ->
{ok, list_to_binary(Secret)}
end.
get_cluster_secret_from_env() ->
case application:get_env(reckon_db, cluster_secret) of
{ok, Secret} when is_binary(Secret), byte_size(Secret) > 0 ->
{ok, Secret};
{ok, Secret} when is_list(Secret), Secret =/= [] ->
{ok, list_to_binary(Secret)};
_ ->
error
end.
%% @private Get config value with default
-spec get_config_value(atom(), term()) -> term().
get_config_value(Key, Default) ->
case application:get_env(reckon_db, Key) of
{ok, Value} -> Value;
undefined -> Default
end.