Packages

Persistent ETS tables backed by DETS — fast in-memory access with automatic disk persistence for the BEAM

Current section

Files

Jump to
shelf src shelf@set.erl
Raw

src/shelf@set.erl

-module(shelf@set).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch, inline]).
-define(FILEPATH, "src/shelf/set.gleam").
-export([open_config/3, open/5, close/1, with_table/6, member/2, to_list/1, fold/3, size/1, insert/3, insert_list/2, delete_key/2, delete_object/3, delete_all/1, save/1, reload/1, sync/1, lookup/2, insert_new/3, update_counter/3]).
-export_type([p_set/2]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
-opaque p_set(MNW, MNX) :: {p_set,
shelf@internal:ets_ref(),
shelf@internal:dets_ref(),
shelf@internal:guardian_ref(),
shelf:write_mode(),
gleam@dynamic@decode:decoder({MNW, MNX})}.
-file("src/shelf/set.gleam", 70).
?DOC(
" Open a persistent set table with full configuration.\n"
"\n"
" If the DETS file exists, its contents are loaded into a fresh ETS\n"
" table after validating each entry through the provided decoders.\n"
" If no file exists, both tables start empty.\n"
"\n"
" The DETS file path is validated against the configured base directory.\n"
"\n"
" ```gleam\n"
" let config =\n"
" shelf.config(name: \"cache\", path: \"cache.dets\",\n"
" base_directory: \"/app/data\")\n"
" |> shelf.write_mode(shelf.WriteThrough)\n"
" let assert Ok(table) =\n"
" set.open_config(config, key: decode.string, value: decode.int)\n"
" ```\n"
).
-spec open_config(
shelf:config(),
gleam@dynamic@decode:decoder(MNY),
gleam@dynamic@decode:decoder(MOA)
) -> {ok, p_set(MNY, MOA)} | {error, shelf:shelf_error()}.
open_config(Config, Key_decoder, Value_decoder) ->
gleam@result:'try'(
shelf@internal:generic_open(
Config,
<<"set"/utf8>>,
Key_decoder,
Value_decoder
),
fun(Result) ->
{ok,
{p_set,
erlang:element(1, Result),
erlang:element(2, Result),
erlang:element(3, Result),
erlang:element(4, Result),
erlang:element(5, Result)}}
end
).
-file("src/shelf/set.gleam", 99).
?DOC(
" Open a persistent set table with defaults (WriteBack mode).\n"
"\n"
" ```gleam\n"
" let assert Ok(table) =\n"
" set.open(name: \"users\", path: \"users.dets\",\n"
" base_directory: \"/app/data\",\n"
" key: decode.string, value: decode.int)\n"
" ```\n"
).
-spec open(
binary(),
binary(),
binary(),
gleam@dynamic@decode:decoder(MOG),
gleam@dynamic@decode:decoder(MOI)
) -> {ok, p_set(MOG, MOI)} | {error, shelf:shelf_error()}.
open(Name, Path, Base_directory, Key_decoder, Value_decoder) ->
open_config(
shelf:config(Name, Path, Base_directory),
Key_decoder,
Value_decoder
).
-file("src/shelf/set.gleam", 124).
?DOC(
" Close the table, saving all data to disk.\n"
"\n"
" Performs a final snapshot of ETS to DETS, closes the DETS file,\n"
" and deletes the ETS table.\n"
"\n"
" On `Ok(Nil)`, the handle must not be used again. If the final save\n"
" fails with a retryable persistence error, `close()` returns\n"
" `Error(...)` and leaves the table open so the caller can retry.\n"
" If close fails terminally, shelf still releases resources and future\n"
" operations on the handle return `Error(TableClosed)`.\n"
).
-spec close(p_set(any(), any())) -> {ok, nil} | {error, shelf:shelf_error()}.
close(Table) ->
shelf_ffi:close(
erlang:element(2, Table),
erlang:element(3, Table),
erlang:element(4, Table)
).
-file("src/shelf/set.gleam", 143).
?DOC(
" Use a table within a callback, ensuring it is closed afterward.\n"
"\n"
" The table is opened before the callback and closed after it returns\n"
" (even if it returns an error). Data is auto-saved on close; if the\n"
" final save fails, `with_table` force-cleans the table to release\n"
" resources. If the callback succeeded, the close error is returned;\n"
" if both the callback and close fail, the callback error is preserved.\n"
"\n"
" ```gleam\n"
" use table <- set.with_table(\"cache\", \"cache.dets\",\n"
" base_directory: \"/app/data\",\n"
" key: decode.string, value: decode.string)\n"
" set.insert(into: table, key: \"key\", value: \"value\")\n"
" ```\n"
).
-spec with_table(
binary(),
binary(),
binary(),
gleam@dynamic@decode:decoder(MOU),
gleam@dynamic@decode:decoder(MOW),
fun((p_set(MOU, MOW)) -> {ok, MPA} | {error, shelf:shelf_error()})
) -> {ok, MPA} | {error, shelf:shelf_error()}.
with_table(Name, Path, Base_directory, Key_decoder, Value_decoder, Fun) ->
shelf@internal:generic_with_table(
fun() ->
open(Name, Path, Base_directory, Key_decoder, Value_decoder)
end,
fun close/1,
Fun
).
-file("src/shelf/set.gleam", 181).
?DOC(" Check if a key exists without returning the value.\n").
-spec member(p_set(MPL, any()), MPL) -> {ok, boolean()} |
{error, shelf:shelf_error()}.
member(Table, Key) ->
shelf_ffi:member(erlang:element(2, Table), Key).
-file("src/shelf/set.gleam", 189).
?DOC(
" Return all key-value pairs as a list.\n"
"\n"
" **Warning**: loads entire table into memory.\n"
).
-spec to_list(p_set(MPR, MPS)) -> {ok, list({MPR, MPS})} |
{error, shelf:shelf_error()}.
to_list(Table) ->
shelf_ffi:to_list(erlang:element(2, Table)).
-file("src/shelf/set.gleam", 195).
?DOC(" Fold over all entries. Order is unspecified.\n").
-spec fold(p_set(MPY, MPZ), MQC, fun((MQC, MPY, MPZ) -> MQC)) -> {ok, MQC} |
{error, shelf:shelf_error()}.
fold(Table, Initial, Fun) ->
shelf@internal:generic_fold(erlang:element(2, Table), Initial, Fun).
-file("src/shelf/set.gleam", 205).
?DOC(" Return the number of entries in the table.\n").
-spec size(p_set(any(), any())) -> {ok, integer()} |
{error, shelf:shelf_error()}.
size(Table) ->
shelf_ffi:size(erlang:element(2, Table)).
-file("src/shelf/set.gleam", 216).
?DOC(
" Insert a key-value pair. Overwrites if key exists.\n"
"\n"
" In WriteBack mode, only ETS is updated — call `save()` to persist.\n"
" In WriteThrough mode, both ETS and DETS are updated.\n"
).
-spec insert(p_set(MQL, MQM), MQL, MQM) -> {ok, nil} |
{error, shelf:shelf_error()}.
insert(Table, Key, Value) ->
shelf@internal:generic_insert(
erlang:element(2, Table),
erlang:element(3, Table),
erlang:element(5, Table),
Key,
Value
).
-file("src/shelf/set.gleam", 226).
?DOC(" Insert multiple key-value pairs at once.\n").
-spec insert_list(p_set(MQR, MQS), list({MQR, MQS})) -> {ok, nil} |
{error, shelf:shelf_error()}.
insert_list(Table, Entries) ->
shelf@internal:generic_insert_list(
erlang:element(2, Table),
erlang:element(3, Table),
erlang:element(5, Table),
Entries
).
-file("src/shelf/set.gleam", 267).
?DOC(" Delete the entry with the given key.\n").
-spec delete_key(p_set(MRE, any()), MRE) -> {ok, nil} |
{error, shelf:shelf_error()}.
delete_key(Table, Key) ->
shelf@internal:generic_delete_key(
erlang:element(2, Table),
erlang:element(3, Table),
erlang:element(5, Table),
Key
).
-file("src/shelf/set.gleam", 279).
?DOC(
" Atomic Compare-and-Delete: delete the entry only if both key and value match.\n"
"\n"
" Unlike `delete_key`, which removes the entry regardless of its value,\n"
" this function checks the full `#(key, value)` tuple. If the stored\n"
" value doesn't match, nothing is deleted — useful for optimistic\n"
" concurrency patterns where you want to avoid clobbering a concurrent\n"
" update.\n"
).
-spec delete_object(p_set(MRK, MRL), MRK, MRL) -> {ok, nil} |
{error, shelf:shelf_error()}.
delete_object(Table, Key, Value) ->
shelf@internal:generic_delete_object(
erlang:element(2, Table),
erlang:element(3, Table),
erlang:element(5, Table),
Key,
Value
).
-file("src/shelf/set.gleam", 298).
?DOC(
" Delete all entries from the table.\n"
"\n"
" The table remains open and usable after this call — only the data\n"
" is removed. To release the table entirely, use `close`.\n"
).
-spec delete_all(p_set(any(), any())) -> {ok, nil} |
{error, shelf:shelf_error()}.
delete_all(Table) ->
shelf@internal:generic_delete_all(
erlang:element(2, Table),
erlang:element(3, Table),
erlang:element(5, Table)
).
-file("src/shelf/set.gleam", 320).
?DOC(
" Snapshot the current ETS contents to DETS.\n"
"\n"
" Uses an atomic save strategy: data is written to a temporary file\n"
" first, then atomically renamed over the original DETS file. This\n"
" prevents data loss if the process is killed mid-save (the original\n"
" file remains intact until the rename succeeds).\n"
"\n"
" See the\n"
" [Durability story](https://shelf.tylerbutler.com/advanced/persistence-operations/#durability-story)\n"
" for the full layer-by-layer breakdown of which crashes this protects against.\n"
"\n"
" ```gleam\n"
" // After a batch of writes...\n"
" let assert Ok(Nil) = set.save(table)\n"
" ```\n"
).
-spec save(p_set(any(), any())) -> {ok, nil} | {error, shelf:shelf_error()}.
save(Table) ->
shelf_ffi:save(erlang:element(2, Table), erlang:element(3, Table)).
-file("src/shelf/set.gleam", 331).
?DOC(
" Discard unsaved ETS changes and reload from DETS.\n"
"\n"
" Clears the ETS table, re-reads all DETS entries, validates them\n"
" through the stored decoders, and loads valid entries into ETS.\n"
" Only useful in WriteBack mode — in WriteThrough mode, ETS and\n"
" DETS are always in sync.\n"
).
-spec reload(p_set(any(), any())) -> {ok, nil} | {error, shelf:shelf_error()}.
reload(Table) ->
shelf@internal:generic_reload(
erlang:element(2, Table),
erlang:element(3, Table),
erlang:element(6, Table)
).
-file("src/shelf/set.gleam", 343).
?DOC(
" Drain the DETS in-memory write buffer into the open DETS file.\n"
"\n"
" Calls `dets:sync/1`. Most useful in WriteThrough mode after a critical\n"
" write, to make sure pending DETS writes are reflected in the file on\n"
" disk. Note: this does not call `fsync(2)` on the file descriptor — see\n"
" the [Durability story](https://shelf.tylerbutler.com/advanced/persistence-operations/#durability-story)\n"
" for what `sync()` does and does not guarantee.\n"
).
-spec sync(p_set(any(), any())) -> {ok, nil} | {error, shelf:shelf_error()}.
sync(Table) ->
shelf_ffi:sync_dets(erlang:element(2, Table), erlang:element(3, Table)).
-file("src/shelf/set.gleam", 175).
?DOC(
" Look up the value for a key.\n"
"\n"
" Reads from ETS — consistent microsecond latency regardless of\n"
" table size or whether the data has been saved to disk.\n"
"\n"
" Returns `Error(NotFound)` if the key does not exist.\n"
).
-spec lookup(p_set(MPF, MPG), MPF) -> {ok, MPG} | {error, shelf:shelf_error()}.
lookup(Table, Key) ->
shelf_ffi:lookup_set(erlang:element(2, Table), Key).
-file("src/shelf/set.gleam", 241).
?DOC(
" Insert a key-value pair only if the key does not already exist.\n"
"\n"
" Returns `Error(KeyAlreadyPresent)` if the key exists.\n"
"\n"
" In WriteThrough mode, uniqueness is checked in ETS first, then DETS\n"
" is written, then ETS. Since writes are owner-only (single process),\n"
" there is no race between the check and write.\n"
).
-spec insert_new(p_set(MQY, MQZ), MQY, MQZ) -> {ok, nil} |
{error, shelf:shelf_error()}.
insert_new(Table, Key, Value) ->
case erlang:element(5, Table) of
write_through ->
gleam@result:'try'(
shelf_ffi:member(erlang:element(2, Table), Key),
fun(Exists) -> case Exists of
true ->
{error, key_already_present};
false ->
gleam@result:'try'(
shelf_ffi:dets_insert(
erlang:element(3, Table),
{Key, Value}
),
fun(_) ->
shelf_ffi:insert(
erlang:element(2, Table),
erlang:element(3, Table),
{Key, Value}
)
end
)
end end
);
write_back ->
shelf_ffi:insert_new(
erlang:element(2, Table),
erlang:element(3, Table),
{Key, Value}
)
end.
-file("src/shelf/set.gleam", 363).
?DOC(
" Atomically increment an integer value by the given amount.\n"
"\n"
" The value associated with the key must be an integer. Returns the\n"
" new value after incrementing. The increment can be negative.\n"
"\n"
" ```gleam\n"
" let assert Ok(Nil) = set.insert(into: table, key: \"hits\", value: 0)\n"
" let assert Ok(1) = set.update_counter(in: table, key: \"hits\", increment: 1)\n"
" let assert Ok(3) = set.update_counter(in: table, key: \"hits\", increment: 2)\n"
" ```\n"
"\n"
" In WriteThrough mode, the ETS atomic increment happens first (only ETS\n"
" supports update_counter), then DETS is updated. If the DETS write fails,\n"
" the ETS increment is rolled back by applying the negated amount.\n"
).
-spec update_counter(p_set(MSO, integer()), MSO, integer()) -> {ok, integer()} |
{error, shelf:shelf_error()}.
update_counter(Table, Key, Amount) ->
gleam@result:'try'(
shelf_ffi:update_counter(erlang:element(2, Table), Key, Amount),
fun(New_val) -> case erlang:element(5, Table) of
write_through ->
case shelf_ffi:dets_insert(
erlang:element(3, Table),
{Key, New_val}
) of
{ok, nil} ->
{ok, New_val};
{error, E} ->
_ = shelf_ffi:update_counter(
erlang:element(2, Table),
Key,
- Amount
),
{error, E}
end;
write_back ->
{ok, New_val}
end end
).