Current section

Files

Jump to
gnat lib gnat jetstream api kv.ex
Raw

lib/gnat/jetstream/api/kv.ex

defmodule Gnat.Jetstream.API.KV do
@moduledoc """
API for interacting with the Key/Value store functionality in Nats Jetstream.
Learn about the Key/Value store: https://docs.nats.io/nats-concepts/jetstream/key-value-store
"""
alias Gnat.Jetstream.API.{Stream}
@stream_prefix "KV_"
@subject_prefix "$KV."
@two_minutes_in_nanoseconds 120_000_000_000
@type bucket_options ::
{:history, non_neg_integer()}
| {:ttl, non_neg_integer()}
| {:max_bucket_size, non_neg_integer()}
| {:max_value_size, non_neg_integer()}
| {:description, binary()}
| {:replicas, non_neg_integer()}
| {:storage, :file | :memory}
| {:placement, Stream.placement()}
@doc """
Create a new Key/Value bucket. Can include the following options
* `:history` - How many historic values to keep per key (defaults to 1, max of 64)
* `:ttl` - How long to keep values for (in nanoseconds)
* `:max_bucket_size` - The max number of bytes the bucket can hold
* `:max_value_size` - The max number of bytes a value may be
* `:description` - A description for the bucket
* `:replicas` - How many replicas of the data to store
* `:storage` - Storage backend to use (:file, :memory)
* `:placement` - A map with :cluster (required) and :tags (optional)
## Examples
iex> {:ok, info} = Jetstream.API.KV.create_bucket(:gnat, "my_bucket")
"""
@spec create_bucket(conn :: Gnat.t(), bucket_name :: binary(), params :: [bucket_options()]) ::
{:ok, Stream.info()} | {:error, any()}
def create_bucket(conn, bucket_name, params \\ []) do
# The primary NATS docs don't provide information about how to interact
# with Key-Value functionality over the wire. Turns out the KV store is
# just a Stream under-the-hood
# Discovered these settings from looking at the `nats-server -js -DV` logs
# as well as the GoLang implementation https://github.com/nats-io/nats.go/blob/dd91b86bc4f7fa0f061fefe11506aaee413bfafd/kv.go#L339
# If the settings aren't correct, NATS will not consider it a valid KV store
stream = %Stream{
name: stream_name(bucket_name),
subjects: stream_subjects(bucket_name),
description: Keyword.get(params, :description),
max_msgs_per_subject: Keyword.get(params, :history, 1),
discard: :new,
deny_delete: true,
allow_rollup_hdrs: true,
max_age: Keyword.get(params, :ttl, 0),
max_bytes: Keyword.get(params, :max_bucket_size, -1),
max_msg_size: Keyword.get(params, :max_value_size, -1),
num_replicas: Keyword.get(params, :replicas, 1),
storage: Keyword.get(params, :storage, :file),
placement: Keyword.get(params, :placement),
duplicate_window: adjust_duplicate_window(Keyword.get(params, :ttl, 0))
}
Stream.create(conn, stream)
end
# The `duplicate_window` can't be greater than the `max_age`. The default `duplicate_window`
# is 2 minutes. We'll keep the 2 minute window UNLESS the ttl is less than 2 minutes
defp adjust_duplicate_window(ttl) when ttl > 0 and ttl < @two_minutes_in_nanoseconds, do: ttl
defp adjust_duplicate_window(_ttl), do: @two_minutes_in_nanoseconds
@doc """
Delete a Key/Value bucket
## Examples
iex> :ok = Jetstream.API.KV.delete_bucket(:gnat, "my_bucket")
"""
@spec delete_bucket(conn :: Gnat.t(), bucket_name :: binary()) :: :ok | {:error, any()}
def delete_bucket(conn, bucket_name) do
Stream.delete(conn, stream_name(bucket_name))
end
@doc """
Create a Key in a Key/Value Bucket
## Options
* `:timeout` - receive timeout for the request
## Examples
iex> :ok = Jetstream.API.KV.create_key(:gnat, "my_bucket", "my_key", "my_value")
"""
@spec create_key(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
value :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def create_key(conn, bucket_name, key, value, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply = Gnat.request(conn, key_name(bucket_name, key), value, receive_timeout: timeout)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Delete a Key from a K/V Bucket
## Examples
iex> :ok = Jetstream.API.KV.delete_key(:gnat, "my_bucket", "my_key")
"""
@spec delete_key(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def delete_key(conn, bucket_name, key, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply =
Gnat.request(conn, key_name(bucket_name, key), "",
headers: [{"KV-Operation", "DEL"}],
receive_timeout: timeout
)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Purge a Key from a K/V bucket. This will remove any revision history the key had
## Examples
iex> :ok = Jetstream.API.KV.purge_key(:gnat, "my_bucket", "my_key")
"""
@spec purge_key(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def purge_key(conn, bucket_name, key, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply =
Gnat.request(conn, key_name(bucket_name, key), "",
headers: [{"KV-Operation", "PURGE"}, {"Nats-Rollup", "sub"}],
receive_timeout: timeout
)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Put a value into a Key in a K/V Bucket
## Examples
iex> :ok = Jetstream.API.KV.put_value(:gnat, "my_bucket", "my_key", "my_value")
"""
@spec put_value(
conn :: Gnat.t(),
bucket_name :: binary(),
key :: binary(),
value :: binary(),
opts :: keyword()
) ::
:ok | {:error, any()}
def put_value(conn, bucket_name, key, value, opts \\ []) do
timeout = Keyword.get(opts, :timeout, 5_000)
reply = Gnat.request(conn, key_name(bucket_name, key), value, receive_timeout: timeout)
case reply do
{:ok, _} -> :ok
error -> error
end
end
@doc """
Get the value for a key in a particular K/V bucket
## Examples
iex> "my_value" = Jetstream.API.KV.get_value(:gnat, "my_bucket", "my_key")
"""
@spec get_value(conn :: Gnat.t(), bucket_name :: binary(), key :: binary()) ::
binary() | {:error, any()} | nil
def get_value(conn, bucket_name, key) do
case Stream.get_message(conn, stream_name(bucket_name), %{
last_by_subj: key_name(bucket_name, key)
}) do
{:ok, message} -> message.data
error -> error
end
end
@doc """
Get all the non-deleted key-value pairs for a Bucket
## Examples
iex> {:ok, %{"key1" => "value1"}} = Jetstream.API.KV.contents(:gnat, "my_bucket")
"""
@spec contents(conn :: Gnat.t(), bucket_name :: binary(), domain :: nil | binary()) ::
{:ok, map()} | {:error, binary()}
def contents(conn, bucket_name, domain \\ nil) do
alias Gnat.Jetstream.Pager
stream = stream_name(bucket_name)
Pager.reduce(conn, stream, [domain: domain], %{}, fn msg, acc ->
case msg do
%{topic: key, body: body, headers: headers} ->
if {"kv-operation", "DEL"} in headers do
acc
else
Map.put(acc, subject_to_key(key, bucket_name), body)
end
%{topic: key, body: body} ->
Map.put(acc, subject_to_key(key, bucket_name), body)
end
end)
end
@doc """
Information about the state of the bucket's Stream.
## Opts
* `:domain` - (default `nil`) the domain of the bucket
"""
@spec info(conn :: Gnat.t(), bucket_name :: binary(), keyword()) ::
{:ok, Stream.info()} | {:error, any()}
def info(conn, bucket_name, opts \\ []) do
Stream.info(conn, @stream_prefix <> bucket_name, Keyword.get(opts, :domain))
end
@doc ~S"""
Starts a monitor for key changes in a given bucket. Supply a handler that will receive
key change notifications.
## Examples
iex> {:ok, _pid} = Jetstream.API.KV.watch(:gnat, "my_bucket", fn action, key, value ->
...> IO.puts("#{action} taken on #{key}")
...> end)
"""
def watch(conn, bucket_name, handler) do
Gnat.Jetstream.API.KV.Watcher.start_link(
conn: conn,
bucket_name: bucket_name,
handler: handler
)
end
@doc ~S"""
Stops a previously running monitor. This will unsubscribe from the key changes and remove the
ephemeral consumer
## Examples
iex> :ok = Jetstream.API.KV.unwatch(pid)
"""
def unwatch(pid) do
Gnat.Jetstream.API.KV.Watcher.stop(pid)
end
@spec is_kv_bucket_stream?(stream_name :: binary()) :: boolean()
@deprecated "Use Gnat.Jetstream.API.KV.kv_bucket_stream?/1 instead"
def is_kv_bucket_stream?(stream_name) do
kv_bucket_stream?(stream_name)
end
@doc """
Returns true if the provided stream is a KV bucket, false otherwise
## Parameters
* `stream_name` - the stream name to test
"""
@spec kv_bucket_stream?(stream_name :: binary()) :: boolean()
def kv_bucket_stream?(stream_name) do
String.starts_with?(stream_name, "KV_")
end
@doc """
Returns a list of all the buckets in the KV
"""
@spec list_buckets(conn :: Gnat.t()) :: {:error, term()} | {:ok, list(String.t())}
def list_buckets(conn) do
with {:ok, %{streams: streams}} <- Stream.list(conn) do
stream_names =
for bucket <- streams, kv_bucket_stream?(bucket) do
String.trim_leading(bucket, @stream_prefix)
end
{:ok, stream_names}
else
{:error, reason} ->
{:error, reason}
end
end
@doc false
def stream_name(bucket_name) do
"#{@stream_prefix}#{bucket_name}"
end
defp stream_subjects(bucket_name) do
["#{@subject_prefix}#{bucket_name}.>"]
end
defp key_name(bucket_name, key) do
"#{@subject_prefix}#{bucket_name}.#{key}"
end
@doc false
def subject_to_key(subject, bucket_name) do
String.replace(subject, "#{@subject_prefix}#{bucket_name}.", "")
end
end