Current section
Files
Jump to
Current section
Files
lib/klife.ex
defmodule Klife do
@moduledoc false
alias Klife.Record
alias Klife.Producer
alias Klife.Consumer.Fetcher
alias Klife.TxnProducerPool
alias Klife.Connection.Controller, as: ConnController
alias Klife.MetadataCache
@produce_opts [
producer: [
type: :atom,
required: false,
doc:
"Producer's name that will override the `default_producer` configuration. Ignored inside transactions."
],
partitioner: [
type: :atom,
required: false,
doc: "Module that will override `default_partitioner` configuration."
]
]
@async_opts [
callback: [
type: :any,
required: false,
doc:
"MFA or function/1 that will be called with the produce result. The result is injected as the first argument on MFA and is the only argument for anonymous functions"
]
]
@txn_opts [
pool_name: [
type: :atom,
required: false,
doc: "Txn pool's name that will override the `default_txn_pool` configuration."
]
]
@fetch_opts [
fetcher: [
type: :atom,
required: false,
doc: "Fetcher's name that will override the `default_fetcher` configuration."
],
isolation_level: [
type: {:in, [:read_committed, :read_uncommitted]},
required: false,
doc:
"Controls whether the fetch response includes uncommitted transactional records. Defaults to `:read_committed`."
],
max_bytes: [
type: :non_neg_integer,
required: false,
doc:
"Maximum number of bytes to return per fetch request. Defaults to `100_000` for `fetch/4` and `500_000` for `fetch_async/4`."
]
]
@doc false
def get_produce_opts(), do: @produce_opts
@doc false
def get_txn_opts(), do: @txn_opts
@doc false
def get_async_opts(), do: @async_opts
@doc false
def get_fetch_opts(), do: @fetch_opts
@doc false
def produce(%Record{} = record, client, opts \\ []) do
[resp] = produce_batch([record], client, opts)
resp
end
@doc false
def produce_batch([%Record{} | _] = records, client, opts \\ []) do
records = prepare_records(records, client, opts)
if TxnProducerPool.in_txn?(client) do
TxnProducerPool.produce(records, client, opts)
else
if ConnController.disabled_feature?(client, :producer) do
raise "Produce API called but producer feature is disabled (client=#{inspect(client)}). Check logs for details."
end
Producer.produce(records, client, opts)
end
end
@doc false
def produce_async(%Record{} = record, client, opts \\ []) do
if ConnController.disabled_feature?(client, :producer) do
raise "Produce API called but producer feature is disabled (client=#{inspect(client)}). Check logs for details."
end
prepared_rec = prepare_records(record, client, opts)
Producer.produce_async([prepared_rec], client, opts)
end
@doc false
def produce_batch_async([%Record{} | _] = records, client, opts \\ []) do
if ConnController.disabled_feature?(client, :producer) do
raise "Produce API called but producer feature is disabled (client=#{inspect(client)}). Check logs for details."
end
case opts[:callback] do
nil ->
records = prepare_records(records, client, opts)
Producer.produce_async(records, client, opts)
{m, f, args} ->
Task.start(fn -> apply(m, f, [produce_batch(records, client, opts) | args]) end)
:ok
fun when is_function(fun, 1) ->
Task.start(fn -> fun.(produce_batch(records, client, opts)) end)
:ok
end
end
@doc false
def produce_batch_txn([%Record{} | _] = records, client, opts \\ []) do
transaction(
fn -> records |> produce_batch(client, opts) |> Record.verify_batch() end,
client,
opts
)
end
@doc false
def transaction(fun, client, opts \\ []) do
if ConnController.disabled_feature?(client, :txn_producer) do
raise "Transaction API called but txn_producer feature is disabled (client=#{inspect(client)}). Check logs for details."
end
TxnProducerPool.run_txn(client, get_txn_pool(client, opts), fun)
end
def fetch(topic, partition, offset, client, opts \\ []) do
tpo = {topic, partition, offset}
Fetcher.fetch(tpo, client, opts)
end
def fetch(tpo_list, client, opts) when is_list(tpo_list) do
Fetcher.fetch(tpo_list, client, opts)
end
@doc false
def fetch_async(topic, partition, offset, client, opts \\ []) do
tpo = {topic, partition, offset}
Fetcher.fetch_async(tpo, client, opts)
end
defp get_txn_pool(client, opts) do
Keyword.get(opts, :pool_name) || client.get_default_txn_pool()
end
def add_partition(%Record{} = record, client, opts \\ []) do
case record do
%Record{partition: nil, topic: topic} ->
case MetadataCache.get_metadata(client, topic, 0) do
{:ok,
%{
default_partitioner: default_partitioner_mod,
max_partition: max_partition
}} ->
partitioner_mod = Keyword.get(opts, :partitioner, default_partitioner_mod)
%{record | partition: partitioner_mod.get_partition(record, max_partition)}
{:error, :not_found} ->
{:error, :unkown_metadata_for_topic}
end
record ->
record
end
end
def add_partition!(%Record{} = record, client, opts \\ []) do
case add_partition(record, client, opts) do
%Record{} = rec -> rec
err -> raise "Error on add partition: #{inspect(err)}"
end
end
def log_metadata do
[
:client,
:broker_id,
:host,
:topic,
:partition,
:offset,
:group,
:producer,
:pool,
:consumer_group_mod,
:coordinator_id,
:error_code,
:error_message,
:error,
:reason,
:message,
:feature,
:api_key,
:requested_offset,
:reset_offset,
:latest_committed_offset,
:latest_processed_offset,
:stacktrace
]
end
defp prepare_records(%Record{} = rec, client, opts) do
[new_rec] = prepare_records([rec], client, opts)
new_rec
end
defp prepare_records(recs, client, opts) when is_list(recs) do
recs
|> Enum.with_index(1)
|> Enum.map(fn {rec, idx} ->
rec
|> Map.replace!(:__estimated_size, Record.estimate_size(rec))
|> Map.replace!(:__batch_index, idx)
|> add_partition!(client, opts)
end)
end
end