Packages

Framework for consuming Kafka events

Current section

Files

Jump to
kaffeine lib kaffeine.ex
Raw

lib/kaffeine.ex

defmodule Kaffeine do
require Logger
import Kaffeine.Util, only: [opts_or_application: 3, opts_or_application: 4]
alias Kaffeine.{
Consumer,
Partitions,
Producer,
TopicSupervisor,
}
@moduledoc """
Documentation for Kaffeine.
"""
@doc """
"""
@spec start(list(Consumer.t), Keyword.t) :: Supervisor.supervisor
def start(consumers, opts \\ []) do
import Supervisor.Spec
super_opts = [{:strategy, :one_for_one} | Keyword.take(opts, [:name])]
with {:ok, brokers} <- opts_or_application(opts, :kafka_ex, :brokers),
{:ok, kafka_version} <- opts_or_application(opts, :kafka_ex, :kafka_version),
{:ok, kafka_impl} <- opts_or_application(opts, :kafka_impl, :impl, :kafka_impl),
{:ok, partitions} <- Partitions.partition_counts(brokers, kafka_version, kafka_impl)
do
consumers
|> Enum.reduce([],
fn
%Consumer{} = consumer, acc ->
case Map.get(partitions, consumer.topic, 0) do
0 ->
Logger.warn "No partitions found for `#{consumer.topic}`"
acc
partition_count ->
Logger.info "Found #{partition_count} partitions for '#{consumer.topic}'"
child = supervisor(
TopicSupervisor,
[
partition_count,
%{consumer | brokers: brokers, kafka_impl: kafka_impl, kafka_version: kafka_version},
],
[
id: :"Kaffeine.TopicPartitionSupervisor-#{consumer.topic}"
]
)
[child | acc]
end
%Producer{} = producer, acc ->
child = worker(Producer, [
%{ producer |
brokers: brokers,
kafka_impl: kafka_impl,
kafka_version: kafka_version,
max_partitions: Map.get(partitions, producer.topic, 1),
}
],
[
id: :"Kaffeine.Producer.#{producer.topic}",
])
[child | acc]
end
)
|> Supervisor.start_link(super_opts)
end
end
@doc """
Receives a topic name to consume, and the module, function, and additional arguments of the
handler.
Returns a Kaffeine.Consumer struct of the consumer defintion.
"""
@spec consumer(String.t, Consumer.mfa_t | Consumer.handler_fun_t, Keyword.t) :: Consumer.t | :error
def consumer(topic, mfa, opts \\ []) do
with {:ok, consumer_group} <- opts_or_application(opts, :kafka_ex, :consumer_group),
{:ok, consumer_wait_ms} <- opts_or_application(opts, :kaffeine, :consumer_wait_ms, fn app, key -> EnvConfig.get_integer(app, key) end)
do
%Consumer{
topic: topic,
handler: mfa,
consumer_group: consumer_group,
consumer_wait_ms: consumer_wait_ms,
}
end
end
@doc """
Build the configuration for a produce worker for a specific topic.
ex: `producer("NewMessage",
encoder: &Poison.encode/1,
partitioner: fn message, max_partitions -> {:ok, rem(message.user_id, max_partitions)} end)`
opts:
* `encoder`
`(any -> {:ok, any} | {:error, String.t})`
An anonymous function that is used when producing messages to encode for Kafka.
ex: `fn message -> {:ok, Poison.encode(message)} end`
* `partitioner`
`(any -> {:ok, integer} | {:error, String.t})`
An anonymous function that is used to determine which partition to put the message in. Is
passed the message and the max number of partitions for the topic.
ex: `fn message, max_partitions -> {:ok, rem(message.user_id, max_partitions)} end`
* `required_acks`
Required acknowledgements by kafka brokers before considered successful.
* `timeout`
Request timeout in milliseconds.
"""
def producer(topic, opts \\ []) do
struct(%Producer{topic: topic}, opts)
end
@doc """
Produce a message to a given topic. A Producer must already be setup for the topic you wish to
write to.
## Examples
iex> Kaffeine.produce("hello world", "test")
:ok
iex> Kaffeine.produce(%{id: 501, name: "Bob"}, "Users")
:ok
"""
@spec produce(any, String.t, String.t | nil) :: :ok
def produce(message, topic, key \\ nil) do
Producer.produce(message, topic, key)
end
end