genstage_kafka

defmodule GenStageKafka do
  use GenStage

  @kafka_client_id :gs_kafka_producer_client

  @moduledoc """

  A module that is a GenStage Consumer, and publishes the records received
  into a Kafka topic

  """

  @doc """

  Starts the GenStageKafka with the options.

  The options handled by the module arei (defaults shown).

  :hosts - [localhost: 9092]
  :topic - "no_topic"
  :partition - 0

  Given as a keyword lists, i.e.

  KafkaGenStage.start_link( topic: "my_topic", hosts: [my_broker.example.com: 9092] )

  """
  def start_link( kafka_options ) do
    GenStage.start_link( GenStageKafka, kafka_options )
  end

  @doc """

  The init function for the GenStage server.

  Setup the Kafka produce and indicate that we are a GenStage consumer

  """
  def init(kafka_options) do
    # Setup the kafka producer.
    bootstrapProducer( kafka_options )

    # Indicate that we are a GenStage consumer
    {:consumer, %{kafka_options: kafka_options}}
  end

  @doc """

  The GenStage consumer event handler.

  The events have to be a list of maps where each map
  has 'key' and 'value'. Every map that conforms to these
  requirements are produced to the configured kafka partition

  """
  def handle_events(events, _from, state) do
    # Produce to kafka
    results = Enum.map(events, fn(e) ->
      case e do
        %{key: key, value: value} ->
          :brod.produce_sync(@kafka_client_id, state.kafka_options[:topic], state.kafka_options[:partition], key, value)
        _ ->
          :error_invalid_data
      end
    end)

    # We are a consumer, so we would never emit items.
    {:noreply, [], state}
  end

  defp bootstrapProducer(kafka_options) do
    hosts = Keyword.get(kafka_options, :host, [localhost: 9092])
    topic = Keyword.get(kafka_options, :topic, "no_topic")

    :ok = :brod.start_client(hosts, @kafka_client_id, _client_config=[])
    :ok = :brod.start_producer(@kafka_client_id, [topic], _producer_config=[])
  end

end