kflow

%%%===================================================================
%%% @copyright 2019 Klarna Bank AB (publ)
%%%
%%% @doc This stream processing node commits fully processed offsets
%%% to Kafka
%%% @end
%%%===================================================================
-module(kflow_kafka_commit).

-behavior(kflow_gen).

-include("kflow_int.hrl").

%% kflow_gen callbacks:
-export([ init/2
        , handle_message/3
        , handle_flush/2
        , terminate/2
        ]).

%%%===================================================================
%%% Types
%%%===================================================================

-type commit_fun() :: fun((brod:offset()) -> _).

-type state() ::
        #{ commit_fun            := commit_fun()
         , topic                 := brod:topic()
         , partition             := brod:partition()
         , group_id              := brod:group_id()
         , last_committed_offset := brod:offset()
         }.

%%%===================================================================
%%% `kflow_gen' callbacks
%%%===================================================================

%% @private
init(_NodeId, {_CbModule, InitInfo}) ->
  prometheus_gauge:declare([ {name, <<"kflow_kafka_offset_committed">>}
                           , {help, "Group subscriber offset that has been committed to Kafka"}
                           , {labels, [group_id, topic, partition]}
                           ]),
  prometheus_gauge:declare([ {name, <<"kflow_kafka_offset_commit_time">>}
                           , {help, "POSIX time when the offset was committed to Kafka"}
                           , {labels, [group_id, topic, partition]}
                           ]),
  {ok, InitInfo#{last_committed_offset => -1}}.

%% @private
handle_message( Msg = #kflow_msg{ offset                 = Offset
                                , fully_processed_offset = FPO
                                , route                  = Route
                                }
              , State
              , _
              ) ->
  #{ commit_fun            := CommitFun
   , topic                 := Topic
   , partition             := Partition
   , group_id              := GroupId
   , last_committed_offset := LastCommittedOffset
   } = State,
  SafeOffset = case FPO of
                 undefined -> Offset;
                 _         -> FPO
               end,
  %% Assert:
  SafeOffset =< Offset orelse
    ?slog(alert, #{ what           => "Offset tracking bug!"
                  , safe_offset    => SafeOffset
                  , message_offset => Offset
                  }),
  if SafeOffset >= LastCommittedOffset ->
      ok = CommitFun(SafeOffset),
      prometheus_gauge:set( <<"kflow_kafka_offset_committed">>
                          , [GroupId, Topic, Partition]
                          , SafeOffset
                          ),
      prometheus_gauge:set( <<"kflow_kafka_offset_commit_time">>
                          , [GroupId, Topic, Partition]
                          , erlang:system_time(second)
                          ),
      ?tp(info, kflow_kafka_commit_offset, #{ group_id  => GroupId
                                            , topic     => Topic
                                            , partition => Partition
                                            , offset    => SafeOffset
                                            });
     SafeOffset =:= LastCommittedOffset ->
      ok;
     %% Bug:
     true ->
      ?slog(alert, #{ what                   => "Offset tracking bug!"
                    , message_offset         => Offset
                    , fully_processed_offset => FPO
                    , last_committed_offset  => LastCommittedOffset
                    , group_id               => GroupId
                    , route                  => Route
                    })
  end,
  {ok, [Msg], State#{last_committed_offset => SafeOffset}}.

%% @private
handle_flush(State, _) ->
  {ok, [], State}.

%% @private
terminate(_, _) ->
  ok.

%%%===================================================================
%%% Internal functions
%%%===================================================================