Packages
brod
3.0.0
4.5.7
4.5.6
4.5.5
4.5.4
4.5.3
4.5.2
4.5.1
4.5.0
4.4.7
4.4.6
4.4.5
4.4.4
4.4.3
4.4.2
4.4.1
4.4.0
4.3.3
4.3.2
4.3.1
4.3.0
4.2.0
4.1.1
4.1.0
4.0.0
3.19.1
3.19.0
3.18.0
3.17.1
3.17.0
3.16.5
3.16.4
3.16.3
3.16.2
3.16.1
3.16.0
3.15.6
3.15.5
3.15.4
3.15.3
3.15.1
3.15.0
3.14.0
3.13.0
3.12.0
3.11.0
3.10.0
3.9.5
3.9.3
3.9.2
3.9.1
3.9.0
3.8.1
3.8.0
3.7.11
3.7.10
3.7.9
3.7.8
3.7.7
3.7.6
3.7.5
3.7.4
3.7.3
3.7.2
3.7.1
3.7.0
3.6.2
3.6.1
3.6.0
3.5.2
3.5.1
3.5.0
3.4.0
3.3.5
3.3.4
3.3.3
3.3.2
3.3.1
3.3.0
3.2.0
3.0.0
2.5.0
2.4.1
2.4.0
2.3.7
2.3.6
2.3.5
2.3.4
2.3.3
2.3.1
2.2.16
2.2.15
2.2.14
2.2.12
2.2.11
2.2.10
2.2.9
2.2.8
2.2.7
2.2.6
2.2.5
2.2.4
2.2.3
2.2.2
2.2.1
2.2.0
2.1.12
2.1.11
2.1.10
2.1.8
2.1.7
2.1.4
2.1.2
2.0.0
Apache Kafka Erlang client library
Current section
113 Versions
Jump to
Current section
113 Versions
Compare versions
25
files changed
+1640
additions
-1109
deletions
| @@ -1,6 +1,6 @@ | |
| 1 1 | PROJECT = brod |
| 2 2 | PROJECT_DESCRIPTION = Kafka client library in Erlang |
| 3 | - PROJECT_VERSION = 2.5.0 |
| 3 | + PROJECT_VERSION = 3.0.0 |
| 4 4 | |
| 5 5 | DEPS = supervisor3 kafka_protocol |
| 6 6 | TEST_DEPS = docopt jsone meck proper |
| @@ -10,7 +10,7 @@ ERLC_OPTS = -Werror +warn_unused_vars +warn_shadow_vars +warn_unused_import +war | |
| 10 10 | TEST_ERLC_OPTS = -Werror +warn_unused_vars +warn_shadow_vars +warn_unused_import +warn_obsolete_guard +debug_info |
| 11 11 | |
| 12 12 | dep_supervisor3_commit = 1.1.5 |
| 13 | - dep_kafka_protocol_commit = 0.9.2 |
| 13 | + dep_kafka_protocol_commit = 1.0.0 |
| 14 14 | dep_docopt = git https://github.com/zmstone/docopt-erl.git 0.1.3 |
| 15 15 | |
| 16 16 | ESCRIPT_FILE = scripts/brod |
| @@ -1,9 +1,9 @@ | |
| 1 1 | {<<"name">>,<<"brod">>}. |
| 2 | - {<<"version">>,<<"2.5.0">>}. |
| 2 | + {<<"version">>,<<"3.0.0">>}. |
| 3 3 | {<<"requirements">>, |
| 4 4 | #{<<"kafka_protocol">> => #{<<"app">> => <<"kafka_protocol">>, |
| 5 5 | <<"optional">> => false, |
| 6 | - <<"requirement">> => <<"0.9.2">>}, |
| 6 | + <<"requirement">> => <<"1.0.0">>}, |
| 7 7 | <<"supervisor3">> => #{<<"app">> => <<"supervisor3">>, |
| 8 8 | <<"optional">> => false, |
| 9 9 | <<"requirement">> => <<"1.1.5">>}}}. |
| @@ -19,8 +19,8 @@ | |
| 19 19 | <<"docker/docker-compose-kafka-2.yml">>,<<"docker/docker-entrypoint.sh">>, |
| 20 20 | <<"docker/jaas.conf">>,<<"docker/server.jks">>, |
| 21 21 | <<"docker/server.properties">>,<<"docker/truststore.jks">>,<<"erlang.mk">>, |
| 22 | - <<"include/brod.hrl">>,<<"rebar.config">>,<<"rebar.config.script">>, |
| 23 | - <<"scripts/.gitignore">>,<<"scripts/brod">>, |
| 22 | + <<"include/brod.hrl">>,<<"include/brod_int.hrl">>,<<"rebar.config">>, |
| 23 | + <<"rebar.config.script">>,<<"scripts/.gitignore">>,<<"scripts/brod">>, |
| 24 24 | <<"scripts/cover-print-not-covered-lines.escript">>, |
| 25 25 | <<"scripts/cover-summary.escript">>,<<"scripts/generate-certificates.sh">>, |
| 26 26 | <<"scripts/line-width-check.sh">>,<<"scripts/setup-test-env.sh">>, |
| @@ -29,12 +29,11 @@ | |
| 29 29 | <<"src/brod_cli_pipe.erl">>,<<"src/brod_client.erl">>, |
| 30 30 | <<"src/brod_consumer.erl">>,<<"src/brod_consumers_sup.erl">>, |
| 31 31 | <<"src/brod_group_coordinator.erl">>,<<"src/brod_group_member.erl">>, |
| 32 | - <<"src/brod_group_subscriber.erl">>,<<"src/brod_int.hrl">>, |
| 33 | - <<"src/brod_kafka_requests.erl">>,<<"src/brod_producer.erl">>, |
| 34 | - <<"src/brod_producer_buffer.erl">>,<<"src/brod_producers_sup.erl">>, |
| 35 | - <<"src/brod_sock.erl">>,<<"src/brod_sup.erl">>, |
| 36 | - <<"src/brod_topic_subscriber.erl">>,<<"src/brod_utils.erl">>, |
| 37 | - <<"src/rebar.config">>,<<"sys.config.example">>]}. |
| 32 | + <<"src/brod_group_subscriber.erl">>,<<"src/brod_kafka_requests.erl">>, |
| 33 | + <<"src/brod_producer.erl">>,<<"src/brod_producer_buffer.erl">>, |
| 34 | + <<"src/brod_producers_sup.erl">>,<<"src/brod_sock.erl">>, |
| 35 | + <<"src/brod_sup.erl">>,<<"src/brod_topic_subscriber.erl">>, |
| 36 | + <<"src/brod_utils.erl">>,<<"src/rebar.config">>,<<"sys.config.example">>]}. |
| 38 37 | {<<"licenses">>,[<<"Apache License 2.0">>]}. |
| 39 38 | {<<"links">>,[{<<"Github">>,<<"https://github.com/klarna/brod">>}]}. |
| 40 39 | {<<"build_tools">>,[<<"make">>,<<"rebar">>,<<"rebar3">>]}. |
| @@ -1,5 +1,5 @@ | |
| 1 1 | %%% |
| 2 | - %%% Copyright (c) 2014-2016, Klarna AB |
| 2 | + %%% Copyright (c) 2014-2017, Klarna AB |
| 3 3 | %%% |
| 4 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 5 | %%% you may not use this file except in compliance with the License. |
| @@ -17,104 +17,64 @@ | |
| 17 17 | -ifndef(__BROD_HRL). |
| 18 18 | -define(__BROD_HRL, true). |
| 19 19 | |
| 20 | - -type kafka_key() :: kpro:key(). |
| 21 | - -type kafka_value() :: kpro:value(). |
| 22 | - -type kafka_kv_list() :: kpro:kv_list(). |
| 23 | - -type kafka_topic() :: kpro:topic(). |
| 24 | - -type kafka_partition() :: kpro:partition(). |
| 25 | - -type kafka_offset() :: kpro:offset(). |
| 26 | - -type kafka_error_code() :: kpro:error_code(). |
| 27 | - -type kafka_group_id() :: binary(). |
| 28 | - -type kafka_group_member_id() :: binary(). |
| 29 | - -type kafka_group_generation_id() :: non_neg_integer(). |
| 30 | - -type kafka_compression() :: no_compression | gzip | snappy. |
| 31 | - -type brod_client_id() :: atom(). |
| 20 | + %% -record(kafka_message, {...}). |
| 21 | + -include_lib("kafka_protocol/include/kpro_public.hrl"). |
| 32 22 | |
| 33 | - -record(kafka_message, |
| 34 | - { offset :: kafka_offset() |
| 35 | - , magic_byte :: integer() |
| 36 | - , attributes :: integer() |
| 37 | - , key :: binary() |
| 38 | - , value :: binary() |
| 39 | - , crc :: integer() |
| 40 | - }). |
| 41 | - |
| 42 | - -type kafka_message() :: kafka_message(). |
| 23 | + -define(BROD_DEFAULT_CLIENT_ID, brod_default_client). |
| 24 | + -define(BROD_CONSUMER_GROUP_PROTOCOL_VERSION, 0). |
| 43 25 | |
| 44 26 | -record(kafka_message_set, |
| 45 | - { topic :: kafka_topic() |
| 46 | - , partition :: kafka_partition() |
| 27 | + { topic :: brod:topic() |
| 28 | + , partition :: brod:partition() |
| 47 29 | , high_wm_offset :: integer() %% max offset of the partition |
| 48 | - , messages :: kpro:incomplete_message() | [kafka_message()] |
| 30 | + , messages :: [brod:message()] %% exposed to brod user |
| 31 | + | kpro:incomplete_message() %% this union member |
| 32 | + %% is internal only |
| 49 33 | }). |
| 50 34 | |
| 51 35 | -record(kafka_fetch_error, |
| 52 | - { topic :: kafka_topic() |
| 53 | - , partition :: kafka_partition() |
| 54 | - , error_code :: kafka_error_code() |
| 36 | + { topic :: brod:topic() |
| 37 | + , partition :: brod:partition() |
| 38 | + , error_code :: brod:error_code() |
| 55 39 | , error_desc :: binary() |
| 56 40 | }). |
| 57 41 | |
| 58 | - -define(BROD_DEFAULT_CLIENT_ID, brod_default_client). |
| 59 | - |
| 60 42 | -record(brod_call_ref, { caller :: pid() |
| 61 43 | , callee :: pid() |
| 62 44 | , ref :: reference() |
| 63 45 | }). |
| 64 46 | |
| 65 | - -type brod_call_ref() :: #brod_call_ref{}. |
| 66 | - |
| 67 | - -type brod_produce_result() :: brod_produce_req_buffered |
| 68 | - | brod_produce_req_acked. |
| 69 | - |
| 70 | - -record(brod_produce_reply, { call_ref :: brod_call_ref() |
| 71 | - , result :: brod_produce_result() |
| 47 | + -record(brod_produce_reply, { call_ref :: brod:call_ref() |
| 48 | + , result :: brod:produce_result() |
| 72 49 | }). |
| 73 50 | |
| 74 | - -type brod_produce_reply() :: #brod_produce_reply{}. |
| 75 | - |
| 76 | - -type brod_client_config() :: proplists:proplist(). |
| 77 | - -type brod_producer_config() :: proplists:proplist(). |
| 78 | - -type brod_consumer_config() :: proplists:proplist(). |
| 79 | - -type brod_group_config() :: proplists:proplist(). |
| 80 | - -type brod_offset_commit_policy() :: commit_to_kafka_v2 % default |
| 81 | - | consumer_managed. |
| 82 | - -type brod_partition_assignment_strategy() :: roundrobin |
| 83 | - | callback_implemented. |
| 84 | - |
| 85 51 | -record(kafka_group_member_metadata, |
| 86 52 | { version :: non_neg_integer() |
| 87 | - , topics :: [kafka_topic()] |
| 53 | + , topics :: [brod:topic()] |
| 88 54 | , user_data :: binary() |
| 89 55 | }). |
| 90 56 | |
| 91 | - -type kafka_group_member() :: {kafka_group_member_id(), |
| 92 | - #kafka_group_member_metadata{}}. |
| 93 | - |
| 94 | - -type brod_partition_assignment() :: |
| 95 | - {kafka_topic(), [kafka_partition()]}. |
| 96 | - |
| 97 57 | -record(brod_received_assignment, |
| 98 | - { topic :: kafka_topic() |
| 99 | - , partition :: kafka_partition() |
| 100 | - , begin_offset :: undefined | kafka_offset() |
| 58 | + { topic :: brod:topic() |
| 59 | + , partition :: brod:partition() |
| 60 | + , begin_offset :: undefined | brod:offset() |
| 101 61 | }). |
| 102 62 | |
| 103 63 | -type brod_received_assignments() :: [#brod_received_assignment{}]. |
| 104 64 | |
| 105 | - -define(BROD_CONSUMER_GROUP_PROTOCOL_VERSION, 0). |
| 106 | - |
| 107 | - -type brod_partition_fun() :: fun(( Topic :: kafka_topic() |
| 65 | + -type brod_partition_fun() :: fun(( Topic :: brod:topic() |
| 108 66 | , PartitionsCnt :: integer() |
| 109 | - , Key :: binary() |
| 110 | - , Value :: binary()) -> |
| 111 | - {ok, Partition :: integer()}). |
| 67 | + , Key :: brod:key() |
| 68 | + , Value :: brod:value()) -> |
| 69 | + {ok, Partition :: brod:partition()}). |
| 112 70 | |
| 113 71 | |
| 72 | + -record(brod_cg, { id :: brod:group_id() |
| 73 | + , protocol_type :: brod:cg_protocol_type() |
| 74 | + }). |
| 114 75 | |
| 115 76 | -endif. % include brod.hrl |
| 116 77 | |
| 117 78 | %%% Local Variables: |
| 118 79 | %%% erlang-indent-level: 2 |
| 119 80 | %%% End: |
| 120 | - |
| @@ -0,0 +1,54 @@ | |
| 1 | + %%% |
| 2 | + %%% Copyright (c) 2014-2017, Klarna AB |
| 3 | + %%% |
| 4 | + %%% Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + %%% you may not use this file except in compliance with the License. |
| 6 | + %%% You may obtain a copy of the License at |
| 7 | + %%% |
| 8 | + %%% http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + %%% |
| 10 | + %%% Unless required by applicable law or agreed to in writing, software |
| 11 | + %%% distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + %%% See the License for the specific language governing permissions and |
| 14 | + %%% limitations under the License. |
| 15 | + %%% |
| 16 | + |
| 17 | + -ifndef(__BROD_INT_HRL). |
| 18 | + -define(__BROD_INT_HRL, true). |
| 19 | + |
| 20 | + -include("brod.hrl"). |
| 21 | + -include_lib("kafka_protocol/include/kpro.hrl"). |
| 22 | + |
| 23 | + -define(undef, undefined). |
| 24 | + |
| 25 | + -define(OFFSET_EARLIEST, earliest). |
| 26 | + -define(OFFSET_LATEST, latest). |
| 27 | + -define(IS_SPECIAL_OFFSET(O), (O =:= ?OFFSET_EARLIEST orelse |
| 28 | + O =:= ?OFFSET_LATEST orelse |
| 29 | + O =:= -2 orelse |
| 30 | + O =:= -1)). |
| 31 | + |
| 32 | + -record(socket, { pid :: pid() |
| 33 | + , host :: string() |
| 34 | + , port :: integer() |
| 35 | + , node_id :: integer() |
| 36 | + }). |
| 37 | + |
| 38 | + -type sasl_opt() :: {plain, User :: string() | binary(), |
| 39 | + Pass :: string() | binary() | |
| 40 | + fun(() -> string() | binary())} |
| 41 | + | {plain, File :: file:name_all()} |
| 42 | + | {callback, module(), term()} |
| 43 | + | ?undef. |
| 44 | + |
| 45 | + %% Is kafka error code |
| 46 | + -define(IS_ERROR(EC), kpro_error_code:is_error(EC)). |
| 47 | + |
| 48 | + -endif. % include brod_int.hrl |
| 49 | + |
| 50 | + %%%_* Emacs ==================================================================== |
| 51 | + %%% Local Variables: |
| 52 | + %%% allout-layout: t |
| 53 | + %%% erlang-indent-level: 2 |
| 54 | + %%% End: |
| @@ -1,4 +1,4 @@ | |
| 1 1 | {deps, [ {supervisor3, "1.1.5"} |
| 2 | - , {kafka_protocol, "0.9.2"} |
| 2 | + , {kafka_protocol, "1.0.0"} |
| 3 3 | ]}. |
| 4 4 | {erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import,warn_obsolete_guard,debug_info]}. |
Loading more files…