Packages
brod
3.16.4
4.5.7
4.5.6
4.5.5
4.5.4
4.5.3
4.5.2
4.5.1
4.5.0
4.4.7
4.4.6
4.4.5
4.4.4
4.4.3
4.4.2
4.4.1
4.4.0
4.3.3
4.3.2
4.3.1
4.3.0
4.2.0
4.1.1
4.1.0
4.0.0
3.19.1
3.19.0
3.18.0
3.17.1
3.17.0
3.16.5
3.16.4
3.16.3
3.16.2
3.16.1
3.16.0
3.15.6
3.15.5
3.15.4
3.15.3
3.15.1
3.15.0
3.14.0
3.13.0
3.12.0
3.11.0
3.10.0
3.9.5
3.9.3
3.9.2
3.9.1
3.9.0
3.8.1
3.8.0
3.7.11
3.7.10
3.7.9
3.7.8
3.7.7
3.7.6
3.7.5
3.7.4
3.7.3
3.7.2
3.7.1
3.7.0
3.6.2
3.6.1
3.6.0
3.5.2
3.5.1
3.5.0
3.4.0
3.3.5
3.3.4
3.3.3
3.3.2
3.3.1
3.3.0
3.2.0
3.0.0
2.5.0
2.4.1
2.4.0
2.3.7
2.3.6
2.3.5
2.3.4
2.3.3
2.3.1
2.2.16
2.2.15
2.2.14
2.2.12
2.2.11
2.2.10
2.2.9
2.2.8
2.2.7
2.2.6
2.2.5
2.2.4
2.2.3
2.2.2
2.2.1
2.2.0
2.1.12
2.1.11
2.1.10
2.1.8
2.1.7
2.1.4
2.1.2
2.0.0
Apache Kafka Erlang client library
Current section
113 Versions
Jump to
Current section
113 Versions
Compare versions
10
files changed
+69
additions
-56
deletions
| @@ -73,7 +73,7 @@ SubscriberCallbackFun = fun(Partition, Msg, ShellPid = CallbackState) -> ShellPi | |
| 73 73 | Receive = fun() -> receive Msg -> Msg after 1000 -> timeout end end, |
| 74 74 | brod_topic_subscriber:start_link(client1, Topic, Partitions=[Partition], |
| 75 75 | _ConsumerConfig=[{begin_offset, FirstOffset}], |
| 76 | - _CommittdOffsets=[], message, SubscriberCallbackFun, |
| 76 | + _CommittedOffsets=[], message, SubscriberCallbackFun, |
| 77 77 | _CallbackState=self()), |
| 78 78 | AckCb = fun(Partition, BaseOffset) -> io:format(user, "\nProduced to partition ~p at base-offset ~p\n", [Partition, BaseOffset]) end, |
| 79 79 | ok = brod:produce_cb(client1, Topic, Partition, <<>>, [{<<"key3">>, <<"value3">>}], AckCb). |
| @@ -188,7 +188,7 @@ The `Value` arg in these APIs can be: | |
| 188 188 | - `[{K, V} | {T, K, V}]`: A batch, where `V` could be a nested list of such representation. |
| 189 189 | - `[#{key => K, value => V, ts => T, headers => [{_, _}]}]`: A batch. |
| 190 190 | |
| 191 | - When `Value` is a batch, the `Key` argument is only used as partitioner input. |
| 191 | + When `Value` is a batch, the `Key` argument is only used as partitioner input and all messages are written on the same partition. |
| 192 192 | All messages are unified into a batch format of below spec: |
| 193 193 | `[#{key => K, value => V, ts => T, headers => [{_, _}]}]`. |
| 194 194 | `ts` field is dropped for kafka prior to version `0.10` (produce API version 0, magic version 0) |
| @@ -229,10 +229,7 @@ Topic = <<"brod-test-topic-1">>, | |
| 229 229 | ```erlang |
| 230 230 | Client = brod_client_1, |
| 231 231 | Topic = <<"brod-test-topic-1">>, |
| 232 | - PartitionFun = fun(_Topic, PartitionsCount, _Key, _Value) -> |
| 233 | - {ok, crypto:rand_uniform(0, PartitionsCount)} |
| 234 | - end, |
| 235 | - ok = brod:produce_sync(Client, Topic, PartitionFun, Key, Value). |
| 232 | + ok = brod:produce_sync(Client, Topic, random, Key, Value). |
| 236 233 | ``` |
| 237 234 | |
| 238 235 | ## Produce a Batch |
| @@ -313,9 +310,10 @@ ack individual offsets). | |
| 313 310 | ## Topic subscriber (`brod_topic_subscriber`) |
| 314 311 |  |
| 315 312 | |
| 316 | - A topic subscriber provides the easiest way to receive and process |
| 317 | - messages from ALL partitions of a given topic. See `brod_demo_cg_collector` |
| 318 | - and `brod_demo_topic_subscriber` for example. |
| 313 | + A topic subscriber provides the easiest way to receive and process messages from |
| 314 | + ALL partitions of a given topic. See |
| 315 | + [brod_demo_cg_collector](test/brod_demo_cg_collector.erl) and |
| 316 | + [brod_demo_topic_subscriber](test/brod_demo_topic_subscriber.erl) for example. |
| 319 317 | |
| 320 318 | Users may choose to implement the `brod_topic_subscriber` behaviour callbacks |
| 321 319 | in a module, or simply provide an anonymous callback function to have the |
| @@ -324,10 +322,11 @@ individual messages processed. | |
| 324 322 | ## Group subscriber (`brod_group_subscriber`) |
| 325 323 |  |
| 326 324 | |
| 327 | - Similar to topic subscriber, the `brod_group_subscriber` behaviour callbacks |
| 328 | - are to be implemented to process individual messages. See |
| 329 | - `brod_demo_group_subscriber_koc` and `brod_demo_group_subscriber_loc` |
| 330 | - for example. |
| 325 | + Similar to topic subscriber, the `brod_group_subscriber` behaviour callbacks are |
| 326 | + to be implemented to process individual messages. See |
| 327 | + [brod_demo_group_subscriber_koc](test/brod_demo_group_subscriber_koc.erl) and |
| 328 | + [brod_demo_group_subscriber_loc](test/brod_demo_group_subscriber_loc.erl) for |
| 329 | + example. |
| 331 330 | |
| 332 331 | A group subscriber is started by giving a set of topics, some |
| 333 332 | (maybe none, or maybe all) of the partitions in the topic set will be |
| @@ -418,12 +417,11 @@ Partition = 0. | |
| 418 417 | Timeout = 1000. |
| 419 418 | TopicConfigs = [ |
| 420 419 | #{ |
| 421 | - config_entries => [ #{ config_name => <<"cleanup.policy">> |
| 422 | - , config_value => "compact"}], |
| 420 | + configs => [ #{name => <<"cleanup.policy">>, value => "compact"}], |
| 423 421 | num_partitions => 1, |
| 424 | - replica_assignment => [], |
| 422 | + assignments => [], |
| 425 423 | replication_factor => 1, |
| 426 | - topic => Topic |
| 424 | + name => Topic |
| 427 425 | } |
| 428 426 | ]. |
| 429 427 | brod:get_metadata(Hosts). |
| @@ -2,10 +2,10 @@ | |
| 2 2 | {<<"build_tools">>,[<<"rebar3">>]}. |
| 3 3 | {<<"description">>,<<"Apache Kafka Erlang client library">>}. |
| 4 4 | {<<"files">>, |
| 5 | - [<<"LICENSE">>,<<"Makefile">>,<<"NOTICE">>,<<"README.md">>, |
| 5 | + [<<"LICENSE">>,<<"Makefile">>,<<"NOTICE">>,<<"README.md">>,<<"include">>, |
| 6 6 | <<"include/brod.hrl">>,<<"include/brod_int.hrl">>,<<"rebar.config">>, |
| 7 | - <<"rebar.config.script">>,<<"src/brod.app.src">>,<<"src/brod.erl">>, |
| 8 | - <<"src/brod_cg_commits.erl">>,<<"src/brod_cli.erl">>, |
| 7 | + <<"rebar.config.script">>,<<"src">>,<<"src/brod.app.src">>, |
| 8 | + <<"src/brod.erl">>,<<"src/brod_cg_commits.erl">>,<<"src/brod_cli.erl">>, |
| 9 9 | <<"src/brod_cli_pipe.erl">>,<<"src/brod_client.erl">>, |
| 10 10 | <<"src/brod_consumer.erl">>,<<"src/brod_consumers_sup.erl">>, |
| 11 11 | <<"src/brod_group_coordinator.erl">>,<<"src/brod_group_member.erl">>, |
| @@ -22,7 +22,7 @@ | |
| 22 22 | [{<<"kafka_protocol">>, |
| 23 23 | [{<<"app">>,<<"kafka_protocol">>}, |
| 24 24 | {<<"optional">>,false}, |
| 25 | - {<<"requirement">>,<<"4.0.3">>}]}, |
| 25 | + {<<"requirement">>,<<"4.1.0">>}]}, |
| 26 26 | {<<"snappyer">>, |
| 27 27 | [{<<"app">>,<<"snappyer">>}, |
| 28 28 | {<<"optional">>,false}, |
| @@ -31,4 +31,4 @@ | |
| 31 31 | [{<<"app">>,<<"supervisor3">>}, |
| 32 32 | {<<"optional">>,false}, |
| 33 33 | {<<"requirement">>,<<"1.1.11">>}]}]}. |
| 34 | - {<<"version">>,<<"3.16.3">>}. |
| 34 | + {<<"version">>,<<"3.16.4">>}. |
| @@ -60,7 +60,7 @@ | |
| 60 60 | -record(brod_received_assignment, |
| 61 61 | { topic :: brod:topic() |
| 62 62 | , partition :: brod:partition() |
| 63 | - , begin_offset :: undefined | brod:offset() |
| 63 | + , begin_offset :: undefined | brod:offset() | {begin_offset, brod:offset_time()} |
| 64 64 | }). |
| 65 65 | |
| 66 66 | -type brod_received_assignments() :: [#brod_received_assignment{}]. |
| @@ -1,5 +1,5 @@ | |
| 1 1 | {deps, [ {supervisor3, "1.1.11"} |
| 2 | - , {kafka_protocol, "4.0.3"} |
| 2 | + , {kafka_protocol, "4.1.0"} |
| 3 3 | , {snappyer, "1.2.8"} |
| 4 4 | ]}. |
| 5 5 | {project_plugins, [{rebar3_lint, "~> 1.0.2"}]}. |
| @@ -1,6 +1,6 @@ | |
| 1 1 | {application,brod, |
| 2 2 | [{description,"Apache Kafka Erlang client library"}, |
| 3 | - {vsn,"3.16.3"}, |
| 3 | + {vsn,"3.16.4"}, |
| 4 4 | {registered,[]}, |
| 5 5 | {applications,[kernel,stdlib,kafka_protocol,supervisor3, |
| 6 6 | snappyer]}, |
Loading more files…