Packages
brod
3.7.1
4.5.7
4.5.6
4.5.5
4.5.4
4.5.3
4.5.2
4.5.1
4.5.0
4.4.7
4.4.6
4.4.5
4.4.4
4.4.3
4.4.2
4.4.1
4.4.0
4.3.3
4.3.2
4.3.1
4.3.0
4.2.0
4.1.1
4.1.0
4.0.0
3.19.1
3.19.0
3.18.0
3.17.1
3.17.0
3.16.5
3.16.4
3.16.3
3.16.2
3.16.1
3.16.0
3.15.6
3.15.5
3.15.4
3.15.3
3.15.1
3.15.0
3.14.0
3.13.0
3.12.0
3.11.0
3.10.0
3.9.5
3.9.3
3.9.2
3.9.1
3.9.0
3.8.1
3.8.0
3.7.11
3.7.10
3.7.9
3.7.8
3.7.7
3.7.6
3.7.5
3.7.4
3.7.3
3.7.2
3.7.1
3.7.0
3.6.2
3.6.1
3.6.0
3.5.2
3.5.1
3.5.0
3.4.0
3.3.5
3.3.4
3.3.3
3.3.2
3.3.1
3.3.0
3.2.0
3.0.0
2.5.0
2.4.1
2.4.0
2.3.7
2.3.6
2.3.5
2.3.4
2.3.3
2.3.1
2.2.16
2.2.15
2.2.14
2.2.12
2.2.11
2.2.10
2.2.9
2.2.8
2.2.7
2.2.6
2.2.5
2.2.4
2.2.3
2.2.2
2.2.1
2.2.0
2.1.12
2.1.11
2.1.10
2.1.8
2.1.7
2.1.4
2.1.2
2.0.0
Apache Kafka Erlang client library
Current section
113 Versions
Jump to
Current section
113 Versions
Compare versions
14
files changed
+211
additions
-148
deletions
| @@ -1,14 +1,14 @@ | |
| 1 1 | KAFKA_VERSION ?= 1.1 |
| 2 2 | PROJECT = brod |
| 3 3 | PROJECT_DESCRIPTION = Kafka client library in Erlang |
| 4 | - PROJECT_VERSION = 3.7.0 |
| 4 | + PROJECT_VERSION = 3.7.1 |
| 5 5 | |
| 6 6 | DEPS = supervisor3 kafka_protocol |
| 7 7 | |
| 8 8 | ERLC_OPTS = -Werror +warn_unused_vars +warn_shadow_vars +warn_unused_import +warn_obsolete_guard +debug_info -Dbuild_brod_cli |
| 9 9 | |
| 10 10 | dep_supervisor3_commit = 1.1.7 |
| 11 | - dep_kafka_protocol_commit = 2.1.2 |
| 11 | + dep_kafka_protocol_commit = 2.2.2 |
| 12 12 | dep_kafka_protocol = git https://github.com/klarna/kafka_protocol.git $(dep_kafka_protocol_commit) |
| 13 13 | |
| 14 14 | EDOC_OPTS = preprocess, {macros, [{build_brod_cli, true}]} |
| @@ -1,9 +1,9 @@ | |
| 1 1 | {<<"name">>,<<"brod">>}. |
| 2 | - {<<"version">>,<<"3.7.0">>}. |
| 2 | + {<<"version">>,<<"3.7.1">>}. |
| 3 3 | {<<"requirements">>, |
| 4 4 | #{<<"kafka_protocol">> => |
| 5 5 | #{<<"app">> => <<"kafka_protocol">>,<<"optional">> => false, |
| 6 | - <<"requirement">> => <<"2.1.2">>}, |
| 6 | + <<"requirement">> => <<"2.2.2">>}, |
| 7 7 | <<"supervisor3">> => |
| 8 8 | #{<<"app">> => <<"supervisor3">>,<<"optional">> => false, |
| 9 9 | <<"requirement">> => <<"1.1.7">>}}}. |
| @@ -1,5 +1,5 @@ | |
| 1 1 | {deps, [ {supervisor3, "1.1.7"} |
| 2 | - , {kafka_protocol, "2.1.2"} |
| 2 | + , {kafka_protocol, "2.2.2"} |
| 3 3 | ]}. |
| 4 4 | {edoc_opts, [{preprocess, true}, {macros, [{build_brod_cli, true}]}]}. |
| 5 5 | {erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import,warn_obsolete_guard,debug_info]}. |
| @@ -1,6 +1,6 @@ | |
| 1 1 | {application,brod, |
| 2 2 | [{description,"Apache Kafka Erlang client library"}, |
| 3 | - {vsn,"3.7.0"}, |
| 3 | + {vsn,"3.7.1"}, |
| 4 4 | {registered,[]}, |
| 5 5 | {applications,[kernel,stdlib,kafka_protocol,supervisor3]}, |
| 6 6 | {env,[]}, |
| @@ -141,6 +141,7 @@ | |
| 141 141 | , partition/0 |
| 142 142 | , partition_assignment/0 |
| 143 143 | , partition_fun/0 |
| 144 | + , partitioner/0 |
| 144 145 | , portnum/0 |
| 145 146 | , produce_ack_cb/0 |
| 146 147 | , producer_config/0 |
| @@ -187,8 +188,9 @@ | |
| 187 188 | %% producers |
| 188 189 | -type produce_reply() :: #brod_produce_reply{}. |
| 189 190 | -type producer_config() :: brod_producer:config(). |
| 190 | - -type partition_fun() :: fun((topic(), partition(), key(), value()) -> |
| 191 | + -type partition_fun() :: fun((topic(), pos_integer(), key(), value()) -> |
| 191 192 | {ok, partition()}). |
| 193 | + -type partitioner() :: partition_fun() | random | hash. |
| 192 194 | -type produce_ack_cb() :: fun((partition(), offset()) -> _). |
| 193 195 | -type compression() :: no_compression | gzip | snappy. |
| 194 196 | -type call_ref() :: #brod_call_ref{}. |
| @@ -433,20 +435,21 @@ produce(ProducerPid, Key, Value) -> | |
| 433 435 | %% so the caller can used it to expect (match) a |
| 434 436 | %% `#brod_produce_reply{result = brod_produce_req_acked}' |
| 435 437 | %% message after the produce request has been acked by kafka. |
| 436 | - -spec produce(client(), topic(), partition() | partition_fun(), |
| 438 | + -spec produce(client(), topic(), partition() | partitioner(), |
| 437 439 | key(), value()) -> {ok, call_ref()} | {error, any()}. |
| 438 | - produce(Client, Topic, PartFun, Key, Value) when is_function(PartFun) -> |
| 440 | + produce(Client, Topic, Partition, Key, Value) when is_integer(Partition) -> |
| 441 | + case get_producer(Client, Topic, Partition) of |
| 442 | + {ok, Pid} -> produce(Pid, Key, Value); |
| 443 | + {error, Reason} -> {error, Reason} |
| 444 | + end; |
| 445 | + produce(Client, Topic, Partitioner, Key, Value) -> |
| 446 | + PartFun = brod_utils:make_part_fun(Partitioner), |
| 439 447 | case brod_client:get_partitions_count(Client, Topic) of |
| 440 448 | {ok, PartitionsCnt} -> |
| 441 449 | {ok, Partition} = PartFun(Topic, PartitionsCnt, Key, Value), |
| 442 450 | produce(Client, Topic, Partition, Key, Value); |
| 443 451 | {error, Reason} -> |
| 444 452 | {error, Reason} |
| 445 | - end; |
| 446 | - produce(Client, Topic, Partition, Key, Value) when is_integer(Partition) -> |
| 447 | - case get_producer(Client, Topic, Partition) of |
| 448 | - {ok, Pid} -> produce(Pid, Key, Value); |
| 449 | - {error, Reason} -> {error, Reason} |
| 450 453 | end. |
| 451 454 | |
| 452 455 | %% @doc Same as `produce/3' only the ack is not delivered as a message, |
| @@ -461,11 +464,16 @@ produce_cb(ProducerPid, Key, Value, AckCb) -> | |
| 461 464 | %% instead, the callback is evaluated by producer worker when ack is received |
| 462 465 | %% from kafka. Return the partition to caller as `{ok, Partition}' for caller |
| 463 466 | %% to correlate the callback when the 3rd arg is not a partition number. |
| 464 | - -spec produce_cb(client(), topic(), partition() | partition_fun(), |
| 467 | + -spec produce_cb(client(), topic(), partition() | partitioner(), |
| 465 468 | key(), value(), produce_ack_cb()) -> |
| 466 469 | ok | {ok, partition()} | {error, any()}. |
| 467 | - produce_cb(Client, Topic, PartFun, Key, Value, AckCb) |
| 468 | - when is_function(PartFun) -> |
| 470 | + produce_cb(Client, Topic, Part, Key, Value, AckCb) when is_integer(Part) -> |
| 471 | + case get_producer(Client, Topic, Part) of |
| 472 | + {ok, Pid} -> produce_cb(Pid, Key, Value, AckCb); |
| 473 | + {error, Reason} -> {error, Reason} |
| 474 | + end; |
| 475 | + produce_cb(Client, Topic, Partitioner, Key, Value, AckCb) -> |
| 476 | + PartFun = brod_utils:make_part_fun(Partitioner), |
| 469 477 | case brod_client:get_partitions_count(Client, Topic) of |
| 470 478 | {ok, PartitionsCnt} -> |
| 471 479 | {ok, Partition} = PartFun(Topic, PartitionsCnt, Key, Value), |
| @@ -475,11 +483,6 @@ produce_cb(Client, Topic, PartFun, Key, Value, AckCb) | |
| 475 483 | end; |
| 476 484 | {error, Reason} -> |
| 477 485 | {error, Reason} |
| 478 | - end; |
| 479 | - produce_cb(Client, Topic, Partition, Key, Value, AckCb) -> |
| 480 | - case get_producer(Client, Topic, Partition) of |
| 481 | - {ok, Pid} -> produce_cb(Pid, Key, Value, AckCb); |
| 482 | - {error, Reason} -> {error, Reason} |
| 483 486 | end. |
| 484 487 | |
| 485 488 | %% @doc Send the message to partition worker without any ack. |
| @@ -492,9 +495,15 @@ produce_no_ack(ProducerPid, Key, Value) -> | |
| 492 495 | %% @doc Find the partition worker and send message without any ack. |
| 493 496 | %% NOTE: This call has no back-pressure to the caller, |
| 494 497 | %% excessive usage may cause beam to run out of memory. |
| 495 | - -spec produce_no_ack(client(), topic(), partition() | partition_fun(), |
| 498 | + -spec produce_no_ack(client(), topic(), partition() | partitioner(), |
| 496 499 | key(), value()) -> ok | {error, any()}. |
| 497 | - produce_no_ack(Client, Topic, PartFun, Key, Value) when is_function(PartFun) -> |
| 500 | + produce_no_ack(Client, Topic, Part, Key, Value) when is_integer(Part) -> |
| 501 | + case get_producer(Client, Topic, Part) of |
| 502 | + {ok, Pid} -> produce_no_ack(Pid, Key, Value); |
| 503 | + {error, Reason} -> {error, Reason} |
| 504 | + end; |
| 505 | + produce_no_ack(Client, Topic, Partitioner, Key, Value) -> |
| 506 | + PartFun = brod_utils:make_part_fun(Partitioner), |
| 498 507 | case brod_client:get_partitions_count(Client, Topic) of |
| 499 508 | {ok, PartitionsCnt} -> |
| 500 509 | {ok, Partition} = PartFun(Topic, PartitionsCnt, Key, Value), |
| @@ -502,11 +511,6 @@ produce_no_ack(Client, Topic, PartFun, Key, Value) when is_function(PartFun) -> | |
| 502 511 | {error, _Reason} -> |
| 503 512 | %% error ignored |
| 504 513 | ok |
| 505 | - end; |
| 506 | - produce_no_ack(Client, Topic, Partition, Key, Value) -> |
| 507 | - case get_producer(Client, Topic, Partition) of |
| 508 | - {ok, Pid} -> produce_no_ack(Pid, Key, Value); |
| 509 | - {error, Reason} -> {error, Reason} |
| 510 514 | end. |
| 511 515 | |
| 512 516 | %% @doc Same as `produce/5' only the ack is not d |
| @@ -534,7 +538,7 @@ produce_sync(Pid, Key, Value) -> | |
| 534 538 | %% This function will not return until a response is received from kafka, |
| 535 539 | %% however if producer is started with required_acks set to 0, this function |
| 536 540 | %% will return once the messages are buffered in the producer process. |
| 537 | - -spec produce_sync(client(), topic(), partition() | partition_fun(), |
| 541 | + -spec produce_sync(client(), topic(), partition() | partitioner(), |
| 538 542 | key(), value()) -> ok | {error, any()}. |
| 539 543 | produce_sync(Client, Topic, Partition, Key, Value) -> |
| 540 544 | case produce_sync_offset(Client, Topic, Partition, Key, Value) of |
| @@ -545,7 +549,7 @@ produce_sync(Client, Topic, Partition, Key, Value) -> | |
| 545 549 | %% @doc Version of produce_sync/5 that returns the offset assigned by Kafka |
| 546 550 | %% If producer is started with required_acks set to 0, the offset will be |
| 547 551 | %% `?BROD_PRODUCE_UNKNOWN_OFFSET'. |
| 548 | - -spec produce_sync_offset(client(), topic(), partition() | partition_fun(), |
| 552 | + -spec produce_sync_offset(client(), topic(), partition() | partitioner(), |
| 549 553 | key(), value()) -> {ok, offset()} | {error, any()}. |
| 550 554 | produce_sync_offset(Client, Topic, Partition, Key, Value) -> |
| 551 555 | case produce(Client, Topic, Partition, Key, Value) of |
Loading more files…