Packages
brod
4.3.0
4.5.7
4.5.6
4.5.5
4.5.4
4.5.3
4.5.2
4.5.1
4.5.0
4.4.7
4.4.6
4.4.5
4.4.4
4.4.3
4.4.2
4.4.1
4.4.0
4.3.3
4.3.2
4.3.1
4.3.0
4.2.0
4.1.1
4.1.0
4.0.0
3.19.1
3.19.0
3.18.0
3.17.1
3.17.0
3.16.5
3.16.4
3.16.3
3.16.2
3.16.1
3.16.0
3.15.6
3.15.5
3.15.4
3.15.3
3.15.1
3.15.0
3.14.0
3.13.0
3.12.0
3.11.0
3.10.0
3.9.5
3.9.3
3.9.2
3.9.1
3.9.0
3.8.1
3.8.0
3.7.11
3.7.10
3.7.9
3.7.8
3.7.7
3.7.6
3.7.5
3.7.4
3.7.3
3.7.2
3.7.1
3.7.0
3.6.2
3.6.1
3.6.0
3.5.2
3.5.1
3.5.0
3.4.0
3.3.5
3.3.4
3.3.3
3.3.2
3.3.1
3.3.0
3.2.0
3.0.0
2.5.0
2.4.1
2.4.0
2.3.7
2.3.6
2.3.5
2.3.4
2.3.3
2.3.1
2.2.16
2.2.15
2.2.14
2.2.12
2.2.11
2.2.10
2.2.9
2.2.8
2.2.7
2.2.6
2.2.5
2.2.4
2.2.3
2.2.2
2.2.1
2.2.0
2.1.12
2.1.11
2.1.10
2.1.8
2.1.7
2.1.4
2.1.2
2.0.0
Apache Kafka Erlang client library
Current section
113 Versions
Jump to
Current section
113 Versions
Compare versions
8
files changed
+5
additions
-1845
deletions
| @@ -35,13 +35,6 @@ hex-publish: clean | |
| 35 35 | ## tests that require kafka running at localhost |
| 36 36 | INTEGRATION_CTS = brod_cg_commits brod_client brod_compression brod_consumer brod_producer brod_group_subscriber brod_topic_subscriber brod |
| 37 37 | |
| 38 | - ## build escript and a release, and copy escript to release bin dir |
| 39 | - brod-cli: |
| 40 | - @rebar3 as brod_cli do compile,escriptize,release |
| 41 | - @cp _build/brod_cli/bin/brod_cli _build/brod_cli/rel/brod/bin/ |
| 42 | - @cp scripts/brod _build/brod_cli/rel/brod/bin/ |
| 43 | - @cp scripts/brod.escript _build/brod_cli/rel/brod/bin/ |
| 44 | - |
| 45 38 | cover: |
| 46 39 | @rebar3 cover -v |
| @@ -490,102 +490,3 @@ brod:delete_topics(Hosts, [Topic], Timeout). | |
| 490 490 | ``` |
| 491 491 | |
| 492 492 | Caution the above delete_topics can fail if you do not have `delete.topic.enable` set to true in your kafka config |
| 493 | - |
| 494 | - ## brod-cli: A command line tool to interact with Kafka |
| 495 | - |
| 496 | - This will build a self-contained binary with brod application |
| 497 | - |
| 498 | - ```bash |
| 499 | - make brod-cli |
| 500 | - _build/brod_cli/rel/brod/bin/brod -h |
| 501 | - ``` |
| 502 | - |
| 503 | - Disclaimer: This script is NOT designed for use cases where fault-tolerance is a hard requirement. |
| 504 | - As it may crash when e.g. kafka cluster is temporarily unreachable, |
| 505 | - or (for fetch command) when the partition leader migrates to another broker in the cluster. |
| 506 | - |
| 507 | - ### brod-cli examples (with `alias brod=_build/brod_cli/rel/brod/bin/brod`): |
| 508 | - |
| 509 | - #### Fetch and print metadata |
| 510 | - |
| 511 | - ```sh |
| 512 | - brod meta -b localhost |
| 513 | - ``` |
| 514 | - |
| 515 | - #### Produce a Message |
| 516 | - |
| 517 | - ```sh |
| 518 | - brod send -b localhost -t test-topic -p 0 -k "key" -v "value" |
| 519 | - |
| 520 | - ``` |
| 521 | - |
| 522 | - #### Fetch a Message |
| 523 | - |
| 524 | - ```sh |
| 525 | - brod fetch -b localhost -t test-topic -p 0 --fmt 'io:format("offset=~p, ts=~p, key=~s, value=~s\n", [Offset, Ts, Key, Value])' |
| 526 | - ``` |
| 527 | - |
| 528 | - Bound variables to be used in `--fmt` expression: |
| 529 | - |
| 530 | - - `Offset`: Message offset |
| 531 | - - `Key`: Kafka key |
| 532 | - - `Value`: Kafka Value |
| 533 | - - `TsType`: Timestamp type either `create` or `append` |
| 534 | - - `Ts`: Timestamp, `-1` as no value |
| 535 | - |
| 536 | - #### Stream Messages to Kafka |
| 537 | - |
| 538 | - Send `README.md` to kafka one line per kafka message |
| 539 | - |
| 540 | - ```sh |
| 541 | - brod pipe -b localhost:9092 -t test-topic -p 0 -s @./README.md |
| 542 | - ``` |
| 543 | - |
| 544 | - #### Resolve Offset |
| 545 | - |
| 546 | - ```sh |
| 547 | - brod offset -b localhost:9092 -t test-topic -p 0 |
| 548 | - ``` |
| 549 | - |
| 550 | - #### List or Describe Groups |
| 551 | - |
| 552 | - ```sh |
| 553 | - # List all groups |
| 554 | - brod groups -b localhost:9092 |
| 555 | - |
| 556 | - # Describe groups |
| 557 | - brod groups -b localhost:9092 --ids group-1,group-2 |
| 558 | - ``` |
| 559 | - |
| 560 | - #### Display Committed Offsets |
| 561 | - |
| 562 | - ```sh |
| 563 | - # all topics |
| 564 | - brod commits -b localhost:9092 --id the-group-id --describe |
| 565 | - |
| 566 | - # a specific topic |
| 567 | - brod commits -b localhost:9092 --id the-group-id --describe --topic topic-name |
| 568 | - ``` |
| 569 | - |
| 570 | - #### Commit Offsets |
| 571 | - |
| 572 | - NOTE: This feature is designed for force overwriting commits, not for regular use of offset commit. |
| 573 | - |
| 574 | - ```sh |
| 575 | - # Commit 'latest' offsets of all partitions with 2 days retention |
| 576 | - brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 2d |
| 577 | - |
| 578 | - # Commit offset=100 for partition 0 and 200 for partition 1 |
| 579 | - brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets "0:100,1:200" |
| 580 | - |
| 581 | - # Use --retention 0 to delete commits (may linger in kafka before cleaner does its job) |
| 582 | - brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 0 |
| 583 | - |
| 584 | - # Try join an active consumer group using 'range' protocol and steal one partition assignment then commit offset=10000 |
| 585 | - brod commits -b localhost:9092 -i the-group-id -t topic-name -o "0:10000" --protocol range |
| 586 | - ``` |
| 587 | - |
| 588 | - ### TODOs |
| 589 | - |
| 590 | - - Support scram-sasl in brod-cli |
| 591 | - - Transactional produce APIs |
| @@ -5,8 +5,7 @@ | |
| 5 5 | [<<"LICENSE">>,<<"Makefile">>,<<"NOTICE">>,<<"README.md">>,<<"include">>, |
| 6 6 | <<"include/brod.hrl">>,<<"include/brod_int.hrl">>,<<"rebar.config">>, |
| 7 7 | <<"rebar.config.script">>,<<"src">>,<<"src/brod.app.src">>, |
| 8 | - <<"src/brod.erl">>,<<"src/brod_cg_commits.erl">>,<<"src/brod_cli.erl">>, |
| 9 | - <<"src/brod_cli_pipe.erl">>,<<"src/brod_client.erl">>, |
| 8 | + <<"src/brod.erl">>,<<"src/brod_cg_commits.erl">>,<<"src/brod_client.erl">>, |
| 10 9 | <<"src/brod_consumer.erl">>,<<"src/brod_consumers_sup.erl">>, |
| 11 10 | <<"src/brod_group_coordinator.erl">>,<<"src/brod_group_member.erl">>, |
| 12 11 | <<"src/brod_group_subscriber.erl">>,<<"src/brod_group_subscriber_v2.erl">>, |
| @@ -25,4 +24,4 @@ | |
| 25 24 | [{<<"app">>,<<"kafka_protocol">>}, |
| 26 25 | {<<"optional">>,false}, |
| 27 26 | {<<"requirement">>,<<"4.1.9">>}]}]}. |
| 28 | - {<<"version">>,<<"4.2.0">>}. |
| 27 | + {<<"version">>,<<"4.3.0">>}. |
| @@ -1,26 +1,11 @@ | |
| 1 1 | {deps, [{kafka_protocol, "4.1.9"}]}. |
| 2 2 | {project_plugins, [{rebar3_lint, "~> 3.2.5"}]}. |
| 3 | - {edoc_opts, [{preprocess, true}, {macros, [{build_brod_cli, true}]}]}. |
| 3 | + {edoc_opts, [{preprocess, true}]}. |
| 4 4 | {erl_opts, [warnings_as_errors, warn_unused_vars,warn_shadow_vars,warn_obsolete_guard,debug_info]}. |
| 5 5 | {xref_checks, [undefined_function_calls, undefined_functions, |
| 6 6 | locals_not_used, deprecated_function_calls, |
| 7 7 | deprecated_functions]}. |
| 8 8 | {profiles, [ |
| 9 | - {brod_cli, [ |
| 10 | - {deps, [ {docopt, {git, "https://github.com/zmstone/docopt-erl.git", {tag, "0.1.3"}}} |
| 11 | - , {jsone, "1.7.0"} |
| 12 | - , {snappyer, "1.2.9"} |
| 13 | - ]}, |
| 14 | - {erl_opts, [warnings_as_errors, {d, build_brod_cli}]}, |
| 15 | - {escript_name, brod_cli}, |
| 16 | - {relx, [{release, {brod, "i"}, % release the interactive shell as brod-i |
| 17 | - [brod, jsone, docopt]}, |
| 18 | - {include_erts, true}, |
| 19 | - {overlay, [{copy, "scripts/brod", "bin/"}, |
| 20 | - {copy, "{{lib_dirs}}/crc32cer/priv/crc32cer*.so", "bin/"}, |
| 21 | - {copy, "{{lib_dirs}}/snappyer/priv/snappyer.so", "bin/"} |
| 22 | - ]} |
| 23 | - ]}]}, |
| 24 9 | {test, [ |
| 25 10 | {deps, [ {docopt, {git, "https://github.com/zmstone/docopt-erl.git", {tag, "0.1.3"}}} |
| 26 11 | , {hut, "1.3.0"} |
| @@ -1,6 +1,6 @@ | |
| 1 1 | {application,brod, |
| 2 2 | [{description,"Apache Kafka Erlang client library"}, |
| 3 | - {vsn,"4.2.0"}, |
| 3 | + {vsn,"4.3.0"}, |
| 4 4 | {registered,[]}, |
| 5 5 | {applications,[kernel,stdlib,kafka_protocol]}, |
| 6 6 | {env,[]}, |
Loading more files…