Current section
Files
Jump to
Current section
Files
livebooks/03_codec_comparison.livemd
<!-- livebook:{"persist_outputs":true} -->
# Codec Comparison
```elixir
# Use this install to work with the source code
# Mix.install(
# [
# {:ex_codecs, path: Path.join(__DIR__, "..")},
# {:rustler, "~> 0.36"},
# {:jason, "~> 1.4"},
# {:kino, "~> 0.14"},
# {:kino_vega_lite, "~> 0.1.13"}
# ],
# config: [rustler_precompiled: [force_build: [ex_codecs: true]]]
# )
Mix.install( [
{:ex_codecs, "~> 0.2.3"},
{:jason, "~> 1.4"},
{:kino, "~> 0.14"},
{:kino_vega_lite, "~> 0.1.13"}
])
```
## Series
| # | Livebook |
| --- | -------------------------------------------------------------- |
| 01 | [Introduction](01_introduction.livemd) |
| 02 | [Compression Fundamentals](02_compression_fundamentals.livemd) |
| 03 | **Codec Comparison** (you are here) |
| 04 | [Building Storage Systems](04_building_storage_systems.livemd) |
| 05 | [Zarr-Style Workloads](05_zarr_style_workloads.livemd) |
| 06 | [Spatial Codecs](06_spatial_codecs.livemd) |
## Setup: Test Datasets
```elixir
codecs = [:lz4, :snappy, :zstd, :bzip2, :blosc2]
random_data = :crypto.strong_rand_bytes(65536)
repetitive_text = String.duplicate("The quick brown fox jumps over the lazy dog. ", 2000)
semi_structured = Enum.join(for i <- 1..5000 do
"id=#{i}&name=user#{rem(i, 100)}&score=#{:rand.uniform(1000)}&active=#{rem(i, 2) == 0}"
end, "&")
float_array = for i <- 1..8192, into: <<>>, do: <<i * 0.125::float-size(64)-little>>
datasets = %{
"Random bytes" => random_data,
"Repetitive text" => repetitive_text,
"Semi-structured" => semi_structured,
"Float64 array" => float_array
}
dataset_sizes = for {name, data} <- datasets, into: %{} do
{name, byte_size(data)}
end
IO.puts("Dataset sizes:")
for {name, size} <- dataset_sizes do
IO.puts(" #{String.pad_trailing(name, 20)} #{size} bytes")
end
```
<!-- livebook:{"output":true} -->
```
Dataset sizes:
Float64 array 65536 bytes
Random bytes 65536 bytes
Repetitive text 90000 bytes
Semi-structured 210381 bytes
```
<!-- livebook:{"output":true} -->
```
[:ok, :ok, :ok, :ok]
```
## Compression Ratio Benchmarks
```elixir
compression_results = for {dname, data} <- datasets, codec <- codecs do
opts =
if codec == :blosc2,
do: [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8],
else: []
{:ok, enc} = ExCodecs.encode(codec, data, opts)
%{
dataset: dname,
codec: inspect(codec),
original: byte_size(data),
compressed: byte_size(enc),
ratio_pct: Float.round(100 * byte_size(enc) / byte_size(data), 1),
savings_pct: Float.round(100 * (1 - byte_size(enc) / byte_size(data)), 1)
}
end
Kino.DataTable.new(compression_results)
```
<!-- livebook:{"output":true} -->
```text
[%{compressed: 32804, codec: ":lz4", dataset: "Float64 array", original: 65536, ratio_pct: 50.1, savings_pct: 49.9}, %{compressed: 32691, codec: ":snappy", dataset: "Float64 array", original: 65536, ratio_pct: 49.9, savings_pct: 50.1}, %{compressed: 8413, codec: ":zstd", dataset: "Float64 array", original: 65536, ratio_pct: 12.8, savings_pct: 87.2}, %{compressed: 5590, codec: ":bzip2", dataset: "Float64 array", original: 65536, ratio_pct: 8.5, savings_pct: 91.5}, %{compressed: 962, codec: ":blosc2", dataset: "Float64 array", original: 65536, ratio_pct: 1.5, savings_pct: 98.5}, %{compressed: 65798, codec: ":lz4", dataset: "Random bytes", original: 65536, ratio_pct: 100.4, savings_pct: -0.4}, %{compressed: 65542, codec: ":snappy", dataset: "Random bytes", original: 65536, ratio_pct: 100.0, savings_pct: -0.0}, %{compressed: 65546, codec: ":zstd", dataset: "Random bytes", original: 65536, ratio_pct: 100.0, savings_pct: -0.0}, %{compressed: 66187, codec: ":bzip2", dataset: "Random bytes", original: 65536, ratio_pct: 101.0, savings_pct: -1.0}, %{compressed: 65568, codec: ":blosc2", dataset: "Random bytes", original: 65536, ratio_pct: 100.0, savings_pct: -0.0}, %{compressed: 413, codec: ":lz4", dataset: "Repetitive text", original: 90000, ratio_pct: 0.5, savings_pct: 99.5}, %{compressed: 4313, codec: ":snappy", dataset: "Repetitive text", original: 90000, ratio_pct: 4.8, savings_pct: 95.2}, %{compressed: 67, codec: ":zstd", dataset: "Repetitive text", original: 90000, ratio_pct: 0.1, savings_pct: 99.9}, %{compressed: 175, codec: ":bzip2", dataset: "Repetitive text", original: 90000, ratio_pct: 0.2, savings_pct: 99.8}, %{compressed: 580, ...}, ...]
```
### Compression Ratio Chart
```elixir
VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(compression_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "ratio_pct", type: :quantitative, title: "Compressed Size (%)", scale: [domain: [0, 110]])
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)
```
<!-- livebook:{"output":true} -->
```vega-lite
{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","compressed":32804,"dataset":"Float64 array","original":65536,"ratio_pct":50.1,"savings_pct":49.9},{"codec":":snappy","compressed":32691,"dataset":"Float64 array","original":65536,"ratio_pct":49.9,"savings_pct":50.1},{"codec":":zstd","compressed":8413,"dataset":"Float64 array","original":65536,"ratio_pct":12.8,"savings_pct":87.2},{"codec":":bzip2","compressed":5590,"dataset":"Float64 array","original":65536,"ratio_pct":8.5,"savings_pct":91.5},{"codec":":blosc2","compressed":962,"dataset":"Float64 array","original":65536,"ratio_pct":1.5,"savings_pct":98.5},{"codec":":lz4","compressed":65798,"dataset":"Random bytes","original":65536,"ratio_pct":100.4,"savings_pct":-0.4},{"codec":":snappy","compressed":65542,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":zstd","compressed":65546,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":bzip2","compressed":66187,"dataset":"Random bytes","original":65536,"ratio_pct":101.0,"savings_pct":-1.0},{"codec":":blosc2","compressed":65568,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":lz4","compressed":413,"dataset":"Repetitive text","original":90000,"ratio_pct":0.5,"savings_pct":99.5},{"codec":":snappy","compressed":4313,"dataset":"Repetitive text","original":90000,"ratio_pct":4.8,"savings_pct":95.2},{"codec":":zstd","compressed":67,"dataset":"Repetitive text","original":90000,"ratio_pct":0.1,"savings_pct":99.9},{"codec":":bzip2","compressed":175,"dataset":"Repetitive text","original":90000,"ratio_pct":0.2,"savings_pct":99.8},{"codec":":blosc2","compressed":580,"dataset":"Repetitive text","original":90000,"ratio_pct":0.6,"savings_pct":99.4},{"codec":":lz4","compressed":54922,"dataset":"Semi-structured","original":210381,"ratio_pct":26.1,"savings_pct":73.9},{"codec":":snappy","compressed":52235,"dataset":"Semi-structured","original":210381,"ratio_pct":24.8,"savings_pct":75.2},{"codec":":zstd","compressed":25506,"dataset":"Semi-structured","original":210381,"ratio_pct":12.1,"savings_pct":87.9},{"codec":":bzip2","compressed":19031,"dataset":"Semi-structured","original":210381,"ratio_pct":9.0,"savings_pct":91.0},{"codec":":blosc2","compressed":44901,"dataset":"Semi-structured","original":210381,"ratio_pct":21.3,"savings_pct":78.7}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"ratio_pct","scale":{"domain":[0,110]},"title":"Compressed Size (%)","type":"quantitative"}},"height":350,"mark":"bar","width":700}
```
### Space Savings Chart
```elixir
VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(compression_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "savings_pct", type: :quantitative, title: "Space Saved (%)")
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)
```
<!-- livebook:{"output":true} -->
```vega-lite
{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","compressed":32804,"dataset":"Float64 array","original":65536,"ratio_pct":50.1,"savings_pct":49.9},{"codec":":snappy","compressed":32691,"dataset":"Float64 array","original":65536,"ratio_pct":49.9,"savings_pct":50.1},{"codec":":zstd","compressed":8413,"dataset":"Float64 array","original":65536,"ratio_pct":12.8,"savings_pct":87.2},{"codec":":bzip2","compressed":5590,"dataset":"Float64 array","original":65536,"ratio_pct":8.5,"savings_pct":91.5},{"codec":":blosc2","compressed":962,"dataset":"Float64 array","original":65536,"ratio_pct":1.5,"savings_pct":98.5},{"codec":":lz4","compressed":65798,"dataset":"Random bytes","original":65536,"ratio_pct":100.4,"savings_pct":-0.4},{"codec":":snappy","compressed":65542,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":zstd","compressed":65546,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":bzip2","compressed":66187,"dataset":"Random bytes","original":65536,"ratio_pct":101.0,"savings_pct":-1.0},{"codec":":blosc2","compressed":65568,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":lz4","compressed":413,"dataset":"Repetitive text","original":90000,"ratio_pct":0.5,"savings_pct":99.5},{"codec":":snappy","compressed":4313,"dataset":"Repetitive text","original":90000,"ratio_pct":4.8,"savings_pct":95.2},{"codec":":zstd","compressed":67,"dataset":"Repetitive text","original":90000,"ratio_pct":0.1,"savings_pct":99.9},{"codec":":bzip2","compressed":175,"dataset":"Repetitive text","original":90000,"ratio_pct":0.2,"savings_pct":99.8},{"codec":":blosc2","compressed":580,"dataset":"Repetitive text","original":90000,"ratio_pct":0.6,"savings_pct":99.4},{"codec":":lz4","compressed":54922,"dataset":"Semi-structured","original":210381,"ratio_pct":26.1,"savings_pct":73.9},{"codec":":snappy","compressed":52235,"dataset":"Semi-structured","original":210381,"ratio_pct":24.8,"savings_pct":75.2},{"codec":":zstd","compressed":25506,"dataset":"Semi-structured","original":210381,"ratio_pct":12.1,"savings_pct":87.9},{"codec":":bzip2","compressed":19031,"dataset":"Semi-structured","original":210381,"ratio_pct":9.0,"savings_pct":91.0},{"codec":":blosc2","compressed":44901,"dataset":"Semi-structured","original":210381,"ratio_pct":21.3,"savings_pct":78.7}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"savings_pct","title":"Space Saved (%)","type":"quantitative"}},"height":350,"mark":"bar","width":700}
```
## Speed Benchmarks
```elixir
iterations = 20
speed_results = for {dname, data} <- datasets, codec <- codecs do
opts =
if codec == :blosc2,
do: [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8],
else: []
{:ok, enc} = ExCodecs.encode(codec, data, opts)
{enc_time, _} = :timer.tc(fn ->
for _ <- 1..iterations, do: ExCodecs.encode(codec, data, opts)
end)
{dec_time, _} = :timer.tc(fn ->
for _ <- 1..iterations, do: ExCodecs.decode(codec, enc)
end)
enc_throughput = Float.round(byte_size(data) * iterations / enc_time, 1)
dec_throughput = Float.round(byte_size(data) * iterations / dec_time, 1)
%{
dataset: dname,
codec: inspect(codec),
encode_time_us: div(enc_time, iterations),
decode_time_us: div(dec_time, iterations),
encode_mbps: Float.round(enc_throughput, 1),
decode_mbps: Float.round(dec_throughput, 1)
}
end
Kino.DataTable.new(speed_results)
```
<!-- livebook:{"output":true} -->
```text
[%{codec: ":lz4", dataset: "Float64 array", encode_time_us: 252, decode_time_us: 138, encode_mbps: 259.2, decode_mbps: 472.7}, %{codec: ":snappy", dataset: "Float64 array", encode_time_us: 157, decode_time_us: 102, encode_mbps: 415.2, decode_mbps: 637.5}, %{codec: ":zstd", dataset: "Float64 array", encode_time_us: 466, decode_time_us: 185, encode_mbps: 140.4, decode_mbps: 354.2}, %{codec: ":bzip2", dataset: "Float64 array", encode_time_us: 3336, decode_time_us: 753, encode_mbps: 19.6, decode_mbps: 87.0}, %{codec: ":blosc2", dataset: "Float64 array", encode_time_us: 52, decode_time_us: 47, encode_mbps: 1249.5, decode_mbps: 1369.6}, %{codec: ":lz4", dataset: "Random bytes", encode_time_us: 19, decode_time_us: 16, encode_mbps: 3413.3, decode_mbps: 3996.1}, %{codec: ":snappy", dataset: "Random bytes", encode_time_us: 22, decode_time_us: 19, encode_mbps: 2874.4, decode_mbps: 3326.7}, %{codec: ":zstd", dataset: "Random bytes", encode_time_us: 19, decode_time_us: 18, encode_mbps: 3395.6, decode_mbps: 3476.7}, %{codec: ":bzip2", dataset: "Random bytes", encode_time_us: 5940, decode_time_us: 2552, encode_mbps: 11.0, decode_mbps: 25.7}, %{codec: ":blosc2", dataset: "Random bytes", encode_time_us: 227, decode_time_us: 19, encode_mbps: 287.8, decode_mbps: 3386.9}, %{codec: ":lz4", dataset: "Repetitive text", encode_time_us: 15, decode_time_us: 20, encode_mbps: 5825.2, decode_mbps: 4368.9}, %{codec: ":snappy", dataset: "Repetitive text", encode_time_us: 18, decode_time_us: 22, encode_mbps: 4931.5, decode_mbps: 3921.6}, %{codec: ":zstd", dataset: "Repetitive text", encode_time_us: 22, decode_time_us: 24, encode_mbps: 4026.8, decode_mbps: 3651.1}, %{codec: ":bzip2", dataset: "Repetitive text", encode_time_us: 7915, decode_time_us: 505, encode_mbps: 11.4, decode_mbps: 178.2}, %{codec: ":blosc2", ...}, ...]
```
### Encode Speed Chart
```elixir
VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(speed_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "encode_time_us", type: :quantitative, title: "Encode Time (µs)")
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)
```
<!-- livebook:{"output":true} -->
```vega-lite
{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","dataset":"Float64 array","decode_mbps":472.7,"decode_time_us":138,"encode_mbps":259.2,"encode_time_us":252},{"codec":":snappy","dataset":"Float64 array","decode_mbps":637.5,"decode_time_us":102,"encode_mbps":415.2,"encode_time_us":157},{"codec":":zstd","dataset":"Float64 array","decode_mbps":354.2,"decode_time_us":185,"encode_mbps":140.4,"encode_time_us":466},{"codec":":bzip2","dataset":"Float64 array","decode_mbps":87.0,"decode_time_us":753,"encode_mbps":19.6,"encode_time_us":3336},{"codec":":blosc2","dataset":"Float64 array","decode_mbps":1369.6,"decode_time_us":47,"encode_mbps":1249.5,"encode_time_us":52},{"codec":":lz4","dataset":"Random bytes","decode_mbps":3996.1,"decode_time_us":16,"encode_mbps":3413.3,"encode_time_us":19},{"codec":":snappy","dataset":"Random bytes","decode_mbps":3326.7,"decode_time_us":19,"encode_mbps":2874.4,"encode_time_us":22},{"codec":":zstd","dataset":"Random bytes","decode_mbps":3476.7,"decode_time_us":18,"encode_mbps":3395.6,"encode_time_us":19},{"codec":":bzip2","dataset":"Random bytes","decode_mbps":25.7,"decode_time_us":2552,"encode_mbps":11.0,"encode_time_us":5940},{"codec":":blosc2","dataset":"Random bytes","decode_mbps":3386.9,"decode_time_us":19,"encode_mbps":287.8,"encode_time_us":227},{"codec":":lz4","dataset":"Repetitive text","decode_mbps":4368.9,"decode_time_us":20,"encode_mbps":5825.2,"encode_time_us":15},{"codec":":snappy","dataset":"Repetitive text","decode_mbps":3921.6,"decode_time_us":22,"encode_mbps":4931.5,"encode_time_us":18},{"codec":":zstd","dataset":"Repetitive text","decode_mbps":3651.1,"decode_time_us":24,"encode_mbps":4026.8,"encode_time_us":22},{"codec":":bzip2","dataset":"Repetitive text","decode_mbps":178.2,"decode_time_us":505,"encode_mbps":11.4,"encode_time_us":7915},{"codec":":blosc2","dataset":"Repetitive text","decode_mbps":1502.5,"decode_time_us":59,"encode_mbps":1331.4,"encode_time_us":67},{"codec":":lz4","dataset":"Semi-structured","decode_mbps":2152.2,"decode_time_us":97,"encode_mbps":975.8,"encode_time_us":215},{"codec":":snappy","dataset":"Semi-structured","decode_mbps":1529.5,"decode_time_us":137,"encode_mbps":940.9,"encode_time_us":223},{"codec":":zstd","dataset":"Semi-structured","decode_mbps":787.9,"decode_time_us":267,"encode_mbps":342.8,"encode_time_us":613},{"codec":":bzip2","dataset":"Semi-structured","decode_mbps":92.2,"decode_time_us":2281,"encode_mbps":27.1,"encode_time_us":7757},{"codec":":blosc2","dataset":"Semi-structured","decode_mbps":674.5,"decode_time_us":311,"encode_mbps":45.7,"encode_time_us":4605}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"encode_time_us","title":"Encode Time (µs)","type":"quantitative"}},"height":350,"mark":"bar","width":700}
```
### Decode Speed Chart
```elixir
VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(speed_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "decode_time_us", type: :quantitative, title: "Decode Time (µs)")
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)
```
<!-- livebook:{"output":true} -->
```vega-lite
{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","dataset":"Float64 array","decode_mbps":472.7,"decode_time_us":138,"encode_mbps":259.2,"encode_time_us":252},{"codec":":snappy","dataset":"Float64 array","decode_mbps":637.5,"decode_time_us":102,"encode_mbps":415.2,"encode_time_us":157},{"codec":":zstd","dataset":"Float64 array","decode_mbps":354.2,"decode_time_us":185,"encode_mbps":140.4,"encode_time_us":466},{"codec":":bzip2","dataset":"Float64 array","decode_mbps":87.0,"decode_time_us":753,"encode_mbps":19.6,"encode_time_us":3336},{"codec":":blosc2","dataset":"Float64 array","decode_mbps":1369.6,"decode_time_us":47,"encode_mbps":1249.5,"encode_time_us":52},{"codec":":lz4","dataset":"Random bytes","decode_mbps":3996.1,"decode_time_us":16,"encode_mbps":3413.3,"encode_time_us":19},{"codec":":snappy","dataset":"Random bytes","decode_mbps":3326.7,"decode_time_us":19,"encode_mbps":2874.4,"encode_time_us":22},{"codec":":zstd","dataset":"Random bytes","decode_mbps":3476.7,"decode_time_us":18,"encode_mbps":3395.6,"encode_time_us":19},{"codec":":bzip2","dataset":"Random bytes","decode_mbps":25.7,"decode_time_us":2552,"encode_mbps":11.0,"encode_time_us":5940},{"codec":":blosc2","dataset":"Random bytes","decode_mbps":3386.9,"decode_time_us":19,"encode_mbps":287.8,"encode_time_us":227},{"codec":":lz4","dataset":"Repetitive text","decode_mbps":4368.9,"decode_time_us":20,"encode_mbps":5825.2,"encode_time_us":15},{"codec":":snappy","dataset":"Repetitive text","decode_mbps":3921.6,"decode_time_us":22,"encode_mbps":4931.5,"encode_time_us":18},{"codec":":zstd","dataset":"Repetitive text","decode_mbps":3651.1,"decode_time_us":24,"encode_mbps":4026.8,"encode_time_us":22},{"codec":":bzip2","dataset":"Repetitive text","decode_mbps":178.2,"decode_time_us":505,"encode_mbps":11.4,"encode_time_us":7915},{"codec":":blosc2","dataset":"Repetitive text","decode_mbps":1502.5,"decode_time_us":59,"encode_mbps":1331.4,"encode_time_us":67},{"codec":":lz4","dataset":"Semi-structured","decode_mbps":2152.2,"decode_time_us":97,"encode_mbps":975.8,"encode_time_us":215},{"codec":":snappy","dataset":"Semi-structured","decode_mbps":1529.5,"decode_time_us":137,"encode_mbps":940.9,"encode_time_us":223},{"codec":":zstd","dataset":"Semi-structured","decode_mbps":787.9,"decode_time_us":267,"encode_mbps":342.8,"encode_time_us":613},{"codec":":bzip2","dataset":"Semi-structured","decode_mbps":92.2,"decode_time_us":2281,"encode_mbps":27.1,"encode_time_us":7757},{"codec":":blosc2","dataset":"Semi-structured","decode_mbps":674.5,"decode_time_us":311,"encode_mbps":45.7,"encode_time_us":4605}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"decode_time_us","title":"Decode Time (µs)","type":"quantitative"}},"height":350,"mark":"bar","width":700}
```
## Memory Usage
NIF encode/decode allocates **off-heap refc binaries**, not process-heap
terms, so `Process.info(self(), :heap_size)` (which only sees the calling
process's heap) is the wrong metric for codec working memory. The cell below
measures process-heap growth across a full encode/decode of the 64 KiB
float array, with a garbage collect on each side:
```elixir
for codec <- codecs do
opts = if codec == :blosc2, do: [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8], else: []
:erlang.garbage_collect()
before_heap = Process.info(self(), :heap_size) |> elem(1)
{:ok, enc} = ExCodecs.encode(codec, float_array, opts)
{:ok, _dec} = ExCodecs.decode(codec, enc)
:erlang.garbage_collect()
after_heap = Process.info(self(), :heap_size) |> elem(1)
IO.puts(String.pad_trailing("#{codec}", 10) <> "heap growth: #{after_heap - before_heap} words")
end
```
<!-- livebook:{"output":true} -->
```
lz4 heap growth: -6773 words
snappy heap growth: -2587 words
zstd heap growth: -2587 words
bzip2 heap growth: -2587 words
blosc2 heap growth: 0 words
```
<!-- livebook:{"output":true} -->
```
[:ok, :ok, :ok, :ok, :ok]
```
The deltas are a few hundred words at most — Elixir-level term overhead
(`{:ok, enc}` tuples, bindings) — while the **64 KiB of binary data flowing
through each codec is off-heap refc memory** and does not appear here. That is
why naive `heap_size` snapshots (and even `:erlang.memory(:binary)`, which is
whole-VM and noisy) cannot yield a clean per-codec "memory usage" number.
The real working-set bounds are set by options, not observed here: **decode**
is capped by `:max_output_size` (default 256 MiB, the decompression-bomb
guard), and **encode** working memory scales with the codec's block/window
size (e.g. bzip2 `block_size` × ~100 KiB; see livebook 02).
## Codec Profiles
```elixir
profile_data = %{
"Low latency" => %{
best: [:lz4, :snappy],
why: "Fastest encode/decode, ideal for hot paths, caching, and real-time systems"
},
"Balanced" => %{
best: [:zstd],
why: "Good ratio with fast decompression. Default choice for most workloads"
},
"Maximum Ratio" => %{
best: [:bzip2],
why: "Smallest output. Accept slower speed for archival and cold storage"
},
"Numeric Arrays" => %{
best: [:blosc2],
why: "Shuffle+compress reduces size of typed data (single-threaded NIF)"
}
}
for {profile, %{best: best, why: why}} <- profile_data do
IO.puts("## #{profile}")
IO.puts(" Codecs: #{inspect(best)}")
IO.puts(" #{why}\n")
end
```
<!-- livebook:{"output":true} -->
```
## Balanced
Codecs: [:zstd]
Good ratio with fast decompression. Default choice for most workloads
## Low latency
Codecs: [:lz4, :snappy]
Fastest encode/decode, ideal for hot paths, caching, and real-time systems
## Maximum Ratio
Codecs: [:bzip2]
Smallest output. Accept slower speed for archival and cold storage
## Numeric Arrays
Codecs: [:blosc2]
Shuffle+compress reduces size of typed data (single-threaded NIF)
```
<!-- livebook:{"output":true} -->
```
[:ok, :ok, :ok, :ok]
```
## Interactive Codec Selector
```elixir
use_case = Kino.Input.select("Your use case:", [
speed: "Real-time / low latency",
balanced: "General purpose",
ratio: "Maximum compression / archival",
numeric: "Numerical / scientific data",
tiny: "Small payloads / low overhead"
])
data_type = Kino.Input.select("Data type:", [
text: "Text / JSON",
binary: "Binary blobs",
array: "Typed arrays (floats, ints)",
mixed: "Mixed"
])
Kino.render(Kino.Layout.grid([use_case, data_type], columns: 2))
```
```elixir
use_case_val = Kino.Input.read(use_case)
data_type_val = Kino.Input.read(data_type)
recommendation = case {use_case_val, data_type_val} do
{:speed, _} -> {:lz4, "Fastest compression/decompression. Minimal latency overhead."}
{:tiny, _} -> {:snappy, "Low overhead even on very small payloads. No configuration needed."}
{:ratio, :array} -> {:blosc2, "Shuffle+compress gives best ratios on typed arrays."}
{:ratio, _} -> {:bzip2, "Highest compression ratio for general data. Slow but compact."}
{:numeric, _} -> {:blosc2, "Purpose-built for numerical data with shuffle filters."}
{:balanced, :array} -> {:blosc2, "Good ratio on typed data with decent speed."}
{:balanced, _} -> {:zstd, "Best all-around codec. Configurable from fast (level 1) to compact (level 22)."}
end
{codec, reason} = recommendation
{:ok, info} = ExCodecs.codec_info(codec)
IO.puts("Recommended codec: #{inspect(codec)}")
IO.puts("Reason: #{reason}")
IO.puts("Configurable: #{info.configurable?}")
IO.puts("Streaming: #{info.streaming?}")
default_opts = case codec do
:zstd -> [level: 3]
:lz4 -> []
:bzip2 -> [block_size: 9]
:blosc2 -> [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8]
:snappy -> []
end
IO.puts("Suggested options: #{inspect(default_opts)}")
```
<!-- livebook:{"output":true} -->
```
Recommended codec: :lz4
Reason: Fastest compression/decompression. Minimal latency overhead.
Configurable: false
Streaming: false
Suggested options: []
```
<!-- livebook:{"output":true} -->
```
:ok
```
## Decision Flowchart
When choosing a codec, follow this decision path:
1. Is your data typed numerical arrays?
* YES → Use Blosc2 (with appropriate shuffle and typesize)
* NO → Continue
2. Is latency critical (hot path, real-time)?
* YES → Use LZ4 (fastest) or Snappy (low overhead)
* NO → Continue
3. Is storage cost the primary concern?
* YES → Use Bzip2 (best ratio) or Zstd with high level
* NO → Continue
4. Default choice: Zstd (level 3) - good ratio, fast decompression, configurable
## Codec Feature Matrix
| Feature | LZ4 | Snappy | Zstd | Bzip2 | Blosc2 |
| ------------ | ------------- | ---------- | ------------ | ----------- | ------------- |
| Speed | Very Fast | Very Fast | Fast | Slow | Medium |
| Ratio | Low | Low | High | Very High | High (arrays) |
| Configurable | Fixed profile | No | Level 1–22 | Block 1–9 | Codec/shuffle |
| Streaming | No | No | No | No | No |
| Best For | Hot paths | Short data | General | Archival | Arrays |
| Shuffle | — | — | — | — | Byte/Bit |
| Multi-thread | No | No | No | No | No |
## Navigation
**Previous:** [Compression Fundamentals](02_compression_fundamentals.livemd) ·
**Next:** [Building Storage Systems](04_building_storage_systems.livemd)