Codec Comparison

# Use this install to work with the source code
# Mix.install(
#   [
#     {:ex_codecs, path: Path.join(__DIR__, "..")}, 
#     {:rustler, "~> 0.36"},
#     {:jason, "~> 1.4"}, 
#     {:kino, "~> 0.14"}, 
#     {:kino_vega_lite, "~> 0.1.13"}
#   ],
#   config:  [rustler_precompiled: [force_build: [ex_codecs: true]]]
# )

Mix.install( [
    {:ex_codecs, "~> 0.2.3"}, 
    {:jason, "~> 1.4"}, 
    {:kino, "~> 0.14"}, 
    {:kino_vega_lite, "~> 0.1.13"}
  ])

Series

Setup: Test Datasets

codecs = [:lz4, :snappy, :zstd, :bzip2, :blosc2]

random_data = :crypto.strong_rand_bytes(65536)

repetitive_text = String.duplicate("The quick brown fox jumps over the lazy dog. ", 2000)

semi_structured = Enum.join(for i <- 1..5000 do
  "id=#{i}&name=user#{rem(i, 100)}&score=#{:rand.uniform(1000)}&active=#{rem(i, 2) == 0}"
end, "&")

float_array = for i <- 1..8192, into: <<>>, do: <<i * 0.125::float-size(64)-little>>

datasets = %{
  "Random bytes" => random_data,
  "Repetitive text" => repetitive_text,
  "Semi-structured" => semi_structured,
  "Float64 array" => float_array
}

dataset_sizes = for {name, data} <- datasets, into: %{} do
  {name, byte_size(data)}
end

IO.puts("Dataset sizes:")
for {name, size} <- dataset_sizes do
  IO.puts("  #{String.pad_trailing(name, 20)} #{size} bytes")
end

Dataset sizes:
  Float64 array        65536 bytes
  Random bytes         65536 bytes
  Repetitive text      90000 bytes
  Semi-structured      210381 bytes

[:ok, :ok, :ok, :ok]

Compression Ratio Benchmarks

compression_results = for {dname, data} <- datasets, codec <- codecs do
  opts =
    if codec == :blosc2,
      do: [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8],
      else: []
  {:ok, enc} = ExCodecs.encode(codec, data, opts)
  %{
    dataset: dname,
    codec: inspect(codec),
    original: byte_size(data),
    compressed: byte_size(enc),
    ratio_pct: Float.round(100 * byte_size(enc) / byte_size(data), 1),
    savings_pct: Float.round(100 * (1 - byte_size(enc) / byte_size(data)), 1)
  }
end

Kino.DataTable.new(compression_results)

[%{compressed: 32804, codec: ":lz4", dataset: "Float64 array", original: 65536, ratio_pct: 50.1, savings_pct: 49.9}, %{compressed: 32691, codec: ":snappy", dataset: "Float64 array", original: 65536, ratio_pct: 49.9, savings_pct: 50.1}, %{compressed: 8413, codec: ":zstd", dataset: "Float64 array", original: 65536, ratio_pct: 12.8, savings_pct: 87.2}, %{compressed: 5590, codec: ":bzip2", dataset: "Float64 array", original: 65536, ratio_pct: 8.5, savings_pct: 91.5}, %{compressed: 962, codec: ":blosc2", dataset: "Float64 array", original: 65536, ratio_pct: 1.5, savings_pct: 98.5}, %{compressed: 65798, codec: ":lz4", dataset: "Random bytes", original: 65536, ratio_pct: 100.4, savings_pct: -0.4}, %{compressed: 65542, codec: ":snappy", dataset: "Random bytes", original: 65536, ratio_pct: 100.0, savings_pct: -0.0}, %{compressed: 65546, codec: ":zstd", dataset: "Random bytes", original: 65536, ratio_pct: 100.0, savings_pct: -0.0}, %{compressed: 66187, codec: ":bzip2", dataset: "Random bytes", original: 65536, ratio_pct: 101.0, savings_pct: -1.0}, %{compressed: 65568, codec: ":blosc2", dataset: "Random bytes", original: 65536, ratio_pct: 100.0, savings_pct: -0.0}, %{compressed: 413, codec: ":lz4", dataset: "Repetitive text", original: 90000, ratio_pct: 0.5, savings_pct: 99.5}, %{compressed: 4313, codec: ":snappy", dataset: "Repetitive text", original: 90000, ratio_pct: 4.8, savings_pct: 95.2}, %{compressed: 67, codec: ":zstd", dataset: "Repetitive text", original: 90000, ratio_pct: 0.1, savings_pct: 99.9}, %{compressed: 175, codec: ":bzip2", dataset: "Repetitive text", original: 90000, ratio_pct: 0.2, savings_pct: 99.8}, %{compressed: 580, ...}, ...]

Compression Ratio Chart

VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(compression_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "ratio_pct", type: :quantitative, title: "Compressed Size (%)", scale: [domain: [0, 110]])
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)

{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","compressed":32804,"dataset":"Float64 array","original":65536,"ratio_pct":50.1,"savings_pct":49.9},{"codec":":snappy","compressed":32691,"dataset":"Float64 array","original":65536,"ratio_pct":49.9,"savings_pct":50.1},{"codec":":zstd","compressed":8413,"dataset":"Float64 array","original":65536,"ratio_pct":12.8,"savings_pct":87.2},{"codec":":bzip2","compressed":5590,"dataset":"Float64 array","original":65536,"ratio_pct":8.5,"savings_pct":91.5},{"codec":":blosc2","compressed":962,"dataset":"Float64 array","original":65536,"ratio_pct":1.5,"savings_pct":98.5},{"codec":":lz4","compressed":65798,"dataset":"Random bytes","original":65536,"ratio_pct":100.4,"savings_pct":-0.4},{"codec":":snappy","compressed":65542,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":zstd","compressed":65546,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":bzip2","compressed":66187,"dataset":"Random bytes","original":65536,"ratio_pct":101.0,"savings_pct":-1.0},{"codec":":blosc2","compressed":65568,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":lz4","compressed":413,"dataset":"Repetitive text","original":90000,"ratio_pct":0.5,"savings_pct":99.5},{"codec":":snappy","compressed":4313,"dataset":"Repetitive text","original":90000,"ratio_pct":4.8,"savings_pct":95.2},{"codec":":zstd","compressed":67,"dataset":"Repetitive text","original":90000,"ratio_pct":0.1,"savings_pct":99.9},{"codec":":bzip2","compressed":175,"dataset":"Repetitive text","original":90000,"ratio_pct":0.2,"savings_pct":99.8},{"codec":":blosc2","compressed":580,"dataset":"Repetitive text","original":90000,"ratio_pct":0.6,"savings_pct":99.4},{"codec":":lz4","compressed":54922,"dataset":"Semi-structured","original":210381,"ratio_pct":26.1,"savings_pct":73.9},{"codec":":snappy","compressed":52235,"dataset":"Semi-structured","original":210381,"ratio_pct":24.8,"savings_pct":75.2},{"codec":":zstd","compressed":25506,"dataset":"Semi-structured","original":210381,"ratio_pct":12.1,"savings_pct":87.9},{"codec":":bzip2","compressed":19031,"dataset":"Semi-structured","original":210381,"ratio_pct":9.0,"savings_pct":91.0},{"codec":":blosc2","compressed":44901,"dataset":"Semi-structured","original":210381,"ratio_pct":21.3,"savings_pct":78.7}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"ratio_pct","scale":{"domain":[0,110]},"title":"Compressed Size (%)","type":"quantitative"}},"height":350,"mark":"bar","width":700}

Space Savings Chart

VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(compression_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "savings_pct", type: :quantitative, title: "Space Saved (%)")
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)

{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","compressed":32804,"dataset":"Float64 array","original":65536,"ratio_pct":50.1,"savings_pct":49.9},{"codec":":snappy","compressed":32691,"dataset":"Float64 array","original":65536,"ratio_pct":49.9,"savings_pct":50.1},{"codec":":zstd","compressed":8413,"dataset":"Float64 array","original":65536,"ratio_pct":12.8,"savings_pct":87.2},{"codec":":bzip2","compressed":5590,"dataset":"Float64 array","original":65536,"ratio_pct":8.5,"savings_pct":91.5},{"codec":":blosc2","compressed":962,"dataset":"Float64 array","original":65536,"ratio_pct":1.5,"savings_pct":98.5},{"codec":":lz4","compressed":65798,"dataset":"Random bytes","original":65536,"ratio_pct":100.4,"savings_pct":-0.4},{"codec":":snappy","compressed":65542,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":zstd","compressed":65546,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":bzip2","compressed":66187,"dataset":"Random bytes","original":65536,"ratio_pct":101.0,"savings_pct":-1.0},{"codec":":blosc2","compressed":65568,"dataset":"Random bytes","original":65536,"ratio_pct":100.0,"savings_pct":-0.0},{"codec":":lz4","compressed":413,"dataset":"Repetitive text","original":90000,"ratio_pct":0.5,"savings_pct":99.5},{"codec":":snappy","compressed":4313,"dataset":"Repetitive text","original":90000,"ratio_pct":4.8,"savings_pct":95.2},{"codec":":zstd","compressed":67,"dataset":"Repetitive text","original":90000,"ratio_pct":0.1,"savings_pct":99.9},{"codec":":bzip2","compressed":175,"dataset":"Repetitive text","original":90000,"ratio_pct":0.2,"savings_pct":99.8},{"codec":":blosc2","compressed":580,"dataset":"Repetitive text","original":90000,"ratio_pct":0.6,"savings_pct":99.4},{"codec":":lz4","compressed":54922,"dataset":"Semi-structured","original":210381,"ratio_pct":26.1,"savings_pct":73.9},{"codec":":snappy","compressed":52235,"dataset":"Semi-structured","original":210381,"ratio_pct":24.8,"savings_pct":75.2},{"codec":":zstd","compressed":25506,"dataset":"Semi-structured","original":210381,"ratio_pct":12.1,"savings_pct":87.9},{"codec":":bzip2","compressed":19031,"dataset":"Semi-structured","original":210381,"ratio_pct":9.0,"savings_pct":91.0},{"codec":":blosc2","compressed":44901,"dataset":"Semi-structured","original":210381,"ratio_pct":21.3,"savings_pct":78.7}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"savings_pct","title":"Space Saved (%)","type":"quantitative"}},"height":350,"mark":"bar","width":700}

Speed Benchmarks

iterations = 20

speed_results = for {dname, data} <- datasets, codec <- codecs do
  opts =
    if codec == :blosc2,
      do: [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8],
      else: []
  {:ok, enc} = ExCodecs.encode(codec, data, opts)

  {enc_time, _} = :timer.tc(fn ->
    for _ <- 1..iterations, do: ExCodecs.encode(codec, data, opts)
  end)

  {dec_time, _} = :timer.tc(fn ->
    for _ <- 1..iterations, do: ExCodecs.decode(codec, enc)
  end)

  enc_throughput = Float.round(byte_size(data) * iterations / enc_time, 1)
  dec_throughput = Float.round(byte_size(data) * iterations / dec_time, 1)

  %{
    dataset: dname,
    codec: inspect(codec),
    encode_time_us: div(enc_time, iterations),
    decode_time_us: div(dec_time, iterations),
    encode_mbps: Float.round(enc_throughput, 1),
    decode_mbps: Float.round(dec_throughput, 1)
  }
end

Kino.DataTable.new(speed_results)

[%{codec: ":lz4", dataset: "Float64 array", encode_time_us: 252, decode_time_us: 138, encode_mbps: 259.2, decode_mbps: 472.7}, %{codec: ":snappy", dataset: "Float64 array", encode_time_us: 157, decode_time_us: 102, encode_mbps: 415.2, decode_mbps: 637.5}, %{codec: ":zstd", dataset: "Float64 array", encode_time_us: 466, decode_time_us: 185, encode_mbps: 140.4, decode_mbps: 354.2}, %{codec: ":bzip2", dataset: "Float64 array", encode_time_us: 3336, decode_time_us: 753, encode_mbps: 19.6, decode_mbps: 87.0}, %{codec: ":blosc2", dataset: "Float64 array", encode_time_us: 52, decode_time_us: 47, encode_mbps: 1249.5, decode_mbps: 1369.6}, %{codec: ":lz4", dataset: "Random bytes", encode_time_us: 19, decode_time_us: 16, encode_mbps: 3413.3, decode_mbps: 3996.1}, %{codec: ":snappy", dataset: "Random bytes", encode_time_us: 22, decode_time_us: 19, encode_mbps: 2874.4, decode_mbps: 3326.7}, %{codec: ":zstd", dataset: "Random bytes", encode_time_us: 19, decode_time_us: 18, encode_mbps: 3395.6, decode_mbps: 3476.7}, %{codec: ":bzip2", dataset: "Random bytes", encode_time_us: 5940, decode_time_us: 2552, encode_mbps: 11.0, decode_mbps: 25.7}, %{codec: ":blosc2", dataset: "Random bytes", encode_time_us: 227, decode_time_us: 19, encode_mbps: 287.8, decode_mbps: 3386.9}, %{codec: ":lz4", dataset: "Repetitive text", encode_time_us: 15, decode_time_us: 20, encode_mbps: 5825.2, decode_mbps: 4368.9}, %{codec: ":snappy", dataset: "Repetitive text", encode_time_us: 18, decode_time_us: 22, encode_mbps: 4931.5, decode_mbps: 3921.6}, %{codec: ":zstd", dataset: "Repetitive text", encode_time_us: 22, decode_time_us: 24, encode_mbps: 4026.8, decode_mbps: 3651.1}, %{codec: ":bzip2", dataset: "Repetitive text", encode_time_us: 7915, decode_time_us: 505, encode_mbps: 11.4, decode_mbps: 178.2}, %{codec: ":blosc2", ...}, ...]

Encode Speed Chart

VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(speed_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "encode_time_us", type: :quantitative, title: "Encode Time (µs)")
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)

{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","dataset":"Float64 array","decode_mbps":472.7,"decode_time_us":138,"encode_mbps":259.2,"encode_time_us":252},{"codec":":snappy","dataset":"Float64 array","decode_mbps":637.5,"decode_time_us":102,"encode_mbps":415.2,"encode_time_us":157},{"codec":":zstd","dataset":"Float64 array","decode_mbps":354.2,"decode_time_us":185,"encode_mbps":140.4,"encode_time_us":466},{"codec":":bzip2","dataset":"Float64 array","decode_mbps":87.0,"decode_time_us":753,"encode_mbps":19.6,"encode_time_us":3336},{"codec":":blosc2","dataset":"Float64 array","decode_mbps":1369.6,"decode_time_us":47,"encode_mbps":1249.5,"encode_time_us":52},{"codec":":lz4","dataset":"Random bytes","decode_mbps":3996.1,"decode_time_us":16,"encode_mbps":3413.3,"encode_time_us":19},{"codec":":snappy","dataset":"Random bytes","decode_mbps":3326.7,"decode_time_us":19,"encode_mbps":2874.4,"encode_time_us":22},{"codec":":zstd","dataset":"Random bytes","decode_mbps":3476.7,"decode_time_us":18,"encode_mbps":3395.6,"encode_time_us":19},{"codec":":bzip2","dataset":"Random bytes","decode_mbps":25.7,"decode_time_us":2552,"encode_mbps":11.0,"encode_time_us":5940},{"codec":":blosc2","dataset":"Random bytes","decode_mbps":3386.9,"decode_time_us":19,"encode_mbps":287.8,"encode_time_us":227},{"codec":":lz4","dataset":"Repetitive text","decode_mbps":4368.9,"decode_time_us":20,"encode_mbps":5825.2,"encode_time_us":15},{"codec":":snappy","dataset":"Repetitive text","decode_mbps":3921.6,"decode_time_us":22,"encode_mbps":4931.5,"encode_time_us":18},{"codec":":zstd","dataset":"Repetitive text","decode_mbps":3651.1,"decode_time_us":24,"encode_mbps":4026.8,"encode_time_us":22},{"codec":":bzip2","dataset":"Repetitive text","decode_mbps":178.2,"decode_time_us":505,"encode_mbps":11.4,"encode_time_us":7915},{"codec":":blosc2","dataset":"Repetitive text","decode_mbps":1502.5,"decode_time_us":59,"encode_mbps":1331.4,"encode_time_us":67},{"codec":":lz4","dataset":"Semi-structured","decode_mbps":2152.2,"decode_time_us":97,"encode_mbps":975.8,"encode_time_us":215},{"codec":":snappy","dataset":"Semi-structured","decode_mbps":1529.5,"decode_time_us":137,"encode_mbps":940.9,"encode_time_us":223},{"codec":":zstd","dataset":"Semi-structured","decode_mbps":787.9,"decode_time_us":267,"encode_mbps":342.8,"encode_time_us":613},{"codec":":bzip2","dataset":"Semi-structured","decode_mbps":92.2,"decode_time_us":2281,"encode_mbps":27.1,"encode_time_us":7757},{"codec":":blosc2","dataset":"Semi-structured","decode_mbps":674.5,"decode_time_us":311,"encode_mbps":45.7,"encode_time_us":4605}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"encode_time_us","title":"Encode Time (µs)","type":"quantitative"}},"height":350,"mark":"bar","width":700}

Decode Speed Chart

VegaLite.new(width: 700, height: 350)
|> VegaLite.data_from_values(speed_results)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "codec", type: :nominal, sort: ["lz4", "snappy", "zstd", "bzip2", "blosc2"])
|> VegaLite.encode_field(:y, "decode_time_us", type: :quantitative, title: "Decode Time (µs)")
|> VegaLite.encode_field(:color, "codec", type: :nominal)
|> VegaLite.encode_field(:column, "dataset", type: :nominal)

{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"codec":":lz4","dataset":"Float64 array","decode_mbps":472.7,"decode_time_us":138,"encode_mbps":259.2,"encode_time_us":252},{"codec":":snappy","dataset":"Float64 array","decode_mbps":637.5,"decode_time_us":102,"encode_mbps":415.2,"encode_time_us":157},{"codec":":zstd","dataset":"Float64 array","decode_mbps":354.2,"decode_time_us":185,"encode_mbps":140.4,"encode_time_us":466},{"codec":":bzip2","dataset":"Float64 array","decode_mbps":87.0,"decode_time_us":753,"encode_mbps":19.6,"encode_time_us":3336},{"codec":":blosc2","dataset":"Float64 array","decode_mbps":1369.6,"decode_time_us":47,"encode_mbps":1249.5,"encode_time_us":52},{"codec":":lz4","dataset":"Random bytes","decode_mbps":3996.1,"decode_time_us":16,"encode_mbps":3413.3,"encode_time_us":19},{"codec":":snappy","dataset":"Random bytes","decode_mbps":3326.7,"decode_time_us":19,"encode_mbps":2874.4,"encode_time_us":22},{"codec":":zstd","dataset":"Random bytes","decode_mbps":3476.7,"decode_time_us":18,"encode_mbps":3395.6,"encode_time_us":19},{"codec":":bzip2","dataset":"Random bytes","decode_mbps":25.7,"decode_time_us":2552,"encode_mbps":11.0,"encode_time_us":5940},{"codec":":blosc2","dataset":"Random bytes","decode_mbps":3386.9,"decode_time_us":19,"encode_mbps":287.8,"encode_time_us":227},{"codec":":lz4","dataset":"Repetitive text","decode_mbps":4368.9,"decode_time_us":20,"encode_mbps":5825.2,"encode_time_us":15},{"codec":":snappy","dataset":"Repetitive text","decode_mbps":3921.6,"decode_time_us":22,"encode_mbps":4931.5,"encode_time_us":18},{"codec":":zstd","dataset":"Repetitive text","decode_mbps":3651.1,"decode_time_us":24,"encode_mbps":4026.8,"encode_time_us":22},{"codec":":bzip2","dataset":"Repetitive text","decode_mbps":178.2,"decode_time_us":505,"encode_mbps":11.4,"encode_time_us":7915},{"codec":":blosc2","dataset":"Repetitive text","decode_mbps":1502.5,"decode_time_us":59,"encode_mbps":1331.4,"encode_time_us":67},{"codec":":lz4","dataset":"Semi-structured","decode_mbps":2152.2,"decode_time_us":97,"encode_mbps":975.8,"encode_time_us":215},{"codec":":snappy","dataset":"Semi-structured","decode_mbps":1529.5,"decode_time_us":137,"encode_mbps":940.9,"encode_time_us":223},{"codec":":zstd","dataset":"Semi-structured","decode_mbps":787.9,"decode_time_us":267,"encode_mbps":342.8,"encode_time_us":613},{"codec":":bzip2","dataset":"Semi-structured","decode_mbps":92.2,"decode_time_us":2281,"encode_mbps":27.1,"encode_time_us":7757},{"codec":":blosc2","dataset":"Semi-structured","decode_mbps":674.5,"decode_time_us":311,"encode_mbps":45.7,"encode_time_us":4605}]},"encoding":{"color":{"field":"codec","type":"nominal"},"column":{"field":"dataset","type":"nominal"},"x":{"field":"codec","sort":["lz4","snappy","zstd","bzip2","blosc2"],"type":"nominal"},"y":{"field":"decode_time_us","title":"Decode Time (µs)","type":"quantitative"}},"height":350,"mark":"bar","width":700}

Memory Usage

NIF encode/decode allocates off-heap refc binaries, not process-heap terms, so Process.info(self(), :heap_size) (which only sees the calling process’s heap) is the wrong metric for codec working memory. The cell below measures process-heap growth across a full encode/decode of the 64 KiB float array, with a garbage collect on each side:

for codec <- codecs do
  opts = if codec == :blosc2, do: [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8], else: []
  :erlang.garbage_collect()
  before_heap = Process.info(self(), :heap_size) |> elem(1)
  {:ok, enc} = ExCodecs.encode(codec, float_array, opts)
  {:ok, _dec} = ExCodecs.decode(codec, enc)
  :erlang.garbage_collect()
  after_heap = Process.info(self(), :heap_size) |> elem(1)
  IO.puts(String.pad_trailing("#{codec}", 10) <> "heap growth: #{after_heap - before_heap} words")
end

lz4       heap growth: -6773 words
snappy    heap growth: -2587 words
zstd      heap growth: -2587 words
bzip2     heap growth: -2587 words
blosc2    heap growth: 0 words

[:ok, :ok, :ok, :ok, :ok]

The deltas are a few hundred words at most — Elixir-level term overhead ({:ok, enc} tuples, bindings) — while the 64 KiB of binary data flowing through each codec is off-heap refc memory and does not appear here. That is why naive heap_size snapshots (and even :erlang.memory(:binary), which is whole-VM and noisy) cannot yield a clean per-codec “memory usage” number.

The real working-set bounds are set by options, not observed here: decode is capped by :max_output_size (default 256 MiB, the decompression-bomb guard), and encode working memory scales with the codec’s block/window size (e.g. bzip2 block_size × ~100 KiB; see livebook 02).

Codec Profiles

profile_data = %{
  "Low latency" => %{
    best: [:lz4, :snappy],
    why: "Fastest encode/decode, ideal for hot paths, caching, and real-time systems"
  },
  "Balanced" => %{
    best: [:zstd],
    why: "Good ratio with fast decompression. Default choice for most workloads"
  },
  "Maximum Ratio" => %{
    best: [:bzip2],
    why: "Smallest output. Accept slower speed for archival and cold storage"
  },
  "Numeric Arrays" => %{
    best: [:blosc2],
    why: "Shuffle+compress reduces size of typed data (single-threaded NIF)"
  }
}

for {profile, %{best: best, why: why}} <- profile_data do
  IO.puts("## #{profile}")
  IO.puts("  Codecs: #{inspect(best)}")
  IO.puts("  #{why}\n")
end

## Balanced
  Codecs: [:zstd]
  Good ratio with fast decompression. Default choice for most workloads

## Low latency
  Codecs: [:lz4, :snappy]
  Fastest encode/decode, ideal for hot paths, caching, and real-time systems

## Maximum Ratio
  Codecs: [:bzip2]
  Smallest output. Accept slower speed for archival and cold storage

## Numeric Arrays
  Codecs: [:blosc2]
  Shuffle+compress reduces size of typed data (single-threaded NIF)

[:ok, :ok, :ok, :ok]

Interactive Codec Selector

use_case = Kino.Input.select("Your use case:", [
  speed: "Real-time / low latency",
  balanced: "General purpose",
  ratio: "Maximum compression / archival",
  numeric: "Numerical / scientific data",
  tiny: "Small payloads / low overhead"
])

data_type = Kino.Input.select("Data type:", [
  text: "Text / JSON",
  binary: "Binary blobs",
  array: "Typed arrays (floats, ints)",
  mixed: "Mixed"
])

Kino.render(Kino.Layout.grid([use_case, data_type], columns: 2))

use_case_val = Kino.Input.read(use_case)
data_type_val = Kino.Input.read(data_type)

recommendation = case {use_case_val, data_type_val} do
  {:speed, _} -> {:lz4, "Fastest compression/decompression. Minimal latency overhead."}
  {:tiny, _} -> {:snappy, "Low overhead even on very small payloads. No configuration needed."}
  {:ratio, :array} -> {:blosc2, "Shuffle+compress gives best ratios on typed arrays."}
  {:ratio, _} -> {:bzip2, "Highest compression ratio for general data. Slow but compact."}
  {:numeric, _} -> {:blosc2, "Purpose-built for numerical data with shuffle filters."}
  {:balanced, :array} -> {:blosc2, "Good ratio on typed data with decent speed."}
  {:balanced, _} -> {:zstd, "Best all-around codec. Configurable from fast (level 1) to compact (level 22)."}
end

{codec, reason} = recommendation
{:ok, info} = ExCodecs.codec_info(codec)

IO.puts("Recommended codec: #{inspect(codec)}")
IO.puts("Reason: #{reason}")
IO.puts("Configurable: #{info.configurable?}")
IO.puts("Streaming: #{info.streaming?}")

default_opts = case codec do
  :zstd -> [level: 3]
  :lz4 -> []
  :bzip2 -> [block_size: 9]
  :blosc2 -> [cname: :zstd, clevel: 5, shuffle: :byte, typesize: 8]
  :snappy -> []
end
IO.puts("Suggested options: #{inspect(default_opts)}")

Recommended codec: :lz4
Reason: Fastest compression/decompression. Minimal latency overhead.
Configurable: false
Streaming: false
Suggested options: []

:ok

Decision Flowchart

When choosing a codec, follow this decision path:

1. Is your data typed numerical arrays?
   • YES → Use Blosc2 (with appropriate shuffle and typesize)
   • NO → Continue
2. Is latency critical (hot path, real-time)?
   • YES → Use LZ4 (fastest) or Snappy (low overhead)
   • NO → Continue
3. Is storage cost the primary concern?
   • YES → Use Bzip2 (best ratio) or Zstd with high level
   • NO → Continue
4. Default choice: Zstd (level 3) - good ratio, fast decompression, configurable

Codec Feature Matrix

Navigation

Previous: Compression Fundamentals · Next: Building Storage Systems