Cookbook: Scientific Computing Patterns

Mix.install([
  {:ex_zarr, "~> 1.2"}
  # {:ex_zarr, path: Path.join(__DIR__, "../../..")},
])

Map-reduce over chunks

alias ExZarr.Array
alias ExZarr.Gallery.Pack

{:ok, array} =
  Array.create(
    shape: {80, 80},
    chunks: {20, 20},
    dtype: :float64,
    compressor: :zlib,
    storage: :memory
  )

vals = for i <- 0..(80 * 80 - 1), do: i * 1.0
:ok = Array.set_slice(array, Pack.pack(vals, :float64), start: {0, 0}, stop: {80, 80})

partial_sums =
  array
  |> Array.stream_chunks(concurrency: System.schedulers_online())
  |> Enum.map(fn {_idx, data} ->
    for <<val::float-little-64 <- data>>, reduce: 0.0 do
      acc -> acc + val
    end
  end)

%{partials: length(partial_sums), total: Enum.sum(partial_sums)}

GenStage backpressure

When downstream is slower than I/O, use ExZarr.GenStage.start_chunk_producer/2 so the producer only reads what consumers ask for. See the GenStage module docs and docs/livebooks/broadway_pipeline.livemd for related pipeline patterns.