Evision Example - High Dynamic Range Imaging

Mix.install([
  {:evision, "~> 0.2"},
  {:kino, "~> 0.7"},
  {:req, "~> 0.5"}
], system_env: [
  # optional, defaults to `true`
  # set `EVISION_PREFER_PRECOMPILED` to `false`
  # if you prefer `:evision` to be compiled from source
  # note that to compile from source, you may need at least 1GB RAM
  {"EVISION_PREFER_PRECOMPILED", true},

  # optional, defaults to `true`
  # set `EVISION_ENABLE_CONTRIB` to `false`
  # if you don't need modules from `opencv_contrib`
  {"EVISION_ENABLE_CONTRIB", true},

  # optional, defaults to `false`
  # set `EVISION_ENABLE_CUDA` to `true`
  # if you wish to use CUDA related functions
  # note that `EVISION_ENABLE_CONTRIB` also has to be `true`
  # because cuda related modules come from the `opencv_contrib` repo
  {"EVISION_ENABLE_CUDA", false},

  # required when
  # - `EVISION_ENABLE_CUDA` is `true`
  # - and `EVISION_PREFER_PRECOMPILED` is `true`
  #
  # set `EVISION_CUDA_VERSION` to the version that matches
  # your local CUDA runtime version
  #
  # current available versions are
  # - 118
  # - 121
  {"EVISION_CUDA_VERSION", "118"},

  # require for Windows users when
  # - `EVISION_ENABLE_CUDA` is `true`
  # set `EVISION_CUDA_RUNTIME_DIR` to the directory that contains
  # CUDA runtime libraries
  {"EVISION_CUDA_RUNTIME_DIR", "C:/PATH/TO/CUDA/RUNTIME"}
])

:ok

Define Some Helper Functions

defmodule Helper do
  def download!(url, save_as, overwrite? \\ false) do
    unless File.exists?(save_as) do
      Req.get!(url, http_errors: :raise, into: File.stream!(save_as), cache: not overwrite?)
    end

    :ok
  end

  @doc """
  This function chunks binary data by every requested `chunk_size`

  To make it more general, this function allows the length of the last chunk
  to be less than the request `chunk_size`.

  For example, if you have a 7-byte binary data, and you'd like to chunk it by every
  4 bytes, then this function will return two chunks with the first gives you the
  byte 0 to 3, and the second one gives byte 4 to 6.
  """
  def chunk_binary(binary, chunk_size) when is_binary(binary) do
    total_bytes = byte_size(binary)
    full_chunks = div(total_bytes, chunk_size)

    chunks =
      if full_chunks > 0 do
        for i <- 0..(full_chunks - 1), reduce: [] do
          acc -> [:binary.part(binary, chunk_size * i, chunk_size) | acc]
        end
      else
        []
      end

    remaining = rem(total_bytes, chunk_size)

    chunks =
      if remaining > 0 do
        [:binary.part(binary, chunk_size * full_chunks, remaining) | chunks]
      else
        chunks
      end

    Enum.reverse(chunks)
  end
end

{:module, Helper, <<70, 79, 82, 49, 0, 0, 16, ...>>, {:chunk_binary, 2}}

Download the Test Images

alias Evision, as: Cv

# change to the file's directory
# or somewhere you have write permission
File.cd!(__DIR__)

# create a directory for storing the test images
File.mkdir_p!("photo_hdr_test")

exposure_filenames =
  0..15
  |> Enum.map(&Integer.to_string(&1))
  |> Enum.map(&String.pad_leading(&1, 2, "0"))
  |> Enum.map(&amp;("memorial" <> &amp;1 <> ".png"))

exposure_file_urls =
  exposure_filenames
  |> Enum.map(
    &amp;("https://raw.githubusercontent.com/opencv/opencv_extra/4.x/testdata/cv/hdr/exposures/" <> &amp;1)
  )

exposure_file_save_paths =
  exposure_filenames
  |> Enum.map(&amp;Path.join([__DIR__, "photo_hdr_test", &amp;1]))

exposure_file_urls
|> Enum.zip(exposure_file_save_paths)
|> Enum.map(fn {url, save_as} -> Helper.download!(url, save_as) end)
|> Enum.all?(&amp;(:ok = &amp;1))

true

Download list.txt that Stores the Exposure Sequences

list_txt_file = Path.join([__DIR__, "photo_hdr_test", "list.txt"])

Helper.download!(
  "https://raw.githubusercontent.com/opencv/opencv_extra/4.x/testdata/cv/hdr/exposures/list.txt",
  list_txt_file
)

:ok

Load the Exposure Sequences

# Firstly we load input images and exposure times from user-defined folder.
# The folder should contain images and list.txt - file that contains file names and inverse exposure times.
exposure_sequences =
  list_txt_file
  |> File.read!()
  |> String.split("\n")
  |> Enum.reject(&(String.length(&1) == 0))
  |> Enum.map(&String.split(&1, " "))
  |> Enum.map(&List.to_tuple(&1))
  |> Enum.map(fn {image_filename, times} ->
    mat = Cv.imread(Path.join([__DIR__, "photo_hdr_test", image_filename]))
    {val, ""} = Float.parse(times)
    {mat, 1 / val}
  end)

images =
  exposure_sequences
  |> Enum.map(&elem(&1, 0))

# `times` HAS to be float32, otherwise OpenCV will crash
times =
  exposure_sequences
  |> Enum.map(&elem(&1, 1))
  |> Enum.into(<<>>, fn d -> <> end)
  |> Cv.Mat.from_binary_by_shape({:f, 32}, {1, Enum.count(images)})

%Evision.Mat{
  channels: 1,
  dims: 2,
  type: {:f, 32},
  raw_type: 5,
  shape: {1, 16},
  ref: #Reference<0.741743058.1175846932.38571>
}

Estimate Camera Response

# It is necessary to know camera response function (CRF) for a lot of HDR construction algorithms.
# We use one of the calibration algorithms to estimate inverse CRF for all 256 pixel values.
calibrate = Cv.createCalibrateDebevec()
response = Cv.CalibrateDebevec.process(calibrate, images, times)

%Evision.Mat{
  channels: 3,
  dims: 2,
  type: {:f, 32},
  raw_type: 21,
  shape: {256, 1, 3},
  ref: #Reference<0.741743058.1175846932.38575>
}

Process and Get the HDR Image

# We use Debevec's weighting scheme to construct HDR image
# using response calculated in the previous item.
merge_debevec = Cv.createMergeDebevec()
hdr = Cv.MergeDebevec.process(merge_debevec, images, times, response: response)

%Evision.Mat{
  channels: 3,
  dims: 2,
  type: {:f, 32},
  raw_type: 21,
  shape: {714, 484, 3},
  ref: #Reference<0.741743058.1175846932.38579>
}

Tonemap the HDR image

# Since we want to see our results on common LDR display we have to map our HDR image to 8-bit range
# preserving most details.
# It is the main goal of tonemapping methods.
# We use tonemapper with bilateral filtering and set 2.2 as the value for gamma correction.
tonemap = Cv.createTonemap(gamma: 2.2)
ldr = Cv.Tonemap.process(tonemap, hdr)

%Evision.Mat{
  channels: 3,
  dims: 2,
  type: {:f, 32},
  raw_type: 21,
  shape: {714, 484, 3},
  ref: #Reference<0.741743058.1175846932.38583>
}

Perform Exposure Fusions

# There is an alternative way to merge our exposures in case when we don't need HDR image.
# This process is called exposure fusion and produces LDR image that doesn't require gamma correction.
# It also doesn't use exposure values of the photographs.
merge_mertens = Cv.createMergeMertens()
fusion = Cv.MergeMertens.process(merge_mertens, images)

%Evision.Mat{
  channels: 3,
  dims: 2,
  type: {:f, 32},
  raw_type: 21,
  shape: {714, 484, 3},
  ref: #Reference<0.741743058.1175846932.38587>
}

Write Fusion

output_fusion_file = Path.join([__DIR__, "photo_hdr_test", "fusion.png"])

result =
  fusion
  |> Cv.Mat.to_nx(Nx.BinaryBackend)
  |> Nx.multiply(255)
  |> Nx.clip(0, 255)
  |> Nx.as_type({:u, 8})
  |> Cv.Mat.from_nx_2d()
  |> then(fn result ->
    Cv.imwrite(output_fusion_file, result)
    result
  end)

%Evision.Mat{
  channels: 3,
  dims: 2,
  type: {:u, 8},
  raw_type: 16,
  shape: {714, 484, 3},
  ref: #Reference<0.741743058.1175846932.38590>
}

result = Cv.imencode(".png", result)

Kino.Image.new(result, :png)

Write LDR Image

output_ldr_file = Path.join([__DIR__, "photo_hdr_test", "ldr.png"])
f32_shape = Cv.Mat.shape(ldr)
nan = <<0, 0, 192, 255>>
positive_inf = <<0, 0, 128, 127>>
negative_inf = <<0, 0, 128, 255>>

result =
  ldr
  |> Cv.Mat.to_binary()
  |> Helper.chunk_binary(4)
  |> Enum.map(fn f32 ->
    case f32 do
      ^nan ->
        <<0, 0, 0, 0>>

      ^positive_inf ->
        <<0, 0, 0, 0>>

      ^negative_inf ->
        <<0, 0, 0, 0>>

      # legal value
      _ ->
        f32
    end
  end)
  |> IO.iodata_to_binary()
  |> Nx.from_binary({:f, 32})
  |> Nx.reshape(f32_shape)
  |> Nx.multiply(255)
  |> Nx.clip(0, 255)
  |> Nx.as_type({:u, 8})
  |> Cv.Mat.from_nx_2d()

result = Cv.imencode(".png", result)

Kino.Image.new(result, :png)

Write HDR Image

output_hdr_file = Path.join([__DIR__, "photo_hdr_test", "hdr.hdr"])
Cv.imwrite(output_hdr_file, hdr)

true