Monocular Depth Estimation: SC-DepthV3

File.cd!(__DIR__)
# for windows JP
# System.shell("chcp 65001")

Mix.install([
  {:onnx_interp, "~> 0.1.8"},
  {:cimg, "~> 0.1.18"},
  {:nx, "~> 0.4.2"},
  {:kino, "~> 0.8.0"}
])

0.Original work

SC-DepthV3: Robust Self-supervised Monocular Depth Estimation for Dynamic Scenes

• https://arxiv.org/abs/2211.03660

GitHub: SC_Depth

• https://github.com/JiawangBian/sc_depth_pl

This note usee the pretraind model converted from above project ;-)

Thanks a lot!!!

Implementation with OnnxInterp in Elixir

1.Defining the inference module: ScDepth

• Model
  sc_depth-epoch=99-val_loss=0.1438.onnx

• Pre-processing
  Resize the input image to the size {640,384}, gauss {{114.75,57.375},{114.75,57.375},{114.75,57.375}}} and transpose NCHW.

• Post-processing
  normalize depth map and color-mapping.

defmodule ScDepth do
  @width 640
  @height 384

  alias OnnxInterp, as: NNInterp

  use NNInterp,
    model: "model/sc_depth-epoch=99-val_loss=0.1438.onnx",
    url:
      "https://github.com/shoz-f/onnx_interp/releases/download/models/sc_depth-epoch.99-val_loss.0.1438.onnx",
    inputs: [f32: {1, 3, @height, @width}],
    outputs: [f32: {1, 1, @height, @width}]

  def apply(img) do
    # preprocess
    input0 =
      CImg.builder(img)
      |> CImg.resize({@width, @height})
      |> CImg.to_binary([{:gauss, {{114.75, 57.375}, {114.75, 57.375}, {114.75, 57.375}}}, :nchw])

    # prediction
    output0 =
      session()
      |> NNInterp.set_input_tensor(0, input0)
      |> NNInterp.invoke()
      |> NNInterp.get_output_tensor(0)

    # postprocess
    {w, h, _, _} = CImg.shape(img)

    output0
    |> CImg.from_binary(@width, @height, 1, 1, range: min_max(output0), dtype: " CImg.resize({w, h})
  end

  defp min_max(bin) do
    t = Nx.from_binary(bin, :f32)

    {
      Nx.reduce_min(t) |> Nx.to_number(),
      Nx.reduce_max(t) |> Nx.to_number()
    }
  end
end

Launch ScDepth.

OnnxInterp.stop(ScDepth)
ScDepth.start_link([])

Display the properties of the ScDepth model.

OnnxInterp.info(ScDepth)

2.Defining execution module ScDepth

defmodule LiveScDepth do
  def run(path) do
    img = CImg.load(path)

    depth =
      ScDepth.apply(img)
      |> CImg.color_mapping(:jet)

    Kino.Layout.grid(
      Enum.map([img, depth], &CImg.display_kino(&1, :jpeg)),
      columns: 2
    )
  end
end

3.Let’s try it

LiveScDepth.run("sample.jpg")

Appendix

[1] export_onnx.py: python script to convert Pytorch checkpoint to ONNX.

import torch
import torch.onnx
from path import Path
import os

from config import get_opts, get_training_size

from SC_Depth import SC_Depth
from SC_DepthV2 import SC_DepthV2
from SC_DepthV3 import SC_DepthV3


@torch.no_grad()
def main():
    hparams = get_opts()

    if hparams.model_version == 'v1':
        system = SC_Depth(hparams)
    elif hparams.model_version == 'v2':
        system = SC_DepthV2(hparams)
    elif hparams.model_version == 'v3':
        system = SC_DepthV3(hparams)

    output_dir = Path(hparams.output_dir)
    output_dir.makedirs_p()

    system = system.load_from_checkpoint(hparams.ckpt_path, strict=False)

    model = system.depth_net
    model.eval()

    # training size
    training_size = get_training_size(hparams.dataset_name)
    dummy_input = torch.randn(1, 3, *training_size)

    # export the model
    torch.onnx.export(model,
        dummy_input,
        output_dir / Path(hparams.ckpt_path).stem + ".onnx",
        export_params=True,
        #opset_version=10,
        do_constant_folding=True,
        input_names=["input.0"],
        output_names=["output.0"],
        #dynamic_axes={}
        )


if __name__ == '__main__':
    main()

□