Object Detection: YOLOv8

File.cd!(__DIR__)
# for windows JP
# System.shell("chcp 65001")

Mix.install([
  {:onnx_interp, "~> 0.1.8"},
  {:cimg, "~> 0.1.14"},
  {:postdnn, "~> 0.1.5"},
  {:kino, "~> 0.7.0"}
])

0.Original work

Ultralytics YOLOv8

• https://github.com/ultralytics/ultralytics

Thanks a lot!!!

Implementation with OnnxInterp in Elixir

1.Defining the inference module: YOLOv8

• Model
  yolov8n.onnx

• Pre-processing
  Resize the input image to the size {640,640}, normalize to {0.0,1.0} and transpose NCHW.

• Post-processing
  Filtering Boxes and Scores(output[0]) with Multi-class Non Maximum Suppression.

defmodule YOLOv8 do
  @moduledoc """
  Original work:
    Ultralytics YOLOv8 - https://github.com/ultralytics/ultralytics
  """

  @width 640
  @height 640

  alias OnnxInterp, as: NNInterp

  use NNInterp,
    label: "./model/coco.label",
    model: "./model/yolov8n.onnx",
    url: "https://github.com/shoz-f/onnx_interp/releases/download/models/yolov8n.onnx",
    inputs: [f32: {1, 3, @width, @height}],
    outputs: [f32: {1, 84, 8400}]

  def apply(img) do
    # preprocess
    input0 =
      CImg.builder(img)
      |> CImg.resize({@width, @height})
      |> CImg.to_binary([{:range, {0.0, 1.0}}, :nchw])

    # prediction
    output0 =
      session()
      |> NNInterp.set_input_tensor(0, input0)
      |> NNInterp.invoke()
      |> NNInterp.get_output_tensor(0)
      |> Nx.from_binary(:f32)
      |> Nx.reshape({84, 8400})

    # postprocess
    scores = Nx.transpose(output0[4..-1//1])
    boxes = Nx.transpose(output0[0..3])

    NNInterp.non_max_suppression_multi_class(
      __MODULE__,
      Nx.shape(scores),
      Nx.to_binary(boxes),
      Nx.to_binary(scores)
    )
    |> ratio_box()
  end

  defp ratio_box({:ok, result}) do
    clamp = fn x -> min(max(x, 0.0), 1.0) end

    {
      :ok,
      Enum.reduce(Map.keys(result), result, fn key, map ->
        Map.update!(
          map,
          key,
          &Enum.map(&1, fn [score, x1, y1, x2, y2, index] ->
            [
              score,
              clamp.(x1 / @width),
              clamp.(y1 / @height),
              clamp.(x2 / @width),
              clamp.(y2 / @height),
              index
            ]
          end)
        )
      end)
    }
  end
end

Launch YOLOv8.

# OnnxInterp.stop(YOLOv8)
YOLOv8.start_link([])

Display the properties of the YOLOv8 model.

OnnxInterp.info(YOLOv8)

2.Defining execution module LiveYOLOv8

defmodule LiveYOLOv8 do
  @palette CImg.Util.rand_palette("./model/coco.label")

  def run(path) do
    img = CImg.load(path)

    with {:ok, res} <- YOLOv8.apply(img) do
      IO.inspect(res)

      Enum.reduce(res, CImg.builder(img), &amp;draw_item(&amp;1, &amp;2))
      |> CImg.display_kino(:jpeg)
    end
  end

  defp draw_item({name, boxes}, canvas) do
    color = @palette[name]

    Enum.reduce(boxes, canvas, fn [_score, x1, y1, x2, y2, _index], canvas ->
      [x1, y1, x2, y2] = PostDNN.clamp([x1, y1, x2, y2], {0.0, 1.0})

      CImg.fill_rect(canvas, x1, y1, x2, y2, color, 0.4)
    end)
  end
end

3.Let’s try it

LiveYOLOv8.run("bus.jpg")

Appendix

□