Powered by AppSignal & Oban Pro

Evision.DNN Example - Object Detection with SSD MobileNet v2

examples/dnn-detection-model.livemd

Evision.DNN Example - Object Detection with SSD MobileNet v2

# set `EVISION_PREFER_PRECOMPILED` to `false`
# if you prefer `:evision` to be compiled from source
# note that to compile from source, you may need at least 1GB RAM
# System.put_env("EVISION_PREFER_PRECOMPILED", "false")

Mix.install([
  {:evision, "~> 1.0"},
  {:req, "~> 0.5"},
  {:kino, "~> 0.7"}
], system_env: [
  # optional, defaults to `true`
  # set `EVISION_PREFER_PRECOMPILED` to `false`
  # if you prefer `:evision` to be compiled from source
  # note that to compile from source, you may need at least 1GB RAM
  {"EVISION_PREFER_PRECOMPILED", true},

  # optional, defaults to `true`
  # set `EVISION_ENABLE_CONTRIB` to `false`
  # if you don't need modules from `opencv_contrib`
  {"EVISION_ENABLE_CONTRIB", true},

  # optional, defaults to `false`
  # set `EVISION_ENABLE_CUDA` to `true`
  # if you wish to use CUDA related functions
  # note that `EVISION_ENABLE_CONTRIB` also has to be `true`
  # because cuda related modules come from the `opencv_contrib` repo
  {"EVISION_ENABLE_CUDA", false},

  # required when
  # - `EVISION_ENABLE_CUDA` is `true`
  # - and `EVISION_PREFER_PRECOMPILED` is `true`
  #
  # set `EVISION_CUDA_VERSION` to the version that matches
  # your local CUDA runtime version
  #
  # current available versions are
  # - 118
  # - 121
  {"EVISION_CUDA_VERSION", "118"},

  # require for Windows users when
  # - `EVISION_ENABLE_CUDA` is `true`
  # set `EVISION_CUDA_RUNTIME_DIR` to the directory that contains
  # CUDA runtime libraries
  {"EVISION_CUDA_RUNTIME_DIR", "C:/PATH/TO/CUDA/RUNTIME"}
])
:ok

Define Some Helper Functions

defmodule Helper do
  def download!(url, save_as) do
    unless File.exists?(save_as) do
      Req.get!(url, http_errors: :raise, into: File.stream!(save_as))
    end

    :ok
  end
end
{:module, Helper, <<70, 79, 82, 49, 0, 0, 10, ...>>, {:download!, 3}}

Write the Detection Module

The model is OpenCV's ONNX export of TensorFlow's ssd_mobilenet_v2_coco_2018_03_29. It takes a 300x300 RGB uint8 image in NHWC layout and returns four tensors: normalized {y1, x1, y2, x2} boxes, scores, 1-based COCO class ids, and the number of detections.

# change to the file's directory
# or somewhere you have write permission
File.cd!(__DIR__)

defmodule SSDMobileNetV2 do
  @model_url "https://huggingface.co/opencv/opencv_contribution/resolve/main/ssd_mobilenet_v2_coco_2018_03_29/ssd_mobilenet_v2_coco_2018_03_29_2026jul.onnx?download=true"
  @labels_url "https://raw.githubusercontent.com/cocoa-xu/evision/main/test/testdata/models/coco_names.txt"
  @out_names [
    "detection_boxes:0",
    "detection_scores:0",
    "detection_classes:0",
    "num_detections:0"
  ]

  def load do
    Helper.download!(@model_url, "ssd_mobilenet_v2_coco_2018_03_29.onnx")
    Helper.download!(@labels_url, "coco_names.txt")

    net = Evision.DNN.readNet("ssd_mobilenet_v2_coco_2018_03_29.onnx")
    labels = "coco_names.txt" |> File.read!() |> String.split("\n")
    {net, labels}
  end

  def detect({net, labels}, image, confidence_threshold \\ 0.5) do
    {h, w, _} = image.shape

    input =
      image
      |> Evision.cvtColor(Evision.Constant.cv_COLOR_BGR2RGB())
      |> Evision.resize({300, 300})
      |> Evision.Mat.to_binary()
      |> Evision.Mat.from_binary_by_shape({:u, 8}, {1, 300, 300, 3})

    [boxes, scores, classes, count] =
      net
      |> Evision.DNN.Net.setInput(input)
      |> Evision.DNN.Net.forward(outBlobNames: @out_names)
      |> Enum.map(&Evision.Mat.to_nx(&1, Nx.BinaryBackend))

    count = count |> Nx.to_flat_list() |> hd() |> trunc()
    boxes = boxes |> Nx.reshape({:auto, 4}) |> Nx.to_list()

    [boxes, Nx.to_flat_list(scores), Nx.to_flat_list(classes)]
    |> Enum.zip()
    |> Enum.take(count)
    |> Enum.filter(fn {_box, score, _class} -> score > confidence_threshold end)
    |> Enum.map(fn {[y1, x1, y2, x2], score, class} ->
      {Enum.at(labels, trunc(class) - 1), score,
       {trunc(x1 * w), trunc(y1 * h), trunc(x2 * w), trunc(y2 * h)}}
    end)
  end

  def draw(image, detections) do
    Enum.reduce(detections, image, fn {label, score, {l, t, r, b}}, image ->
      text = "#{label}: #{Float.round(score, 2)}"

      image
      |> Evision.rectangle({l, t}, {r, b}, {255, 0, 0}, thickness: 2)
      |> Evision.putText(
        text,
        {l, max(t - 5, 12)},
        Evision.Constant.cv_FONT_HERSHEY_SIMPLEX(),
        0.5,
        {0, 0, 255}
      )
    end)
  end
end
{:module, SSDMobileNetV2, <<70, 79, 82, 49, 0, 0, 22, ...>>, {:draw, 2}}

Detect Objects in An Image

Helper.download!(
  "https://raw.githubusercontent.com/cocoa-xu/evision/main/test/testdata/dnn_detection_test.jpg",
  "dnn_detection_test.jpg"
)

model = SSDMobileNetV2.load()
image = Evision.imread("dnn_detection_test.jpg")
detections = SSDMobileNetV2.detect(model, image)
[
  {"car", 0.9736444354057312, {464, 205, 602, 339}},
  {"car", 0.8993234634399414, {280, 214, 480, 352}},
  {"car", 0.882705807685852, {557, 208, 803, 364}},
  {"car", 0.6523396968841553, {765, 198, 872, 350}},
  {"person", 0.5624266862869263, {157, 133, 301, 372}},
  {"car", 0.5576940774917603, {99, 162, 212, 316}}
]
image
|> SSDMobileNetV2.draw(detections)
|> then(&Evision.imencode(".png", &1))
|> Kino.Image.new(:png)

Detect Objects in a Video Stream

# if you have a camera available
# uncomment the line below
# video = Evision.VideoCapture.videoCapture(0)

# or the OpenCV library you compiled can decode video files
# uncomment the line below
# video = Evision.VideoCapture.videoCapture("/path/to/your/video/file")
nil
defmodule VideoDetection do
  def detect(video, widget, max_frames \\ 30 * 60) when is_integer(max_frames) do
    loop(SSDMobileNetV2.load(), video, widget, max_frames)
  end

  defp loop(_model, _video, _widget, 0), do: :ok

  defp loop(model, video, widget, frames_left) do
    case Evision.VideoCapture.read(video) do
      %Evision.Mat{} = frame ->
        frame
        |> SSDMobileNetV2.draw(SSDMobileNetV2.detect(model, frame))
        |> then(&Evision.imencode(".png", &1))
        |> Kino.Image.new(:png)
        |> then(&Kino.Frame.render(widget, &1))

        loop(model, video, widget, frames_left - 1)

      _ ->
        :ok
    end
  end
end
{:module, VideoDetection, <<70, 79, 82, 49, 0, 0, 12, ...>>, {:loop, 4}}

Run the detection

widget = Kino.Frame.new() |> Kino.render()
# read 1,800 frames at most
# change the number to negative values to
# detect objects until OpenCV cannot read
# new frame from the video stream
VideoDetection.detect(video, widget, 1800)
:ok