Evision.DNN Example - Object Detection with SSD MobileNet v2
# set `EVISION_PREFER_PRECOMPILED` to `false`
# if you prefer `:evision` to be compiled from source
# note that to compile from source, you may need at least 1GB RAM
# System.put_env("EVISION_PREFER_PRECOMPILED", "false")
Mix.install([
{:evision, "~> 1.0"},
{:req, "~> 0.5"},
{:kino, "~> 0.7"}
], system_env: [
# optional, defaults to `true`
# set `EVISION_PREFER_PRECOMPILED` to `false`
# if you prefer `:evision` to be compiled from source
# note that to compile from source, you may need at least 1GB RAM
{"EVISION_PREFER_PRECOMPILED", true},
# optional, defaults to `true`
# set `EVISION_ENABLE_CONTRIB` to `false`
# if you don't need modules from `opencv_contrib`
{"EVISION_ENABLE_CONTRIB", true},
# optional, defaults to `false`
# set `EVISION_ENABLE_CUDA` to `true`
# if you wish to use CUDA related functions
# note that `EVISION_ENABLE_CONTRIB` also has to be `true`
# because cuda related modules come from the `opencv_contrib` repo
{"EVISION_ENABLE_CUDA", false},
# required when
# - `EVISION_ENABLE_CUDA` is `true`
# - and `EVISION_PREFER_PRECOMPILED` is `true`
#
# set `EVISION_CUDA_VERSION` to the version that matches
# your local CUDA runtime version
#
# current available versions are
# - 118
# - 121
{"EVISION_CUDA_VERSION", "118"},
# require for Windows users when
# - `EVISION_ENABLE_CUDA` is `true`
# set `EVISION_CUDA_RUNTIME_DIR` to the directory that contains
# CUDA runtime libraries
{"EVISION_CUDA_RUNTIME_DIR", "C:/PATH/TO/CUDA/RUNTIME"}
])
:ok
Define Some Helper Functions
defmodule Helper do
def download!(url, save_as) do
unless File.exists?(save_as) do
Req.get!(url, http_errors: :raise, into: File.stream!(save_as))
end
:ok
end
end
{:module, Helper, <<70, 79, 82, 49, 0, 0, 10, ...>>, {:download!, 3}}
Write the Detection Module
The model is OpenCV's ONNX export of TensorFlow's ssd_mobilenet_v2_coco_2018_03_29. It takes a 300x300 RGB uint8 image in NHWC layout and returns four tensors: normalized {y1, x1, y2, x2} boxes, scores, 1-based COCO class ids, and the number of detections.
# change to the file's directory
# or somewhere you have write permission
File.cd!(__DIR__)
defmodule SSDMobileNetV2 do
@model_url "https://huggingface.co/opencv/opencv_contribution/resolve/main/ssd_mobilenet_v2_coco_2018_03_29/ssd_mobilenet_v2_coco_2018_03_29_2026jul.onnx?download=true"
@labels_url "https://raw.githubusercontent.com/cocoa-xu/evision/main/test/testdata/models/coco_names.txt"
@out_names [
"detection_boxes:0",
"detection_scores:0",
"detection_classes:0",
"num_detections:0"
]
def load do
Helper.download!(@model_url, "ssd_mobilenet_v2_coco_2018_03_29.onnx")
Helper.download!(@labels_url, "coco_names.txt")
net = Evision.DNN.readNet("ssd_mobilenet_v2_coco_2018_03_29.onnx")
labels = "coco_names.txt" |> File.read!() |> String.split("\n")
{net, labels}
end
def detect({net, labels}, image, confidence_threshold \\ 0.5) do
{h, w, _} = image.shape
input =
image
|> Evision.cvtColor(Evision.Constant.cv_COLOR_BGR2RGB())
|> Evision.resize({300, 300})
|> Evision.Mat.to_binary()
|> Evision.Mat.from_binary_by_shape({:u, 8}, {1, 300, 300, 3})
[boxes, scores, classes, count] =
net
|> Evision.DNN.Net.setInput(input)
|> Evision.DNN.Net.forward(outBlobNames: @out_names)
|> Enum.map(&Evision.Mat.to_nx(&1, Nx.BinaryBackend))
count = count |> Nx.to_flat_list() |> hd() |> trunc()
boxes = boxes |> Nx.reshape({:auto, 4}) |> Nx.to_list()
[boxes, Nx.to_flat_list(scores), Nx.to_flat_list(classes)]
|> Enum.zip()
|> Enum.take(count)
|> Enum.filter(fn {_box, score, _class} -> score > confidence_threshold end)
|> Enum.map(fn {[y1, x1, y2, x2], score, class} ->
{Enum.at(labels, trunc(class) - 1), score,
{trunc(x1 * w), trunc(y1 * h), trunc(x2 * w), trunc(y2 * h)}}
end)
end
def draw(image, detections) do
Enum.reduce(detections, image, fn {label, score, {l, t, r, b}}, image ->
text = "#{label}: #{Float.round(score, 2)}"
image
|> Evision.rectangle({l, t}, {r, b}, {255, 0, 0}, thickness: 2)
|> Evision.putText(
text,
{l, max(t - 5, 12)},
Evision.Constant.cv_FONT_HERSHEY_SIMPLEX(),
0.5,
{0, 0, 255}
)
end)
end
end
{:module, SSDMobileNetV2, <<70, 79, 82, 49, 0, 0, 22, ...>>, {:draw, 2}}
Detect Objects in An Image
Helper.download!(
"https://raw.githubusercontent.com/cocoa-xu/evision/main/test/testdata/dnn_detection_test.jpg",
"dnn_detection_test.jpg"
)
model = SSDMobileNetV2.load()
image = Evision.imread("dnn_detection_test.jpg")
detections = SSDMobileNetV2.detect(model, image)
[
{"car", 0.9736444354057312, {464, 205, 602, 339}},
{"car", 0.8993234634399414, {280, 214, 480, 352}},
{"car", 0.882705807685852, {557, 208, 803, 364}},
{"car", 0.6523396968841553, {765, 198, 872, 350}},
{"person", 0.5624266862869263, {157, 133, 301, 372}},
{"car", 0.5576940774917603, {99, 162, 212, 316}}
]
image
|> SSDMobileNetV2.draw(detections)
|> then(&Evision.imencode(".png", &1))
|> Kino.Image.new(:png)
Detect Objects in a Video Stream
# if you have a camera available
# uncomment the line below
# video = Evision.VideoCapture.videoCapture(0)
# or the OpenCV library you compiled can decode video files
# uncomment the line below
# video = Evision.VideoCapture.videoCapture("/path/to/your/video/file")
nil
defmodule VideoDetection do
def detect(video, widget, max_frames \\ 30 * 60) when is_integer(max_frames) do
loop(SSDMobileNetV2.load(), video, widget, max_frames)
end
defp loop(_model, _video, _widget, 0), do: :ok
defp loop(model, video, widget, frames_left) do
case Evision.VideoCapture.read(video) do
%Evision.Mat{} = frame ->
frame
|> SSDMobileNetV2.draw(SSDMobileNetV2.detect(model, frame))
|> then(&Evision.imencode(".png", &1))
|> Kino.Image.new(:png)
|> then(&Kino.Frame.render(widget, &1))
loop(model, video, widget, frames_left - 1)
_ ->
:ok
end
end
end
{:module, VideoDetection, <<70, 79, 82, 49, 0, 0, 12, ...>>, {:loop, 4}}
Run the detection
widget = Kino.Frame.new() |> Kino.render()
# read 1,800 frames at most
# change the number to negative values to
# detect objects until OpenCV cannot read
# new frame from the video stream
VideoDetection.detect(video, widget, 1800)
:ok