Driving Perception: YOLOP
File.cd!(__DIR__)
# for windows JP
System.shell("chcp 65001")
Mix.install([
{:onnx_interp, path: ".."},
{:cimg, "~> 0.1.18"},
{:nx, "~> 0.4.1"},
{:kino, "~> 0.8.0"}
])
0.Original work
-
YOLOP: You Only Look Once for Panoptic Driving Perception
https://arxiv.org/abs/2108.11250
1.Implementation with OnnxInterp in Elixir
defmodule YOLOP do
@width 640
@height 640
alias OnnxInterp, as: NNInterp
use OnnxInterp,
model: "model/yolop-640-640.onnx",
url: "https://github.com/hustvl/YOLOP/raw/main/weights/yolop-640-640.onnx",
inputs: [f32: {1, 3, @height, @width}],
outputs: [f32: {1, 25200, 6}, f32: {1, 2, @height, @width}, f32: {1, 2, @height, @width}]
def apply(img) do
# preprocess
input0 =
img
|> CImg.resize({@width, @height})
|> CImg.to_binary([{:gauss, {{103.53, 57.375}, {116.28, 57.12}, {123.675, 58.395}}}, :nchw])
# prediction
outputs =
session()
|> OnnxInterp.set_input_tensor(0, input0)
|> OnnxInterp.invoke()
det_out =
OnnxInterp.get_output_tensor(outputs, 0) |> Nx.from_binary(:f32) |> Nx.reshape({:auto, 6})
drive_area_seg =
OnnxInterp.get_output_tensor(outputs, 1) |> Nx.from_binary(:f32) |> Nx.reshape({2, :auto})
lane_line_seg =
OnnxInterp.get_output_tensor(outputs, 2) |> Nx.from_binary(:f32) |> Nx.reshape({2, :auto})
# postprocess
img_size = CImg.shape(img) |> then(fn {w, h, _, _} -> {w, h} end)
{
:ok,
decode_bbox(det_out, img_size),
decode_segments(drive_area_seg, img_size),
decode_segments(lane_line_seg, img_size)
}
end
def decode_bbox(t, {w, h}) do
boxes = Nx.slice_along_axis(t, 0, 4, axis: 1)
scores =
Nx.multiply(Nx.slice_along_axis(t, 4, 1, axis: 1), Nx.slice_along_axis(t, 5, 1, axis: 1))
{:ok, res} =
NNInterp.non_max_suppression_multi_class(
__MODULE__,
Nx.shape(scores),
Nx.to_binary(boxes),
Nx.to_binary(scores),
iou_threshold: 0.45,
score_threshold: 0.25
)
scale_x = fn x -> round(x * w / @width) end
scale_y = fn y -> round(y * h / @height) end
Enum.map(res["0"], fn [score, x1, y1, x2, y2, index] ->
[score, scale_x.(x1), scale_y.(y1), scale_x.(x2), scale_y.(y2), index]
end)
end
defp decode_segments(t, img_size) do
Nx.greater(t[1], t[0])
|> Nx.to_binary()
|> CImg.from_binary(640, 640, 1, 1, dtype: " CImg.resize(img_size)
end
end
Launch YOLOP
.
YOLOP.start_link([])
2.Let’s try it
defmodule DemoYOLOP do
def run(path) do
img = CImg.load(path)
with {:ok, cars, drive_area, lane_line} = YOLOP.apply(img) do
CImg.builder(img)
|> CImg.paint_mask(drive_area, {255, 255, 0})
|> CImg.paint_mask(lane_line, {255, 0, 0})
|> draw_item(cars, {0, 192, 0})
|> CImg.display_kino(:jpeg)
end
end
defp draw_item(canvas, bbox, color) do
Enum.reduce(bbox, canvas, fn [_score, x1, y1, x2, y2, _index], canvas ->
CImg.fill_rect(canvas, x1, y1, x2, y2, color, 0.3)
end)
end
end
DemoYOLOP.run("01.jpg")
3.TIL ;-)
Appendix
□