YoloV8 on a single image

Mix.install([
  {:yolo, ">= 0.0.0"},
  {:nx, "~> 0.9.1"},
  {:exla, "~> 0.9.0"},
  {:kino, "~> 0.14.0"},
  {:image, "~> 0.54"},
  {:evision, "~> 0.2"}
],[
  config: [
    nx: [default_backend: EXLA.Backend]
  ]
])

Define Paths

image_path = Kino.FS.file_path("traffic.jpg")
model_path = Kino.FS.file_path("yolov8n.onnx")
classes_path = Kino.FS.file_path("yolov8n_classes.json")
:ok

:ok

Load image

mat = Evision.imread(image_path)

%Evision.Mat{
  channels: 3,
  dims: 2,
  type: {:u, 8},
  raw_type: 16,
  shape: {1080, 1920, 3},
  ref: #Reference<0.133361986.436862999.157550>
}

Load and run YoloV8n

model = YOLO.load(model_path: model_path, classes_path: classes_path)

detected_objects = 
  model
  |> YOLO.detect(mat, iou_threshold: 0.45, prob_threshold: 0.25)
  |> YOLO.to_detected_objects(model.classes)

[
  %{
    class: "person",
    prob: 0.2571249008178711,
    bbox: %{h: 237, w: 22, cx: 1908, cy: 468},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.3452834486961365,
    bbox: %{h: 119, w: 45, cx: 697, cy: 396},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.3655376434326172,
    bbox: %{h: 213, w: 98, cx: 674, cy: 862},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.37552228569984436,
    bbox: %{h: 79, w: 33, cx: 867, cy: 208},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.4229397177696228,
    bbox: %{h: 108, w: 38, cx: 1500, cy: 364},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.43063610792160034,
    bbox: %{h: 159, w: 55, cx: 1770, cy: 501},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.4697304964065552,
    bbox: %{h: 117, w: 95, cx: 730, cy: 651},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.5778378248214722,
    bbox: %{h: 117, w: 46, cx: 556, cy: 403},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.6154893040657043,
    bbox: %{h: 150, w: 72, cx: 47, cy: 525},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.6168261766433716,
    bbox: %{h: 161, w: 53, cx: 800, cy: 579},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.6266347169876099,
    bbox: %{h: 136, w: 44, cx: 1614, cy: 419},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.6296634078025818,
    bbox: %{h: 156, w: 49, cx: 1686, cy: 489},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.655760645866394,
    bbox: %{h: 113, w: 40, cx: 1531, cy: 416},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.6911468505859375,
    bbox: %{h: 237, w: 79, cx: 39, cy: 799},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.7227481603622437,
    bbox: %{h: 152, w: 59, cx: 298, cy: 557},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.742075502872467,
    bbox: %{h: 225, w: 89, cx: 468, cy: 848},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.7557895183563232,
    bbox: %{h: 214, w: 80, cx: 609, cy: 776},
    class_idx: 0
  },
  %{
    class: "person",
    prob: 0.7682790756225586,
    bbox: %{h: 146, w: 69, cx: 699, cy: 578},
    class_idx: 0
  },
  %{
    class: "bicycle",
    prob: 0.3721795678138733,
    bbox: %{h: 123, w: 63, cx: 461, cy: 977},
    class_idx: 1
  },
  %{
    class: "bicycle",
    prob: 0.47977668046951294,
    bbox: %{h: 106, w: 71, cx: 723, cy: 738},
    class_idx: 1
  },
  %{
    class: "bicycle",
    prob: 0.500577449798584,
    bbox: %{h: 127, w: 56, cx: 590, cy: 862},
    class_idx: 1
  },
  %{class: "car", prob: 0.35978999733924866, bbox: %{h: 66, w: 73, cx: 1169, cy: 145}, class_idx: 2},
  %{class: "car", prob: 0.39499765634536743, bbox: %{h: 84, w: 93, cx: 1028, cy: 211}, class_idx: 2},
  %{
    class: "car",
    prob: 0.42516815662384033,
    bbox: %{h: 81, w: 100, cx: 1203, cy: 243},
    class_idx: 2
  },
  %{class: "car", prob: 0.6461262106895447, bbox: %{h: 91, w: 102, cx: 1037, cy: 268}, class_idx: 2},
  %{
    class: "car",
    prob: 0.6885334253311157,
    bbox: %{h: 108, w: 124, cx: 1251, cy: 326},
    class_idx: 2
  },
  %{
    class: "car",
    prob: 0.6895469427108765,
    bbox: %{h: 114, w: 141, cx: 1305, cy: 403},
    class_idx: 2
  },
  %{
    class: "truck",
    prob: 0.4962051510810852,
    bbox: %{h: 246, w: 191, cx: 1100, cy: 473},
    class_idx: 7
  },
  %{
    class: "traffic light",
    prob: 0.3155118525028229,
    bbox: %{h: 100, w: 49, cx: 194, cy: 398},
    class_idx: 9
  },
  %{
    class: "traffic light",
    prob: 0.4702877998352051,
    bbox: %{h: 140, w: 76, cx: 1334, cy: 103},
    class_idx: 9
  }
]

Showing Detected Objects

Just run the cell below to have the YOLODraw util.

##
defmodule YOLODraw do
  @font_size 18
  @stroke_width 3
  def draw_detected_objects(mat, detected_objects) do
    {:ok, image} = Image.from_evision(mat)

    detected_objects
    |> Enum.reduce(image, fn %{bbox: bbox}=od, image ->
      left = max(round(bbox.cx - bbox.w/2), 0)
      top = max(round(bbox.cy - bbox.h/2), 0)
      prob = round(od.prob * 100)
      color = class_color(od.class_idx)
      
      text_image = 
        Image.Text.simple_text!("#{od.class} #{prob}%", text_fill_color: "white", font_size: @font_size)
        |> Image.Text.add_background_padding!(background_fill_color: color, padding: [5, 5])
        |> Image.Text.add_background!(background_fill_color: color)
        |> Image.split_alpha()
        |> elem(0)
      {_, text_hight, _} = Image.shape(text_image)
      
      image
      |> Image.Draw.rect!(left,top,bbox.w,bbox.h,[
        stroke_width: @stroke_width, color: color, fill: false
      ])
      |> Image.Draw.image!(text_image, left, max(top - text_hight - 2, 0))
      
    end)
  end
  
  @class_colors  [
    "#FF0000", "#00FF00", "#0000FF", "#FFFF00", "#FF00FF", "#00FFFF", 
    "#800000", "#008000", "#000080", "#FF00FF", "#800080", "#008080", 
    "#C0C0C0", "#FFA500", "#A52A2A", "#8A2BE2", "#5F9EA0", "#7FFF00", 
    "#D2691E", "#FF7F50", "#6495ED", "#DC143C", "#00FFFF", "#00008B", 
    "#008B8B", "#B8860B", "#A9A9A9", "#006400", "#BDB76B", "#8B008B", 
    "#556B2F", "#FF8C00", "#9932CC", "#8B0000", "#E9967A", "#8FBC8F", 
    "#483D8B", "#2F4F4F", "#00CED1", "#9400D3", "#FF1493", "#00BFFF", 
    "#696969", "#1E90FF", "#B22222", "#FFFAF0", "#228B22", "#FF00FF", 
    "#DCDCDC", "#F8F8FF", "#FFD700", "#DAA520", "#808080", "#ADFF2F", 
    "#F0FFF0", "#FF69B4", "#CD5C5C", "#4B0082", "#FFFFF0", "#F0E68C", 
    "#E6E6FA", "#FFF0F5", "#7CFC00", "#FFFACD", "#ADD8E6", "#F08080", 
    "#E0FFFF", "#FAFAD2", "#D3D3D3", "#90EE90", "#FFB6C1", "#FFA07A", 
    "#20B2AA", "#87CEFA", "#778899", "#B0C4DE", "#FFFFE0", "#00FF7F", 
    "#4682B4", "#D2B48C", "#008080", "#D8BFD8", "#FF6347", "#40E0D0", 
    "#EE82EE", "#F5DEB3", "#FFFFFF", "#F5F5F5"
  ]
  |> Enum.with_index(&{&2, &1})
  |> Map.new()  

  def class_color(class_idx) do
    Map.get(@class_colors, class_idx, "#FF0000")
  end
end

{:module, YOLODraw, <<70, 79, 82, 49, 0, 0, 22, ...>>, {:class_color, 1}}

YOLODraw.draw_detected_objects(mat, detected_objects)

YoloV8x

Use the python script to generate the yolov8x.onnx model!

Simply run

python python/yolov8_to_onnx.py x

large_model_path = Kino.FS.file_path("yolov8x.onnx")

model_x = YOLO.load(model_path: large_model_path, classes_path: classes_path)

detected_objects =
  model_x
  |> YOLO.detect(mat)
  |> YOLO.to_detected_objects(model_x.classes)    


YOLODraw.draw_detected_objects(mat, detected_objects)