HailoAI Concurrent Inference

Setup

This notebook runs two YOLOv8 models concurrently on a single Hailo-8 device using the HailoRT round_robin scheduler. It requires the same Attached Node runtime as remote_device_inference.livemd.

Start the node on the device:

./scripts/start_node.exs

Then in Livebook go to Runtime → Attached Node and enter the node name and cookie printed by the script.

Run livebooks/download_models.livemd twice — once for yolov8m and once for yolov8s (both with device = "hailo8") — before proceeding.

Load Models on a Shared VDevice

Both models share a single VDevice created with scheduling_algorithm: :round_robin. The HailoRT scheduler time-slices the neural engine between the two pipelines automatically.

alias NxHailo.API
priv = to_string(:code.priv_dir(:nx_hailo))

# A single VDevice shared by both models. round_robin lets the HailoRT scheduler
# interleave inference requests from both pipelines on the same hardware.
{:ok, vdevice} = API.create_vdevice(%{scheduling_algorithm: :round_robin})

{:ok, ng_m} = API.configure_network_group(vdevice, "#{priv}/yolov8m.hef")
{:ok, pipeline_m} = API.create_pipeline(ng_m)
model_m = %NxHailo.Model{pipeline: pipeline_m, name: "yolov8m"}

{:ok, ng_s} = API.configure_network_group(vdevice, "#{priv}/yolov8s.hef")
{:ok, pipeline_s} = API.create_pipeline(ng_s)
model_s = %NxHailo.Model{pipeline: pipeline_s, name: "yolov8s"}

[model_m, model_s]

Load Classes and Helpers

classes =
  File.read!("#{priv}/yolov8m_classes.json")
  |> Jason.decode!()
  |> Enum.with_index()
  |> Map.new(fn {v, k} -> {k, v} end)

[input_m] = model_m.pipeline.input_vstream_infos
[output_m] = model_m.pipeline.output_vstream_infos
[input_s] = model_s.pipeline.input_vstream_infos
[output_s] = model_s.pipeline.output_vstream_infos

evision_resize_and_pad =
  fn image, {h, w}, target_size ->
    size = max(h, w)
    pad_h = div(size - h, 2)
    pad_w = div(size - w, 2)

    Evision.copyMakeBorder(
      image,
      pad_h,
      pad_h + rem(size - h, 2),
      pad_w,
      pad_w + rem(size - w, 2),
      Evision.Constant.cv_BORDER_CONSTANT(),
      value: {114, 114, 114}
    )
    |> Evision.resize({target_size, target_size})
  end

defmodule YOLODraw do
  @moduledoc """
  Draws detections onto an Evision `Mat`.

  Coordinates have to be in pixels, so run detections through
  `NxHailo.Parsers.YoloV8.postprocess/2` first.
  """

  alias NxHailo.Palette

  @font Evision.Constant.cv_FONT_HERSHEY_SIMPLEX()
  @line_type Evision.Constant.cv_LINE_AA()
  @filled -1

  @label_scale 0.5
  @caption_scale 0.7
  @stroke_width 2
  @padding 5

  @white {255, 255, 255}
  @black {0, 0, 0}
  @blue {255, 0, 0}

  def draw_detected_objects(mat, detections, caption) do
    # Boxes first, then labels, so a label is never covered by the box of a
    # detection drawn after it.
    mat = Enum.reduce(detections, mat, &draw_box(&2, &1))
    mat = Enum.reduce(detections, mat, &draw_label(&2, &1))

    draw_caption(mat, caption)
  end

  defp draw_box(mat, detection) do
    Evision.rectangle(
      mat,
      {detection.xmin, detection.ymin},
      {detection.xmax, detection.ymax},
      Palette.bgr(detection.class_id),
      thickness: @stroke_width
    )
  end

  defp draw_label(mat, detection) do
    text = "#{detection.class_name} #{round(detection.score * 100)}%"
    {{width, height}, baseline} = Evision.getTextSize(text, @font, @label_scale, 1)

    top_left = {detection.xmin, max(detection.ymin - height - 2 * @padding - baseline, 0)}
    bottom_right = {detection.xmin + width + 2 * @padding, max(detection.ymin - baseline, 0)}
    baseline_at = max(detection.ymin - @padding - baseline, height + @padding)

    mat
    |> Evision.rectangle(top_left, bottom_right, @black, thickness: @filled)
    |> put_text(text, {detection.xmin + @padding, baseline_at}, @label_scale)
  end

  defp draw_caption(mat, caption) do
    # Evision reports shape as {height, width, channels}.
    {frame_height, frame_width, _channels} = Evision.Mat.shape(mat)
    {{width, height}, _baseline} = Evision.getTextSize(caption, @font, @caption_scale, 1)

    box_width = width + 2 * @padding
    box_height = height + 2 * @padding

    mat
    |> Evision.rectangle(
      {frame_width - box_width, frame_height - box_height},
      {frame_width, frame_height},
      @blue,
      thickness: @filled
    )
    |> put_text(
      caption,
      {frame_width - box_width + @padding, frame_height - @padding},
      @caption_scale
    )
  end

  defp put_text(mat, text, origin, scale) do
    Evision.putText(mat, text, origin, @font, scale, @white, thickness: 1, lineType: @line_type)
  end
end

Camera Setup

Choose one option below, then proceed to the Concurrent Inference section.

Option A — Standard V4L2 Camera

find_working_cap = fn ->
  Enum.find_value(Path.wildcard("/dev/video*"), fn device ->
    cap = Evision.VideoCapture.videoCapture(device)

    cond do
      not Evision.VideoCapture.isOpened(cap) ->
        Evision.VideoCapture.release(cap)
        false

      not Evision.VideoCapture.grab(cap) ->
        Evision.VideoCapture.release(cap)
        false

      true ->
        {cap, device}
    end
  end)
end

{capture, device} = find_working_cap.()
IO.puts("Using camera: #{device}")

capture_frame = fn ->
  Evision.VideoCapture.set(capture, Evision.Constant.cv_CAP_PROP_BUFFERSIZE(), 1)
  true = Evision.VideoCapture.grab(capture)
  Evision.VideoCapture.read(capture)
end

sample_frame = capture_frame.()
{frame_h, frame_w, _} = Evision.Mat.shape(sample_frame)
input_shape = {frame_h, frame_w}

Option B — Raspberry Pi Camera (PiCam3 / libcamera)

Start the camera on the device (separate terminal):

rpicam-still --nopreview -t 0 --timelapse 50 --width 640 --height 480 \
  -o /tmp/frame.jpg 2>/dev/null &

Then run this cell:

# capture_frame = fn ->
#   Evision.imread("/tmp/frame.jpg", flags: Evision.Constant.cv_IMREAD_COLOR())
# end

# sample_frame = capture_frame.()
# {frame_h, frame_w, _} = Evision.Mat.shape(sample_frame)
# input_shape = {frame_h, frame_w}

# IO.puts("Frame size: #{frame_w}×#{frame_h}")

Concurrent Inference

Both models run inference in parallel on each frame using Task.async_stream. The HailoRT round_robin scheduler distributes the neural engine time between them. Results are displayed side-by-side: yolov8m (left) vs yolov8s (right).

padded_shape = 640
score_threshold = 0.5
frame_interval_ms = 20

Kino.animate(frame_interval_ms, fn _ ->
  raw_frame = capture_frame.()

  input_tensor =
    evision_resize_and_pad.(raw_frame, input_shape, padded_shape)
    |> Evision.Mat.to_nx()
    |> Nx.reshape({padded_shape, padded_shape, 3})
    |> Nx.backend_transfer()

  # Both models are handed a frame at the same time. Each has its own pipeline,
  # so the two calls really do overlap; the round_robin scheduler decides how
  # the accelerator is split between them.
  [objects_m, objects_s] =
    [{model_m, input_m.name, output_m.name}, {model_s, input_s.name, output_s.name}]
    |> Task.async_stream(
      fn {model, in_name, out_name} ->
        {:ok, raw} =
          NxHailo.infer(
            model,
            %{in_name => input_tensor},
            NxHailo.Parsers.YoloV8,
            classes: classes,
            key: out_name
          )

        raw
        |> Enum.reject(&(&1.score < score_threshold))
        |> NxHailo.Parsers.YoloV8.postprocess(input_shape)
      end,
      max_concurrency: 2,
      ordered: true,
      timeout: 5_000
    )
    |> Enum.map(fn {:ok, detections} -> detections end)

  n_m = length(objects_m)
  n_s = length(objects_s)

  frame_m = YOLODraw.draw_detected_objects(raw_frame, objects_m, "yolov8m  #{n_m} objects")
  frame_s = YOLODraw.draw_detected_objects(raw_frame, objects_s, "yolov8s  #{n_s} objects")

  encode = fn mat ->
    buf = Evision.imencode(".jpg", mat)
    Kino.Image.new(IO.iodata_to_binary(buf), :jpeg)
  end

  Kino.Layout.grid([encode.(frame_m), encode.(frame_s)], columns: 2)
end)