HailoAI Remote Inference

Setup

This notebook runs inference on a Hailo device via Livebook's Attached Node runtime. The node must be running on the device before connecting.

Start the node on the device:

./scripts/start_node.exs

Then in Livebook go to Runtime → Attached Node and enter the node name and cookie printed by the script.

See livebooks/download_models.livemd to download a .hef model file before running inference.

Load Model

priv = to_string(:code.priv_dir(:nx_hailo))
{:ok, hailo_model} = NxHailo.load("#{priv}/yolov8m.hef")

[%{name: name}] = hailo_model.pipeline.input_vstream_infos
[%{name: output_key}] = hailo_model.pipeline.output_vstream_infos

hailo_model

Image Preprocessing

Resizes and letterbox-pads an image to the model's square input size (640×640 for YOLOv8), filling borders with neutral grey.

evision_resize_and_pad =
  fn image, {h, w}, {target_size, target_size} ->
    size = max(h, w)
    pad_h = div(size - h, 2)
    pad_w = div(size - w, 2)

    padded =
      Evision.copyMakeBorder(
        image,
        pad_h,
        pad_h + rem(size - h, 2),
        pad_w,
        pad_w + rem(size - w, 2),
        Evision.Constant.cv_BORDER_CONSTANT(),
        value: {114, 114, 114}
      )

    Evision.resize(padded, {target_size, target_size})
  end

Drawing Utilities

Draws bounding boxes and confidence labels on a frame.

defmodule YOLODraw do
  @moduledoc """
  Draws detections onto an Evision `Mat`.

  Coordinates have to be in pixels, so run detections through
  `NxHailo.Parsers.YoloV8.postprocess/2` first.
  """

  alias NxHailo.Palette

  @font Evision.Constant.cv_FONT_HERSHEY_SIMPLEX()
  @line_type Evision.Constant.cv_LINE_AA()
  @filled -1

  @label_scale 0.5
  @caption_scale 0.7
  @stroke_width 2
  @padding 5

  @white {255, 255, 255}
  @black {0, 0, 0}
  @blue {255, 0, 0}

  def draw_detected_objects(mat, detections, caption) do
    # Boxes first, then labels, so a label is never covered by the box of a
    # detection drawn after it.
    mat = Enum.reduce(detections, mat, &draw_box(&2, &1))
    mat = Enum.reduce(detections, mat, &draw_label(&2, &1))

    draw_caption(mat, caption)
  end

  defp draw_box(mat, detection) do
    Evision.rectangle(
      mat,
      {detection.xmin, detection.ymin},
      {detection.xmax, detection.ymax},
      Palette.bgr(detection.class_id),
      thickness: @stroke_width
    )
  end

  defp draw_label(mat, detection) do
    text = "#{detection.class_name} #{round(detection.score * 100)}%"
    {{width, height}, baseline} = Evision.getTextSize(text, @font, @label_scale, 1)

    top_left = {detection.xmin, max(detection.ymin - height - 2 * @padding - baseline, 0)}
    bottom_right = {detection.xmin + width + 2 * @padding, max(detection.ymin - baseline, 0)}
    baseline_at = max(detection.ymin - @padding - baseline, height + @padding)

    mat
    |> Evision.rectangle(top_left, bottom_right, @black, thickness: @filled)
    |> put_text(text, {detection.xmin + @padding, baseline_at}, @label_scale)
  end

  defp draw_caption(mat, caption) do
    # Evision reports shape as {height, width, channels}.
    {frame_height, frame_width, _channels} = Evision.Mat.shape(mat)
    {{width, height}, _baseline} = Evision.getTextSize(caption, @font, @caption_scale, 1)

    box_width = width + 2 * @padding
    box_height = height + 2 * @padding

    mat
    |> Evision.rectangle(
      {frame_width - box_width, frame_height - box_height},
      {frame_width, frame_height},
      @blue,
      thickness: @filled
    )
    |> put_text(
      caption,
      {frame_width - box_width + @padding, frame_height - @padding},
      @caption_scale
    )
  end

  defp put_text(mat, text, origin, scale) do
    Evision.putText(mat, text, origin, @font, scale, @white, thickness: 1, lineType: @line_type)
  end
end

Camera Setup

Choose one of the sections below depending on your camera type, then proceed to the Inference section.

Option A — Standard V4L2 Camera

Use this for USB webcams or any camera accessible via /dev/video*.

find_working_cap = fn ->
  Enum.find_value(Path.wildcard("/dev/video*"), fn device ->
    cap = Evision.VideoCapture.videoCapture(device)

    cond do
      not Evision.VideoCapture.isOpened(cap) ->
        Evision.VideoCapture.release(cap)
        false

      not Evision.VideoCapture.grab(cap) ->
        Evision.VideoCapture.release(cap)
        false

      true ->
        {cap, device}
    end
  end)
end

{capture, device} = find_working_cap.()
IO.puts("Using camera: #{device}")

capture_frame = fn ->
  Evision.VideoCapture.set(capture, Evision.Constant.cv_CAP_PROP_BUFFERSIZE(), 1)
  true = Evision.VideoCapture.grab(capture)
  Evision.VideoCapture.read(capture)
end

sample_frame = capture_frame.()
{frame_h, frame_w, _} = Evision.Mat.shape(sample_frame)
input_shape = {frame_h, frame_w}

Option B — Raspberry Pi Camera (PiCam3 / libcamera)

The Pi Camera uses libcamera and is not directly accessible via OpenCV's V4L2 backend. Instead, run rpicam-still in timelapse mode on the device (in a separate terminal) to continuously write frames to a temp file, then read that file here.

Start the camera on the device (separate terminal):

rpicam-still --nopreview -t 0 --timelapse 50 --width 640 --height 480 \
  -o /tmp/frame.jpg 2>/dev/null &

Then run this cell:

capture_frame = fn ->
  Evision.imread("/tmp/frame.jpg", flags: Evision.Constant.cv_IMREAD_COLOR())
end

sample_frame = capture_frame.()
{frame_h, frame_w, _} = Evision.Mat.shape(sample_frame)
input_shape = {frame_h, frame_w}

IO.puts("Frame size: #{frame_w}×#{frame_h}")

Inference

Loads class labels, then runs YOLOv8 inference on each frame and renders the annotated result.

padded_shape = {640, 640}

classes =
  File.read!("#{priv}/yolov8m_classes.json")
  |> Jason.decode!()
  |> Enum.with_index()
  |> Map.new(fn {v, k} -> {k, v} end)

frame_interval_ms = 20

Kino.animate(frame_interval_ms, fn _ ->
  input_image = capture_frame.()
  padded_image = evision_resize_and_pad.(input_image, input_shape, padded_shape)

  input_tensor =
    padded_image
    |> Evision.Mat.to_nx()
    |> Nx.reshape({640, 640, 3})
    |> Nx.backend_transfer()

  {:ok, raw_objects} =
    NxHailo.infer(
      hailo_model,
      %{name => input_tensor},
      NxHailo.Parsers.YoloV8,
      classes: classes,
      key: output_key
    )

  detected_objects =
    raw_objects
    |> Enum.reject(&(&1.score < 0.5))
    |> NxHailo.Parsers.YoloV8.postprocess(input_shape)

  YOLODraw.draw_detected_objects(
    input_image,
    detected_objects,
    "#{length(detected_objects)} objects at #{div(1000, frame_interval_ms)} fps"
  )
end)