Face Detection: CenterFace

File.cd!(__DIR__)
# for windows JP
System.shell("chcp 65001")

Mix.install([
  {:onnx_interp, path: ".."},
  {:cimg, "~> 0.1.16"},
  {:postdnn, "~> 0.1.4"},
  {:nx, "~> 0.4.0"},
  {:kino, "~> 0.6.2"}
])

0.Original work

CenterFace: Joint Face Detection and Alignment Using Face as Point

• https://arxiv.org/abs/1911.03599

CenterFace

• https://github.com/Star-Clouds/CenterFace

Thanks a lot!!!

1.Implementation in Elixir

> [model card] > > inputs:
> [0] f32:{1,3,height,width} - NCHW, RGB, range: {0.0, 255.0} > outputs:
> [0] f32:{1,1,height/4,width/4} - heatmap / classification score
> [1] f32:{1,2,height/4,width/4} - scale / BBox size
> [2] f32:{1,2,height/4,width/4} - offset / BBox offset
> [3] f32:{1,10,height/4,width/4} - landmarks

defmodule CenterFace do
  import Nx.Defn

  @width 640
  @height 640

  alias OnnxInterp, as: NNInterp

  use NNInterp,
    model: "./model/centerface_dynamic.onnx",
    url: "https://github.com/shoz-f/onnx_interp/releases/download/models/centerface_dynamic.onnx",
    inputs: [f32: {1, 3, @height, @width}],
    outputs: [
      f32: {1, 1, div(@height, 4), div(@width, 4)},
      f32: {1, 2, div(@height, 4), div(@width, 4)},
      f32: {1, 2, div(@height, 4), div(@width, 4)},
      f32: {1, 10, div(@height, 4), div(@width, 4)}
    ]

  def apply(img) do
    # preprocess
    bin =
      CImg.builder(img)
      |> CImg.resize({@width, @height}, :ul, 0)
      |> CImg.to_binary([{:range, {0.0, 255.0}}, :nchw])

    # prediction
    outputs =
      session()
      |> NNInterp.set_input_tensor(0, bin)
      |> NNInterp.invoke()

    [heatmap, scale, offset, landm] =
      Enum.with_index([1, 2, 2, 10], fn dim, i ->
        NNInterp.get_output_tensor(outputs, i) |> Nx.from_binary(:f32) |> Nx.reshape({dim, :auto})
      end)

    # postprocess
    scores = Nx.transpose(heatmap)
    boxes = decode_boxes(offset, scale)
    landm = Nx.transpose(landm)

    {:ok, res} =
      NNInterp.non_max_suppression_multi_class(
        __MODULE__,
        Nx.shape(scores),
        Nx.to_binary(boxes),
        Nx.to_binary(scores),
        iou_threshold: 0.2,
        score_threshold: 0.2,
        boxrepr: :corner
      )

    {:ok, fit2image_with_landmark(landm, res["0"], inv_aspect(img))}
  end

  @grid PostDNN.meshgrid({@width, @height}, [4], [:center, :normalize, :transpose])

  defp decode_boxes(offset, size) do
    # decode box center coordinate on {1.0, 1.0}
    center =
      offset
      # swap (y,x) -> (x,y)
      |> Nx.reverse(axes: [0])
      # * grid_pitch(x,y)
      |> Nx.multiply(@grid[2..3])
      # + grid(x,y)
      |> Nx.add(@grid[0..1])

    # decode box half size
    half_size =
      size
      # swap (y,x) -> (x,y)
      |> Nx.reverse(axes: [0])
      |> Nx.exp()
      # * grid_pitch(x,y)
      |> Nx.multiply(@grid[2..3])
      |> Nx.divide(2.0)

    # decode boxes
    [Nx.subtract(center, half_size), Nx.add(center, half_size)]
    |> Nx.concatenate()
    |> PostDNN.clamp({0.0, 1.0})
    |> Nx.transpose()
  end

  defp fit2image_with_landmark(landm, nms_res, {inv_x, inv_y} \\ {1.0, 1.0}) do
    Enum.map(nms_res, fn [score, x1, y1, x2, y2, index] ->
      grid = Nx.slice_along_axis(@grid, index, 1, axis: 1) |> Nx.squeeze()

      landmark =
        landm[index]
        |> Nx.reshape({:auto, 2})
        |> Nx.reverse(axes: [0])
        # * prior_size(x,y)
        |> Nx.multiply(grid[2..3])
        # + grid(x,y)
        |> Nx.add(grid[0..1])
        |> Nx.multiply(Nx.tensor([inv_x, inv_y]))
        |> Nx.to_flat_list()
        |> Enum.chunk_every(2)

      [score, x1 * inv_x, y1 * inv_y, x2 * inv_x, y2 * inv_y, landmark]
    end)
  end

  defp inv_aspect(img) do
    {w, h, _, _} = CImg.shape(img)
    if w > h, do: {1.0, w / h}, else: {h / w, 1.0}
  end
end

Launch CenterFace.

CenterFace.start_link([])

2.Let’s try it

defmodule DemoCenterFace do
  def run(path) do
    img = CImg.load(path)

    with {:ok, res} = CenterFace.apply(img) do
      res
      |> draw_item(CImg.builder(img), {0, 255, 0})
      |> CImg.display_kino(:jpeg)
    end
  end

  defp draw_item(boxes, canvas, color \\ {255, 255, 255}) do
    Enum.reduce(boxes, canvas, fn [_score, x1, y1, x2, y2, _landmark], canvas ->
      CImg.fill_rect(canvas, x1, y1, x2, y2, color, 0.3)
    end)
  end
end

DemoCenterFace.run("10.jpg")

3.TIL ;-)

Appendix

A) How to get ONNX model

Donwload the model from “https://github.com/Star-Clouds/CenterFace/raw/master/models/onnx/centerface.onnx”, and modify it’s input layer by following python script.

import onnx

model = onnx.load_model("centerface.onnx")
d = model.graph.input[0].type.tensor_type.shape.dim
d[0].dim_value = 1
d[2].dim_value = -1  # dynamic dimension
d[3].dim_value = -1  # dynamic dimension
onnx.save_model(model,"centerface_dynamic.onnx" )

□