Face recognition with S3 Vectors

Mix.install([
  {:kino, "~> 0.15"},
  {:evision, "~> 0.2.13"},
  {:aws, "~> 1.0"},
  {:hackney, "~> 1.24"}
])

画像の準備

image_files = Path.wildcard("/home/livebook/evision/test-images/*.{jpg,png}")

images =
  image_files
  |> Enum.map(fn image_file ->
    Evision.imread(image_file)
  end)

Kino.Layout.grid(images, columns: 4)

顔特徴量の取得

recognizer =
  Evision.Zoo.FaceRecognition.SFace.init(:default_model,
    backend: Evision.Constant.cv_DNN_BACKEND_OPENCV(),
    target: Evision.Constant.cv_DNN_TARGET_CPU(),
    distance_type: :cosine_similarity,
    cosine_threshold: 0.363,
    l2_norm_threshold: 1.128
  )

detector =
  Evision.Zoo.FaceDetection.YuNet.init(:default_model,
    backend: Evision.Constant.cv_DNN_BACKEND_OPENCV(),
    target: Evision.Constant.cv_DNN_TARGET_CPU(),
    nms_threshold: 0.3,
    conf_threshold: 0.8,
    top_k: 5
  )

[feature_list, visualized_list] =
  images
  |> Enum.reduce([[], []], fn image, [feature_acc, visualized_acc] ->
    results = Evision.Zoo.FaceDetection.YuNet.infer(detector, image)

    bbox = Evision.Mat.to_nx(results, Nx.BinaryBackend)[0][0..-2//1]

    feature =
      recognizer
      |> Evision.Zoo.FaceRecognition.SFace.infer(image, bbox)
      |> Evision.Mat.to_nx()
      |> Evision.Mat.from_nx()

    visualized = Evision.Zoo.FaceDetection.YuNet.visualize(image, results[0])

    [[feature | feature_acc], [visualized | visualized_acc]]
  end)
  |> Enum.map(&Enum.reverse/1)

Kino.Layout.grid(visualized_list, columns: 4)

feature_list
|> hd()
|> Evision.Mat.to_nx()

Evision.Zoo.FaceRecognition.SFace.match_feature(
  recognizer,
  Enum.at(feature_list, 8),
  Enum.at(feature_list, 15)
)

Evision.Zoo.FaceRecognition.SFace.match_feature(
  recognizer,
  Enum.at(feature_list, 8),
  Enum.at(feature_list, 0)
)

S3 Vectors

アクセス用関数の準備

access_key_id_input = Kino.Input.password("ACCESS_KEY_ID")
secret_access_key_input = Kino.Input.password("SECRET_ACCESS_KEY")
region_input = Kino.Input.text("REGION")

[
  access_key_id_input,
  secret_access_key_input,
  region_input
]
|> Kino.Layout.grid(columns: 3)

client =
  AWS.Client.create(
    Kino.Input.read(access_key_id_input),
    Kino.Input.read(secret_access_key_input),
    Kino.Input.read(region_input)
  )

request_to_s3_vectors = fn (client, path, payload) ->
  now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
  encoded_payload = AWS.Client.encode!(client, payload, :json)

  headers = [
    {"Host", "s3vectors.us-east-1.api.aws"},
    {"Content-Type", "application/json"}
  ]

  url = "https://s3vectors.us-east-1.api.aws#{path}"

  siged_headers =
    AWS.Signature.sign_v4(
      %{client | service: "s3vectors"}, 
      now,
      :post,
      url,
      headers,
      encoded_payload
    )

  {:ok, %{body: body}} =
    AWS.Client.request(client, :post, url, encoded_payload, siged_headers, [])

  JSON.decode!(body)
end

ベクトルバケットの作成

request_to_s3_vectors.(client, "/ListVectorBuckets", %{"maxResults" => 20})

request_to_s3_vectors.(client, "/CreateVectorBucket", %{
  "vectorBucketName" => "face-vectors",
})

vectorBucket =
  client
  |> request_to_s3_vectors.("/GetVectorBucket", %{"vectorBucketName" => "face-vectors"})
  |> Map.get("vectorBucket")

インデックスの作成

request_to_s3_vectors.(client, "/CreateIndex", %{
  "vectorBucketName" => "face-vectors",
  "indexName" => "face-vectors-index",
  "dataType" => "float32",
  "dimension" => 128,
  "distanceMetric" => "cosine"
})

vectorIndex =
  client
  |> request_to_s3_vectors.("/GetIndex", %{
    "vectorBucketName" => "face-vectors",
    "indexName" => "face-vectors-index"
  })
  |> Map.get("index")

データの登録

vaectors =
  feature_list
  |> Enum.zip(image_files)
  |> Enum.map(fn {feature, image_file} ->
    %{
      "key" => Path.basename(image_file),
      "data" => %{
        "float32" => feature |> Evision.Mat.to_nx() |> Nx.to_flat_list(),
      },
      "metadata" => %{
        "name" => image_file |> Path.basename() |> String.split("-") |> hd()
      }
    }
  end)

request_to_s3_vectors.(client, "/PutVectors", %{
  "indexArn" => Map.get(vectorIndex, "indexArn"),
  "vectors" => vaectors
})

request_to_s3_vectors.(client, "/ListVectors", %{
  "indexArn" => Map.get(vectorIndex, "indexArn"),
  "maxResults" => 20,
  "returnData" => true,
  "returnMetadata" => true
})

データの検索

query_vector =
  feature_list
  |> Enum.at(8)
  |> Evision.Mat.to_nx()
  |> Nx.to_flat_list()

request_to_s3_vectors.(client, "/QueryVectors", %{
  "indexArn" => Map.get(vectorIndex, "indexArn"),
  "queryVector" => %{
    "float32" => query_vector
  },
  "returnDistance" => true,
  "returnMetadata" => true,
  "topK" => 5
})

データの削除

request_to_s3_vectors.(client, "/DeleteVectors", %{
  "indexArn" => Map.get(vectorIndex, "indexArn"),
  "keys" => image_files |> Enum.map(&Path.basename(&1))
})

request_to_s3_vectors.(client, "/ListVectors", %{
  "indexArn" => Map.get(vectorIndex, "indexArn"),
  "maxResults" => 20
})

インデックスの削除

request_to_s3_vectors.(client, "/DeleteIndex", %{
  "indexArn" => Map.get(vectorIndex, "indexArn")
})

request_to_s3_vectors.(client, "/ListIndexes", %{
  "maxResults" => 20,
  "vectorBucketArn" => Map.get(vectorBucket, "vectorBucketArn")
})

ベクトルバケットの削除

request_to_s3_vectors.(client, "/DeleteVectorBucket", %{
  "vectorBucketArn" => Map.get(vectorBucket, "vectorBucketArn")
})

request_to_s3_vectors.(client, "/ListVectorBuckets", %{"maxResults" => 20})