Image Classification

Mix.install(
  [
    {:bumblebee, "~> 0.5"},
    {:nx, "~> 0.9", override: true},
    {:exla, "~> 0.9"},
    {:kino, "~> 0.14"}
  ],
  config: [nx: [default_backend: EXLA.Backend]]
)

Form

cache_dir = "/tmp/bumblebee_cache"

{:ok, resnet} =
  Bumblebee.load_model({
    :hf,
    "microsoft/resnet-50",
    cache_dir: cache_dir
  })

{:ok, featurizer} =
  Bumblebee.load_featurizer({
    :hf,
    "microsoft/resnet-50",
    cache_dir: cache_dir
  })

serving = Bumblebee.Vision.image_classification(resnet, featurizer)

inputs = [
  image: Kino.Input.image("IMAGE", size: {224, 224})
]

form = Kino.Control.form(inputs, submit: "Classify")

frame = Kino.Frame.new()

Kino.listen(form, fn %{data: %{image: image}, origin: origin} ->
  image =
    image
    |> then(fn input ->
      input.file_ref
      |> Kino.Input.file_path()
      |> File.read!()
      |> Nx.from_binary(:u8)
      |> Nx.reshape({input.height, input.width, 3})
    end)

  serving
  |> Nx.Serving.run(image)
  |> Map.get(:predictions)
  |> Kino.DataTable.new()
  |> then(&Kino.Frame.render(frame, &1, to: origin))
end)