Library node

Mix.install(
  [
    {:kino_bumblebee, "~> 0.3.0"},
    {:exla, "~> 0.5.1"}
  ],
  config: [nx: [default_backend: EXLA.Backend]]
)

:ok

Node

[%{node: node() |> Atom.to_string(), node_cookie: Node.get_cookie() |> Atom.to_string()}]
|> Kino.DataTable.new()

Common

defmodule Common do
  def reshape(image) do
    image.data
    |> Nx.from_binary(:u8)
    |> Nx.reshape({image.height, image.width, 3})
  end
end

{:module, Common, <<70, 79, 82, 49, 0, 0, 8, ...>>, {:reshape, 1}}

ResNet Library

defmodule ResNet do
  def build() do
    {:ok, model_info} = Bumblebee.load_model({:hf, "microsoft/resnet-50"})
    {:ok, featurizer} = Bumblebee.load_featurizer({:hf, "microsoft/resnet-50"})

    Bumblebee.Vision.image_classification(model_info, featurizer,
      compile: [batch_size: 1],
      defn_options: [compiler: EXLA]
    )
  end

  def for_render(predict) do
    predict.predictions
    |> Enum.map(&{&1.label, &1.score})
    |> Kino.Bumblebee.ScoredList.new()
  end
end

{:module, ResNet, <<70, 79, 82, 49, 0, 0, 11, ...>>, {:for_render, 1}}

BLIP Library

defmodule BLIP do
  def build() do
    {:ok, model_info} = Bumblebee.load_model({:hf, "Salesforce/blip-image-captioning-base"})
    {:ok, featurizer} = Bumblebee.load_featurizer({:hf, "Salesforce/blip-image-captioning-base"})
    {:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "Salesforce/blip-image-captioning-base"})

    {:ok, generation_config} =
      Bumblebee.load_generation_config({:hf, "Salesforce/blip-image-captioning-base"})

    generation_config = Bumblebee.configure(generation_config, max_new_tokens: 100)

    Bumblebee.Vision.image_to_text(model_info, featurizer, tokenizer, generation_config,
      compile: [batch_size: 1],
      defn_options: [compiler: EXLA]
    )
  end

  def for_render(%{results: [%{text: text}]}) do
    text
    |> Kino.Text.new()
  end
end

{:module, BLIP, <<70, 79, 82, 49, 0, 0, 10, ...>>, {:for_render, 1}}