Clip Text Serving

Mix.install(
  [
    {:bumblebee, github: "elixir-nx/bumblebee"},
    {:exla, "~> 0.5.3"},
    {:livebook_utils, path: "./lib/livebook_utils"}
  ],
  force: false,
  config: [
    logger: [truncate: :infinity],
    nx: [default_backend: EXLA.Backend]
  ]
)

Setup

clip_model_name = "openai/clip-vit-base-patch32"

{:ok, model_info} =
  Bumblebee.load_model({:hf, clip_model_name},
    module: Bumblebee.Text.ClipText,
    architecture: :base
  )

{:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, clip_model_name})

options = [
  embedding_processor: :l2_norm,
  compile: [batch_size: 10, sequence_length: 42],
  defn_options: [compiler: EXLA]
]

serving = Bumblebee.Text.TextEmbedding.text_embedding(model_info, tokenizer, options)

Usage

text = "a photo of a cat"

text = Nx.Serving.run(serving, text) |> Map.get(:embedding)