Streaming Orderbot

Mix.install([
  {:openai_ex, "~> 0.8.6"},
  {:kino, "~> 0.14.2"}
])

alias OpenaiEx
alias OpenaiEx.Chat
alias OpenaiEx.ChatMessage

Setup

This notebook creates an Orderbot, similar to the one in Deeplearning.AI Orderbot, but using the streaming version of the Chat Completion API.

Add the GROQ_API_KEY to your Livebook Personal Secrets then Toggle to allow access

openai =
  System.fetch_env!("LB_GROQ_API_KEY")
  |> OpenaiEx.new()
  |> OpenaiEx.with_base_url("https://api.groq.com/openai/v1")

IO.puts("OpenaiEx instance created.")

# We need to make the request task pid available to the cancel button by sharing state
{:ok, pid} = Agent.start_link(fn -> nil end, name: :shared_task_pid)

defmodule OpenaiEx.Notebooks.StreamingOrderbot do
  alias OpenaiEx
  require Logger

  def set_task_pid(task_pid) do
    Agent.update(:shared_task_pid, fn _ -> task_pid end)
  end

  def get_task_pid do
    Agent.get(:shared_task_pid, fn pid -> pid end)
  end

  def create_chat_req(args = [_ | _]) do
    args
    |> Enum.into(%{
      model: "llama-3.1-8b-instant",
      temperature: 0
    })
    |> Chat.Completions.new()
  end

  def get_stream(openai = %OpenaiEx{}, messages) do
    openai |> Chat.Completions.create!(create_chat_req(messages: messages), stream: true)
  end

  def stream_to_completions(%{body_stream: body_stream}) do
    body_stream
    |> Stream.flat_map(& &1)
    |> Stream.map(fn %{data: d} -> d |> Map.get("choices") |> List.first(%{}) |> Map.get("delta", %{}) end)
    |> Stream.filter(fn map -> map |> Map.has_key?("content") end)
    |> Stream.map(fn map -> map |> Map.get("content") end)
  end

  def stream_completion_to_frame(stream, frame) do
    try do
      result =
        stream
        |> stream_to_completions()
        |> Enum.reduce("", fn token, text ->
          next = (text || "") <> (token || "")          
          Kino.Frame.render(frame, Kino.Text.new(next))
          next
        end)

      {:ok, result}
    rescue
      e in OpenaiEx.Error ->
        case e do
          %{kind: :sse_cancellation} ->
            message = "Request was canceled."
            Kino.Frame.render(frame, Kino.Text.new(message))
            {:canceled, message}

          _ ->
            message = "An error occurred: #{e.message}"
            Kino.Frame.render(frame, Kino.Text.new(message))
            {:error, message}
        end
    end
  end

  def create_orderbot(openai = %OpenaiEx{}, context) do
    chat_frame = Kino.Frame.new()
    last_frame = Kino.Frame.new()
    inputs = [prompt: Kino.Input.textarea("You")]
    form = Kino.Control.form(inputs, submit: "Send", reset_on_submit: [:prompt])
    cancel_button = Kino.Control.button("Cancel")
    Kino.Frame.render(chat_frame, Kino.Markdown.new("### Orderbot Chat"))

    Kino.Layout.grid([chat_frame, cancel_button, last_frame, form], boxed: true, gap: 16)
    |> Kino.render()

    stream_o = openai |> get_stream(context)
    {status, bot_says} = stream_o |> stream_completion_to_frame(last_frame)

    Kino.listen(
      form,
      context ++ if(status == :ok, do: [ChatMessage.assistant(bot_says)], else: []),
      fn %{data: %{prompt: you_say}}, history ->
        Kino.Frame.render(last_frame, Kino.Text.new(""))
        Kino.Frame.append(chat_frame, Kino.Text.new(List.last(history).content))
        Kino.Frame.append(chat_frame, Kino.Markdown.new("**You** #{you_say}"))

        stream = openai |> get_stream(history ++ [ChatMessage.user(you_say)])
        set_task_pid(stream.task_pid)
        {status, bot_says} = stream |> stream_completion_to_frame(last_frame)

        case status do
          :ok -> {:cont, history ++ [ChatMessage.user(you_say), ChatMessage.assistant(bot_says)]}
          _ -> {:cont, history}
        end
      end
    )

    Kino.listen(
      cancel_button,
      fn _event ->
        pid = get_task_pid()
        OpenaiEx.HttpSse.cancel_request(pid)
      end
    )
  end
end

alias OpenaiEx.Notebooks.StreamingOrderbot

Orderbot

context = [
  ChatMessage.system("""
  You are OrderBot, an automated service to collect orders for a pizza restaurant. \
  You first greet the customer, then collects the order, \
  and then asks if it's a pickup or delivery. \
  You wait to collect the entire order, then summarize it and check for a final \
  time if the customer wants to add anything else. \
  If it's a delivery, you ask for an address. \
  Finally you collect the payment.\
  Make sure to clarify all options, extras and sizes to uniquely \
  identify the item from the menu.\
  You respond in a short, very conversational friendly style. \
  The menu includes \
  pepperoni pizza  12.95, 10.00, 7.00 \
  cheese pizza   10.95, 9.25, 6.50 \
  eggplant pizza   11.95, 9.75, 6.75 \
  fries 4.50, 3.50 \
  greek salad 7.25 \
  Toppings: \
  extra cheese 2.00, \
  mushrooms 1.50 \
  sausage 3.00 \
  canadian bacon 3.50 \
  AI sauce 1.50 \
  peppers 1.00 \
  Drinks: \
  coke 3.00, 2.00, 1.00 \
  sprite 3.00, 2.00, 1.00 \
  bottled water 5.00 \
  """)
]

openai |> StreamingOrderbot.create_orderbot(context)