Streaming Orderbot

Mix.install([
  {:openai_ex, "~> 0.8.4"},
  {:kino, "~> 0.13.2"}
])

alias OpenaiEx
alias OpenaiEx.Chat
alias OpenaiEx.ChatMessage

Setup

This notebook creates an Orderbot, similar to the one in Deeplearning.AI Orderbot, but using the streaming version of the Chat Completion API.

openai = System.fetch_env!("LB_OPENAI_API_KEY") |> OpenaiEx.new()

# We need to make the request task pid available to the cancel button by sharing state
{:ok, pid} = Agent.start_link(fn -> nil end, name: :shared_task_pid)

defmodule OpenaiEx.Notebooks.StreamingOrderbot do
  alias OpenaiEx
  require Logger

  def set_task_pid(task_pid) do
    Agent.update(:shared_task_pid, fn _ -> task_pid end)
  end

  def get_task_pid do
    Agent.get(:shared_task_pid, fn pid -> pid end)
  end

  def create_chat_req(args = [_ | _]) do
    args
    |> Enum.into(%{
      model: "gpt-4o-mini",
      temperature: 0
    })
    |> Chat.Completions.new()
  end

  def get_stream(openai = %OpenaiEx{}, messages) do
    openai |> Chat.Completions.create!(create_chat_req(messages: messages), stream: true)
  end

  def stream_to_completions(%{body_stream: body_stream}) do
    body_stream
    |> Stream.flat_map(& &1)
    |> Stream.map(fn %{data: d} -> d |> Map.get("choices") |> List.first() |> Map.get("delta") end)
    |> Stream.filter(fn map -> map |> Map.has_key?("content") end)
    |> Stream.map(fn map -> map |> Map.get("content") end)
  end

  def stream_completion_to_frame(stream, frame) do
    try do
      result =
        stream
        |> stream_to_completions()
        |> Enum.reduce("", fn token, text ->
          next = text <> token
          Kino.Frame.render(frame, Kino.Text.new(next))
          next
        end)

      {:ok, result}
    rescue
      e in OpenaiEx.Error ->
        case e do
          %{kind: :sse_cancellation} ->
            message = "Request was canceled."
            Kino.Frame.render(frame, Kino.Text.new(message))
            {:canceled, message}

          _ ->
            message = "An error occurred: #{e.message}"
            Kino.Frame.render(frame, Kino.Text.new(message))
            {:error, message}
        end
    end
  end

  def create_orderbot(openai = %OpenaiEx{}, context) do
    chat_frame = Kino.Frame.new()
    last_frame = Kino.Frame.new()
    inputs = [prompt: Kino.Input.textarea("You")]
    form = Kino.Control.form(inputs, submit: "Send", reset_on_submit: [:prompt])
    cancel_button = Kino.Control.button("Cancel")
    Kino.Frame.render(chat_frame, Kino.Markdown.new("### Orderbot Chat"))

    Kino.Layout.grid([chat_frame, cancel_button, last_frame, form], boxed: true, gap: 16)
    |> Kino.render()

    stream_o = openai |> get_stream(context)
    {status, bot_says} = stream_o |> stream_completion_to_frame(last_frame)

    Kino.listen(
      form,
      context ++ if(status == :ok, do: [ChatMessage.assistant(bot_says)], else: []),
      fn %{data: %{prompt: you_say}}, history ->
        Kino.Frame.render(last_frame, Kino.Text.new(""))
        Kino.Frame.append(chat_frame, Kino.Text.new(List.last(history).content))
        Kino.Frame.append(chat_frame, Kino.Markdown.new("**You** #{you_say}"))

        stream = openai |> get_stream(history ++ [ChatMessage.user(you_say)])
        set_task_pid(stream.task_pid)
        {status, bot_says} = stream |> stream_completion_to_frame(last_frame)

        case status do
          :ok -> {:cont, history ++ [ChatMessage.user(you_say), ChatMessage.assistant(bot_says)]}
          _ -> {:cont, history}
        end
      end
    )

    Kino.listen(
      cancel_button,
      fn _event ->
        pid = get_task_pid()
        OpenaiEx.HttpSse.cancel_request(pid)
      end
    )
  end
end

alias OpenaiEx.Notebooks.StreamingOrderbot

Orderbot

context = [
  ChatMessage.system("""
  You are OrderBot, an automated service to collect orders for a pizza restaurant. \
  You first greet the customer, then collects the order, \
  and then asks if it's a pickup or delivery. \
  You wait to collect the entire order, then summarize it and check for a final \
  time if the customer wants to add anything else. \
  If it's a delivery, you ask for an address. \
  Finally you collect the payment.\
  Make sure to clarify all options, extras and sizes to uniquely \
  identify the item from the menu.\
  You respond in a short, very conversational friendly style. \
  The menu includes \
  pepperoni pizza  12.95, 10.00, 7.00 \
  cheese pizza   10.95, 9.25, 6.50 \
  eggplant pizza   11.95, 9.75, 6.75 \
  fries 4.50, 3.50 \
  greek salad 7.25 \
  Toppings: \
  extra cheese 2.00, \
  mushrooms 1.50 \
  sausage 3.00 \
  canadian bacon 3.50 \
  AI sauce 1.50 \
  peppers 1.00 \
  Drinks: \
  coke 3.00, 2.00, 1.00 \
  sprite 3.00, 2.00, 1.00 \
  bottled water 5.00 \
  """)
]

openai |> StreamingOrderbot.create_orderbot(context)