Powered by AppSignal & Oban Pro

Ticket Routing

guides/examples/ticket-routing.livemd

Ticket Routing

In this notebook you improve a bank support ticket classifier with labeled examples. :labeled_few_shot copies a seeded sample of training tickets into the prompt as demonstrations, and you then compare the original and tuned classifier on held-out tickets. The notebook runs offline with no credentials: a scripted model stands in for the provider.

Setup

Mix.install([
  {:tune, "~> 0.5.0"},
  {:plug, "~> 1.20"}
])

Script the model

Scripted serves OpenAI-compatible replies through an in-process Plug, so ReqLLM runs its real request path without a network call. options/1 takes a function of the decoded request body, which lets the reply depend on what the prompt contains. In real use you pass a real model spec and the host's credentials instead.

defmodule Scripted do
  @moduledoc "Serves scripted OpenAI-compatible replies to ReqLLM through an in-process Plug."

  def model, do: %{provider: :openai, id: "gpt-4-turbo", extra: %{wire: %{protocol: "openai_chat"}}}

  def options(reply) when is_function(reply, 1) do
    plug = fn conn ->
      {:ok, body, conn} = Plug.Conn.read_body(conn)
      respond(conn, reply.(JSON.decode!(body)))
    end

    [api_key: "scripted", req_http_options: [plug: plug], max_retries: 0]
  end

  def chat(fields) do
    Enum.map_join(fields, "\n\n", fn {name, value} -> "[[ ## #{name} ## ]]\n#{value}" end) <>
      "\n\n[[ ## completed ## ]]"
  end

  def field(body, name) do
    text = body["messages"] |> Enum.filter(&(&1["role"] == "user")) |> List.last() |> Map.fetch!("content")

    case Regex.run(~r/\[\[ ## #{name} ## \]\]\n(.*?)(?=\n\n\[\[ ## |\n\nRespond with|\z)/s, text) do
      [_, value] -> value
      nil -> nil
    end
  end

  def demos?(body), do: Enum.any?(body["messages"], &(&1["role"] == "assistant"))

  defp respond(conn, text) do
    send_json(conn, 200, %{
      id: "scripted",
      model: "gpt-4-turbo",
      choices: [%{message: %{role: "assistant", content: text}, finish_reason: "stop"}],
      usage: %{prompt_tokens: 10, completion_tokens: 5, total_tokens: 15}
    })
  end

  defp send_json(conn, status, body) do
    conn
    |> Plug.Conn.put_resp_content_type("application/json")
    |> Plug.Conn.send_resp(status, JSON.encode!(body))
  end
end

The scripted router answers "unrouted" when the prompt has no demonstrations. With demonstrations, it routes by keywords. This mimics a model that needs examples to learn the label set.

route = fn ticket ->
  cond do
    ticket =~ ~r/lost|missing|find my/i -> "lost_card"
    ticket =~ ~r/atm|cash/i -> "cash_withdrawal"
    ticket =~ ~r/fee/i -> "card_payment_fee"
    true -> "unrouted"
  end
end

reply = fn body ->
  label = if Scripted.demos?(body), do: route.(Scripted.field(body, "ticket")), else: "unrouted"
  Scripted.chat(label: label)
end

model = [
  model: Scripted.model(),
  adapter: Tune.Adapter.Chat,
  request_options: Scripted.options(reply)
]

Define the classifier and data

The output is a Zoi.enum/1, so a reply outside the label set fails validation instead of reaching your code.

{:ok, classifier} =
  Tune.Predictor.new(
    signature: [
      instructions: "Route the support ticket.",
      inputs: [ticket: Zoi.string()],
      outputs: [label: Zoi.enum(["lost_card", "cash_withdrawal", "card_payment_fee", "unrouted"])]
    ]
  )

rows = fn pairs ->
  Enum.map(pairs, fn {id, ticket, label} ->
    %{id: id, input: %{ticket: ticket}, expected: %{label: label}}
  end)
end

train =
  rows.([
    {"train-1", "I lost my debit card on the train", "lost_card"},
    {"train-2", "My card is missing after a trip", "lost_card"},
    {"train-3", "The ATM debited me and dispensed no cash", "cash_withdrawal"},
    {"train-4", "Cash withdrawal failed at the station ATM", "cash_withdrawal"},
    {"train-5", "There is a fee on my card purchase abroad", "card_payment_fee"},
    {"train-6", "Why was there a fee on my card payment?", "card_payment_fee"}
  ])

test =
  rows.([
    {"lost-card", "I cannot find my card", "lost_card"},
    {"no-cash", "The ATM did not give me cash", "cash_withdrawal"},
    {"purchase-fee", "Why is there a purchase fee?", "card_payment_fee"}
  ])

:ok

Build the study

The :route score gives 1 for the expected label and 0 otherwise. :labeled_few_shot with k: 4 samples four training tickets. The seed fixes the sample, so the same seed gives the same demonstrations.

study =
  Tune.new(classifier, model: model)
  |> Tune.data(train: train, test: test)
  |> Tune.score(:route, fn row, output ->
    if output["label"] == row.expected["label"], do: 1, else: 0
  end)
  |> Tune.search(:labeled_few_shot, k: 4)
  |> Tune.limit(seed: 5, model_calls: 12)

{:ok, original_prompt} = Tune.preview(study, %{ticket: "I cannot find my card"})
Enum.map(original_prompt.context.messages, & &1.role)

Tune.preview/2 renders the request without a model call. The original prompt has only a system message and the ticket:

[:system, :user]

Select demonstrations

{:ok, winner} = Tune.tune(study)

for demo <- winner.value.demos do
  IO.puts("#{demo.inputs["ticket"]} -> #{demo.outputs["label"]}")
end

IO.puts("Search model calls: #{Enum.sum(Map.values(winner.calls))}")
Why was there a fee on my card payment? -> card_payment_fee
The ATM debited me and dispensed no cash -> cash_withdrawal
I lost my debit card on the train -> lost_card
Cash withdrawal failed at the station ATM -> cash_withdrawal
Search model calls: 0

:labeled_few_shot makes no model calls and computes no score during search. It only copies complete labeled rows. winner.calls is empty, and winner.metadata.notes records how many demonstrations each predictor received.

Enum.sort(winner.metadata.notes)
[
  demos: %{"root" => 4},
  settings: %{"k" => 4, "sample" => true},
  train_duplicates_removed: 0
]

Training rows with the same input count once. train_duplicates_removed reports how many were dropped.

Inspect the tuned prompt

winner.value is the tuned classifier. Preview it with the same model configuration. Each demonstration adds one user and one assistant message before the ticket.

{:ok, tuned_prompt} = Tune.new(winner.value, model: model) |> Tune.preview(%{ticket: "I cannot find my card"})

Enum.map(tuned_prompt.context.messages, & &1.role)
[:system, :user, :assistant, :user, :assistant, :user, :assistant, :user,
 :assistant, :user]

Compare on held-out tickets

Tune.compare/3 runs both classifiers on the test split. It pairs the results by case ID.

{:ok, comparison} = Tune.compare(study, winner, split: :test)

Enum.sort(comparison.metrics.route)
[
  baseline_mean: 0.0,
  challenger_mean: 1.0,
  counts: %{improved: 3},
  delta: 1.0,
  direction: :maximize,
  paired_count: 3,
  status: :improved
]

Each row holds both observations. The original classifier routed the lost card ticket to "unrouted"; the tuned one routed it correctly.

row = comparison.rows["lost-card"]

IO.puts("original: #{row.baseline.output["label"]}")
IO.puts("tuned:    #{row.challenger.output["label"]}")
row.metrics.route
original: unrouted
tuned:    lost_card
%{status: :improved, delta: 1, baseline: 0, challenger: 1}

The held-out tickets never appear in the prompt. The tuned classifier scores 1.0 on them because the scripted model needs examples of each label, not copies of the test tickets.

Next steps

  • Search API lists the other demonstration strategies.
  • Artifacts saves the selected demonstrations.
  • QA prompts tunes instructions and demonstrations together.