Import, Eval, And Trace

Purpose And Boundary

This notebook shows a data-authored agent imported from YAML, explicit registries for executable references, deterministic evals, and trace recording. Imported specs contain definition data, not executable handlers. The notebook therefore supplies the retained action adapter as an advanced runtime capability. Jidoka.Kino is development and test support only.

Mix.install(
  [
    {:kino, "~> 0.14"},
    {:jidoka, path: Path.expand("../..", __DIR__)}
  ],
  consolidate_protocols: false
)

Jidoka.Kino.setup_notebook(model: "test:notebook-model", check_provider?: false)

Register Executable References

defmodule NotebookEcho do
  use Jidoka.Action,
    name: "echo_value",
    description: "Echoes a deterministic value.",
    schema:
      Zoi.object(%{
        value: Zoi.string()
      })

  @impl true
  def run(params, _context) do
    value = Map.get(params, :value) || Map.get(params, "value")
    {:ok, %{value: value, source: "notebook"}}
  end
end

Import A YAML Agent

yaml = """
agent:
  id: imported_eval_agent
  model:
    provider: test
    id: notebook-model
  instructions: Use echo_value before answering.
  runtime_defaults:
    max_model_turns: 4
tools:
  actions:
    - echo
controls:
  max_turns: 4
"""

{:ok, spec} =
  Jidoka.import(yaml,
    actions: %{"echo" => NotebookEcho}
  )

{:ok, _inspection} = Jidoka.Kino.debug_agent(spec)
{:ok, _diagram} = Jidoka.Kino.agent_diagram(spec)

Run And Trace The Turn

fake_llm = fn _intent, journal, _context ->
  llm_calls =
    journal.results
    |> Map.values()
    |> Enum.count(&(&1.kind == :llm))

  case llm_calls do
    0 -> {:ok, %{type: :operation, name: "echo_value", arguments: %{"value" => "Ada"}}}
    1 -> {:ok, %{type: :final, content: "Echo returned Ada from the notebook."}}
  end
end

{:ok, result} =
  Jidoka.turn(spec, "Echo Ada.",
    llm: fake_llm,
    operations: Jidoka.Adapter.Jido.Actions.operations([NotebookEcho])
  )

{:ok, trace_pid} = Jidoka.Trace.Sink.InMemory.start_link()

:ok =
  Jidoka.Trace.record(result.events, {Jidoka.Trace.Sink.InMemory, pid: trace_pid},
    policy: [redact_keys: [], omit_keys: []]
  )

Jidoka.Trace.Sink.InMemory.list(trace_pid)
|> Enum.map(& &1.event)

{:ok, _timeline} = Jidoka.Kino.timeline(result)
{:ok, _graph} = Jidoka.Kino.call_graph(result)

Turn The Flow Into An Eval

{:ok, eval_run} =
  Jidoka.Eval.run_case(
    [
      id: "imported_echo_eval",
      agent: spec,
      input: "Echo Ada.",
      assertions: %{
        contains: "Ada",
        operation_called: "echo_value"
      }
    ],
    llm: fake_llm,
    operations: Jidoka.Adapter.Jido.Actions.operations([NotebookEcho])
  )

Jidoka.inspect(eval_run)