Lesson 07 Livebook: Drift Recovery

The bridge has two kinds of bad news now: stale telemetry and no telemetry at all.

Mix.install([
  {:helios_fleet, path: "../07_drift_recovery"}
])

Application.ensure_all_started(:helios_fleet)

Helpers

defmodule Lesson07Helpers do
  alias HeliosFleet.Runtime

  def unique_agent_id(prefix) do
    "#{prefix}-#{System.unique_integer([:positive])}"
  end

  def safe_stop_family(parent_id) do
    child_ids = [
      "#{parent_id}/khepri/port_relay",
      "#{parent_id}/khepri/starboard_relay",
      "#{parent_id}/menhit/port_relay",
      "#{parent_id}/menhit/starboard_relay",
      "#{parent_id}/khepri",
      "#{parent_id}/menhit"
    ]

    Enum.each(child_ids, fn child_id ->
      case Runtime.whereis(child_id) do
        nil -> :ok
        _pid -> Runtime.stop_agent(child_id)
      end
    end)

    case Runtime.whereis(parent_id) do
      nil -> :ok
      _pid -> Runtime.stop_agent(parent_id)
    end
  end
end

Let The Mesh Drift

alias HeliosFleet
alias Jido.AgentServer
alias Jido.Signal

vessel_id = Lesson07Helpers.unique_agent_id("orpheus")
{:ok, pid} = HeliosFleet.start_command_vessel(vessel_id)

{:ok, _vessel} =
  AgentServer.call(
    pid,
    Signal.new!(
      "fleet.command.mesh_deployment_requested",
      %{
        mesh_id: "mesh-icefall",
        target: "ice fragment 6R"
      },
      source: "/orpheus/bridge"
    )
  )

Process.sleep(50)
{:ok, command_state} = AgentServer.state(pid)
khepri_pid = command_state.children.khepri.pid

{:ok, _probe} =
  AgentServer.call(
    khepri_pid,
    Signal.new!(
      "fleet.probe.telemetry_drift_detected",
      %{
        telemetry_age_s: 480,
        ephemeris_error_km: 23
      },
      source: "/nav/core"
    )
  )

{:ok, _probe} =
  AgentServer.call(
    khepri_pid,
    Signal.new!("fleet.probe.health_report_requested", %{}, source: "/orpheus/bridge")
  )

Process.sleep(200)
{:ok, updated_command_state} = AgentServer.state(pid)

Lesson07Helpers.safe_stop_family(vessel_id)

%{
  probe_health_reports: updated_command_state.agent.state.probe_health_reports,
  recovery_snapshots: updated_command_state.agent.state.recovery_snapshots
}