Interactive Browser with Bibbidi

bibbidi_dep =
  if path = System.get_env("LB_BBD_PATH") do
    {:bibbidi, path: path}
  else
    {:bibbidi, "~> 0.3.0"}
  end

Mix.install([
  bibbidi_dep,
  {:kino, "~> 0.14"}
])

UI Module

defmodule UI do
  @behaviour Kino.Screen
  import Kino.{Control, Input, Screen}

  alias Bibbidi.Connection
  alias Bibbidi.Commands.BrowsingContext.{
    CaptureScreenshot,
    GetTree,
    Navigate,
    Reload,
    SetViewport,
    TraverseHistory
  }
  alias Bibbidi.Commands.Script.{AddPreloadScript, Evaluate}
  alias Bibbidi.Commands.Session.Subscribe

  @viewports %{
    desktop: {1280, 800},
    tablet: {768, 1024},
    phone: {375, 667}
  }

  def new do
    screenshot_frame = Kino.Frame.new()
    log_frame = Kino.Frame.new()

    state = %{
      browser: nil,
      conn: nil,
      context: nil,
      log_listener: nil,
      screenshot_frame: screenshot_frame,
      log_frame: log_frame,
      elements: [],
      page_url: nil,
      page_title: nil,
      viewport: :desktop,
      js_result: nil,
      status: nil
    }

    Kino.Layout.grid(
      [
        Kino.Screen.new(__MODULE__, state),
        Kino.Markdown.new("---\n### Screenshots"),
        screenshot_frame,
        Kino.Markdown.new("---\n### Console Log"),
        log_frame
      ],
      gap: 8
    )
  end

  # -- Render ------------------------------------------------------------------

  @impl true
  def render(%{conn: nil} = state) do
    start_btn = button("Start Browser") |> control(&handle_start/2)

    [
      state.status && Kino.Markdown.new(state.status),
      start_btn
    ]
    |> Enum.reject(&is_nil/1)
    |> Kino.Layout.grid(gap: 16)
  end

  def render(state) do
    page_info =
      if state.page_url do
        Kino.Markdown.new("**#{state.page_title || "Untitled"}** | `#{state.page_url}`")
      end

    status = state.status && Kino.Markdown.new(state.status)

    # Navigation
    back_btn = button("Back") |> control(&handle_back/2)
    fwd_btn = button("Forward") |> control(&handle_forward/2)
    reload_btn = button("Reload") |> control(&handle_reload/2)

    nav_form =
      form([url: text("URL", default: state.page_url || "https://example.com")], submit: "Go")
      |> control(&handle_navigate/2)

    # Actions
    click_form =
      form([click_text: text("Button/Link text")], submit: "Click")
      |> control(&handle_click/2)

    viewport_form =
      form(
        [
          size:
            select("Viewport",
              [desktop: "Desktop (1280x800)", tablet: "Tablet (768x1024)", phone: "Phone (375x667)"],
              default: state.viewport
            )
        ],
        report_changes: true
      )
      |> control(&handle_viewport/2)

    # JS Console
    js_form =
      form([expression: textarea("JavaScript", default: "document.title")], submit: "Run")
      |> control(&handle_js/2)

    js_output = state.js_result && Kino.Markdown.new(state.js_result)

    # Tools
    rescan_btn = button("Rescan Page") |> control(&handle_rescan/2)
    screenshot_btn = button("Screenshot") |> control(&handle_screenshot/2)
    stop_btn = button("Stop Browser") |> control(&handle_stop/2)

    # Elements
    elements_table =
      if state.elements != [] do
        Kino.DataTable.new(state.elements,
          keys: [:tag, :id, :text],
          name: "Buttons & Links"
        )
      end

    [
      page_info,
      status,
      Kino.Layout.grid([back_btn, fwd_btn, reload_btn], columns: 3),
      nav_form,
      Kino.Layout.grid([click_form, viewport_form], columns: 2),
      js_form,
      js_output,
      Kino.Layout.grid([rescan_btn, screenshot_btn, stop_btn], columns: 3),
      elements_table
    ]
    |> Enum.reject(&is_nil/1)
    |> Kino.Layout.grid(gap: 8)
  end

  # -- Start / Stop ------------------------------------------------------------

  defp handle_start(_, state) do
    {:ok, browser} = Bibbidi.Browser.start_link(headless: false)
    {:ok, conn} = Connection.start_link(browser: browser)
    {:ok, _caps} = Bibbidi.Session.new(conn)
    {:ok, %{"contexts" => [%{"context" => context} | _]}} = Connection.execute(conn, %GetTree{})

    Connection.execute(conn, %SetViewport{context: context, viewport: %{width: 1280, height: 800}})
    install_observers(conn)
    log_listener = start_log_listener(conn, state.log_frame)

    %{state |
      browser: browser,
      conn: conn,
      context: context,
      log_listener: log_listener,
      elements: [],
      page_url: nil,
      page_title: nil,
      viewport: :desktop,
      js_result: nil,
      status: "Browser started — navigate to a page to get started."
    }
  rescue
    e -> %{state | status: "**Error starting browser:** `#{Exception.message(e)}`"}
  end

  defp handle_stop(_, state) do
    if state.log_listener, do: Process.exit(state.log_listener, :kill)
    Connection.close(state.conn)
    Bibbidi.Browser.stop(state.browser)
    Kino.Frame.clear(state.screenshot_frame)
    Kino.Frame.clear(state.log_frame)

    %{state |
      browser: nil,
      conn: nil,
      context: nil,
      log_listener: nil,
      elements: [],
      page_url: nil,
      page_title: nil,
      js_result: nil,
      status: "Browser stopped."
    }
  end

  # -- Navigation --------------------------------------------------------------

  defp handle_navigate(%{data: %{url: url}}, state) do
    case Connection.execute(state.conn, %Navigate{context: state.context, url: url, wait: "complete"}) do
      {:ok, _} ->
        refresh_page(state, "Navigated to `#{url}`")

      {:error, err} ->
        %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_back(_, state) do
    case Connection.execute(state.conn, %TraverseHistory{context: state.context, delta: -1}) do
      {:ok, _} ->
        Process.sleep(500)
        refresh_page(state, "Went back")

      {:error, _} ->
        %{state | status: "No previous page in history"}
    end
  end

  defp handle_forward(_, state) do
    case Connection.execute(state.conn, %TraverseHistory{context: state.context, delta: 1}) do
      {:ok, _} ->
        Process.sleep(500)
        refresh_page(state, "Went forward")

      {:error, _} ->
        %{state | status: "No next page in history"}
    end
  end

  defp handle_reload(_, state) do
    case Connection.execute(state.conn, %Reload{context: state.context, wait: "complete"}) do
      {:ok, _} -> refresh_page(state, "Page reloaded")
      {:error, err} -> %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  # -- Actions -----------------------------------------------------------------

  defp handle_click(%{data: %{click_text: click_text}}, state) do
    js = """
    (() => {
      const els = [...document.querySelectorAll('button, a, [role="button"]')];
      const el = els.find(e => e.textContent.trim().includes(#{JSON.encode!(click_text)}));
      if (el) { el.click(); return true; }
      return false;
    })()
    """

    case Connection.execute(state.conn, %Evaluate{
      expression: js,
      target: %{context: state.context},
      await_promise: false,
      user_activation: true
    }) do
      {:ok, %{"result" => %{"value" => true}}} ->
        Process.sleep(1000)
        refresh_page(state, "Clicked **\"#{click_text}\"**")

      {:ok, %{"result" => %{"value" => false}}} ->
        %{state | status: "No element found matching **\"#{click_text}\"**"}

      {:error, err} ->
        %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_viewport(%{data: %{size: size}}, state) do
    {w, h} = Map.fetch!(@viewports, size)

    case Connection.execute(state.conn, %SetViewport{
      context: state.context,
      viewport: %{width: w, height: h}
    }) do
      {:ok, _} -> %{state | viewport: size, status: "Viewport set to #{w}x#{h}"}
      {:error, err} -> %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_js(%{data: %{expression: expression}}, state) do
    case Connection.execute(state.conn, %Evaluate{
      expression: expression,
      target: %{context: state.context},
      await_promise: false
    }) do
      {:ok, %{"result" => result}} ->
        %{state | js_result: "`-> #{format_js_result(result)}`"}

      {:ok, %{"exceptionDetails" => %{"text" => text}}} ->
        %{state | js_result: "**Exception:** `#{text}`"}

      {:error, err} ->
        %{state | js_result: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_rescan(_, state) do
    refresh_page(state, "Page rescanned")
  end

  defp handle_screenshot(_, state) do
    case Connection.execute(state.conn, %CaptureScreenshot{context: state.context}) do
      {:ok, %{"data" => base64_data}} ->
        image = Kino.Image.new(Base.decode64!(base64_data), :png)
        Kino.Frame.append(state.screenshot_frame, image)
        %{state | status: "Screenshot captured"}

      {:error, err} ->
        %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  # -- Helpers -----------------------------------------------------------------

  defp refresh_page(state, status) do
    {url, title} = get_page_info(state.conn, state.context)
    elements = list_elements(state.conn, state.context)
    %{state | page_url: url, page_title: title, elements: elements, status: status}
  end

  defp get_page_info(conn, context) do
    js = "JSON.stringify({url: location.href, title: document.title})"

    case Connection.execute(conn, %Evaluate{
      expression: js,
      target: %{context: context},
      await_promise: false
    }) do
      {:ok, %{"result" => %{"type" => "string", "value" => json}}} ->
        info = JSON.decode!(json)
        {info["url"], info["title"]}

      _ ->
        {nil, nil}
    end
  end

  defp list_elements(conn, context) do
    js = """
    JSON.stringify(
      [...document.querySelectorAll('button, a, [role="button"]')].map(el => ({
        tag: el.tagName.toLowerCase(),
        id: el.id || "",
        text: (el.textContent || "").trim().substring(0, 20)
      }))
    )
    """

    case Connection.execute(conn, %Evaluate{
      expression: js,
      target: %{context: context},
      await_promise: false
    }) do
      {:ok, %{"result" => %{"type" => "string", "value" => json}}} ->
        json
        |> JSON.decode!()
        |> Enum.map(fn el ->
          %{tag: el["tag"], id: el["id"], text: el["text"]}
        end)

      _ ->
        []
    end
  end

  defp format_js_result(%{"type" => "string", "value" => v}), do: ~s("#{v}")
  defp format_js_result(%{"type" => "number", "value" => v}), do: to_string(v)
  defp format_js_result(%{"type" => "boolean", "value" => v}), do: to_string(v)
  defp format_js_result(%{"type" => "null"}), do: "null"
  defp format_js_result(%{"type" => "undefined"}), do: "undefined"
  defp format_js_result(%{"type" => type} = r), do: "#{type}: #{inspect(Map.get(r, "value", ""))}"

  # -- Observers ---------------------------------------------------------------

  defp install_observers(conn) do
    Connection.execute(conn, %AddPreloadScript{
      function_declaration: """
      () => {
        document.addEventListener('click', (e) => {
          const tag = e.target.tagName.toLowerCase();
          const text = (e.target.textContent || '').trim().substring(0, 40);
          console.log('[click] <' + tag + '> ' + text);
        }, true);

        const attach = () => {
          new MutationObserver((mutations) => {
            const added = mutations.reduce((n, m) => n + m.addedNodes.length, 0);
            const removed = mutations.reduce((n, m) => n + m.removedNodes.length, 0);
            if (added || removed) console.log('[dom] +' + added + ' -' + removed + ' nodes');
          }).observe(document.documentElement, { childList: true, subtree: true });
        };

        if (document.readyState === 'loading') {
          document.addEventListener('DOMContentLoaded', attach);
        } else {
          attach();
        }
      }
      """
    })
  end

  # -- Console log listener ----------------------------------------------------

  @log_events [
    "log.entryAdded",
    "browsingContext.navigationStarted",
    "browsingContext.load",
    "browsingContext.domContentLoaded",
    "browsingContext.fragmentNavigated",
    "browsingContext.userPromptOpened",
    "browsingContext.userPromptClosed"
  ]

  defp start_log_listener(conn, log_frame) do
    Connection.execute(conn, %Subscribe{events: @log_events})

    spawn(fn ->
      for event <- @log_events, do: Connection.subscribe(conn, event)
      log_loop(log_frame)
    end)
  end

  alias Bibbidi.Events.BrowsingContext.{
    NavigationStarted,
    DomContentLoaded,
    Load,
    FragmentNavigated,
    UserPromptOpened,
    UserPromptClosed
  }
  alias Bibbidi.Events.Log.EntryAdded

  defp log_loop(frame) do
    receive do
      {:bibbidi_event, "log.entryAdded", %EntryAdded{level: level, text: text}} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[#{level || "info"}]` #{text}"))

      {:bibbidi_event, "browsingContext.navigationStarted", %NavigationStarted{url: url}} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` started: #{url}"))

      {:bibbidi_event, "browsingContext.domContentLoaded", %DomContentLoaded{url: url}} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` DOM ready: #{url}"))

      {:bibbidi_event, "browsingContext.load", %Load{url: url}} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` loaded: #{url}"))

      {:bibbidi_event, "browsingContext.fragmentNavigated", %FragmentNavigated{url: url}} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` fragment: #{url}"))

      {:bibbidi_event, "browsingContext.userPromptOpened", %UserPromptOpened{type: type, message: msg}} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[prompt]` #{type || "dialog"} opened: #{msg}"))

      {:bibbidi_event, "browsingContext.userPromptClosed", %UserPromptClosed{accepted: accepted}} ->
        status = if accepted, do: "accepted", else: "dismissed"
        Kino.Frame.append(frame, Kino.Markdown.new("`[prompt]` #{status}"))
    end

    log_loop(frame)
  end
end

Controls

UI.new()