Interactive Browser with Bibbidi

Mix.install([
  {:bibbidi, "~> 0.1.0"},
  {:kino, "~> 0.14"}
])

UI Module

defmodule UI do
  @behaviour Kino.Screen
  import Kino.{Control, Input, Screen}

  alias Bibbidi.Commands.BrowsingContext
  alias Bibbidi.Commands.Script

  @viewports %{
    desktop: {1280, 800},
    tablet: {768, 1024},
    phone: {375, 667}
  }

  def new do
    screenshot_frame = Kino.Frame.new()
    log_frame = Kino.Frame.new()

    state = %{
      browser: nil,
      conn: nil,
      context: nil,
      log_listener: nil,
      screenshot_frame: screenshot_frame,
      log_frame: log_frame,
      elements: [],
      page_url: nil,
      page_title: nil,
      viewport: :desktop,
      js_result: nil,
      status: nil
    }

    Kino.Layout.grid(
      [
        Kino.Screen.new(__MODULE__, state),
        Kino.Markdown.new("---\n### Screenshots"),
        screenshot_frame,
        Kino.Markdown.new("---\n### Console Log"),
        log_frame
      ],
      gap: 8
    )
  end

  # -- Render ------------------------------------------------------------------

  @impl true
  def render(%{conn: nil} = state) do
    start_btn = button("Start Browser") |> control(&handle_start/2)

    [
      state.status && Kino.Markdown.new(state.status),
      start_btn
    ]
    |> Enum.reject(&is_nil/1)
    |> Kino.Layout.grid(gap: 16)
  end

  def render(state) do
    page_info =
      if state.page_url do
        Kino.Markdown.new("**#{state.page_title || "Untitled"}** | `#{state.page_url}`")
      end

    status = state.status && Kino.Markdown.new(state.status)

    # Navigation
    back_btn = button("Back") |> control(&handle_back/2)
    fwd_btn = button("Forward") |> control(&handle_forward/2)
    reload_btn = button("Reload") |> control(&handle_reload/2)

    nav_form =
      form([url: text("URL", default: state.page_url || "https://example.com")], submit: "Go")
      |> control(&handle_navigate/2)

    # Actions
    click_form =
      form([click_text: text("Button/Link text")], submit: "Click")
      |> control(&handle_click/2)

    viewport_form =
      form(
        [
          size:
            select("Viewport",
              [desktop: "Desktop (1280x800)", tablet: "Tablet (768x1024)", phone: "Phone (375x667)"],
              default: state.viewport
            )
        ],
        report_changes: true
      )
      |> control(&handle_viewport/2)

    # JS Console
    js_form =
      form([expression: textarea("JavaScript", default: "document.title")], submit: "Run")
      |> control(&handle_js/2)

    js_output = state.js_result && Kino.Markdown.new(state.js_result)

    # Tools
    rescan_btn = button("Rescan Page") |> control(&handle_rescan/2)
    screenshot_btn = button("Screenshot") |> control(&handle_screenshot/2)
    stop_btn = button("Stop Browser") |> control(&handle_stop/2)

    # Elements
    elements_table =
      if state.elements != [] do
        Kino.DataTable.new(state.elements,
          keys: [:tag, :id, :text],
          name: "Buttons & Links"
        )
      end

    [
      page_info,
      status,
      Kino.Layout.grid([back_btn, fwd_btn, reload_btn], columns: 3),
      nav_form,
      Kino.Layout.grid([click_form, viewport_form], columns: 2),
      js_form,
      js_output,
      Kino.Layout.grid([rescan_btn, screenshot_btn, stop_btn], columns: 3),
      elements_table
    ]
    |> Enum.reject(&is_nil/1)
    |> Kino.Layout.grid(gap: 8)
  end

  # -- Start / Stop ------------------------------------------------------------

  defp handle_start(_, state) do
    {:ok, browser} = Bibbidi.Browser.start_link(headless: false)
    {:ok, conn} = Bibbidi.Connection.start_link(browser: browser)
    {:ok, _caps} = Bibbidi.Session.new(conn)
    {:ok, %{"contexts" => [%{"context" => context} | _]}} = BrowsingContext.get_tree(conn)

    BrowsingContext.set_viewport(conn, context, %{width: 1280, height: 800})
    install_observers(conn)
    log_listener = start_log_listener(conn, state.log_frame)

    %{state |
      browser: browser,
      conn: conn,
      context: context,
      log_listener: log_listener,
      elements: [],
      page_url: nil,
      page_title: nil,
      viewport: :desktop,
      js_result: nil,
      status: "Browser started — navigate to a page to get started."
    }
  rescue
    e -> %{state | status: "**Error starting browser:** `#{Exception.message(e)}`"}
  end

  defp handle_stop(_, state) do
    if state.log_listener, do: Process.exit(state.log_listener, :kill)
    Bibbidi.Connection.close(state.conn)
    Bibbidi.Browser.stop(state.browser)
    Kino.Frame.clear(state.screenshot_frame)
    Kino.Frame.clear(state.log_frame)

    %{state |
      browser: nil,
      conn: nil,
      context: nil,
      log_listener: nil,
      elements: [],
      page_url: nil,
      page_title: nil,
      js_result: nil,
      status: "Browser stopped."
    }
  end

  # -- Navigation --------------------------------------------------------------

  defp handle_navigate(%{data: %{url: url}}, state) do
    case BrowsingContext.navigate(state.conn, state.context, url, wait: "complete") do
      {:ok, _} ->
        refresh_page(state, "Navigated to `#{url}`")

      {:error, err} ->
        %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_back(_, state) do
    case BrowsingContext.traverse_history(state.conn, state.context, -1) do
      {:ok, _} ->
        Process.sleep(500)
        refresh_page(state, "Went back")

      {:error, _} ->
        %{state | status: "No previous page in history"}
    end
  end

  defp handle_forward(_, state) do
    case BrowsingContext.traverse_history(state.conn, state.context, 1) do
      {:ok, _} ->
        Process.sleep(500)
        refresh_page(state, "Went forward")

      {:error, _} ->
        %{state | status: "No next page in history"}
    end
  end

  defp handle_reload(_, state) do
    case BrowsingContext.reload(state.conn, state.context, wait: "complete") do
      {:ok, _} -> refresh_page(state, "Page reloaded")
      {:error, err} -> %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  # -- Actions -----------------------------------------------------------------

  defp handle_click(%{data: %{click_text: click_text}}, state) do
    js = """
    (() => {
      const els = [...document.querySelectorAll('button, a, [role="button"]')];
      const el = els.find(e => e.textContent.trim().includes(#{JSON.encode!(click_text)}));
      if (el) { el.click(); return true; }
      return false;
    })()
    """

    case Script.evaluate(state.conn, js, %{context: state.context}, user_activation: true) do
      {:ok, %{"result" => %{"value" => true}}} ->
        Process.sleep(1000)
        refresh_page(state, "Clicked **\"#{click_text}\"**")

      {:ok, %{"result" => %{"value" => false}}} ->
        %{state | status: "No element found matching **\"#{click_text}\"**"}

      {:error, err} ->
        %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_viewport(%{data: %{size: size}}, state) do
    {w, h} = Map.fetch!(@viewports, size)

    case BrowsingContext.set_viewport(state.conn, state.context, %{width: w, height: h}) do
      {:ok, _} -> %{state | viewport: size, status: "Viewport set to #{w}x#{h}"}
      {:error, err} -> %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_js(%{data: %{expression: expression}}, state) do
    case Script.evaluate(state.conn, expression, %{context: state.context}) do
      {:ok, %{"result" => result}} ->
        %{state | js_result: "`-> #{format_js_result(result)}`"}

      {:ok, %{"exceptionDetails" => %{"text" => text}}} ->
        %{state | js_result: "**Exception:** `#{text}`"}

      {:error, err} ->
        %{state | js_result: "**Error:** `#{inspect(err)}`"}
    end
  end

  defp handle_rescan(_, state) do
    refresh_page(state, "Page rescanned")
  end

  defp handle_screenshot(_, state) do
    case BrowsingContext.capture_screenshot(state.conn, state.context) do
      {:ok, %{"data" => base64_data}} ->
        image = Kino.Image.new(Base.decode64!(base64_data), :png)
        Kino.Frame.append(state.screenshot_frame, image)
        %{state | status: "Screenshot captured"}

      {:error, err} ->
        %{state | status: "**Error:** `#{inspect(err)}`"}
    end
  end

  # -- Helpers -----------------------------------------------------------------

  defp refresh_page(state, status) do
    {url, title} = get_page_info(state.conn, state.context)
    elements = list_elements(state.conn, state.context)
    %{state | page_url: url, page_title: title, elements: elements, status: status}
  end

  defp get_page_info(conn, context) do
    js = "JSON.stringify({url: location.href, title: document.title})"

    case Script.evaluate(conn, js, %{context: context}) do
      {:ok, %{"result" => %{"type" => "string", "value" => json}}} ->
        info = JSON.decode!(json)
        {info["url"], info["title"]}

      _ ->
        {nil, nil}
    end
  end

  defp list_elements(conn, context) do
    js = """
    JSON.stringify(
      [...document.querySelectorAll('button, a, [role="button"]')].map(el => ({
        tag: el.tagName.toLowerCase(),
        id: el.id || "",
        text: (el.textContent || "").trim().substring(0, 20)
      }))
    )
    """

    case Script.evaluate(conn, js, %{context: context}) do
      {:ok, %{"result" => %{"type" => "string", "value" => json}}} ->
        json
        |> JSON.decode!()
        |> Enum.map(fn el ->
          %{tag: el["tag"], id: el["id"], text: el["text"]}
        end)

      _ ->
        []
    end
  end

  defp format_js_result(%{"type" => "string", "value" => v}), do: ~s("#{v}")
  defp format_js_result(%{"type" => "number", "value" => v}), do: to_string(v)
  defp format_js_result(%{"type" => "boolean", "value" => v}), do: to_string(v)
  defp format_js_result(%{"type" => "null"}), do: "null"
  defp format_js_result(%{"type" => "undefined"}), do: "undefined"
  defp format_js_result(%{"type" => type} = r), do: "#{type}: #{inspect(Map.get(r, "value", ""))}"

  # -- Observers ---------------------------------------------------------------

  defp install_observers(conn) do
    Script.add_preload_script(conn, """
    () => {
      document.addEventListener('click', (e) => {
        const tag = e.target.tagName.toLowerCase();
        const text = (e.target.textContent || '').trim().substring(0, 40);
        console.log('[click] <' + tag + '> ' + text);
      }, true);

      const attach = () => {
        new MutationObserver((mutations) => {
          const added = mutations.reduce((n, m) => n + m.addedNodes.length, 0);
          const removed = mutations.reduce((n, m) => n + m.removedNodes.length, 0);
          if (added || removed) console.log('[dom] +' + added + ' -' + removed + ' nodes');
        }).observe(document.documentElement, { childList: true, subtree: true });
      };

      if (document.readyState === 'loading') {
        document.addEventListener('DOMContentLoaded', attach);
      } else {
        attach();
      }
    }
    """)
  end

  # -- Console log listener ----------------------------------------------------

  @log_events [
    "log.entryAdded",
    "browsingContext.navigationStarted",
    "browsingContext.load",
    "browsingContext.domContentLoaded",
    "browsingContext.fragmentNavigated",
    "browsingContext.userPromptOpened",
    "browsingContext.userPromptClosed"
  ]

  defp start_log_listener(conn, log_frame) do
    Bibbidi.Session.subscribe(conn, @log_events)

    spawn(fn ->
      for event <- @log_events, do: Bibbidi.Connection.subscribe(conn, event)
      log_loop(log_frame)
    end)
  end

  defp log_loop(frame) do
    receive do
      {:bibbidi_event, "log.entryAdded", params} ->
        level = params["level"] || "info"
        text = params["text"] || ""
        Kino.Frame.append(frame, Kino.Markdown.new("`[#{level}]` #{text}"))

      {:bibbidi_event, "browsingContext.navigationStarted", params} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` started: #{params["url"]}"))

      {:bibbidi_event, "browsingContext.domContentLoaded", params} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` DOM ready: #{params["url"]}"))

      {:bibbidi_event, "browsingContext.load", params} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` loaded: #{params["url"]}"))

      {:bibbidi_event, "browsingContext.fragmentNavigated", params} ->
        Kino.Frame.append(frame, Kino.Markdown.new("`[nav]` fragment: #{params["url"]}"))

      {:bibbidi_event, "browsingContext.userPromptOpened", params} ->
        type = params["type"] || "dialog"
        msg = params["message"] || ""
        Kino.Frame.append(frame, Kino.Markdown.new("`[prompt]` #{type} opened: #{msg}"))

      {:bibbidi_event, "browsingContext.userPromptClosed", params} ->
        accepted = if params["accepted"], do: "accepted", else: "dismissed"
        Kino.Frame.append(frame, Kino.Markdown.new("`[prompt]` #{accepted}"))
    end

    log_loop(frame)
  end
end

Controls

UI.new()