From 55250b0de55d6109fa4655b255d9fa06b3a79f05 Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Sun, 15 Nov 2026 12:04:00 -0600 Subject: [PATCH 1/3] refactor(extension,vom): seal browser-driver boundary, enforce RAII CdpRunner, and unify DOM predicates & VOM metadata Why: - Upper extension layers (`read`, `extract`, `waits`, `wheel`, `drag`, `action-guard`, `vom/facts`, `AccessibilityLease`, `debug/`, `recording-runtime`, and `geometry/frame-context`) still reached into `browser-driver/frame-graph` internals (`CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, `getFrameGraph`) and transient OOPIF session routing. - `CdpRunner` carried optional `?` methods for handwritten test stubs, while V8 `objectGroup` / `objectId` handles in `handleSelect`, `handleFill`, `handleFocus`/`handleBlur`, `action-checks`, `file-input-transaction`, and `evaluate(return_by_value: false)` risked leaks on throw/abort or could resurrect soft-detached tabs / mask pre-dispatch cooperation errors during `finally` cleanup. - `--selector` resolution, `[data-bsk-overlay]` exclusion, open Shadow DOM walks, sensitive `autocomplete` checks, interactive ARIA roles, and VOM line metadata (`text-delta` regex re-parsing vs `@browser-skill/vom`) had drifted across independent implementations. What: - Seal `CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, and `getFrameGraph` strictly inside `apps/extension/src/browser-driver/` behind `FrameDocumentHandle`, `FrameNodeHandle`, `FrameProcessBoundary`, and `TabEventSource` (`BskDriver.childTargetAttached` / `BskDriver.childTargetDetached`). - Promote all `CdpRunner` methods to a non-optional contract backed by `delegateCdpRunner` and `createStubCdpRunner`, and enforce deterministic RAII `withObjectGroup` / `withRemoteObject` V8 handle release via `cleanup.sendToFrame(..., { allowReattach: false })` without reattaching detached/soft-detached tabs, resetting idle timers, or masking caller errors. - Unify DOM predicates (`apps/extension/src/lib/dom-predicates.ts`) and cross-frame / open Shadow DOM `--selector` resolution (`apps/extension/src/tools/selector-resolution.ts`), fix background-lease `tool.wheel` paint readiness (`lacksForegroundPresentation`), isolate `target`-policy `--selector` refs via `withPinnedSelectorTargets`, require explicit `isAgentWindowId` on `returnBorrowedTab`, and replace module-global download state with instance-scoped `DownloadConcurrencyGate`. - Converge `@browser-skill/vom`, `cooperation/text-delta.ts`, and `crates/bsk-cli/src/cli/recipe/vom.rs` (`vom_contract.json`) onto structured `VomLineMetadata` (`inspectVomLine`, `VOM_DERIVED_NAME_CAP`), post-quote `[expanded]` / `[has-submenu]` tags, budgeted `renderDoubleLayer` occlusion footers, and Canvas `[visual:screenshot]` `@eN` parity on `cooperate sync`. - Sync bilingual `[Unreleased]` entries in `CHANGELOG.md` and `CHANGELOG.zh-CN.md`. --- CHANGELOG.md | 3 + CHANGELOG.zh-CN.md | 3 + .../__tests__/chromium-cdp.test.ts | 102 +- .../__tests__/command-deadline.test.ts | 12 +- .../src/browser-driver/chromium-cdp.ts | 359 +++++- .../src/browser-driver/command-deadline.ts | 61 +- .../src/browser-driver/frame-graph.ts | 77 +- .../src/browser-driver/frame-handles.ts | 239 ++++ .../screencast-coordinator.test.ts | 13 +- .../browser-driver/screencast-coordinator.ts | 2 +- .../cooperation-capture.browser.test.ts | 5 +- .../__tests__/cooperation-capture.test.ts | 14 +- .../src/cooperation/action-checks.ts | 85 +- .../extension/src/cooperation/action-guard.ts | 176 ++- .../cooperation/borrowed-blank-tab.test.ts | 8 +- .../src/cooperation/controller.test.ts | 73 +- apps/extension/src/cooperation/controller.ts | 143 ++- .../src/cooperation/live-text.browser.test.ts | 47 +- .../src/cooperation/observation-evidence.ts | 17 +- .../cooperation/policy-negotiation.test.ts | 12 +- .../src/cooperation/target-policy.test.ts | 195 ++- .../src/cooperation/text-delta.test.ts | 51 + apps/extension/src/cooperation/text-delta.ts | 62 +- .../src/debug/__tests__/debug.browser.test.ts | 4 +- .../src/debug/__tests__/manager.test.ts | 91 +- .../__tests__/network-control.browser.test.ts | 4 +- .../debug/__tests__/network-control.test.ts | 96 +- .../debug/__tests__/network-retention.test.ts | 9 +- .../src/debug/__tests__/network-store.test.ts | 35 +- .../debug/__tests__/observer.browser.test.ts | 51 +- apps/extension/src/debug/manager.ts | 97 +- apps/extension/src/debug/network-control.ts | 56 +- apps/extension/src/debug/network-store.ts | 40 +- apps/extension/src/debug/observer.ts | 3 +- apps/extension/src/entrypoints/background.ts | 32 +- .../src/lib/__tests__/document-settle.test.ts | 13 +- .../lib/__tests__/recording-runtime.test.ts | 4 +- .../lib/__tests__/settle-controller.test.ts | 17 +- apps/extension/src/lib/dom-predicates.ts | 41 + .../src/lib/recording/document-settle.ts | 2 +- .../src/lib/recording/observation-capture.ts | 2 +- .../src/lib/recording/recording-runtime.ts | 50 +- .../src/session-manager/ref-store.ts | 2 +- .../__tests__/background-execution.test.ts | 17 +- ...click-listener-observation.browser.test.ts | 57 +- .../src/tools/__tests__/click.browser.test.ts | 14 +- .../src/tools/__tests__/dispatcher.test.ts | 518 +++++++- .../download-deadline.browser.test.ts | 29 +- .../tools/__tests__/download-deadline.test.ts | 11 +- .../download-effect-state.browser.test.ts | 6 +- .../__tests__/download-effect-state.test.ts | 7 +- .../__tests__/download-lifecycle.test.ts | 20 +- .../src/tools/__tests__/drag.browser.test.ts | 37 +- .../src/tools/__tests__/drag.test.ts | 20 +- .../tools/__tests__/element-geometry.test.ts | 12 +- .../src/tools/__tests__/evaluate.test.ts | 109 +- .../__tests__/file-transfer-readiness.test.ts | 17 +- .../__tests__/file-transfer-trigger.test.ts | 16 +- .../src/tools/__tests__/file-transfer.test.ts | 117 +- .../src/tools/__tests__/fill.browser.test.ts | 70 +- .../src/tools/__tests__/fill.test.ts | 6 +- .../src/tools/__tests__/focus.browser.test.ts | 6 +- .../src/tools/__tests__/focus.test.ts | 32 +- .../tools/__tests__/frame-geometry.test.ts | 211 ++-- .../tools/__tests__/geometry-context.test.ts | 97 +- .../src/tools/__tests__/human-loop.test.ts | 62 +- .../src/tools/__tests__/interaction.test.ts | 100 +- .../navigation-history.browser.test.ts | 9 +- .../__tests__/navigation-recovery.test.ts | 15 +- .../navigation-same-document.browser.test.ts | 11 +- .../navigation-same-document.test.ts | 9 +- .../__tests__/navigation-wait.browser.test.ts | 9 +- .../src/tools/__tests__/navigation.test.ts | 35 +- .../observation-layers.browser.test.ts | 28 +- .../src/tools/__tests__/observation.test.ts | 300 ++--- .../src/tools/__tests__/record-steps.test.ts | 107 +- .../__tests__/screenshot-full-page.test.ts | 57 +- .../tools/__tests__/scroll.browser.test.ts | 79 +- .../src/tools/__tests__/scroll.test.ts | 69 +- .../tools/__tests__/select.browser.test.ts | 6 +- .../snapshot-coordinates.browser.test.ts | 55 +- .../soft-detach-click.browser.test.ts | 4 +- .../stealth-stripper.browser.test.ts | 28 +- .../src/tools/__tests__/tabs.test.ts | 48 +- .../__tests__/target-hit.browser.test.ts | 49 +- .../src/tools/__tests__/target-hit.test.ts | 49 +- .../__tests__/viewport-screenshot.test.ts | 6 +- .../__tests__/visual-geometry.browser.test.ts | 17 +- .../src/tools/__tests__/visual-ref.test.ts | 2 +- .../tools/__tests__/visual-screenshot.test.ts | 52 +- .../src/tools/__tests__/visual-target.test.ts | 4 +- .../src/tools/__tests__/waits.browser.test.ts | 10 +- .../src/tools/__tests__/waits.test.ts | 20 +- .../src/tools/__tests__/wheel.browser.test.ts | 78 +- .../src/tools/__tests__/wheel.test.ts | 69 +- .../tools/__tests__/window.browser.test.ts | 6 +- .../src/tools/__tests__/window.test.ts | 6 +- apps/extension/src/tools/dispatcher.ts | 47 +- apps/extension/src/tools/dom-read-runtime.ts | 20 +- apps/extension/src/tools/download-capture.ts | 28 +- apps/extension/src/tools/download.ts | 69 +- apps/extension/src/tools/drag.ts | 57 +- apps/extension/src/tools/element-geometry.ts | 2 +- apps/extension/src/tools/errors.ts | 2 +- apps/extension/src/tools/evaluate.ts | 52 +- apps/extension/src/tools/extract.ts | 182 ++- apps/extension/src/tools/extract/collector.ts | 41 +- .../src/tools/extract/extract.browser.test.ts | 20 +- .../src/tools/extract/handler.test.ts | 154 ++- apps/extension/src/tools/extract/types.ts | 2 + .../extension/src/tools/extract/validation.ts | 3 + .../src/tools/file-drop-transaction.ts | 44 +- .../src/tools/file-input-transaction.ts | 91 +- apps/extension/src/tools/frame-geometry.ts | 31 +- .../src/tools/geometry/coordinate-types.ts | 7 +- .../src/tools/geometry/frame-context.ts | 137 ++- apps/extension/src/tools/human-loop.ts | 20 +- apps/extension/src/tools/input-readiness.ts | 49 +- apps/extension/src/tools/interaction.ts | 1044 ++++++++--------- apps/extension/src/tools/observation.ts | 166 ++- .../src/tools/pointer-transaction.ts | 170 +-- apps/extension/src/tools/read.ts | 93 +- apps/extension/src/tools/read/collector.ts | 11 +- apps/extension/src/tools/read/read.test.ts | 45 +- apps/extension/src/tools/read/types.ts | 1 + apps/extension/src/tools/read/validation.ts | 9 +- apps/extension/src/tools/record.ts | 93 +- apps/extension/src/tools/scroll-visibility.ts | 16 +- apps/extension/src/tools/scroll.ts | 93 +- .../src/tools/selector-resolution.ts | 151 +++ apps/extension/src/tools/session.ts | 15 +- apps/extension/src/tools/shared.ts | 568 ++++++--- apps/extension/src/tools/snapshot-ref.ts | 2 +- apps/extension/src/tools/tabs.ts | 32 +- apps/extension/src/tools/target-hit.ts | 27 +- .../src/tools/viewport-screenshot.ts | 58 +- apps/extension/src/tools/visual-target.ts | 13 +- .../vom/__tests__/capture-coordinator.test.ts | 858 ++++++++------ .../src/tools/vom/__tests__/capture.test.ts | 49 +- .../vom/__tests__/document-identity.test.ts | 2 +- .../tools/vom/__tests__/form-state.test.ts | 67 +- .../vom/__tests__/frame-document.test.ts | 51 +- .../vom/__tests__/name-enrichment.test.ts | 100 +- .../normalize-form-membership.test.ts | 8 +- .../__tests__/record-safe-observation.test.ts | 11 +- .../vom/__tests__/semantic-graph.test.ts | 15 +- .../vom/__tests__/semantic-resolve.test.ts | 40 +- .../tools/vom/__tests__/visual-dedup.test.ts | 14 +- .../vom/__tests__/visual-discovery.test.ts | 48 +- .../vom/__tests__/visual-observation.test.ts | 47 +- .../src/tools/vom/capture-coordinator.ts | 314 ++--- .../src/tools/vom/document-identity.ts | 2 +- apps/extension/src/tools/vom/facts.ts | 28 +- .../extension/src/tools/vom/frame-document.ts | 56 +- .../src/tools/vom/name-enrichment.ts | 58 +- apps/extension/src/tools/vom/normalize.ts | 57 +- .../src/tools/vom/record-safe-observation.ts | 4 +- .../src/tools/vom/semantic-graph/build.ts | 3 +- .../src/tools/vom/semantic-graph/resolve.ts | 69 +- .../src/tools/vom/semantic-graph/types.ts | 5 +- apps/extension/src/tools/vom/snapshot.ts | 56 +- .../src/tools/vom/visual-discovery.ts | 17 +- .../src/tools/vom/visual-observation.ts | 100 +- apps/extension/src/tools/waits.ts | 116 +- apps/extension/src/tools/wheel.ts | 92 +- apps/extension/src/video/manager.test.ts | 20 +- crates/bsk-cli/src/cli/recipe/vom.rs | 17 +- .../bsk-cli/tests/fixtures/vom_contract.json | 7 +- .../scripts/test_release_audit.py | 25 +- packages/vom/src/__tests__/render.test.ts | 116 ++ packages/vom/src/index.ts | 4 + packages/vom/src/render.ts | 173 ++- packages/vom/src/types.ts | 17 + 173 files changed, 7445 insertions(+), 4122 deletions(-) create mode 100644 apps/extension/src/browser-driver/frame-handles.ts create mode 100644 apps/extension/src/lib/dom-predicates.ts create mode 100644 apps/extension/src/tools/selector-resolution.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 86d3a9545..80c2107d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ determines which versions can connect. ### Changed +- **Complete `browser-driver/` boundary encapsulation, strict `CdpRunner` RAII contract, unified `--selector` DOM predicates, and structured VOM metadata**: Sealed `CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, and `getFrameGraph` inside `apps/extension/src/browser-driver/` behind `FrameDocumentHandle`, `FrameNodeHandle`, `FrameProcessBoundary`, and `TabEventSource` across all remaining upper-layer modules (`read`, `extract`, `waits`, `wheel`, `drag`, `action-guard`, `vom/facts`, `AccessibilityLease`, `debug/`, `recording-runtime`, and `geometry/frame-context`); promoted all `CdpRunner` methods to a non-optional interface backed by `delegateCdpRunner`, `createStubCdpRunner`, and deterministic RAII `withObjectGroup` / `withRemoteObject` V8 handle release (`handleSelect`, `handleFill`, `handleFocus`/`handleBlur`, `action-checks`, `file-input-transaction`, and `evaluate` with `return_by_value: false`, without resurrecting detached tabs or masking pre-dispatch errors); unified cross-frame, open Shadow DOM, `[data-bsk-overlay]` exclusion, sensitive `autocomplete`, and interactive ARIA role predicates across `interaction`, `CooperationController`, `human-loop`, `read`, and `extract`; and converged `cooperate sync` and `observe` onto structured `@browser-skill/vom` `VomLineMetadata` (`inspectVomLine` and `VOM_DERIVED_NAME_CAP`). - **Lightweight `@wxg-prc-cpg/browser-skill-dsh-plugin` on the unified `bsk` CLI runtime**: Replaced the ~22k-LOC bespoke `bsk_*` wrapper and Web UI bundle in `packages/dsh-plugin-browserskill` with a lightweight Cordis plugin (free of `@deepseek-ai/dsh-*` host `rc` peer locks) that packages the canonical `crates/bsk-cli/skill` bundle, automatically heals host `PATH` for `~/.local/bin` (`$BSK_INSTALL_DIR`) and configured `bskPath`, and bundles an on-demand cross-platform `bsk` CLI + browser extension setup workflow (`bin/install-bsk.mjs` + `AGENT_INSTALL.md`) sharing the standard `bsk` CLI and `DSH_SESSION_ID` daemon scoping alongside restored `dsh-plugin-v*` release automation. - **Zero-tax Chrome MV3 `registration: "runtime"` dual-tier content script injection, SW-startup orphan Agent Window reaping, and shared `content/dom-primitives.ts`**: Migrated all four MV3 content-script entrypoints (`content`, `cooperation.content`, `record-frame.content`, and `long-screenshot-page.content`) out of static `manifest.json` `content_scripts` into idempotent runtime injection (`claimIdempotentContentScript`) with `noScriptStartedPostMessage: true` so ordinary browsing incurs zero content-script, `window.postMessage`, or React 19 overhead; injected Tier A (`content.js` and `long-screenshot-page.js`) per `tabId`/`documentId` via idempotent `ensureOverlayInjected` across controlled task tabs, `webNavigation.onCommitted` (including Speculation Rules prerender outermost frames, sequenced strictly after Tier B `cooperation.js` arming/repair), pre-confirmation `tab_borrow` candidate user tabs, `request_help`, `record`, and active `video` overlays while skipping uninjected tabs on `hidden` / `CAPTURE_SUPPRESS` / `INPUT_PASSTHROUGH`; dynamically registered/unregistered pure-TypeScript Tier B (`cooperation.js` and `record-frame.js`) with `persistAcrossSessions: false` when active sessions or recordings transition between `0` and `>= 1`, loading silently at `document_start` with 0 DOM listeners and 0 startup IPC until armed by Background (`COOPERATION_ARM_FRAME` / `RECORD_FRAME_START` plus 50ms per-tab subframe coalescing, `cooperation.js` co-injection in `injectRecordFrameAllFrames`, and non-destructive per-frame `injectCooperationAllFrames` repair across `tab_borrow`, `ack === undefined` frames, and dynamically inserted light-DOM or open/closed `ShadowRoot` `about:blank` / `srcdoc` subframes); tracked dedicated Agent Window `ownedTabIds` in `chrome.storage.session` (`bsk.orphan_agent_windows`) to reap orphaned pure Agent Windows on Service Worker restart without ever closing windows containing user, borrowed, or human-preserved tabs; and extracted `apps/extension/src/content/dom-primitives.ts`. - **Single daemon `tool_pipeline`, 4-module `daemon/ipc` decomposition, non-blocking persistence, and shared `cli::orchestration` kernel**: Decomposed 3,489-line `daemon/ipc.rs` into `daemon/tool_pipeline.rs` (single unified `tool.*` pipeline with RAII `AuditTicketGuard`, `SessionActivityGuard`, `ToolInflightRegistry`, `IntentWal`, and post-settlement scope/run-log reconciliation), `daemon/ipc_sessions.rs`, `daemon/ipc_transport.rs`, and a slim `daemon/ipc.rs` router; promoted exhaustive `OutcomeKind` (`bsk-protocol`) / `Outcome` (`daemon/tool_policy.rs`) routing in `daemon/queue.rs`; moved `TransferRegistry` staging/import/session-teardown I/O, `AuditStore` JSONL `sync_data()` writes (including `session.user_interrupt` markers), and monotonic-sequence `IntentWal` flushes/atomic `sync_data()` compactions out of global state `Mutex`es and Tokio worker threads via `spawn_blocking`; and extracted `cli/orchestration.rs` to share Canvas screenshot point-click validation, `open`/`sync`/action/positive timeout bounds, explicit `--session` + `--browser` conflict checks, cached VOM hydration, and 2-second `CANCEL_RESPONSE_GRACE` settlement across `bsk` CLI and `bsk mcp`. @@ -43,6 +44,7 @@ determines which versions can connect. ### Fixed +- Fixed `tool.wheel` skipping compositor paint readiness (`waitForInputPaint`) under `ToolDispatcher` persistent background leases on unfocused or minimized windows by decoupling `ReadyInput.hidden` (`lacksForegroundPresentation`) from temporary `restoreFocus` and wiring `windowsApi` in `ToolDispatcher`; stopped `target`-policy `--selector` actions in `CooperationController` from polluting `session.refStore`, emitting `used_ref`, or leaking hallucinated `@eN` references in `handleSelect` custom-dropdown error messages (`withPinnedSelectorTargets`); refreshed live frame activity inside `CooperationController.beginDomMutation()` under `target` policy before scripted DOM writes while preserving `state.baseline` (`observation_id`) when pre-write validation fails with `effect_state: "none"`; fixed `clearRecordingForSession` to use the caller/dispatcher's `deps` (`RECORD_CANCEL` + `bypassOverlay(tabId, false)`), required explicit `isAgentWindowId` in `returnBorrowedTab` so returning a borrowed user tab never routes into another session's Agent Window, replaced module-global download locking with instance-scoped `DownloadConcurrencyGate` (`session_busy`, `effect_state: "none"`, `dispatched: false`), and cleaned up `observeIdentityStreaks`, `hoverLatches`, and `hoverBypassTabs` on session teardown and `tabs.onRemoved`; enabled Canvas `[visual:screenshot]` `@eN` parity on `cooperate sync`, budgeted the `renderDoubleLayer` `L2 page … occluded by L1` footer line within `maxTokens`, and moved popup `[expanded]` / `[has-submenu]` tags out of `vom.name` to post-quote VOM tags so custom dropdowns no longer emit duplicate `[expanded]` suffixes. - Eliminated redundant back-to-back frame collection sweeps in `guardedRunner.runBrowserMutation` (`action-guard.ts`), skipped mutation fencing on passive `inspect` probes in `handleFullPageScreenshot`, and raised the per-frame cooperation IPC reply deadline (`FRAME_REPLY_TIMEOUT_MS` / `OWNER_REPLY_MS`) from `1200ms` to `3000ms` so cold high-DPR / software-rasterized full-page captures and Tier-A overlay mounts do not spuriously fail with `FrameUnresponsiveError` (#761). - Added `--max-tokens` (`max_tokens` in MCP) to `bsk step`, `bsk plan`, `bsk_interact` (with `sync_after` or `until_visible`), and `bsk_plan`, forwarding it to the post-action `tool.cooperate` sync instead of hardcoding `None` so `--max-tokens` baselines stay reusable across `delta` and `unchanged` syncs without forcing an unbounded page re-capture (#755). - Ignored internal `data-bsk-*` attributes and `` (`[data-bsk-overlay]`) ShadowHost mutations in `cooperation-capture.ts` so active recording and sensitive-field markers no longer advance `FrameState.structure`, excluded extension ShadowRoot UI controls (`HelpRequestOverlay`, `RecordOverlay`, `BorrowConfirmationOverlay`) across `record-capture.ts`, `record-hover-surface.ts`, and `cooperation-capture.ts` so overlay interactions and hover surfaces are never mis-recorded as webpage steps, and re-evaluated tab control ownership after awaiting `ensureOverlayInjected` so mid-injection tab detachments never flash stale `control` state. @@ -110,6 +112,7 @@ determines which versions can connect. ### Removed +- Removed upper-layer imports of `browser-driver/frame-graph` (`CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, and `getFrameGraph`), optional `?` mock-accommodating methods on `CdpRunner`, module-global `downloadActive` state, the `isAgentWindowId: () => false` fallback default in `returnBorrowedTab`, hardcoded Chinese benchmark heading filters from `@browser-skill/vom`, and regex line re-parsing / duplicated `DERIVED_NAME_CAP` in `cooperation/text-delta.ts`. - Removed dead write-only `apps/extension/src/lib/sessions-live-flag.ts` (`bh.sessions_live` / `SESSIONS_LIVE_FLAG_KEY`) and ineffective Isolated World `history.pushState` / `history.replaceState` monkey-patching in `apps/extension/src/content/record-capture.ts`. - Removed `reattachIfSoftDetached`, `reconcileSoftDetachedFrames`, and `resolveTarget` compensation hooks from `CdpRunner`, `RefStore.remapChildFrameSessions`, partial hand-written `CdpRunner` wrapper object literals (replaced with full `delegateCdpRunner` delegation), and isolated default `new ChromiumCdp()` fallback instances outside `background.ts`. - Removed deprecated `TraceV2` recording types/converters (retaining `TraceV3` only), all 9 legacy `1.x` `Feature::*` compatibility gates, extension/daemon `"1.6"`/`LEGACY_MIN_COMPATIBLE_PEER` shims, duplicate `DragParams.observation_id` / legacy drag modes, and duplicate `wire_*` runtime validators from generated TypeScript artifacts. diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index ec6bb91bf..e9aeecf59 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -17,6 +17,7 @@ CLI 和扩展可以分别发布,能否连接由协议兼容性决定。更新 ### 变更 +- **`browser-driver/` 彻底封盒、必填 `CdpRunner` 与确定性 RAII V8 句柄释放、统一 `--selector` DOM 谓词及结构化 VOM 行元数据**:将 `CdpTarget`、`CdpFrame`、`CdpFrameGraph`、`sendToTarget`、`sendGuarded`、`sendAttached` 与 `getFrameGraph` 彻底封入 `apps/extension/src/browser-driver/` 内部,通过 `FrameDocumentHandle`、`FrameNodeHandle`、`FrameProcessBoundary` 与 `TabEventSource` 统一收口剩余全部上层模块(`read`、`extract`、`waits`、`wheel`、`drag`、`action-guard`、`vom/facts`、`AccessibilityLease`、`debug/`、`recording-runtime` 与 `geometry/frame-context`);将 `CdpRunner` 接口方法全面提升为必填非可选契约,配套 `delegateCdpRunner`、`createStubCdpRunner` 以及确定性 RAII `withObjectGroup` / `withRemoteObject` V8 句柄释放(覆盖 `handleSelect`、`handleFill`、`handleFocus`/`handleBlur`、`action-checks`、`file-input-transaction` 与 `evaluate(return_by_value: false)`,不唤醒已断开标签页且不掩盖派发前错误);在 `interaction`、`CooperationController`、`human-loop`、`read` 与 `extract` 间统一跨帧、open Shadow DOM、`[data-bsk-overlay]` 排除、敏感 `autocomplete` 与交互式 ARIA 角色谓词;并将 `cooperate sync` 与 `observe` 统一收敛至 `@browser-skill/vom` 的结构化 `VomLineMetadata`(`inspectVomLine` 与 `VOM_DERIVED_NAME_CAP`)。 - **基于统一 `bsk` CLI 运行时的轻量 `@wxg-prc-cpg/browser-skill-dsh-plugin` 插件**:将 `packages/dsh-plugin-browserskill` 从原先约 2.2 万行的自定义 `bsk_*` 工具包装器与 Web UI 前端组件重构为极简 Cordis 插件(彻底移除 `@deepseek-ai/dsh-*` 宿主 `rc` 版本对等依赖锁),直接内嵌主仓唯一的 `crates/bsk-cli/skill` 资源包,自动为宿主补齐 `~/.local/bin`(`$BSK_INSTALL_DIR`)与自定义 `bskPath` 的 `PATH` 解析,并打包跨平台按需 `bsk` CLI 安装器与浏览器扩展连接引导流程(`bin/install-bsk.mjs` + `AGENT_INSTALL.md`),底层与其他所有 Agent 100% 共用标准 `bsk` CLI 与 `DSH_SESSION_ID` 自动作用域机制,同时恢复 `dsh-plugin-v*` 独立发版流水线。 - **Chrome MV3 `registration: "runtime"` 零税双层按需内容脚本注入、SW 启动期孤儿 Agent 窗口兜底回收与共享 `content/dom-primitives.ts`**:将全部 4 个 MV3 内容脚本入口(`content`、`cooperation.content`、`record-frame.content` 与 `long-screenshot-page.content`)以 `noScriptStartedPostMessage: true` 与 `claimIdempotentContentScript` 幂等守卫迁出静态 `manifest.json` `content_scripts`,彻底消除日常浏览普通网页时的全局 `` 内容脚本、`window.postMessage` 广播与 React 19 加载税;通过幂等 `ensureOverlayInjected` 对受控任务标签页、`webNavigation.onCommitted`(含 Speculation Rules 预渲染顶层帧,且严格排在 Tier B `cooperation.js` 激活/补注入之后)、确认前 `tab_borrow` 候选用户标签页、`request_help`、`record` 与活跃 `video` 浮层按 `tabId`/`documentId` 单点注入 Tier A(`content.js` 与 `long-screenshot-page.js`),并在 `hidden` / `CAPTURE_SUPPRESS` / `INPUT_PASSTHROUGH` 时跳过未注入标签页;在活跃会话或录制于 `0` 与 `>= 1` 间切换时以 `persistAcrossSessions: false` 动态启停纯 TS 轻量 Tier B(`cooperation.js` 与 `record-frame.js`),使其在 `document_start` 零 DOM 监听、零启动唤醒 IPC 静默加载并由后台按需点名激活(`COOPERATION_ARM_FRAME` / `RECORD_FRAME_START` 配合单标签页 50ms 子帧合并去抖、`injectRecordFrameAllFrames` 协同注入 `cooperation.js`,以及在 `tab_borrow`、`ack === undefined` 帧与动态插入的普通 DOM 或 open/closed `ShadowRoot` 内同源 `about:blank` / `srcdoc` 子帧上进行无损按帧探测补注入);通过 `chrome.storage.session`(`bsk.orphan_agent_windows`)追踪独占 Agent 窗口创建的标签页集合,在 Service Worker 重启后精准回收无活跃会话的纯孤儿 Agent 窗口(严格保留含用户、借用或人工接管标签页的窗口);并抽离共享 DOM 原语模块 `apps/extension/src/content/dom-primitives.ts`。 - **守护进程单一 `tool_pipeline` 工具管线、`daemon/ipc` 四模块拆解、无锁阻塞持久化与 `cli::orchestration` 客户端共享编排内核**:将 3,489 行的 `daemon/ipc.rs` 拆解为 `daemon/tool_pipeline.rs`(统一收口全部 `tool.*` 执行路径的 RAII `AuditTicketGuard`、`SessionActivityGuard`、`ToolInflightRegistry`、`IntentWal` 与作用域/运行日志结算)、`daemon/ipc_sessions.rs`、`daemon/ipc_transport.rs` 及精简路由层 `daemon/ipc.rs`;在 `bsk-protocol` 与 `daemon/tool_policy.rs` / `daemon/queue.rs` 中建立穷尽式 `OutcomeKind` / `Outcome` 结果与超时归因路由;通过局部状态分离与 `spawn_blocking` 将 `TransferRegistry` 大文件暂存/导入复制/会话销毁目录清理、`AuditStore` JSONL `sync_data()` 刷盘(含 `session.user_interrupt` 标记)与单调序号 `IntentWal` 刷盘及 `sync_data()` 原子压缩彻底移出全局状态 `Mutex` 与 Tokio 工作线程阻塞路径;并提取 `cli/orchestration.rs` 统一 `bsk` CLI 与 `bsk mcp` 的 Canvas 截图坐标点击校验、`open`/`sync`/动作/正数超时边界、显式 `--session` + `--browser` 冲突校验、缓存 VOM 回填以及 2 秒 `CANCEL_RESPONSE_GRACE` 优雅取消结算。 @@ -30,6 +31,7 @@ CLI 和扩展可以分别发布,能否连接由协议兼容性决定。更新 ### 修复 +- 修复 `ToolDispatcher` 持久后台执行租约下失焦或最小化窗口上的 `tool.wheel` 因耦合临时 `restoreFocus` 而跳过合成器渲染就绪检查(`waitForInputPaint`)的问题(将 `ReadyInput.hidden` 拆耦为 `lacksForegroundPresentation` 并在 `ToolDispatcher` 注入 `windowsApi`);修复 `CooperationController` 在 `target` 策略下执行 `--selector` 动作时污染 `session.refStore`、误输出 `used_ref` 或在 `handleSelect` 自定义下拉报错引导中泄漏伪造 `@eN` 引用的问题(改用请求级 `withPinnedSelectorTargets`);在 `target` 策略脚本写 DOM 前于 `CooperationController.beginDomMutation()` 内主动刷新实时帧活动,并在写入前校验以 `effect_state: "none"` 失败时保留 `state.baseline`(`observation_id`);修复 `clearRecordingForSession` 未使用调用方/派发器注入 `deps`(`RECORD_CANCEL` + `bypassOverlay(tabId, false)`)的问题,将 `returnBorrowedTab` 的 `isAgentWindowId` 设为必填契约以防归还借用标签页误落入其他会话的 Agent 窗口,以实例级 `DownloadConcurrencyGate`(`session_busy`、`effect_state: "none"`、`dispatched: false`)替换模块级全局下载锁,并在会话销毁及 `tabs.onRemoved` 时完整清理 `observeIdentityStreaks`、`hoverLatches` 与 `hoverBypassTabs`;补齐 `cooperate sync` 对 Canvas `[visual:screenshot]` `@eN` 引用的对齐,将 `renderDoubleLayer` 底部 `L2 page … occluded by L1` 遮挡汇总行纳入 `maxTokens` 预算核算,并将弹出状态标签 `[expanded]` / `[has-submenu]` 从 `vom.name` 迁至引号后 VOM 标签以消除自定义下拉重复输出 `[expanded]`。 - 消除 `guardedRunner.runBrowserMutation`(`action-guard.ts`)中重复的连续整标签页帧状态采集、跳过 `handleFullPageScreenshot` 中只读 `inspect` 探测步骤的变更门禁,并将协作帧 IPC 响应超时阈值(`FRAME_REPLY_TIMEOUT_MS` / `OWNER_REPLY_MS`)从 `1200ms` 放宽至 `3000ms`,防止高分屏(2x DPR)/ 软光栅冷启动与按需浮层挂载期间误报 `FrameUnresponsiveError`(#761)。 - 为 `bsk step`、`bsk plan`、MCP `bsk_interact`(搭配 `sync_after` 或 `until_visible`)及 `bsk_plan` 补齐 `--max-tokens`(MCP 为 `max_tokens`)参数并透传至后置 `tool.cooperate` 同步请求,解决此前硬编码 `None` 导致带预算会话在后续动作触发 `token_budget_changed` 并回退为整页无上限重采的问题(#755)。 - 在 `cooperation-capture.ts` 中忽略扩展内部私有 `data-bsk-*` 属性与 ``(`[data-bsk-overlay]`)ShadowHost 变更以消除录制/敏感字段标记对 `FrameState.structure` 的自污染,在 `record-capture.ts`、`record-hover-surface.ts` 与 `cooperation-capture.ts` 中统一过滤扩展自身 ShadowRoot 浮层控件(`HelpRequestOverlay`、`RecordOverlay`、`BorrowConfirmationOverlay`)防止浮层内点击/输入或悬停面被误录为网页操作,并在 `ensureOverlayInjected` 异步注入完成后二次校验标签页受控归属以杜绝注入途中断开控制导致的陈旧浮层闪现。 @@ -54,6 +56,7 @@ CLI 和扩展可以分别发布,能否连接由协议兼容性决定。更新 ### 移除 +- 移除上层模块对 `browser-driver/frame-graph`(`CdpTarget`、`CdpFrame`、`CdpFrameGraph`、`sendToTarget`、`sendGuarded`、`sendAttached` 与 `getFrameGraph`)的全部直接导入、为迁就手写 Mock 而在 `CdpRunner` 上保留的可选 `?` 方法、模块级全局 `downloadActive` 状态、`returnBorrowedTab` 中的 `isAgentWindowId: () => false` 隐式默认兜底、`@browser-skill/vom` 中硬编码的中文基准测试标题黑名单,以及 `cooperation/text-delta.ts` 中的正则二次解析与重复 `DERIVED_NAME_CAP` 常量。 - 移除已沦为只写不读死代码的 `apps/extension/src/lib/sessions-live-flag.ts`(`bh.sessions_live` / `SESSIONS_LIVE_FLAG_KEY`)以及 `apps/extension/src/content/record-capture.ts` 中在隔离世界无效的 `history.pushState` / `history.replaceState` 重写死代码。 - 从 `CdpRunner` 及上层调用栈净删 `reattachIfSoftDetached`、`reconcileSoftDetachedFrames` 与 `resolveTarget` 三大外泄补偿钩子,移除 `RefStore.remapChildFrameSessions`、手写残缺局部 `CdpRunner` 对象字面量(统一收口为 `delegateCdpRunner` 全量委托代理)以及 `background.ts` 以外所有孤立默认的 `new ChromiumCdp()` 野实例。 - 移除已废弃的 `TraceV2` 录制协议类型与转换器(仅保留 `TraceV3`)、全部 9 套 `1.x` `Feature::*` 历史版本协商门禁、扩展/守护进程 `"1.6"`/`LEGACY_MIN_COMPATIBLE_PEER` 兼容兜底、重复字段 `DragParams.observation_id` / 旧拖拽模式以及生成 TS 产物中重复导出的 `wire_*` 运行时校验器。 diff --git a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts index 71543e59d..86bec1a9d 100644 --- a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts +++ b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts @@ -13,7 +13,7 @@ import { DEFAULT_IDLE_DETACH_MS, } from "../chromium-cdp"; import { CdpReadTimeoutError, READ_TIMEOUT_MS } from "../command-deadline"; -import { StaleFrameHandleError } from "../frame-graph"; +import { StaleFrameHandleError, toRootFrameHandle } from "../frame-handles"; function fakeChromeEvent() { const listeners = new Set<(...args: TArgs) => void>(); @@ -84,7 +84,7 @@ describe("ChromiumCdp", () => { sessionId: "child-session", targetInfo: { type: "iframe", targetId: "child" }, }); - const graph = cdp.getFrameGraph(4, signal.signal); + const graph = cdp.getFrameDocuments(4, signal.signal); await started; if (ending === "abort") signal.abort(); else { @@ -142,7 +142,7 @@ describe("ChromiumCdp", () => { targetInfo: { type: "iframe", targetId: "child" }, }); for (let attempt = 0; attempt < 2; attempt++) { - const capture = cdp.getFrameGraph(4); + const capture = cdp.getFrameDocuments(4); await vi.advanceTimersByTimeAsync(11_000); const graph = await capture; expect(graph.frames.map((frame) => frame.frameId)).toEqual(["main", "healthy"]); @@ -150,18 +150,21 @@ describe("ChromiumCdp", () => { { frameId: "child", parentFrameId: "main", - target: { tabId: 4, sessionId: "child-session" }, + processKey: "oopif:child", + isOutOfProcess: true, }, ]); expect(childReads).toBe(1); } finish({ frameTree: { frame: { id: "child" } } }); await vi.advanceTimersByTimeAsync(0); - const recovered = cdp.getFrameGraph(4); + const recovered = cdp.getFrameDocuments(4); await vi.advanceTimersByTimeAsync(100); - expect( - (await recovered).frames.find((frame) => frame.frameId === "child")?.target.sessionId, - ).toBe("child-session"); + expect((await recovered).frames.find((frame) => frame.frameId === "child")).toMatchObject({ + frameId: "child", + processKey: "oopif:child", + isOutOfProcess: true, + }); expect(childReads).toBe(2); } finally { cdp.dispose(); @@ -174,14 +177,14 @@ describe("ChromiumCdp", () => { const calls = () => vi.mocked(api.sendCommand).mock.calls.filter((c) => c[1] === "Target.setAutoAttach"); await cdp.ensureAttached(4); - await Promise.all([cdp.getFrameGraph(4), cdp.getFrameGraph(4)]); - await cdp.getFrameGraph(4); + await Promise.all([cdp.getFrameDocuments(4), cdp.getFrameDocuments(4)]); + await cdp.getFrameDocuments(4); expect(calls()).toHaveLength(1); onDetach.fire({ tabId: 4 }, "canceled_by_user"); - await cdp.getFrameGraph(4); + await cdp.getFrameDocuments(4); expect(calls()).toHaveLength(2); await cdp.detachAll(); - await cdp.getFrameGraph(4); + await cdp.getFrameDocuments(4); expect(calls()).toHaveLength(3); cdp.dispose(); }); @@ -195,8 +198,8 @@ describe("ChromiumCdp", () => { }); const cdp = new ChromiumCdp(api); await cdp.ensureAttached(4); - await cdp.getFrameGraph(4); - await cdp.getFrameGraph(4); + await cdp.getFrameDocuments(4); + await cdp.getFrameDocuments(4); expect(attempts).toBe(2); cdp.dispose(); }); @@ -239,17 +242,18 @@ describe("ChromiumCdp", () => { }); await vi.advanceTimersByTimeAsync(10_000); expect(childConfigurations).toBe(1); - const graph = cdp.getFrameGraph(4); + const graph = cdp.getFrameDocuments(4); await vi.advanceTimersByTimeAsync(2_000); expect((await graph).unavailableFrames?.map((frame) => frame.frameId)).toEqual(["child"]); expect(childConfigurations).toBe(1); finish({}); await vi.advanceTimersByTimeAsync(0); - const recovered = cdp.getFrameGraph(4); + const recovered = cdp.getFrameDocuments(4); await vi.advanceTimersByTimeAsync(2_000); - expect((await recovered).frames.find((frame) => frame.frameId === "child")?.target).toEqual({ - tabId: 4, - sessionId: "child", + expect((await recovered).frames.find((frame) => frame.frameId === "child")).toMatchObject({ + frameId: "child", + processKey: "oopif:child", + isOutOfProcess: true, }); expect(childConfigurations).toBe(1); } finally { @@ -289,7 +293,7 @@ describe("ChromiumCdp", () => { await cdp.ensureAttached(4); onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" }); } - const graph = cdp.getFrameGraph(4); + const graph = cdp.getFrameDocuments(4); const rejected = expect(graph).rejects.toThrow(`${stuckMethod} timed out`); // Root initialization must not resend the configuration after its deadline. await vi.advanceTimersByTimeAsync(22_000); @@ -434,16 +438,21 @@ describe("ChromiumCdp", () => { sessionId: "nested-session", }); - const graph = await cdp.getFrameGraph(4); + const graph = await cdp.getFrameDocuments(4); expect(graph.frames).toHaveLength(4); - expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 }); - expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe( - "right-session", - ); - expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe( - "nested-session", - ); + expect(graph.frames.find((frame) => frame.frameId === "left")).toMatchObject({ + processKey: "root", + isOutOfProcess: false, + }); + expect(graph.frames.find((frame) => frame.frameId === "right")).toMatchObject({ + processKey: "oopif:right", + isOutOfProcess: true, + }); + expect(graph.frames.find((frame) => frame.frameId === "nested")).toMatchObject({ + processKey: "oopif:nested", + isOutOfProcess: true, + }); expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200); expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300); expect(api.sendCommand).toHaveBeenCalledWith( @@ -505,13 +514,14 @@ describe("ChromiumCdp", () => { ); const cdp = new ChromiumCdp(api); - const graph = await cdp.getFrameGraph(4); + const graph = await cdp.getFrameDocuments(4); expect(graph.frames).toHaveLength(depth + 1); for (let index = 1; index <= depth; index += 1) { - expect( - graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId, - ).toBe(`session-${index}`); + expect(graph.frames.find((frame) => frame.frameId === `frame-${index}`)).toMatchObject({ + processKey: `oopif:frame-${index}`, + isOutOfProcess: true, + }); } }); @@ -1583,8 +1593,8 @@ describe("ChromiumCdp", () => { describe("document-bound references", () => { it.each([ "send", - "sendToTarget", - "sendGuarded", + "sendToFrame", + "sendToFrameGuarded", ] as const)("learns the loaded root from %s without frame discovery, including after reattach", async (method) => { const { api, onEvent } = fakeApi(); let root = "loaded-root"; @@ -1598,9 +1608,10 @@ describe("document-bound references", () => { const cdp = new ChromiumCdp(api, { onDocumentChanged: changed }); const read = () => { if (method === "send") return cdp.send(4, "Page.getFrameTree"); - if (method === "sendToTarget") return cdp.sendToTarget({ tabId: 4 }, "Page.getFrameTree"); - return cdp.sendGuarded( - { tabId: 4 }, + if (method === "sendToFrame") + return cdp.sendToFrame(toRootFrameHandle(4), "Page.getFrameTree"); + return cdp.sendToFrame( + toRootFrameHandle(4), "Page.getFrameTree", {}, { @@ -1614,7 +1625,6 @@ describe("document-bound references", () => { await cdp.ensureAttached(4); changed.mockClear(); refs.set("e1", 42, { tabId: 4, frameId: root }); - await cdp.sendToTarget({ tabId: 4, sessionId: "child-session" }, "Page.getFrameTree"); onEvent.fire({ tabId: 4 }, "Page.navigatedWithinDocument", { frameId: "child" }); expect(changed).not.toHaveBeenCalled(); await read(); @@ -1647,8 +1657,8 @@ describe("document-bound references", () => { const cdp = new ChromiumCdp(api, { onDocumentChanged: changed }); try { await cdp.ensureAttached(4); - const tree = cdp.sendGuarded( - { tabId: 4 }, + const tree = cdp.sendToFrame( + toRootFrameHandle(4), "Page.getFrameTree", {}, { @@ -1682,8 +1692,8 @@ describe("document-bound references", () => { const cdp = new ChromiumCdp(api, { onDocumentChanged: changed }); try { await cdp.ensureAttached(4); - const tree = cdp.sendGuarded( - { tabId: 4 }, + const tree = cdp.sendToFrame( + toRootFrameHandle(4), "Page.getFrameTree", {}, { @@ -2590,8 +2600,8 @@ describe("idle soft-detach and immediate close/return detach", () => { signal, ); expect(prepErr).toBeUndefined(); - await cooperativeRunner.sendGuarded!( - { tabId: 49 }, + await cooperativeRunner.sendToFrame( + toRootFrameHandle(49), "Input.dispatchMouseEvent", { type: "mouseMoved", x: 10, y: 10 }, { signal }, @@ -2662,8 +2672,8 @@ describe("idle soft-detach and immediate close/return detach", () => { await cdp.detach(50); expect(cdp.isAttached(50)).toBe(false); await expect( - cdp.sendGuarded( - { tabId: 50 }, + cdp.sendToFrame( + toRootFrameHandle(50), "Runtime.evaluate", { expression: "1" }, { diff --git a/apps/extension/src/browser-driver/__tests__/command-deadline.test.ts b/apps/extension/src/browser-driver/__tests__/command-deadline.test.ts index 31e6ee422..530e35c29 100644 --- a/apps/extension/src/browser-driver/__tests__/command-deadline.test.ts +++ b/apps/extension/src/browser-driver/__tests__/command-deadline.test.ts @@ -1,4 +1,5 @@ import { afterEach, expect, it, vi } from "vitest"; +import { createStubCdpRunner } from "@/tools/shared"; import { isAbortError, isCaptureTerminalError } from "@/tools/vom/capture-abort"; import { captureObservationFacts } from "@/tools/vom/capture-coordinator"; import { @@ -25,13 +26,14 @@ it("times out stuck snapshots with the actual method and stops fallback reads", method === "DOMSnapshot.captureSnapshot" ? new Promise(() => {}) : Promise.resolve({}), ), ); - const work = captureObservationFacts({ send: send as never }, 4); + const work = captureObservationFacts(createStubCdpRunner({ send: send as never }), 4); const checked = expect(work).rejects.toThrow( "DOMSnapshot.captureSnapshot timed out after 20000ms (tab 4)", ); await vi.advanceTimersByTimeAsync(CAPTURE_READ_TIMEOUT_MS); await checked; expect(send.mock.calls.map((call) => call[1])).toEqual([ + "Page.getFrameTree", "DOMSnapshot.enable", "DOMSnapshot.captureSnapshot", ]); @@ -40,12 +42,12 @@ it("times out stuck snapshots with the actual method and stops fallback reads", it("does not start a snapshot after the frame graph read timed out", async () => { const send = vi.fn(async () => ({})); - const getFrameGraph = vi.fn(async () => { + const getFrameDocuments = vi.fn(async () => { throw new CdpReadTimeoutError("DOM.getFrameOwner", 4, READ_TIMEOUT_MS); }); - await expect(captureObservationFacts({ send, getFrameGraph } as never, 4)).rejects.toThrow( - "DOM.getFrameOwner timed out", - ); + await expect( + captureObservationFacts(createStubCdpRunner({ send: send as never, getFrameDocuments }), 4), + ).rejects.toThrow("DOM.getFrameOwner timed out"); expect(send).not.toHaveBeenCalled(); }); diff --git a/apps/extension/src/browser-driver/chromium-cdp.ts b/apps/extension/src/browser-driver/chromium-cdp.ts index bd25ba9ba..236c0b7ed 100644 --- a/apps/extension/src/browser-driver/chromium-cdp.ts +++ b/apps/extension/src/browser-driver/chromium-cdp.ts @@ -36,23 +36,52 @@ import { type CdpFrameTreeNode, type CdpFrameTreeSource, type CdpTarget, + omitFrameSubtrees, +} from "./frame-graph"; +import { + type FrameDocumentGraph, type FrameDocumentHandle, + type FrameDocumentInfo, type FrameNodeHandle, - omitFrameSubtrees, + isStaleFrameHandleError, StaleFrameHandleError, -} from "./frame-graph"; + type TabEventSource, + toFrameDocumentHandle, + toRootFrameHandle, + type UnavailableFrameDocument, +} from "./frame-handles"; import { NativeInputClock } from "./native-input-clock"; import { sampleNativeInputClock } from "./native-input-probe"; +export { + type FrameDocumentGraph, + type FrameDocumentHandle, + type FrameDocumentInfo, + type FrameNodeHandle, + isStaleFrameHandleError, + StaleFrameHandleError, + type TabEventSource, + toFrameDocumentHandle, + toRootFrameHandle, + type UnavailableFrameDocument, +}; + +/** Internal to `browser-driver/` — never import outside `apps/extension/src/browser-driver/`. */ export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string }; -export type TabEventListener = (source: CdpDebuggee, method: string, params: unknown) => void; +export type TabEventListener = (source: TabEventSource, method: string, params: unknown) => void; /** Guards checked at native dispatch, after any asynchronous connection setup. */ export interface CdpDispatchGuard { signal?: AbortSignal; /** Only use this live (or soft-detached) attachment; never reconnect a hard-detached tab. */ attachmentId?: string; + /** When false, never reattach a soft-detached tab (used by domain/object cleanup). */ + allowReattach?: boolean; + /** When true, route to the frame's live renderer even if `loaderId` navigated (domain cleanup). */ + ignoreFrameNavigation?: boolean; + /** Optional driver read deadline override (ms). */ + readTimeoutMs?: number; /** Called immediately before native send, never for a cancelled or stale command. */ onDispatch?(): void; } @@ -152,6 +181,9 @@ interface FrameDiscoveryState { generation: number; enabled: Map>; frameIds: Map; + targetKeys: Map; + sessionsByTargetKey: Map; + nextTargetSeq: number; } export interface TabLease { @@ -302,6 +334,9 @@ function createFrameDiscoveryState(): FrameDiscoveryState { generation: 0, enabled: new Map(), frameIds: new Map(), + targetKeys: new Map(), + sessionsByTargetKey: new Map(), + nextTargetSeq: 0, }; } @@ -536,19 +571,23 @@ export class ChromiumCdp { } /** True when `source` originated from `handle`'s current live CDP target in `tabId`. */ - isEventFromFrame(source: CdpDebuggee, handle: FrameDocumentHandle): boolean { + isEventFromFrame(source: TabEventSource, handle: FrameDocumentHandle): boolean { if (source.tabId !== handle.tabId) return false; - if (!handle.frameId) return !source.sessionId; + if (!handle.frameId) return source.isRootTarget; const entry = this.tabs.get(handle.tabId); if (isActiveAttachment(entry)) { + const sourceSessionId = source.isRootTarget + ? undefined + : entry.frameDiscovery.sessionsByTargetKey.get(source.targetKey); + if (!source.isRootTarget && !sourceSessionId) return false; const route = entry.frames.get(handle.frameId); - if (route) return (source.sessionId ?? undefined) === route.sessionId; + if (route) return sourceSessionId === route.sessionId; for (const [sessionId, mappedFrameId] of entry.frameDiscovery.frameIds) { - if (mappedFrameId === handle.frameId) return source.sessionId === sessionId; + if (mappedFrameId === handle.frameId) return sourceSessionId === sessionId; } - if (entry.frameDiscovery.rootFrameId === handle.frameId) return !source.sessionId; + if (entry.frameDiscovery.rootFrameId === handle.frameId) return source.isRootTarget; } - return !source.sessionId; + return source.isRootTarget; } /** True when `frameId` and `parentFrameId` (or the root target when omitted) live in distinct CDP targets. */ @@ -904,7 +943,7 @@ export class ChromiumCdp { } } - async sendToTarget( + private async sendToTarget( target: CdpTarget, method: string, params?: object, @@ -918,17 +957,24 @@ export class ChromiumCdp { } } - async sendGuarded( + private async sendGuarded( target: CdpTarget, method: string, params: object | undefined, guard: CdpDispatchGuard, ): Promise { guard.signal?.throwIfAborted(); - if (guard.attachmentId === undefined) await this.ensureAttached(target.tabId); + if (guard.attachmentId === undefined && guard.allowReattach !== false) { + await this.ensureAttached(target.tabId); + } else if (guard.allowReattach === false && !this.isAttached(target.tabId)) { + throw new Error("debugger detached"); + } try { - return (await this.command(target, method, params ?? {}, undefined, () => { + return (await this.command(target, method, params ?? {}, guard.readTimeoutMs, () => { guard.signal?.throwIfAborted(); + if (guard.allowReattach === false && !this.isAttached(target.tabId)) { + throw new Error("debugger detached"); + } if ( guard.attachmentId !== undefined && this.getAttachmentId(target.tabId) !== guard.attachmentId @@ -943,8 +989,8 @@ export class ChromiumCdp { /** * Route a node/frame-scoped CDP command via a stable `FrameDocumentHandle` / `FrameNodeHandle`: - * 1. Transparently reattaches when `SoftDetached` (respecting `guard.attachmentId` without resurrecting hard-detached tabs); - * 2. Resolves `handle.frameId` to its current `{ tabId, sessionId }` route and validates `loaderId` (+ optional `documentElementBackendNodeId`); + * 1. Transparently reattaches when `SoftDetached` (respecting `guard.attachmentId` and `guard.allowReattach`); + * 2. Resolves `handle.frameId` to its current live route and validates `loaderId` (+ optional `documentElementBackendNodeId`); * 3. Deterministically rejects with `StaleFrameHandleError` if the target frame navigated or underwent `document.open()` replacement. */ async sendToFrame( @@ -955,31 +1001,43 @@ export class ChromiumCdp { ): Promise { const { tabId } = handle; guard?.signal?.throwIfAborted(); + const allowReattach = guard?.allowReattach !== false; const currentBefore = this.tabs.get(tabId); if (currentBefore?.kind === "attaching") { + if (!allowReattach) throw new Error("debugger detached"); await currentBefore.attachInFlight; } guard?.signal?.throwIfAborted(); if (guard?.attachmentId !== undefined) { const state = this.tabs.get(tabId); - if (state?.kind === "soft_detached" && state.snapshot.attachmentId === guard.attachmentId) { + if ( + allowReattach && + state?.kind === "soft_detached" && + state.snapshot.attachmentId === guard.attachmentId + ) { await this.ensureAttached(tabId); } else if (state?.kind === "attached") { - await this.verifyRootDocumentAfterDialog(tabId); - this.touchTabActivity(tabId); + if (allowReattach) { + if (state.deferredRootCheck && !state.openDialog) { + await this.verifyRootDocumentAfterDialog(tabId); + } + this.touchTabActivity(tabId); + } } guard.signal?.throwIfAborted(); if (this.getAttachmentId(tabId) !== guard.attachmentId) { throw new Error("Debugger attachment changed before dispatch"); } + } else if (!allowReattach) { + if (!this.isAttached(tabId)) throw new Error("debugger detached"); } else { await this.ensureAttached(tabId); guard?.signal?.throwIfAborted(); } - // Legacy root shorthand (`frameId === ""` and `loaderId === ""`) routes directly to `{ tabId }`. + // Root shorthand (`frameId === ""` and `loaderId === ""`) routes directly to `{ tabId }`. if (!handle.frameId && !handle.loaderId) { if (guard) { return this.sendGuarded({ tabId }, method, params, guard); @@ -987,18 +1045,27 @@ export class ChromiumCdp { return this.sendToTarget({ tabId }, method, params); } - const route = await this.resolveValidatedFrameRoute(handle, method, guard?.signal); + const route = await this.resolveValidatedFrameRoute( + handle, + method, + guard?.signal, + guard?.ignoreFrameNavigation === true, + ); const target: CdpTarget = { tabId, ...(route.sessionId ? { sessionId: route.sessionId } : {}), }; + const ignoreNav = + guard?.ignoreFrameNavigation === true || + method === "Runtime.releaseObjectGroup" || + method === "Runtime.releaseObject"; try { return (await this.command( target, method, params ?? {}, - undefined, + guard?.readTimeoutMs, () => { guard?.signal?.throwIfAborted(); if ( @@ -1008,18 +1075,25 @@ export class ChromiumCdp { throw new Error("Debugger attachment changed before dispatch"); } const liveEntry = this.tabs.get(tabId); - const liveRoute = - liveEntry?.kind === "attached" ? liveEntry.frames.get(handle.frameId) : undefined; + if (liveEntry?.kind !== "attached") { + throw new Error("debugger detached"); + } + if (route.sessionId && !liveEntry.frameDiscovery.sessions.has(route.sessionId)) { + throw new StaleFrameHandleError(tabId, handle.frameId); + } + const liveRoute = liveEntry.frames.get(handle.frameId); + if (liveRoute && liveRoute.sessionId !== route.sessionId) { + throw new StaleFrameHandleError(tabId, handle.frameId); + } if ( - !liveRoute || - liveRoute.sessionId !== route.sessionId || - liveRoute.loaderId !== handle.loaderId || - (handle.documentElementBackendNodeId !== undefined && - method !== "Runtime.releaseObjectGroup" && - method !== "Runtime.releaseObject" && - (liveRoute.documentOpened || - (liveRoute.documentElementBackendNodeId !== undefined && - liveRoute.documentElementBackendNodeId !== handle.documentElementBackendNodeId))) + !ignoreNav && + (!liveRoute || + liveRoute.loaderId !== handle.loaderId || + (handle.documentElementBackendNodeId !== undefined && + (liveRoute.documentOpened || + (liveRoute.documentElementBackendNodeId !== undefined && + liveRoute.documentElementBackendNodeId !== + handle.documentElementBackendNodeId)))) ) { throw new StaleFrameHandleError(tabId, handle.frameId); } @@ -1036,21 +1110,38 @@ export class ChromiumCdp { handle: FrameDocumentHandle | FrameNodeHandle, method: string, signal?: AbortSignal, + ignoreFrameNavigation = false, ): Promise { const { tabId, frameId, loaderId } = handle; - const isCleanup = method === "Runtime.releaseObjectGroup" || method === "Runtime.releaseObject"; + const isCleanup = + ignoreFrameNavigation || + method === "Runtime.releaseObjectGroup" || + method === "Runtime.releaseObject"; let entry = this.tabs.get(tabId); if (entry?.kind !== "attached") throw new Error("debugger detached"); let route = entry.frames.get(frameId); const needsRefresh = - !route || !route.authoritative || (entry.frameDiscovery.pending.size > 0 && !route.sessionId); + !ignoreFrameNavigation && + (!route || + !route.authoritative || + (entry.frameDiscovery.pending.size > 0 && !route.sessionId)); if (needsRefresh) { await this.refreshFrameRoutes(tabId, signal); entry = this.tabs.get(tabId); if (entry?.kind !== "attached") throw new Error("debugger detached"); route = entry.frames.get(frameId); } + if (!route && ignoreFrameNavigation) { + for (const [sessionId, mappedFrameId] of entry.frameDiscovery.frameIds) { + if (mappedFrameId === frameId) { + return { sessionId, loaderId: "", authoritative: false }; + } + } + if (entry.frameDiscovery.rootFrameId === frameId) { + return { loaderId: "", authoritative: false }; + } + } if (!route) { throw new StaleFrameHandleError( @@ -1059,7 +1150,11 @@ export class ChromiumCdp { `Frame ${frameId} is no longer present in tab ${tabId}`, ); } - if (route.loaderId !== loaderId || (route.sessionId !== undefined && !loaderId)) { + const requiresNonEmptyLoader = "backendNodeId" in handle && route.sessionId !== undefined; + if ( + !ignoreFrameNavigation && + (route.loaderId !== loaderId || (requiresNonEmptyLoader && !loaderId)) + ) { throw new StaleFrameHandleError( tabId, frameId, @@ -1375,7 +1470,57 @@ export class ChromiumCdp { }; } - async getFrameGraph(tabId: number, signal?: AbortSignal): Promise { + /** + * Discover all document frames for `tabId` as handle-addressed `FrameDocumentInfo` + * entries without exposing transient CDP `sessionId` strings. + */ + async getFrameDocuments(tabId: number, signal?: AbortSignal): Promise { + const graph = await this.getFrameGraph(tabId, signal); + const attachmentId = this.getAttachmentId(tabId); + const entry = this.tabs.get(tabId); + const processKeyOf = (target: CdpTarget, fallbackFrameId: string): string => { + if (!target.sessionId) return "root"; + if (isActiveAttachment(entry)) { + const mapped = entry.frameDiscovery.frameIds.get(target.sessionId); + if (mapped) return `oopif:${mapped}`; + return `oopif:${this.targetKeyForSession(entry.frameDiscovery, target.sessionId)}`; + } + return `oopif:${fallbackFrameId}`; + }; + const frames: FrameDocumentInfo[] = graph.frames.map((frame) => { + const loaderId = frame.loaderId ?? ""; + return { + handle: toFrameDocumentHandle(tabId, { frameId: frame.frameId, loaderId }), + frameId: frame.frameId, + loaderId, + ...(frame.parentFrameId ? { parentFrameId: frame.parentFrameId } : {}), + ...(frame.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: frame.ownerBackendNodeId } + : {}), + ...(frame.url ? { url: frame.url } : {}), + ...(frame.urlFragment !== undefined ? { urlFragment: frame.urlFragment } : {}), + processKey: processKeyOf(frame.target, frame.frameId), + isOutOfProcess: Boolean(frame.target.sessionId), + }; + }); + const unavailableFrames: UnavailableFrameDocument[] | undefined = graph.unavailableFrames?.map( + (frame) => ({ + frameId: frame.frameId, + ...(frame.parentFrameId ? { parentFrameId: frame.parentFrameId } : {}), + ...(frame.url ? { url: frame.url } : {}), + processKey: processKeyOf(frame.target, frame.frameId), + isOutOfProcess: Boolean(frame.target.sessionId), + }), + ); + return { + ...(attachmentId !== undefined ? { attachmentId } : {}), + rootFrameId: graph.rootFrameId, + frames, + ...(unavailableFrames?.length ? { unavailableFrames } : {}), + }; + } + + private async getFrameGraph(tabId: number, signal?: AbortSignal): Promise { signal?.throwIfAborted(); await this.ensureAttached(tabId); const attachmentId = this.getAttachmentId(tabId); @@ -1420,7 +1565,11 @@ export class ChromiumCdp { target.sessionId && isActiveAttachment(entry) ? entry.frameDiscovery.frameIds.get(target.sessionId) : undefined; - if (!frameId || frameId === sources[0]?.tree.frame.id || error.sessionId !== target.sessionId) + if ( + !frameId || + frameId === sources[0]?.tree.frame.id || + error.processKey !== this.processKeyForTarget(target) + ) throw error; unavailable.set(frameId, target); }; @@ -1766,14 +1915,47 @@ export class ChromiumCdp { } } - /** Debug body reads must never resurrect a detached / returned tab. */ - async sendAttached( - target: CdpDebuggee & { tabId: number }, + /** + * Discover all currently attached out-of-process child `TabEventSource`s on `tabId` + * without exposing transient CDP `sessionId` strings outside `browser-driver/`. + */ + async discoverChildEventSources(tabId: number): Promise { + const graph = await this.getFrameGraph(tabId); + const entry = this.tabs.get(tabId); + if (!isActiveAttachment(entry)) return []; + const seen = new Set(); + const sources: TabEventSource[] = []; + for (const frame of graph.frames) { + const sessionId = frame.target.sessionId; + if (!sessionId || seen.has(sessionId)) continue; + seen.add(sessionId); + const targetKey = this.targetKeyForSession(entry.frameDiscovery, sessionId); + sources.push({ tabId, targetKey, isRootTarget: false }); + } + return sources; + } + + /** + * Dispatch a CDP command on the live target that emitted `source`, without + * resurrecting a detached / soft-detached / returned tab (used by `debug/`). + */ + async sendToEventSource( + source: TabEventSource, method: string, params?: object, ): Promise { - if (!this.isAttached(target.tabId)) throw new Error("debugger detached"); - return this.runTrackedSendCommand(target, method, params) as Promise; + const entry = this.tabs.get(source.tabId); + if (entry?.kind !== "attached") throw new Error("debugger detached"); + if (source.isRootTarget || source.targetKey === "root") { + return this.runTrackedSendCommand({ tabId: source.tabId }, method, params) as Promise; + } + const sessionId = entry.frameDiscovery.sessionsByTargetKey.get(source.targetKey); + if (!sessionId) throw new Error("target session detached"); + return this.runTrackedSendCommand( + { tabId: source.tabId, sessionId }, + method, + params, + ) as Promise; } /** Best-effort detach of every cached tab. Used on session.stop. */ @@ -2279,6 +2461,39 @@ export class ChromiumCdp { await cleanupWait(detachWork); } + private targetKeyForSession(state: FrameDiscoveryState, sessionId: string): string { + let key = state.targetKeys.get(sessionId); + if (!key) { + state.nextTargetSeq += 1; + key = `oopif#${state.nextTargetSeq}`; + state.targetKeys.set(sessionId, key); + state.sessionsByTargetKey.set(key, sessionId); + } + return key; + } + + private processKeyForTarget(target: CdpDebuggee): string { + if (!target.sessionId) return "root"; + const entry = typeof target.tabId === "number" ? this.tabs.get(target.tabId) : undefined; + if (isActiveAttachment(entry)) { + const frameId = entry.frameDiscovery.frameIds.get(target.sessionId); + if (frameId) return `oopif:${frameId}`; + return `oopif:${this.targetKeyForSession(entry.frameDiscovery, target.sessionId)}`; + } + return `oopif:${target.sessionId}`; + } + + private toTabEventSource(tabId: number, source: CdpDebuggee): TabEventSource { + if (!source.sessionId) { + return { tabId, targetKey: "root", isRootTarget: true }; + } + const entry = this.tabs.get(tabId); + const targetKey = isActiveAttachment(entry) + ? this.targetKeyForSession(entry.frameDiscovery, source.sessionId) + : "oopif#detached"; + return { tabId, targetKey, isRootTarget: false }; + } + private async command( target: CdpDebuggee, method: string, @@ -2295,6 +2510,7 @@ export class ChromiumCdp { : undefined; const rootEpochBefore = discoveryBefore?.rootNavigationEpoch; const version = entryBefore?.version ?? 0; + const processKey = this.processKeyForTarget(target); const result = await this.readGate.run( target, method, @@ -2304,6 +2520,7 @@ export class ChromiumCdp { }, readTimeoutMs, signal, + processKey, ); if (discoveryBefore && typeof tabId === "number") { const entryAfter = this.tabs.get(tabId); @@ -2442,17 +2659,38 @@ export class ChromiumCdp { const listener = (source: CdpDebuggee, method: string, params: unknown) => { const tabId = source.tabId; if (typeof tabId !== "number") return; + const eventSource = this.toTabEventSource(tabId, source); this.handleDialogEvent(tabId, source, method, params); this.handleConsoleEvent(tabId, method, params); this.handleNetworkEvent(tabId, method, params); - this.handleFrameTargetEvent(tabId, source, method, params); + const childTargetChange = this.handleFrameTargetEvent(tabId, source, method, params); const entry = this.tabs.get(tabId); if (entry?.eventListeners.size) { - for (const subscriber of [...entry.eventListeners]) { - try { - subscriber(source, method, params); - } catch (err) { - console.debug("[bsk cdp] tab event subscriber threw", { tabId, method, err }); + if (method === "Target.attachedToTarget" || method === "Target.detachedFromTarget") { + if (childTargetChange) { + const driverMethod = + childTargetChange.kind === "attached" + ? "BskDriver.childTargetAttached" + : "BskDriver.childTargetDetached"; + for (const subscriber of [...entry.eventListeners]) { + try { + subscriber(childTargetChange.source, driverMethod, {}); + } catch (err) { + console.debug("[bsk cdp] tab event subscriber threw", { + tabId, + method: driverMethod, + err, + }); + } + } + } + } else { + for (const subscriber of [...entry.eventListeners]) { + try { + subscriber(eventSource, method, params); + } catch (err) { + console.debug("[bsk cdp] tab event subscriber threw", { tabId, method, err }); + } } } } @@ -2468,9 +2706,9 @@ export class ChromiumCdp { source: CdpDebuggee, method: string, params: unknown, - ): void { + ): { kind: "attached" | "detached"; source: TabEventSource } | null { const entry = this.tabs.get(tabId); - if (!isActiveAttachment(entry)) return; + if (!isActiveAttachment(entry)) return null; const raw = (params ?? {}) as Record; const frame = raw.frame as | { @@ -2556,12 +2794,17 @@ export class ChromiumCdp { } const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined; - if (!sessionId) return; + if (!sessionId) return null; if (method === "Target.detachedFromTarget") { const state = entry.frameDiscovery; state.enabled.delete(sessionId); state.frameIds.delete(sessionId); + const targetKey = state.targetKeys.get(sessionId); + if (targetKey) { + state.targetKeys.delete(sessionId); + state.sessionsByTargetKey.delete(targetKey); + } for (const [frameId, route] of entry.frames) { if (route.sessionId === sessionId) { entry.frames.delete(frameId); @@ -2569,13 +2812,16 @@ export class ChromiumCdp { } this.readGate.reset(tabId, sessionId); if (state.sessions.delete(sessionId)) state.generation += 1; - return; + return targetKey + ? { kind: "detached", source: { tabId, targetKey, isRootTarget: false } } + : null; } - if (method !== "Target.attachedToTarget") return; + if (method !== "Target.attachedToTarget") return null; const targetInfo = raw.targetInfo as { type?: string; targetId?: string } | undefined; - if (targetInfo?.type && targetInfo.type !== "iframe") return; + if (targetInfo?.type && targetInfo.type !== "iframe") return null; const state = entry.frameDiscovery; + const targetKey = this.targetKeyForSession(state, sessionId); if (targetInfo?.type === "iframe" && typeof targetInfo.targetId === "string") { state.frameIds.set(sessionId, targetInfo.targetId); const existing = entry.frames.get(targetInfo.targetId); @@ -2583,7 +2829,7 @@ export class ChromiumCdp { entry.frames.delete(targetInfo.targetId); } } - if (state.sessions.has(sessionId)) return; + if (state.sessions.has(sessionId)) return null; state.sessions.add(sessionId); state.generation += 1; @@ -2592,6 +2838,9 @@ export class ChromiumCdp { void task.finally(() => { state.pending.delete(task); }); + return targetInfo?.type === "iframe" + ? { kind: "attached", source: { tabId, targetKey, isRootTarget: false } } + : null; } private retryChildFrameDiscovery(tabId: number): void { diff --git a/apps/extension/src/browser-driver/command-deadline.ts b/apps/extension/src/browser-driver/command-deadline.ts index f3b4ad2b7..55b47dbea 100644 --- a/apps/extension/src/browser-driver/command-deadline.ts +++ b/apps/extension/src/browser-driver/command-deadline.ts @@ -61,7 +61,7 @@ export class CdpReadTimeoutError extends Error { readonly method: string, readonly tabId: number | undefined, timeout: number, - readonly sessionId?: string, + readonly processKey?: string, readonly phase: "deadline" | "blocked" = "deadline", ) { super( @@ -75,9 +75,9 @@ export class CdpReadTimeoutError extends Error { static stillPending( method: string, tabId: number | undefined, - sessionId?: string, + processKey?: string, ): CdpReadTimeoutError { - const error = new CdpReadTimeoutError(method, tabId, 0, sessionId, "blocked"); + const error = new CdpReadTimeoutError(method, tabId, 0, processKey, "blocked"); error.message = `Browser read ${method} refused: an earlier read on tab ${tabId} is still pending in Chrome. ${RECOVERY}`; return error; } @@ -105,28 +105,43 @@ export function readTimeoutDetails(error: unknown): { data?: RpcErrorData } { : {}; } +type ReadGateTarget = CdpDebuggee | { readonly tabId?: number; readonly processKey?: string }; + +function targetGateKey(target: ReadGateTarget): string { + if ("sessionId" in target && target.sessionId) return target.sessionId; + if ("processKey" in target && target.processKey) return target.processKey; + return "root"; +} + +function targetProcessKey(target: ReadGateTarget): string { + if ("processKey" in target && target.processKey) return target.processKey; + if ("sessionId" in target && target.sessionId) return `oopif:${target.sessionId}`; + return "root"; +} + /** Only Chrome settling the original command (or detaching its session) opens * this gate. A caller timing out does not cancel that command. */ export class CdpReadGate { private readonly tabs = new Map>>(); run( - target: CdpDebuggee, + target: ReadGateTarget, method: string, run: () => Promise, timeoutMs?: number, signal?: AbortSignal, + processKey = targetProcessKey(target), ): Promise { signal?.throwIfAborted(); let sessions = this.tabs.get(target.tabId); if (!sessions) this.tabs.set(target.tabId, (sessions = new Map())); - const key = target.sessionId ?? "root"; + const key = targetGateKey(target); let blocked = sessions.get(key); if (!blocked) sessions.set(key, (blocked = new Set())); const error = sessions.get("root")?.values().next().value ?? blocked.values().next().value; if (error && GUARDED_READS.has(method)) return Promise.reject( - CdpReadTimeoutError.stillPending(method, target.tabId, error.sessionId), + CdpReadTimeoutError.stillPending(method, target.tabId, error.processKey), ); let pending = true; let blocker: CdpReadTimeoutError | undefined; @@ -136,17 +151,26 @@ export class CdpReadGate { blocked.add(error); }; // Capture the set, so an old reply cannot modify a replacement attachment. - const work = runCdpCommand(target, method, run, timeoutMs, { - timedOut: blockUntilSettled, - settled: () => { - pending = false; - if (blocker) blocked.delete(blocker); + const work = runCdpCommand( + target, + method, + run, + timeoutMs, + { + timedOut: blockUntilSettled, + settled: () => { + pending = false; + if (blocker) blocked.delete(blocker); + }, }, - }); - if (!signal) return work; - return waitForAbort(work, signal, () => - blockUntilSettled(CdpReadTimeoutError.stillPending(method, target.tabId, target.sessionId)), + processKey, ); + if (!signal) return work; + return waitForAbort(work, signal, () => { + if (GUARDED_READS.has(method)) { + blockUntilSettled(CdpReadTimeoutError.stillPending(method, target.tabId, processKey)); + } + }); } reset(tabId: number, sessionId?: string): void { @@ -184,7 +208,7 @@ async function waitForAbort( } export async function runCdpCommand( - target: CdpDebuggee, + target: ReadGateTarget, method: string, run: () => Promise, timeoutMs = method === "Accessibility.disable" @@ -195,12 +219,13 @@ export async function runCdpCommand( ? READ_TIMEOUT_MS : undefined, lifecycle?: { timedOut(error: CdpReadTimeoutError): void; settled(): void }, + processKey = targetProcessKey(target), ): Promise { const started = Date.now(); let timedOut = false; const details = () => ({ tabId: target.tabId, - frameSessionId: target.sessionId, + frameSessionId: "sessionId" in target ? target.sessionId : undefined, method, elapsedMs: Date.now() - started, }); @@ -236,7 +261,7 @@ export async function runCdpCommand( deadline = setTimeout(() => { timedOut = true; console.warn("[bsk cdp] read timed out", details()); - const error = new CdpReadTimeoutError(method, target.tabId, timeoutMs, target.sessionId); + const error = new CdpReadTimeoutError(method, target.tabId, timeoutMs, processKey); lifecycle?.timedOut(error); reject(error); }, timeoutMs); diff --git a/apps/extension/src/browser-driver/frame-graph.ts b/apps/extension/src/browser-driver/frame-graph.ts index 7383bc056..80f3176c3 100644 --- a/apps/extension/src/browser-driver/frame-graph.ts +++ b/apps/extension/src/browser-driver/frame-graph.ts @@ -1,71 +1,22 @@ +export { + type FrameDocumentGraph, + type FrameDocumentHandle, + type FrameDocumentInfo, + type FrameNodeHandle, + isStaleFrameHandleError, + StaleFrameHandleError, + type TabEventSource, + toFrameDocumentHandle, + toRootFrameHandle, + type UnavailableFrameDocument, +} from "./frame-handles"; + +/** Internal to `browser-driver/` — never import outside `apps/extension/src/browser-driver/`. */ export interface CdpTarget { tabId: number; sessionId?: string; } -/** - * Stable cross-layer document/frame addressing handle. - * Never carries transient CDP `sessionId`; `ChromiumCdp` resolves `frameId` to - * its live session internally and validates `loaderId` (+ optional `documentElementBackendNodeId`). - */ -export interface FrameDocumentHandle { - tabId: number; - frameId: string; - loaderId: string; - documentElementBackendNodeId?: number; -} - -/** - * Stable cross-layer DOM node addressing handle. - * Never carries transient CDP `sessionId`; `ChromiumCdp` routes and validates - * `frameId` + `loaderId` (+ optional `documentElementBackendNodeId`) internally. - */ -export interface FrameNodeHandle { - tabId: number; - frameId: string; - loaderId: string; - backendNodeId: number; - documentElementBackendNodeId?: number; -} - -export class StaleFrameHandleError extends Error { - readonly tabId: number; - readonly frameId: string; - - constructor(handle: { tabId: number; frameId: string }, detail?: string); - constructor(tabId: number, frameId: string, detail?: string); - constructor( - tabOrHandle: number | { tabId: number; frameId: string }, - frameOrDetail?: string, - maybeDetail?: string, - ) { - const tabId = typeof tabOrHandle === "number" ? tabOrHandle : tabOrHandle.tabId; - const frameId = typeof tabOrHandle === "number" ? (frameOrDetail ?? "") : tabOrHandle.frameId; - const detail = typeof tabOrHandle === "number" ? maybeDetail : frameOrDetail; - super(detail ?? `Frame ${frameId} in tab ${tabId} changed or is no longer available`); - this.name = "StaleFrameHandleError"; - this.tabId = tabId; - this.frameId = frameId; - } -} - -export function isStaleFrameHandleError(error: unknown): error is StaleFrameHandleError { - return error instanceof StaleFrameHandleError; -} - -export function toFrameDocumentHandle( - tabId: number, - frame: { frameId: string; loaderId?: string }, - documentElementBackendNodeId?: number, -): FrameDocumentHandle { - return { - tabId, - frameId: frame.frameId, - loaderId: frame.loaderId ?? "", - ...(documentElementBackendNodeId !== undefined ? { documentElementBackendNodeId } : {}), - }; -} - export interface CdpFrame { frameId: string; loaderId?: string; diff --git a/apps/extension/src/browser-driver/frame-handles.ts b/apps/extension/src/browser-driver/frame-handles.ts new file mode 100644 index 000000000..fedee6e21 --- /dev/null +++ b/apps/extension/src/browser-driver/frame-handles.ts @@ -0,0 +1,239 @@ +/** + * Public cross-layer frame/node/event handle types for `browser-driver/`. + * + * Invariant: transient OOPIF CDP `sessionId` strings never appear on any type + * exported from this module. Upper layers address frames, nodes, renderer + * process boundaries, and CDP event sources exclusively via these handles. + */ + +export interface FrameDocumentHandle { + tabId: number; + frameId: string; + loaderId: string; + documentElementBackendNodeId?: number; +} + +export interface FrameNodeHandle { + tabId: number; + frameId: string; + loaderId: string; + backendNodeId: number; + documentElementBackendNodeId?: number; +} + +/** + * Discovered document frame metadata without exposing transient CDP `sessionId`. + * `processKey` identifies the renderer target boundary (`"root"` for the top-level + * renderer process, `"oopif:"` for an out-of-process iframe). + */ +export interface FrameDocumentInfo { + readonly handle: FrameDocumentHandle; + readonly frameId: string; + readonly loaderId: string; + readonly parentFrameId?: string; + readonly ownerBackendNodeId?: number; + readonly url?: string; + readonly urlFragment?: string; + readonly processKey: string; + readonly isOutOfProcess: boolean; +} + +export interface UnavailableFrameDocument { + readonly frameId: string; + readonly parentFrameId?: string; + readonly url?: string; + readonly processKey: string; + readonly isOutOfProcess: boolean; +} + +export interface FrameDocumentGraph { + readonly attachmentId?: string; + readonly rootFrameId: string; + readonly frames: FrameDocumentInfo[]; + /** Unavailable subtrees are excluded from routing, but remain visible as gaps. */ + readonly unavailableFrames?: UnavailableFrameDocument[]; +} + +/** + * Opaque CDP event source emitted by `ChromiumCdp.onEvent(...)`. + * Never exposes transient CDP `sessionId`; `ChromiumCdp` maps `targetKey` + * back to the live target internally. + */ +export interface TabEventSource { + readonly tabId: number; + readonly targetKey: string; + readonly isRootTarget: boolean; +} + +export class StaleFrameHandleError extends Error { + readonly tabId: number; + readonly frameId: string; + + constructor(handle: { tabId: number; frameId: string }, detail?: string); + constructor(tabId: number, frameId: string, detail?: string); + constructor( + tabOrHandle: number | { tabId: number; frameId: string }, + frameOrDetail?: string, + maybeDetail?: string, + ) { + const tabId = typeof tabOrHandle === "number" ? tabOrHandle : tabOrHandle.tabId; + const frameId = typeof tabOrHandle === "number" ? (frameOrDetail ?? "") : tabOrHandle.frameId; + const detail = typeof tabOrHandle === "number" ? maybeDetail : frameOrDetail; + super(detail ?? `Frame ${frameId} in tab ${tabId} changed or is no longer available`); + this.name = "StaleFrameHandleError"; + this.tabId = tabId; + this.frameId = frameId; + } +} + +export function isStaleFrameHandleError(error: unknown): error is StaleFrameHandleError { + return error instanceof StaleFrameHandleError; +} + +export function toFrameDocumentHandle( + tabId: number, + frame: { frameId: string; loaderId?: string }, + documentElementBackendNodeId?: number, +): FrameDocumentHandle { + return { + tabId, + frameId: frame.frameId, + loaderId: frame.loaderId ?? "", + ...(documentElementBackendNodeId !== undefined ? { documentElementBackendNodeId } : {}), + }; +} + +export function toRootFrameHandle(tabId: number): FrameDocumentHandle { + return { tabId, frameId: "", loaderId: "" }; +} + +export function rootEventSource(tabId: number): TabEventSource { + return { tabId, targetKey: "root", isRootTarget: true }; +} + +export function targetEventSource(tabId: number, targetKey = "root"): TabEventSource { + return { tabId, targetKey, isRootTarget: targetKey === "root" }; +} + +export function createFrameDocumentInfo(input: { + tabId: number; + frameId: string; + loaderId?: string; + parentFrameId?: string; + ownerBackendNodeId?: number; + url?: string; + urlFragment?: string; + processKey?: string; + isOutOfProcess?: boolean; + documentElementBackendNodeId?: number; +}): FrameDocumentInfo { + const loaderId = input.loaderId ?? ""; + const isOutOfProcess = + input.isOutOfProcess ?? (input.processKey !== undefined && input.processKey !== "root"); + const processKey = input.processKey ?? (isOutOfProcess ? `oopif:${input.frameId}` : "root"); + return { + handle: toFrameDocumentHandle( + input.tabId, + { frameId: input.frameId, loaderId }, + input.documentElementBackendNodeId, + ), + frameId: input.frameId, + loaderId, + ...(input.parentFrameId ? { parentFrameId: input.parentFrameId } : {}), + ...(input.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: input.ownerBackendNodeId } + : {}), + ...(input.url ? { url: input.url } : {}), + ...(input.urlFragment !== undefined ? { urlFragment: input.urlFragment } : {}), + processKey, + isOutOfProcess, + }; +} + +/** + * Renderer process boundary without exposing transient OOPIF `sessionId`. + * `processKey` is `"root"` for the top-level renderer process or + * `"oopif:"` for an out-of-process iframe target. + */ +export interface FrameProcessBoundary { + readonly tabId: number; + readonly processKey: string; + readonly isOutOfProcess: boolean; + readonly rootHandle: FrameDocumentHandle; +} + +export function rootProcessBoundary(tabId: number): FrameProcessBoundary { + return { + tabId, + processKey: "root", + isOutOfProcess: false, + rootHandle: toRootFrameHandle(tabId), + }; +} + +export function frameProcessBoundary(frame: FrameDocumentInfo): FrameProcessBoundary { + const { tabId } = frame.handle; + if (!frame.isOutOfProcess || frame.processKey === "root") { + return rootProcessBoundary(tabId); + } + return { + tabId, + processKey: frame.processKey, + isOutOfProcess: true, + rootHandle: { + tabId, + frameId: frame.frameId, + loaderId: frame.loaderId, + }, + }; +} + +export function frameProcessIdentityKey(input: { tabId: number; processKey: string }): string { + return `${input.tabId}:${input.processKey}`; +} + +export function omitFrameDocumentSubtrees( + graph: Pick, + unavailable: ReadonlyMap, +): FrameDocumentGraph { + if (!unavailable.size) return graph; + const blocked = new Set(unavailable.keys()); + const children = new Map(); + for (const frame of graph.frames) { + if (!frame.parentFrameId) continue; + const siblings = children.get(frame.parentFrameId) ?? []; + siblings.push(frame.frameId); + children.set(frame.parentFrameId, siblings); + } + const queue = [...blocked]; + for (let i = 0; i < queue.length; i++) { + for (const child of children.get(queue[i]) ?? []) { + if (blocked.has(child)) continue; + blocked.add(child); + queue.push(child); + } + } + const omitted = new Map( + graph.frames + .filter((frame) => blocked.has(frame.frameId)) + .map((frame) => [ + frame.frameId, + { + frameId: frame.frameId, + ...(frame.parentFrameId ? { parentFrameId: frame.parentFrameId } : {}), + ...(frame.url ? { url: frame.url } : {}), + processKey: frame.processKey, + isOutOfProcess: frame.isOutOfProcess, + }, + ]), + ); + for (const [frameId, info] of unavailable) { + const existing = omitted.get(frameId); + omitted.set(frameId, { ...existing, ...info, frameId }); + } + return { + ...graph, + frames: graph.frames.filter((frame) => !blocked.has(frame.frameId)), + unavailableFrames: [...omitted.values()], + }; +} diff --git a/apps/extension/src/browser-driver/screencast-coordinator.test.ts b/apps/extension/src/browser-driver/screencast-coordinator.test.ts index 3013580c3..19ec56624 100644 --- a/apps/extension/src/browser-driver/screencast-coordinator.test.ts +++ b/apps/extension/src/browser-driver/screencast-coordinator.test.ts @@ -1,19 +1,20 @@ import { expect, it, vi } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { guardedRunner } from "@/cooperation/action-guard"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { ScreencastCoordinator, screencasts } from "./screencast-coordinator"; it("shares one stream and ACK, restores remaining consumers and never stops a replacement attachment", async () => { let event!: Parameters>[1]; let attachment = "original"; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({}) as never), getAttachmentId: () => attachment, onEvent: (_tabId, handler) => { event = handler; return { dispose: vi.fn() }; }, - }; + }); const coordinator = screencasts(cdp); const frames = vi.fn(); const video = await coordinator.acquire(7, { @@ -30,7 +31,7 @@ it("shares one stream and ACK, restores remaining consumers and never stops a re expect( vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Page.startScreencast"), ).toHaveLength(1); - event({ tabId: 7 }, "Page.screencastFrame", { data: "test", metadata: {}, sessionId: 9 }); + event(rootEventSource(7), "Page.screencastFrame", { data: "test", metadata: {}, sessionId: 9 }); expect(frames).toHaveBeenCalledOnce(); expect( vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Page.screencastFrameAck"), @@ -57,14 +58,14 @@ it("does not block another tab behind a slow screencast command", async () => { const blocked = new Promise((resolve) => { release = resolve; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (tab, method) => { if (tab === 7 && method === "Page.startScreencast") await blocked; return {} as never; }), getAttachmentId: (tab) => String(tab), onEvent: () => ({ dispose() {} }), - }; + }); const coordinator = new ScreencastCoordinator(cdp); const pending = coordinator.acquire(7, { format: "jpeg", maxWidth: 1280, maxHeight: 1280 }); const independent = await coordinator.acquire(8, { format: "png", maxWidth: 32, maxHeight: 32 }); diff --git a/apps/extension/src/browser-driver/screencast-coordinator.ts b/apps/extension/src/browser-driver/screencast-coordinator.ts index 60742809a..b98a82514 100644 --- a/apps/extension/src/browser-driver/screencast-coordinator.ts +++ b/apps/extension/src/browser-driver/screencast-coordinator.ts @@ -96,7 +96,7 @@ export class ScreencastCoordinator { subscription: this.cdp.onEvent(tabId, (source, method, params) => { if ( source.tabId !== tabId || - source.sessionId || + !source.isRootTarget || method !== "Page.screencastFrame" || this.cdp.getAttachmentId?.(tabId) !== attachment || this.entries.get(tabId) !== created diff --git a/apps/extension/src/content/__tests__/cooperation-capture.browser.test.ts b/apps/extension/src/content/__tests__/cooperation-capture.browser.test.ts index 2ab420ca1..d3570853b 100644 --- a/apps/extension/src/content/__tests__/cooperation-capture.browser.test.ts +++ b/apps/extension/src/content/__tests__/cooperation-capture.browser.test.ts @@ -9,6 +9,7 @@ import { collectCooperationFrames } from "@/cooperation/browser-bridge"; import { CooperationController } from "@/cooperation/controller"; import { COOPERATION_ARM, COOPERATION_READ, type FrameState } from "@/cooperation/types"; import { SessionManager } from "@/session-manager/manager"; +import { createStubCdpRunner } from "@/tools/shared"; type Send = >( method: string, @@ -302,10 +303,10 @@ globalThis.httpCooperationReady = true;`, expression: "document.querySelector('input').focus()", }); const guarded = guardedRunner( - { + createStubCdpRunner({ send: (_tabId: number, method: string, params?: object) => enterSend(method, params), - }, + }), { prepare: async (input) => { if (input) diff --git a/apps/extension/src/content/__tests__/cooperation-capture.test.ts b/apps/extension/src/content/__tests__/cooperation-capture.test.ts index 7f34fb959..d09d3b1cb 100644 --- a/apps/extension/src/content/__tests__/cooperation-capture.test.ts +++ b/apps/extension/src/content/__tests__/cooperation-capture.test.ts @@ -9,7 +9,7 @@ import { type FrameState, type InputExpectation, } from "@/cooperation/types"; -import type { CdpRunner } from "@/tools/shared"; +import { createStubCdpRunner } from "@/tools/shared"; import { attachCooperationCapture } from "../cooperation-capture"; let dispose: (() => void) | undefined; @@ -627,7 +627,7 @@ it("clears text attribution when preparation fails before native dispatch", asyn const target = document.querySelector("input")!; target.focus(); const native = vi.fn(); - const runner = guardedRunner({ send: native } as CdpRunner, { + const runner = guardedRunner(createStubCdpRunner({ send: native }), { prepare: async (input) => { read(COOPERATION_ARM, "first", input); throw new Error("target changed"); @@ -747,13 +747,13 @@ it("attributes a keyless caret command to the Agent while preserving subsequent }); completeInput(document.querySelector("input")!, event); }; - const cdp = { - send: vi.fn(async (_tabId, _method, params) => { + const cdp = createStubCdpRunner({ + send: vi.fn(async (_tabId, _method, params: any) => { // Chromium emits an empty key for native editor commands without a key descriptor. - press(params.key ?? "", params.timestamp * 1000); - return {}; + press(params?.key ?? "", (params?.timestamp ?? 0) * 1000); + return {} as never; }), - } as CdpRunner; + }); const runner = guardedRunner(cdp, { prepare: async (input) => { read(COOPERATION_ARM, "first", input); diff --git a/apps/extension/src/cooperation/action-checks.ts b/apps/extension/src/cooperation/action-checks.ts index c6514d064..179297ab3 100644 --- a/apps/extension/src/cooperation/action-checks.ts +++ b/apps/extension/src/cooperation/action-checks.ts @@ -1,7 +1,4 @@ -import { - RECORD_PASSWORD_ATTRIBUTE, - SENSITIVE_AUTOCOMPLETE_SELECTOR, -} from "@/lib/sensitive-autocomplete"; +import { SENSITIVE_READ_INPUT_SELECTOR } from "@/lib/dom-predicates"; import type { SessionContext } from "@/session-manager/manager"; import { isMissingNode } from "@/tools/errors"; import { FOCUS_CHECK } from "@/tools/focus-state"; @@ -11,6 +8,7 @@ import { isRpcError, type ResolvedTargetTab, sendToFrameHandle, + withObjectGroup, } from "@/tools/shared"; import { buildInPageRowAnchorSnippet } from "@/tools/vom/row-anchor-spec"; import type { RpcError, RpcErrorReason } from "@/transport/types"; @@ -88,7 +86,7 @@ const SAMPLE_TARGET = `function(field, attribute, needRowAnchor) { if (!this.isConnected) return { connected: false }; if (field === 'exists') return { connected: true, value: true }; if (field === 'text') return { connected: true, value: this.textContent }; - const sensitiveSelector = 'input[type="password"],input[type="file"],input[type="hidden"],' + ${JSON.stringify(`[${RECORD_PASSWORD_ATTRIBUTE}],${SENSITIVE_AUTOCOMPLETE_SELECTOR}`)}; + const sensitiveSelector = ${JSON.stringify(SENSITIVE_READ_INPUT_SELECTOR)}; const sensitive = this.matches(sensitiveSelector); if (field === 'attribute') return { connected: true, value: this.getAttribute(attribute), redacted: sensitive && attribute.toLowerCase() === 'value' }; if (field) return { connected: true, value: this[field] ?? null, redacted: sensitive && field === 'value' }; @@ -121,53 +119,48 @@ async function sample( target: ResolvedActionTarget, check?: ActionExpectation, ): Promise { - const objectGroup = `action-check-${crypto.randomUUID()}`; const send = (method: string, params: object) => sendToFrameHandle(cdp, target.handle, method, params); - const cleanup = cdp.resourceCleanup ?? cdp; - try { - const resolved = await send<{ object?: { objectId?: string } }>("DOM.resolveNode", { - backendNodeId: target.backendNodeId, - objectGroup, - }); - if (!resolved.object?.objectId) return { connected: false }; - const reply = await send<{ result?: { value?: Sample }; exceptionDetails?: unknown }>( - "Runtime.callFunctionOn", - { - objectId: resolved.object.objectId, - functionDeclaration: SAMPLE_TARGET, - arguments: [ - { value: check?.field ?? null }, - { value: check?.attribute ?? null }, - { value: !check && Boolean(target.usedRef && target.rowAnchor) }, - ], - returnByValue: true, - }, - ); - if (reply.exceptionDetails || !reply.result?.value) + return withObjectGroup(cdp, target.handle, "action-check", async (objectGroup) => { + try { + const resolved = await send<{ object?: { objectId?: string } }>("DOM.resolveNode", { + backendNodeId: target.backendNodeId, + objectGroup, + }); + if (!resolved.object?.objectId) return { connected: false }; + const reply = await send<{ result?: { value?: Sample }; exceptionDetails?: unknown }>( + "Runtime.callFunctionOn", + { + objectId: resolved.object.objectId, + functionDeclaration: SAMPLE_TARGET, + arguments: [ + { value: check?.field ?? null }, + { value: check?.attribute ?? null }, + { value: !check && Boolean(target.usedRef && target.rowAnchor) }, + ], + returnByValue: true, + }, + ); + if (reply.exceptionDetails || !reply.result?.value) + return actionError( + "target_unavailable", + "Could not inspect the action target", + "target", + "Inspect the target again before deciding on another action.", + ); + return reply.result.value; + } catch (error) { + // CDP can collect an already removed node before it can be sampled. This is + // the same detached identity, not an unavailable browser or a new target. + if (isMissingNode(error)) return { connected: false }; return actionError( "target_unavailable", - "Could not inspect the action target", + error instanceof Error ? error.message : String(error), "target", - "Inspect the target again before deciding on another action.", + "Inspect the target again; the document or node may have been replaced.", ); - return reply.result.value; - } catch (error) { - // CDP can collect an already removed node before it can be sampled. This is - // the same detached identity, not an unavailable browser or a new target. - if (isMissingNode(error)) return { connected: false }; - return actionError( - "target_unavailable", - error instanceof Error ? error.message : String(error), - "target", - "Inspect the target again; the document or node may have been replaced.", - ); - } finally { - // Navigation may already have released the execution context. - await sendToFrameHandle(cleanup, target.handle, "Runtime.releaseObjectGroup", { - objectGroup, - }).catch(() => {}); - } + } + }); } export async function checkActionTarget( diff --git a/apps/extension/src/cooperation/action-guard.ts b/apps/extension/src/cooperation/action-guard.ts index 1fc4a4d06..aa7392ace 100644 --- a/apps/extension/src/cooperation/action-guard.ts +++ b/apps/extension/src/cooperation/action-guard.ts @@ -1,11 +1,14 @@ import type { CdpDispatchGuard } from "@/browser-driver/chromium-cdp"; -import type { CdpTarget, FrameDocumentHandle, FrameNodeHandle } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentHandle, + type FrameNodeHandle, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import { NativeInputClockError, type NativeInputTimestamp, } from "@/browser-driver/native-input-clock"; -import { shareScreencasts } from "@/browser-driver/screencast-coordinator"; -import { type CdpRunner, delegateCdpRunner, sendToFrameHandle } from "@/tools/shared"; +import { type CdpRunner, delegateCdpRunner } from "@/tools/shared"; import type { InputExpectation } from "./types"; interface ActionGuard { @@ -20,6 +23,25 @@ interface ActionGuard { scriptedEffect?: boolean; } +const STRIPPED_BACKGROUND_EXECUTION_KEYS = new Set([ + "acquireBackgroundExecution", + "ownsBackgroundExecution", + "restoreBackgroundExecution", +]); + +const BOUND_EMULATION_METHODS = new Set([ + "setDeviceMetricsOverride", + "clearDeviceMetricsOverride", + "setUserAgentOverride", + "setTouchEmulationEnabled", +]); + +const RESOURCE_CLEANUP_METHODS = new Set([ + "Runtime.callFunctionOn", + "Runtime.releaseObjectGroup", + "Runtime.releaseObject", +]); + /** * True when a CDP step is an internal preparation, focus, geometry, or isolated-world * helper call that cannot synthesize renderer `event.isTrusted === true` input events. @@ -49,47 +71,39 @@ export function guardedRunner(cdp: T, options: ActionGuard const { prepare, dispatch, verifyPointerTarget, beginDomMutation, detach } = options; const leases = new Map(); const attachments = new Map(); - const cleanupSend = (target: CdpTarget, method: string, params?: object) => { - if (!["Runtime.callFunctionOn", "Runtime.releaseObjectGroup"].includes(method)) - throw new Error("Resource cleanup cannot dispatch page input"); - if (!attachments.has(target.tabId)) throw new Error("No resources owned on this tab"); - return cdp.sendGuarded - ? cdp.sendGuarded(target, method, params, { attachmentId: attachments.get(target.tabId) }) - : target.sessionId && cdp.sendToTarget - ? cdp.sendToTarget(target, method, params) - : cdp.send(target.tabId, method, params); - }; const cleanupSendFrame = ( handle: FrameDocumentHandle | FrameNodeHandle, method: string, params?: object, + guard?: CdpDispatchGuard, ) => { - if (!["Runtime.callFunctionOn", "Runtime.releaseObjectGroup"].includes(method)) + if (!RESOURCE_CLEANUP_METHODS.has(method)) throw new Error("Resource cleanup cannot dispatch page input"); - if (!attachments.has(handle.tabId)) throw new Error("No resources owned on this tab"); - return sendToFrameHandle(cdp, handle, method, params, { + if (!attachments.has(handle.tabId)) + return Promise.reject(new Error("No resources owned on this tab")); + return cdp.sendToFrame(handle, method, params, { + ...guard, + allowReattach: false, attachmentId: attachments.get(handle.tabId), }); }; const resourceCleanup: CdpRunner = delegateCdpRunner( - { send: (tabId, method, params) => cleanupSend({ tabId }, method, params) }, + cdp, { - sendToTarget: cleanupSend, + send: (tabId: number, method: string, params?: object) => + cleanupSendFrame(toRootFrameHandle(tabId), method, params), sendToFrame: cleanupSendFrame, }, + { omit: STRIPPED_BACKGROUND_EXECUTION_KEYS }, ); - type SendRoute = - | { kind: "target"; target: CdpTarget } - | { kind: "frame"; handle: FrameDocumentHandle | FrameNodeHandle }; - const sendRoute = async ( - route: SendRoute, + const sendToFrame = async ( + handle: FrameDocumentHandle | FrameNodeHandle, method: string, params?: object, guard?: CdpDispatchGuard, - ) => { - const tabId = route.kind === "target" ? route.target.tabId : route.handle.tabId; - const leaseScope = - route.kind === "target" ? (route.target.sessionId ?? "") : `frame:${route.handle.frameId}`; + ): Promise => { + const { tabId } = handle; + const leaseScope = handle.frameId ? `frame:${handle.frameId}` : ""; const p = params as Record | undefined; const mouse = method === "Input.dispatchMouseEvent"; const key = method === "Input.dispatchKeyEvent"; @@ -144,7 +158,7 @@ export function guardedRunner(cdp: T, options: ActionGuard !!input || method === "Input.insertText" || method === "DOM.setFileInputFiles" || drag, method, ); - if (!attachments.has(tabId)) attachments.set(tabId, cdp.getAttachmentId?.(tabId)); + if (!attachments.has(tabId)) attachments.set(tabId, cdp.getAttachmentId(tabId)); guard?.onDispatch?.(); if ( input?.type === "pointerdown" || @@ -153,37 +167,18 @@ export function guardedRunner(cdp: T, options: ActionGuard (chooser && p?.enabled === true) || (drag && p?.type === "dragEnter") ) - leases.set(identity, cdp.getAttachmentId?.(tabId)); + leases.set(identity, cdp.getAttachmentId(tabId)); if (release && !cancelDrag) leases.delete(identity); }; - const sendNative = () => { - if (route.kind === "frame") { - if (cdp.sendToFrame && cdp.sendGuarded) { - return cdp.sendToFrame(route.handle, method, params, { - ...guard, - ...(release ? { signal: undefined, attachmentId: leases.get(identity) } : {}), - onDispatch: check, - }); - } - check(); - return sendToFrameHandle(cdp, route.handle, method, params, guard); - } - const target = route.target; - if (cdp.sendGuarded) - return cdp.sendGuarded(target, method, params, { - ...guard, - ...(release ? { signal: undefined, attachmentId: leases.get(identity) } : {}), - onDispatch: check, - }); - // Test/adaptor runners must provide the same synchronous pre-send check. - check(); - return target.sessionId && cdp.sendToTarget - ? cdp.sendToTarget(target, method, params) - : cdp.send(target.tabId, method, params); - }; + const sendNative = () => + cdp.sendToFrame(handle, method, params, { + ...guard, + ...(release ? { signal: undefined, attachmentId: leases.get(identity) } : {}), + onDispatch: check, + }); try { if (input && input.type !== "beforeinput") { - timing = await cdp.prepareInputTimestamp?.(); + timing = await cdp.prepareInputTimestamp(); input.timestamp = timing?.timestamp ?? Date.now(); params = { ...params, timestamp: timing?.nativeTimestamp ?? input.timestamp / 1000 }; } @@ -207,8 +202,8 @@ export function guardedRunner(cdp: T, options: ActionGuard "location", ]) if (p?.[field] !== undefined) up[field] = p[field]; - await sendRoute(route, method, up) - .catch(() => cdp.detach?.(tabId, attachment)) + await sendToFrame(handle, method, up) + .catch(() => cdp.detach(tabId, attachment)) .catch(() => {}); } } @@ -217,51 +212,26 @@ export function guardedRunner(cdp: T, options: ActionGuard if (input?.type === "beforeinput") await options.finishInput?.(input); } }; - const send = (target: CdpTarget, method: string, params?: object, guard?: CdpDispatchGuard) => - sendRoute({ kind: "target", target }, method, params, guard); - const sendToFrame = ( - handle: FrameDocumentHandle | FrameNodeHandle, - method: string, - params?: object, - guard?: CdpDispatchGuard, - ) => sendRoute({ kind: "frame", handle }, method, params, guard); - const runner = new Proxy(cdp, { - get(target, key, receiver) { - if (key === "resourceCleanup") return resourceCleanup; - if (key === "runBrowserMutation") - return async (operation: () => Promise): Promise => { - if (!prepare) throw new Error("Shared-tab observation cannot mutate the browser"); - await prepare(undefined, "Browser.mutate"); - dispatch?.(true, "Browser.mutate"); - return operation(); - }; - if (key === "beginDomMutation") return beginDomMutation; - if (key === "verifyPointerTarget") return verifyPointerTarget; - if (key === "detach" && detach) return detach; - if ( - key === "acquireBackgroundExecution" || - key === "ownsBackgroundExecution" || - key === "restoreBackgroundExecution" - ) - return undefined; - if (key === "send") - return (tabId: number, method: string, params?: object) => send({ tabId }, method, params); - if (key === "sendToTarget" || key === "sendGuarded") return send; - if (key === "sendToFrame") return sendToFrame; - const value = Reflect.get(target, key); - // Typed emulation helpers call this.send; keep those sends behind the same fence. - if ( - [ - "setDeviceMetricsOverride", - "clearDeviceMetricsOverride", - "setUserAgentOverride", - "setTouchEmulationEnabled", - ].includes(String(key)) - ) - return typeof value === "function" ? value.bind(receiver) : value; - return typeof value === "function" ? value.bind(target) : value; + return delegateCdpRunner( + cdp, + { + resourceCleanup, + runBrowserMutation: async (operation: () => Promise): Promise => { + if (!prepare) throw new Error("Shared-tab observation cannot mutate the browser"); + await prepare(undefined, "Browser.mutate"); + dispatch?.(true, "Browser.mutate"); + return operation(); + }, + beginDomMutation, + verifyPointerTarget, + ...(detach ? { detach } : {}), + send: (tabId: number, method: string, params?: object, signal?: AbortSignal) => + sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined), + sendToFrame, }, - }); - shareScreencasts(cdp, runner); - return runner; + { + omit: STRIPPED_BACKGROUND_EXECUTION_KEYS, + bindReceiver: BOUND_EMULATION_METHODS, + }, + ); } diff --git a/apps/extension/src/cooperation/borrowed-blank-tab.test.ts b/apps/extension/src/cooperation/borrowed-blank-tab.test.ts index 6e052912b..9f7f5b7bf 100644 --- a/apps/extension/src/cooperation/borrowed-blank-tab.test.ts +++ b/apps/extension/src/cooperation/borrowed-blank-tab.test.ts @@ -1,6 +1,6 @@ import { afterEach, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { CooperationController } from "./controller"; import { FrameCollectionError } from "./frame-error"; import type { CooperationFrames, FrameState } from "./types"; @@ -68,10 +68,10 @@ async function fixture( return { frames: new Map([[0, structuredClone(frame)]]), gaps: [] }; }); const send = vi.fn(async () => ({})); - const cdp = { - send, + const cdp: CdpRunner = createStubCdpRunner({ + send: send as CdpRunner["send"], runBrowserMutation: async (op: () => Promise) => op(), - } as unknown as CdpRunner; + }); const controller = new CooperationController({ sessions, readFrames: readFrames as never, diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 6248e39ac..0f5a8f9d9 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -1,16 +1,22 @@ import type { VomNode, VomScene } from "@browser-skill/vom"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { - type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp, DEFAULT_IDLE_DETACH_MS, } from "@/browser-driver/chromium-cdp"; +import { toRootFrameHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleEmulate } from "@/tools/emulate"; import { resolveActionTarget } from "@/tools/interaction"; import { captureVomObservation, handleSnapshot } from "@/tools/observation"; -import { type CdpRunner, sessionCdpRunner } from "@/tools/shared"; +import { + type CdpRunner, + createStubCdpRunner, + sessionCdpRunner, + withObjectGroup, + withRemoteObject, +} from "@/tools/shared"; import { handleTabBorrow, handleTabCreate, handleTabReturn, handleTabSelect } from "@/tools/tabs"; import { validateToolResult } from "@/transport/protocol"; import { CooperationController } from "./controller"; @@ -72,16 +78,16 @@ async function fixture(options: { url?: string; sync?: boolean } = {}) { pendingUrl: undefined as string | undefined, }; vi.stubGlobal("chrome", { tabs: { get: async () => tab, query: async () => [tab] } }); - const native = vi.fn(async () => ({})); + const native = vi.fn(async (): Promise => ({})); let beforeDispatch: (() => Promise) | undefined; - const cdp = { - send: native, - sendGuarded: async (_target, _method, _params, guard) => { + const cdp: CdpRunner = createStubCdpRunner({ + send: native as CdpRunner["send"], + sendToFrame: async (_handle, _method, _params, guard) => { await beforeDispatch?.(); - guard.onDispatch?.(); + guard?.onDispatch?.(); return native(); }, - } as CdpRunner; + }); const readFrames = vi.fn( async (): Promise => ({ frames: new Map([[0, structuredClone(frame)]]), @@ -3752,14 +3758,13 @@ describe("tool.record_start cooperation guard", () => { it("remaps unchanged OOPIF @eN sessions across idle soft-detach and refreshes sync snapshot when an OOPIF changes or disappears", async () => { vi.useFakeTimers(); - const listeners = new Set<(source: CdpDebuggee, method: string, params: unknown) => void>(); + type DebuggeeEvent = Parameters[0]; + const listeners = new Set(); const onEvent = { - addListener: (cb: (source: CdpDebuggee, method: string, params: unknown) => void) => - listeners.add(cb), - removeListener: (cb: (source: CdpDebuggee, method: string, params: unknown) => void) => - listeners.delete(cb), - fire: (source: CdpDebuggee, method: string, params: unknown) => { - for (const cb of listeners) cb(source, method, params); + addListener: (cb: DebuggeeEvent) => listeners.add(cb), + removeListener: (cb: DebuggeeEvent) => listeners.delete(cb), + fire: (...args: Parameters) => { + for (const cb of listeners) cb(...args); }, }; const detachListeners = new Set<(source: chrome.debugger.Debuggee, reason: string) => void>(); @@ -3950,7 +3955,7 @@ describe("tool.record_start cooperation guard", () => { truncated: false, })); - // 1. Strict policy: `fixture()` syncs under `strict` policy; fail AFTER `beginDomMutation` / `sendGuarded`. + // 1. Strict policy: `fixture()` syncs under `strict` policy; fail AFTER `beginDomMutation` / guarded `sendToFrame`. const f = await fixture(); expect(f.first.control_policy).toBe("strict"); @@ -3970,8 +3975,8 @@ describe("tool.record_start cooperation guard", () => { f.cdp, f.abort.signal, async (runner) => { - await runner.sendGuarded?.( - { tabId: 7 }, + await runner.sendToFrame( + toRootFrameHandle(7), "Input.insertText", { text: "secret" }, { signal: f.abort.signal }, @@ -4108,4 +4113,36 @@ describe("tool.record_start cooperation guard", () => { data: { effect_state: "none" }, }); }); + + it("preserves pre-dispatch cooperation rejections inside withObjectGroup and withRemoteObject when no tab resources were owned yet", async () => { + const f = await fixture(); + f.change({ human: 1, lastInput: 10000 }); + const groupOutcome = await f.act(f.first.context_id, async (cdp) => + withObjectGroup(cdp, toRootFrameHandle(7), "pre-dispatch", async (objectGroup) => + cdp.send(7, "Runtime.callFunctionOn", { + objectId: "never-dispatched", + objectGroup, + functionDeclaration: "function() {}", + }), + ), + ); + expect(groupOutcome).toMatchObject({ + code: "permission_denied", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + + const remoteOutcome = await f.act(f.first.context_id, async (cdp) => + withRemoteObject(cdp, toRootFrameHandle(7), "never-owned-obj", async (objectId) => + cdp.send(7, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: "function() {}", + }), + ), + ); + expect(remoteOutcome).toMatchObject({ + code: "permission_denied", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + }); }); diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index 8cd1aa881..b89825f13 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -1,3 +1,4 @@ +import { inspectVomLine } from "@browser-skill/vom"; import { injectCooperationAllFrames } from "@/lib/runtime-content-scripts"; import { isAgentControlledTab, @@ -12,8 +13,10 @@ import { cdpBlockedUrlReason, chromeTabsApi, isRpcError, + type PinnedSelectorTarget, resolveTargetTab, sessionCdpRunner, + withPinnedSelectorTargets, } from "@/tools/shared"; import { lookupSnapshotRef } from "@/tools/snapshot-ref"; import { AccessibilityLease } from "@/tools/vom/capture-coordinator"; @@ -1027,6 +1030,7 @@ export class CooperationController implements ControlAuthority { tabsApi: chromeTabsApi, refStore, accessibilityLease: state.accessibilityLease, + includeVisualFacts: true, onScene: (scene) => { evidence = observationEvidence(scene); }, @@ -1082,9 +1086,15 @@ export class CooperationController implements ControlAuthority { text: observation.text, truncated: observation.truncated, maxTokens: params.max_tokens, + lineMeta: observation.text.split("\n").map((line) => inspectVomLine(line)), } : text && baseline?.held - ? { ...baseline.held, text: text.text, patched: true } + ? { + ...baseline.held, + text: text.text, + lineMeta: text.lineMeta, + patched: true, + } : baseline?.held, } : {}), @@ -2174,20 +2184,21 @@ export class CooperationController implements ControlAuthority { assert(); }; // Resolve selectors once per action. Handlers use the same concrete DOM node - // that passed the checks, even if a list reorders during preparation. - const pin = (node: ResolvedActionTarget) => { - const identity = { - tabId: target.tabId, + // that passed the checks, even if a list reorders during preparation, without + // polluting session.refStore or rewriting selector params into fake @eN refs. + const pinnedSelectors = new Map(); + const pinSelector = (selector: string, node: ResolvedActionTarget) => { + pinnedSelectors.set(selector, { + handle: node.handle ?? { + tabId: target.tabId, + frameId: node.frameId ?? "", + loaderId: "", + backendNodeId: node.backendNodeId, + }, backendNodeId: node.backendNodeId, - frameId: node.handle?.frameId || node.frameId, - loaderId: node.handle?.loaderId, - ...(node.handle?.documentElementBackendNodeId !== undefined - ? { documentElementBackendNodeId: node.handle.documentElementBackendNodeId } - : {}), - }; - const ref = session.refStore.allocateRef(identity); - session.refStore.set(ref, node.backendNodeId, identity); - return ref; + ...(node.frameId ? { frameId: node.frameId } : {}), + usedSelector: selector, + }); }; const primaryEffect = (method: string) => (req.method === "tool.focus" && method === "DOM.focus") || @@ -2219,6 +2230,28 @@ export class CooperationController implements ControlAuthority { }, }; let fallbackBootstrap = false; + let restoreStrictObservation = false; + const maybeRestoreObservation = (effect: string) => { + if ( + restoreStrictObservation && + context && + effect === "none" && + !nativeInputStarted && + !preparationSent && + !detachedByHandler && + !postRefreshFailed && + !revoked && + checkFailure?.data?.reason !== "observation_changed" && + !state.stopped && + state.baseline === context && + state.context !== undefined && + context.epoch === state.epoch && + !interruption(false) + ) { + context.consumed = false; + this.changed(state); + } + }; try { await this.refresh(state); assert(); @@ -2263,17 +2296,11 @@ export class CooperationController implements ControlAuthority { target, primaryTarget, "shared action", - targetPolicy || req.method === "tool.drag", + true, ); if (isRpcError(resolved)) return targetError(resolved); actionTarget = resolved; - if (targeted && primaryTarget.selector) { - const pinned = { ...primaryTarget, ref: pin(resolved), selector: undefined }; - req.params = - req.method === "tool.drag" - ? { ...(req.params as object), source: pinned } - : { ...(req.params as object), ...pinned }; - } + if (primaryTarget.selector) pinSelector(primaryTarget.selector, resolved); } if (req.method === "tool.drag" && (params.target?.ref || params.target?.selector)) { const resolved = await resolveActionTarget( @@ -2286,11 +2313,7 @@ export class CooperationController implements ControlAuthority { ); if (isRpcError(resolved)) return targetError(resolved); destinationTarget = resolved; - if (targeted && params.target?.selector) - req.params = { - ...(req.params as object), - target: { ...params.target, ref: pin(resolved), selector: undefined }, - }; + if (params.target.selector) pinSelector(params.target.selector, resolved); } if (params.depends_on !== undefined) { if ( @@ -2316,6 +2339,7 @@ export class CooperationController implements ControlAuthority { target, /^@?e\d+$/.test(dependency) ? { ref: dependency } : { selector: dependency }, "action dependency", + true, ); if (isRpcError(resolved)) return targetError(resolved); evidenceTargets.push(resolved); @@ -2352,10 +2376,15 @@ export class CooperationController implements ControlAuthority { if (targeted && params.timeout_ms !== undefined) req.params = { ...(req.params as object), timeout_ms: Math.max(1, deadline - Date.now()) }; if (!targetPolicy) state.context = undefined; - else if (!targeted) context.consumed = true; + else if (!targeted && !context.consumed) { + context.consumed = true; + restoreStrictObservation = true; + } this.changed(state); - let result = await invoke( - guardedRunner(cdp, { + const actionCdp = + pinnedSelectors.size > 0 ? withPinnedSelectorTargets(cdp, pinnedSelectors) : cdp; + const result = await invoke( + guardedRunner(actionCdp, { onInputClockError: (error) => { checkFailure = error.toRpcError(); }, @@ -2424,16 +2453,15 @@ export class CooperationController implements ControlAuthority { } }, beginDomMutation: async () => { + await this.refresh(state); + assert(); if (targeted) { - assert(); if (!nativeInputStarted && expectations.length > 0) { await verifyExpectations(); } nativeInputStarted = true; return; } - await this.refresh(state); - assert(); if (!nativeInputStarted) await verifyDecision(); nativeInputStarted = true; }, @@ -2464,16 +2492,6 @@ export class CooperationController implements ControlAuthority { operationSignal, completionSignal, ); - // Pinning preserves the chosen node; the receipt also preserves how the - // caller selected it. This applies to download's early return as well. - if ( - targeted && - params.selector && - result && - typeof result === "object" && - !isRpcError(result) - ) - result = { ...result, used_selector: params.selector }; // A completed download transfers cleanup ownership to the daemon. Keep its // receipt even if control changed during final cleanup; dropping it would // orphan the file. The consumed context still requires a fresh next step. @@ -2502,12 +2520,14 @@ export class CooperationController implements ControlAuthority { // Even a successful CDP reply does not authorize a stale next step. if (state.stopped) return stopped(sent ? "unknown" : "none"); if (signal.aborted) { + const effect = sent ? "unknown" : "none"; if (targetPolicy && sent) state.context = undefined; + else maybeRestoreObservation(effect); return { code: "cancelled", message: "Shared-tab action cancelled", data: { - effect_state: sent ? "unknown" : "none", + effect_state: effect, phase: sent ? "execution" : "precondition", ...(sent ? { tab_id: state.tabId } : {}), }, @@ -2519,12 +2539,16 @@ export class CooperationController implements ControlAuthority { (!completed && !isRpcError(result) && detachedByHandler ? "The observed page or control state changed; synchronize" : undefined); - if (checkFailure || reason) - return reportFailure(checkFailure?.message ?? reason!, sent ? "unknown" : "none"); + if (checkFailure || reason) { + const effect = sent ? "unknown" : "none"; + maybeRestoreObservation(effect); + return reportFailure(checkFailure?.message ?? reason!, effect); + } if (isRpcError(result)) { const effect = result.data?.effect_state ?? (sent ? "unknown" : "none"); - if (targetPolicy && (effect === "unknown" || detachedByHandler || postRefreshFailed)) + if (targetPolicy && (effect !== "none" || detachedByHandler || postRefreshFailed)) state.context = undefined; + else maybeRestoreObservation(effect); return { ...result, data: { @@ -2540,16 +2564,29 @@ export class CooperationController implements ControlAuthority { }, }; } + if ( + primaryTarget?.selector && + result && + typeof result === "object" && + (result as Record).used_selector === undefined + ) { + return { + ...(result as Record), + used_selector: primaryTarget.selector, + }; + } return result; } catch (error) { if (state.stopped) return stopped(sent ? "unknown" : "none"); if (signal.aborted) { + const effect = sent ? "unknown" : "none"; if (targetPolicy && sent) state.context = undefined; + else maybeRestoreObservation(effect); return { code: "cancelled", message: "Shared-tab action cancelled", data: { - effect_state: sent ? "unknown" : "none", + effect_state: effect, phase: sent ? "execution" : "precondition", ...(sent ? { tab_id: state.tabId } : {}), }, @@ -2582,12 +2619,10 @@ export class CooperationController implements ControlAuthority { fallbackBootstrap = true; } else { if (error instanceof ActionCheckError) checkFailure = error.rpcError; - if (error instanceof FrameCollectionError) - return error.toRpcError(sent ? "unknown" : "none"); - return reportFailure( - error instanceof Error ? error.message : String(error), - sent ? "unknown" : "none", - ); + const effect = sent ? "unknown" : "none"; + maybeRestoreObservation(effect); + if (error instanceof FrameCollectionError) return error.toRpcError(effect); + return reportFailure(error instanceof Error ? error.message : String(error), effect); } } finally { finished = true; diff --git a/apps/extension/src/cooperation/live-text.browser.test.ts b/apps/extension/src/cooperation/live-text.browser.test.ts index d07c4913c..5a2475a95 100644 --- a/apps/extension/src/cooperation/live-text.browser.test.ts +++ b/apps/extension/src/cooperation/live-text.browser.test.ts @@ -7,13 +7,13 @@ import { createServer } from "node:http"; import { afterEach, describe, expect, it, vi } from "vitest"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; -import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { createFrameDocumentInfo, type FrameDocumentInfo } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { type EmulateCdpRunner, handleEmulate } from "@/tools/emulate"; import { handleEvaluate } from "@/tools/evaluate"; import { handleBlur, handleClick, handleFill, handlePress } from "@/tools/interaction"; import { handleObserve, handleScreenshot, handleSnapshot } from "@/tools/observation"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import type { RequestFrame } from "@/transport/types"; import { CooperationController } from "./controller"; import type { FrameState } from "./types"; @@ -163,15 +163,25 @@ async function withCooperation( }; await local("Page.enable"); await local("Page.bringToFront"); - const graph = async (): Promise => { + const getFrameDocuments = async (): Promise<{ + rootFrameId: string; + frames: FrameDocumentInfo[]; + }> => { interface Tree { - frame: { id: string }; + frame: { id: string; loaderId?: string }; childFrames?: Tree[]; } const { frameTree } = await local<{ frameTree: Tree }>("Page.getFrameTree"); - const frames: CdpFrame[] = []; + const frames: FrameDocumentInfo[] = []; const visit = (tree: Tree, parentFrameId?: string) => { - frames.push({ frameId: tree.frame.id, parentFrameId, target: { tabId: 4 } }); + frames.push( + createFrameDocumentInfo({ + tabId: 4, + frameId: tree.frame.id, + loaderId: tree.frame.loaderId ?? "", + parentFrameId, + }), + ); for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); }; visit(frameTree); @@ -183,16 +193,21 @@ async function withCooperation( if (method === "Accessibility.getFullAXTree") captures.accessibility++; return local(method, params); }; - const cdp: CdpRunner & EmulateCdpRunner = { - send: (_tabId, method, params) => instrumented(method, params), - sendToTarget: (_target, method, params) => instrumented(method, params), - getAttachmentId: () => sessionId, - getFrameGraph: graph, - setDeviceMetricsOverride: ChromiumCdp.prototype.setDeviceMetricsOverride, - clearDeviceMetricsOverride: ChromiumCdp.prototype.clearDeviceMetricsOverride, - setUserAgentOverride: ChromiumCdp.prototype.setUserAgentOverride, - setTouchEmulationEnabled: ChromiumCdp.prototype.setTouchEmulationEnabled, - }; + const cdp: CdpRunner & EmulateCdpRunner = Object.assign( + createStubCdpRunner({ + send: (_tabId, method, params) => instrumented(method, params), + sendToFrame: (_handle, method, params) => instrumented(method, params), + getAttachmentId: () => sessionId, + getFrameDocuments, + getMainFrameHandle: async () => (await getFrameDocuments()).frames[0].handle, + }), + { + setDeviceMetricsOverride: ChromiumCdp.prototype.setDeviceMetricsOverride, + clearDeviceMetricsOverride: ChromiumCdp.prototype.clearDeviceMetricsOverride, + setUserAgentOverride: ChromiumCdp.prototype.setUserAgentOverride, + setTouchEmulationEnabled: ChromiumCdp.prototype.setTouchEmulationEnabled, + }, + ); const sessions = new SessionManager({ agentWindow: { create: async () => ({ windowId: 100, initialTabIds: [4] }), diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index f42c06e04..32f57724b 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -1,4 +1,9 @@ -import { selectionMarker, type VomNode, type VomScene } from "@browser-skill/vom"; +import { + selectionMarker, + VOM_DERIVED_NAME_CAP, + type VomNode, + type VomScene, +} from "@browser-skill/vom"; import { playbackProgressMedia } from "./media-progress"; /** Reuse the observation's semantic nodes. Only normalized selection hints from @@ -92,6 +97,8 @@ const factFields = [ "visible", "pointer_events", "attributes", + "has_popup", + "expanded", "src", ] as const; // Stands in for learned text inside a derived name; HTML text never contains it. @@ -386,7 +393,7 @@ export function preparationEvidence( // Keep every known character exact; never shorten the old observation // when an added tooltip hides previously observed text instead. if ( - original?.name?.length === 240 && + original?.name?.length === VOM_DERIVED_NAME_CAP && original.name !== prior?.name && prior?.name && node.name?.startsWith(prior.name) @@ -521,6 +528,8 @@ function signature( semanticAttributes.map((key) => tolerate && contentAttributes.has(key) ? undefined : node.attrs?.[key], ), + node.hasPopup, + node.expanded, ]; }; const rows = (list: VomNode[], relax: boolean): Row[] => @@ -581,7 +590,9 @@ function sameUnrelaxedNodeFacts(a: VomNode, b: VomNode): boolean { !jsonPrimitiveEq(a.checked, b.checked) || !jsonPrimitiveEq(a.disabled, b.disabled) || !jsonPrimitiveEq(a.inert, b.inert) || - !jsonPrimitiveEq(a.modal, b.modal) + !jsonPrimitiveEq(a.modal, b.modal) || + !jsonPrimitiveEq(a.hasPopup, b.hasPopup) || + !jsonPrimitiveEq(a.expanded, b.expanded) ) { return false; } diff --git a/apps/extension/src/cooperation/policy-negotiation.test.ts b/apps/extension/src/cooperation/policy-negotiation.test.ts index 1e8dd4714..71ff42499 100644 --- a/apps/extension/src/cooperation/policy-negotiation.test.ts +++ b/apps/extension/src/cooperation/policy-negotiation.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { handleSnapshot } from "@/tools/observation"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { CooperationController } from "./controller"; import type { CooperationParams, FrameState } from "./types"; @@ -51,9 +51,9 @@ async function fixture() { changed: () => {}, now: () => 10_000, }); - const cdp = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tabId, method) => { - if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {} as never; if (method === "Accessibility.getFullAXTree") return { nodes: [ @@ -71,16 +71,16 @@ async function fixture() { backendDOMNodeId: 200, }, ], - }; + } as never; if (method === "Page.getLayoutMetrics") return { cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, - }; + } as never; // Exercise the existing AX fallback with a minimal, stable document. throw new Error(`Unsupported fixture method: ${method}`); }), - } as CdpRunner; + }); const abort = new AbortController(); return { session, diff --git a/apps/extension/src/cooperation/target-policy.test.ts b/apps/extension/src/cooperation/target-policy.test.ts index e3ccff702..8be844c37 100644 --- a/apps/extension/src/cooperation/target-policy.test.ts +++ b/apps/extension/src/cooperation/target-policy.test.ts @@ -2,9 +2,9 @@ import type { VomScene } from "@browser-skill/vom"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { scrollNodeIntoView } from "@/tools/element-geometry"; -import { resolveActionTarget } from "@/tools/interaction"; +import { handleSelect, resolveActionTarget } from "@/tools/interaction"; import { captureVomObservation, handleSnapshot } from "@/tools/observation"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner, lookupPinnedSelectorTarget } from "@/tools/shared"; import { ActionCheckError, actionError, @@ -18,7 +18,10 @@ vi.mock("@/tools/observation", () => ({ captureVomObservation: vi.fn(), handleSnapshot: vi.fn(async () => ({ text: "@e1 button Continue", tab_id: 7 })), })); -vi.mock("@/tools/interaction", () => ({ resolveActionTarget: vi.fn() })); +vi.mock("@/tools/interaction", async (original) => ({ + ...(await original()), + resolveActionTarget: vi.fn(), +})); vi.mock("./action-checks", async (original) => ({ ...(await original()), checkActionTarget: vi.fn(async () => undefined), @@ -55,8 +58,15 @@ async function fixture(policy: "strict" | "target" = "target") { handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 4 }, usedRef: "e1", }); - const send = vi.fn(async () => ({})); - const cdp = { send } as CdpRunner; + const send = vi.fn( + async ( + _tabId: number, + _method: string, + _params?: object, + _signal?: AbortSignal, + ): Promise => ({}), + ); + const cdp: CdpRunner = createStubCdpRunner({ send: send as CdpRunner["send"] }); const readFrames = vi.fn(async () => ({ frames: new Map([[0, structuredClone(frame)]]), gaps: [], @@ -1083,7 +1093,7 @@ describe("negotiated target control", () => { "tool.download", "tool.upload", "tool.wheel", - ])("preserves the selector in a pinned %s receipt", async (method) => { + ])("preserves the selector in a pinned %s receipt without polluting refStore", async (method) => { const f = await fixture(); const request = { id: "selector-action", @@ -1096,9 +1106,9 @@ describe("negotiated target control", () => { } as Record, }; const result = await f.controller.run(request, f.cdp, f.signal, async (runner) => { - expect(request.params.selector).toBeUndefined(); - expect(request.params.ref).toMatch(/^e\d+$/); - expect(f.session.refStore.resolveEntry(request.params.ref as string)).toMatchObject({ + expect(request.params.selector).toBe("#confirm"); + expect(request.params.ref).toBeUndefined(); + expect(lookupPinnedSelectorTarget(runner, 7, "#confirm")).toMatchObject({ backendNodeId: 4, }); await runner.send(7, "Input.dispatchMouseEvent", { @@ -1107,11 +1117,10 @@ describe("negotiated target control", () => { x: 10, y: 20, }); - return { tab_id: 7, used_ref: request.params.ref }; + return { tab_id: 7, used_selector: "#confirm" }; }); expect(result).toEqual({ tab_id: 7, - used_ref: request.params.ref, used_selector: "#confirm", }); expect(resolveActionTarget).toHaveBeenCalledTimes(1); @@ -1284,13 +1293,14 @@ describe("targeted optimistic fast-path", () => { // Entry preflight runs checkActionTarget & checkExpectations once. expect(checkActionTarget).toHaveBeenCalledTimes(1); expect(checkExpectations).toHaveBeenCalledTimes(1); - // readFrames is called at entry refresh (undefined), once to arm beforeinput (not twice), - // plus non-blocking clear cleanup for beforeinput. + // readFrames is called at entry refresh (undefined), once at beginDomMutation() to + // re-fence human input after scroll/focus preparation, once to arm beforeinput, + // plus non-blocking clear cleanup for beforeinput. Internal runner.send steps emit 0 polls. const armCalls = f.readFrames.mock.calls.filter( ([, input]) => input !== undefined && input.type !== "clear", ); const pollCalls = f.readFrames.mock.calls.filter(([, input]) => input === undefined); - expect(pollCalls).toHaveLength(1); + expect(pollCalls).toHaveLength(2); expect(armCalls).toHaveLength(1); expect(armCalls[0]?.[1]).toMatchObject({ type: "beforeinput", text: "hello" }); }); @@ -1796,4 +1806,161 @@ describe("text deltas under target control", () => { observation: { text: VOM, truncated: false }, }); }); + + it("pins --selector without polluting session.refStore, emitting used_ref, or leaking @eN in handleSelect errors", async () => { + const f = await fixture("target"); + const initialRefs = [...f.session.refStore.entries()]; + vi.mocked(resolveActionTarget).mockResolvedValueOnce({ + tab: { tabId: 7, windowId: 100, active: true, url: "https://fixture.test" }, + backendNodeId: 77, + handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 77 }, + usedSelector: "#custom-combo", + }); + + f.send.mockImplementation(async (_tabId: number, method: string) => { + if (method === "DOM.describeNode") { + return { node: { nodeName: "DIV", localName: "div", attributes: [] } }; + } + return {}; + }); + + const sessions = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [7] }), + remove: async () => {}, + ensureActiveTab: async () => 7, + }, + }); + // Reuse the controller's real session manager via a wrapper calling real handleSelect. + const selectReq = { + id: "sel-custom", + method: "tool.select", + params: { + session_id: "s", + tab_id: 7, + context_id: f.first.context_id, + observation_id: f.first.observation_id, + selector: "#custom-combo", + values: ["opt-1"], + }, + }; + const fakeSm = { + get: (id: string) => (id === "s" ? f.session : sessions.get(id)), + findByWindowId: (wid: number) => (wid === 100 ? f.session : null), + } as unknown as SessionManager; + const tab = { + id: 7, + windowId: 100, + active: true, + url: "https://fixture.test", + } as chrome.tabs.Tab; + const errRes = await f.controller.run(selectReq, f.cdp, f.signal, (runner, sig) => + handleSelect(fakeSm, selectReq.params, { + cdp: runner, + tabsApi: { get: async () => tab, query: async () => [tab] }, + signal: sig, + }), + ); + expect(errRes).toMatchObject({ + code: "invalid_params", + message: expect.stringContaining("Click #custom-combo first"), + }); + expect((errRes as { message: string }).message).not.toMatch(/@e\d+/); + expect([...f.session.refStore.entries()]).toEqual(initialRefs); + + // Valid selector action returns used_selector without used_ref and leaves refStore untouched. + vi.mocked(resolveActionTarget).mockResolvedValueOnce({ + tab: { tabId: 7, windowId: 100, active: true, url: "https://fixture.test" }, + backendNodeId: 78, + handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 78 }, + usedSelector: "#save-btn", + }); + const clickReq = { + id: "clk-sel", + method: "tool.click", + params: { + session_id: "s", + tab_id: 7, + context_id: f.first.context_id, + observation_id: f.first.observation_id, + selector: "#save-btn", + }, + }; + const okRes = (await f.controller.run(clickReq, f.cdp, f.signal, async (runner) => { + const pinned = lookupPinnedSelectorTarget(runner, 7, "#save-btn"); + expect(pinned?.backendNodeId).toBe(78); + await runner.send(7, "Input.dispatchMouseEvent", { + type: "mousePressed", + button: "left", + x: 10, + y: 20, + }); + return { tab_id: 7, used_selector: "#save-btn", x: 10, y: 20 }; + })) as Record; + expect(okRes).toMatchObject({ tab_id: 7, used_selector: "#save-btn" }); + expect(okRes).not.toHaveProperty("used_ref"); + expect([...f.session.refStore.entries()]).toEqual(initialRefs); + }); + + it("refreshes live frame activity inside beginDomMutation under target policy and blocks DOM write on human intervention", async () => { + const f = await fixture("target"); + let domMutated = false; + const res = await f.run( + { ref: "e1", observation_id: f.first.observation_id, value: "opt-a" }, + "tool.select", + async (runner) => { + // Simulate human input occurring during scroll/focus preparation without push notification yet. + f.change({ human: 1, editing: true }, false); + await runner.beginDomMutation?.(); + domMutated = true; + return { tab_id: 7, used_ref: "e1", selected_values: ["opt-a"], selected_labels: ["A"] }; + }, + ); + expect(domMutated).toBe(false); + expect(res).toMatchObject({ + code: "permission_denied", + data: { reason: "human_intervention", effect_state: "none" }, + }); + }); + + it("does not consume target-policy baseline when an untargeted action fails pre-write validation with effect_state: none", async () => { + const f = await fixture("target"); + const preWriteErr = await f.run( + { key: "InvalidKey", observation_id: f.first.observation_id }, + "tool.press", + async () => ({ + code: "invalid_params", + message: "unsupported key", + data: { effect_state: "none", phase: "precondition" }, + }), + ); + expect(preWriteErr).toMatchObject({ + code: "invalid_params", + data: { effect_state: "none" }, + }); + + // Retrying with the same observation_id succeeds without requiring a redundant sync. + const retryOk = await f.run( + { key: "Enter", observation_id: f.first.observation_id }, + "tool.press", + async (runner) => { + await runner.send(7, "Input.dispatchKeyEvent", { type: "keyDown", key: "Enter" }); + return { tab_id: 7, key: "Enter" }; + }, + ); + expect(retryOk).toMatchObject({ tab_id: 7, key: "Enter" }); + }); + + it("passes includeVisualFacts: true during cooperate sync so Canvas @eN visual refs stay in parity with observe", async () => { + const f = await fixture("target"); + snapshots.mockClear(); + f.change({ structure: 5, revision: 5 }); + await f.sync("target"); + expect(snapshots).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ session_id: "s", tab_id: 7 }), + expect.objectContaining({ includeVisualFacts: true }), + expect.anything(), + ); + }); }); diff --git a/apps/extension/src/cooperation/text-delta.test.ts b/apps/extension/src/cooperation/text-delta.test.ts index 3dd085f27..fd9968525 100644 --- a/apps/extension/src/cooperation/text-delta.test.ts +++ b/apps/extension/src/cooperation/text-delta.test.ts @@ -200,4 +200,55 @@ describe("textDelta", () => { }); expect(delta(after(grown), { ...held, maxTokens: 120 }, 120)).toBe("token_budget_shift"); }); + + it("uses structured VomLineMetadata so placeholder attributes and landmark names containing 'canvas ' are classified accurately", () => { + const trickyVom = [ + "@vom 1", + "@view 1280x720", + "@layers 1 focus=L1", + "L1 page", + ' RootWebArea "Studio"', + ' @e1 textbox placeholder="Version: rev-0"', + ' banner "Paint on canvas Version: rev-0"', + ' StaticText "Paint on canvas Version: rev-0"', + ].join("\n"); + const trickyHeld: HeldText = { text: trickyVom, truncated: false }; + + // A landmark whose label happens to contain `canvas ` is NOT a `[visual:screenshot]` canvas line; + // structured VOM metadata patches both the landmark's derived name and its StaticText cleanly. + const res = delta( + after([text(1, "Paint on canvas Version: rev-0", "Paint on canvas Version: rev-1")]), + trickyHeld, + ); + expect(res).toMatchObject({ + lines: [ + { + line: 7, + previous: ' banner "Paint on canvas Version: rev-0"', + text: ' banner "Paint on canvas Version: rev-1"', + }, + { + line: 8, + previous: ' StaticText "Paint on canvas Version: rev-0"', + text: ' StaticText "Paint on canvas Version: rev-1"', + }, + ], + }); + + // Meanwhile, an unnamed textbox with `placeholder="Version: rev-0"` has no quoted node name + // or StaticText line, so changing `"Version: rev-0"` refuses rather than mutating placeholder attributes. + expect( + delta(after([text(1, "Version: rev-0", "Version: rev-1")]), { + text: [ + "@vom 1", + "@view 1280x720", + "@layers 1 focus=L1", + "L1 page", + ' RootWebArea "Studio"', + ' @e1 textbox placeholder="Version: rev-0"', + ].join("\n"), + truncated: false, + }), + ).toBe("text_ambiguous"); + }); }); diff --git a/apps/extension/src/cooperation/text-delta.ts b/apps/extension/src/cooperation/text-delta.ts index 68867d182..cae42a947 100644 --- a/apps/extension/src/cooperation/text-delta.ts +++ b/apps/extension/src/cooperation/text-delta.ts @@ -1,4 +1,10 @@ -import { estimateLineTokens } from "@browser-skill/vom"; +import { + encodeVomNameContent, + estimateLineTokens, + inspectVomLine, + VOM_DERIVED_NAME_CAP, + type VomLineMetadata, +} from "@browser-skill/vom"; import type { FrameState } from "./types"; type Frames = ReadonlyMap; @@ -8,6 +14,7 @@ export interface HeldText { text: string; truncated: boolean; maxTokens?: number; + lineMeta?: readonly VomLineMetadata[]; /** A delta rewrote this text after the capture that published the refs. */ patched?: boolean; } @@ -21,6 +28,7 @@ export interface LinePatch { export interface TextDelta { text: string; + lineMeta: VomLineMetadata[]; lines: LinePatch[]; changes: { frame_id: number; @@ -52,16 +60,8 @@ export type TextDeltaRefusal = | "text_cut_by_truncation" | "token_budget_shift"; -/** A name derived from content is cut here without a mark (semantic-graph/resolve.ts). */ -const DERIVED_NAME_CAP = 240; /** Headroom for the header and notice lines, which a capture costs as one block. */ const BUDGET_MARGIN = 64; -/** Inner text of a rendered name: the renderer collapses whitespace and writes JSON strings. */ -const rendered = (value: string) => JSON.stringify(value.replace(/\s+/g, " ").trim()).slice(1, -1); -const depth = (line: string) => line.length - line.trimStart().length; -const QUOTED = /"((?:[^"\\]|\\.)*)"/; -/** `role "name"` and nothing else: no ref, state, value or context label. */ -const nameOnly = (line: string) => /^ *[A-Za-z][\w-]* "((?:[^"\\]|\\.)*)"$/.exec(line)?.[1]; /** * Replaces one text in the held lines. Only two kinds of line may carry it: the @@ -71,20 +71,22 @@ const nameOnly = (line: string) => /^ *[A-Za-z][\w-]* "((?:[^"\\]|\\.)*)"$/.exec */ function replaceText( lines: string[], + meta: VomLineMetadata[], touched: Map, before: string, after: string, ): TextDeltaRefusal | number[] { - const old = rendered(before); - const next = rendered(after); + const old = encodeVomNameContent(before); + const next = encodeVomNameContent(after); if (old === next) return []; const hits = lines.flatMap((line, index) => (line.includes(old) ? [index] : [])); const leaf = hits[hits.length - 1]; if (leaf === undefined) return "text_outside_observation"; - if (nameOnly(lines[leaf]!) !== old) return "text_ambiguous"; + const leafMeta = meta[leaf]!; + if (!leafMeta.plainNamedNode || leafMeta.escapedName !== old) return "text_ambiguous"; const ancestors = new Set(); - for (let index = leaf - 1, limit = depth(lines[leaf]!); index >= 0 && limit > 0; index--) { - const level = depth(lines[index]!); + for (let index = leaf - 1, limit = leafMeta.indent; index >= 0 && limit > 0; index--) { + const level = meta[index]!.indent; if (level >= limit) continue; ancestors.add(index); limit = level; @@ -92,14 +94,15 @@ function replaceText( for (const hit of hits) { if (hit === leaf) continue; if (!ancestors.has(hit)) return "text_ambiguous"; - const line = lines[hit]!; - const name = nameOnly(line); - if (name === undefined || name.endsWith("…")) return "text_names_other_content"; - const level = depth(line); + const ancestorMeta = meta[hit]!; + const name = ancestorMeta.escapedName; + if (!ancestorMeta.plainNamedNode || name === undefined || name.endsWith("…")) + return "text_names_other_content"; + const level = ancestorMeta.indent; const children: string[] = []; - for (let index = hit + 1; index < lines.length && depth(lines[index]!) > level; index++) { - if (depth(lines[index]!) !== level + 2) continue; - const child = QUOTED.exec(lines[index]!)?.[1]; + for (let index = hit + 1; index < lines.length && meta[index]!.indent > level; index++) { + if (meta[index]!.indent !== level + 2) continue; + const child = meta[index]!.escapedName; if (child === undefined) return "text_names_other_content"; children.push(child); } @@ -107,8 +110,8 @@ function replaceText( children.join(" ") !== name || name.split(old).length !== 2 || // Escaped lengths are never shorter than the names themselves. - name.length >= DERIVED_NAME_CAP || - name.length - old.length + next.length >= DERIVED_NAME_CAP + name.length >= VOM_DERIVED_NAME_CAP || + name.length - old.length + next.length >= VOM_DERIVED_NAME_CAP ) return "text_names_other_content"; } @@ -118,8 +121,9 @@ function replaceText( while (common < old.length && old[common] === next[common]) common++; if (common + 1 < old.length) { const probe = old.slice(0, common + 1); - for (const line of lines) { - const silent = /^ *@context /.test(line) || line.includes(" canvas "); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]!; + const silent = meta[lineIndex]!.silentlyTruncatesLabel; for (let at = line.indexOf(probe); at >= 0; at = line.indexOf(probe, at + 1)) { let length = probe.length; while (length < old.length && line[at + length] === old[length]) length++; @@ -132,6 +136,7 @@ function replaceText( for (const index of hits) { if (!touched.has(index)) touched.set(index, lines[index]!); lines[index] = lines[index]!.replace(old, () => next); + meta[index] = inspectVomLine(lines[index]!); } return hits.map((index) => index + 1); } @@ -175,10 +180,14 @@ export function textDelta( // Activity that no frame can account for keeps requiring a fresh observation. if (!logged.length) return "unlogged_activity"; const lines = held.text.split("\n"); + const meta = + held.lineMeta && held.lineMeta.length === lines.length + ? [...held.lineMeta] + : lines.map((line) => inspectVomLine(line)); const touched = new Map(); const changes: TextDelta["changes"] = []; for (const change of logged) { - const result = replaceText(lines, touched, change.previous_value, change.value); + const result = replaceText(lines, meta, touched, change.previous_value, change.value); if (typeof result === "string") return result; if (!result.length) continue; // Report an element that changed twice once, from its first text to its last. @@ -207,6 +216,7 @@ export function textDelta( return "token_budget_shift"; return { text: lines.join("\n"), + lineMeta: meta, lines: patch, changes: changes.filter((change) => change.previous_value !== change.value), focusMoved, diff --git a/apps/extension/src/debug/__tests__/debug.browser.test.ts b/apps/extension/src/debug/__tests__/debug.browser.test.ts index a31fe06d8..fc1ab4d2e 100644 --- a/apps/extension/src/debug/__tests__/debug.browser.test.ts +++ b/apps/extension/src/debug/__tests__/debug.browser.test.ts @@ -4,7 +4,7 @@ import { writeFile } from "node:fs/promises"; import { createServer } from "node:http"; import { describe, expect, it, vi } from "vitest"; -import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; import { handleClick } from "@/tools/interaction"; import { DebugManager } from "../manager"; @@ -15,7 +15,7 @@ type Send = >( sessionId?: string, ) => Promise; type Event = { sessionId?: string; method: string; params?: Record }; -type Listener = (source: CdpDebuggee, method: string, params: unknown) => void; +type Listener = Parameters[0]; describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser website debugging", () => { it("captures and verifies an HTTP 200 business failure, redirects, iframe requests and cleanup", async () => { diff --git a/apps/extension/src/debug/__tests__/manager.test.ts b/apps/extension/src/debug/__tests__/manager.test.ts index 9943d2da1..b06c31830 100644 --- a/apps/extension/src/debug/__tests__/manager.test.ts +++ b/apps/extension/src/debug/__tests__/manager.test.ts @@ -1,7 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpDebuggee } from "@/browser-driver/chromium-cdp"; +import { rootEventSource, type TabEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleDebug, validateDebugParams } from "@/tools/debug"; +import { createStubCdpRunner } from "@/tools/shared"; import type { DebugArchive } from "../archive"; import { mergeRequest } from "../journal"; import { DEBUG_START_TIMEOUT_MS, DebugManager } from "../manager"; @@ -20,12 +21,12 @@ async function fixture(archive?: DebugArchive) { const context = await sessions.start("s1"); context.agentCreatedTabs.add(7); let now = 10_000; - let listener: ((source: CdpDebuggee, method: string, params: unknown) => void) | undefined; + let listener: ((source: TabEventSource, method: string, params: unknown) => void) | undefined; const dispose = vi.fn(() => { listener = undefined; }); - const sendAttached = vi.fn( - async (_target: CdpDebuggee, method: string): Promise => + const sendToEventSource = vi.fn( + async (_target: TabEventSource, method: string): Promise => method === "Accessibility.getFullAXTree" ? { nodes: [ @@ -37,15 +38,25 @@ async function fixture(archive?: DebugArchive) { ? { body: '{"ok":false}' } : {}, ); - const cdp = { - send: vi.fn(), - sendAttached: sendAttached as never, - ensureNetworkCapture: vi.fn(async () => {}), - onEvent: vi.fn((_tabId: number, fn: NonNullable) => { - listener = fn; - return { dispose }; + const ensureNetworkCapture = vi.fn(async () => {}); + const onEvent = vi.fn((_tabId: number, fn: NonNullable) => { + listener = fn; + return { dispose }; + }); + const cdp = Object.assign( + createStubCdpRunner({ + send: vi.fn(), + sendToEventSource: sendToEventSource as never, }), - }; + { + sendToEventSource: sendToEventSource as never, + discoverChildEventSources: undefined as + | ((tabId: number) => Promise) + | undefined, + ensureNetworkCapture, + onEvent, + }, + ); const tabs = { get: vi.fn( async (id: number) => @@ -60,7 +71,8 @@ async function fixture(archive?: DebugArchive) { query: vi.fn(async () => [{ id: 7, windowId: 100, active: true }] as chrome.tabs.Tab[]), }; const manager = new DebugManager(sessions, cdp, tabs, () => now, archive); - const event = (method: string, data: object, tabId = 7) => listener?.({ tabId }, method, data); + const event = (method: string, data: object, tabId = 7) => + listener?.(rootEventSource(tabId), method, data); const request = (id: string) => event("Network.requestWillBeSent", { requestId: id, @@ -71,7 +83,7 @@ async function fixture(archive?: DebugArchive) { sessions, context, cdp, - sendAttached, + sendToEventSource, tabs, dispose, event, @@ -120,8 +132,8 @@ describe("task-scoped debug lifecycle", () => { it("enables Runtime on the root tab before installing the observer", async () => { const f = await fixture(); active.push(f.manager); - const normal = f.sendAttached.getMockImplementation()!; - f.sendAttached.mockImplementation(async (target, method) => + const normal = f.sendToEventSource.getMockImplementation()!; + f.sendToEventSource.mockImplementation(async (target, method) => method === "Page.getFrameTree" ? { frameTree: { frame: { id: "root" } } } : method === "Page.createIsolatedWorld" @@ -129,8 +141,8 @@ describe("task-scoped debug lifecycle", () => { : normal(target, method), ); await f.manager.start("s1", 7); - const rootCalls = f.sendAttached.mock.calls - .filter(([target]) => target.sessionId === undefined) + const rootCalls = f.sendToEventSource.mock.calls + .filter(([target]) => target.isRootTarget) .map(([, method]) => method); expect(rootCalls).toContain("Runtime.enable"); expect(rootCalls).toContain("Log.enable"); @@ -159,8 +171,8 @@ describe("task-scoped debug lifecycle", () => { it("keeps debug capture usable when browser log capture is unavailable", async () => { const f = await fixture(); active.push(f.manager); - const normal = f.sendAttached.getMockImplementation()!; - f.sendAttached.mockImplementation(async (target, method) => { + const normal = f.sendToEventSource.getMockImplementation()!; + f.sendToEventSource.mockImplementation(async (target, method) => { if (method === "Log.enable") throw new Error("restricted"); return normal(target, method); }); @@ -196,7 +208,7 @@ describe("task-scoped debug lifecycle", () => { expect(f.dispose).toHaveBeenCalledTimes(1); resolve(); await Promise.resolve(); - expect(f.sendAttached).not.toHaveBeenCalled(); + expect(f.sendToEventSource).not.toHaveBeenCalled(); expect((await f.manager.start("s1", 7)).state).toBe("capturing"); }); @@ -207,14 +219,15 @@ describe("task-scoped debug lifecycle", () => { f.cdp.ensureNetworkCapture.mockImplementationOnce( () => new Promise((done) => setTimeout(done, 8000)), ); + const prevDiscover = f.cdp.discoverChildEventSources; const graph = vi.fn(() => new Promise(() => {})); - Object.assign(f.cdp, { getFrameGraph: graph }); + f.cdp.discoverChildEventSources = graph; const starting = expect(f.manager.start("s1", 7)).rejects.toThrow("debug start timeout"); await vi.advanceTimersByTimeAsync(DEBUG_START_TIMEOUT_MS); await starting; expect(graph).toHaveBeenCalledTimes(1); expect((await f.manager.read({ session_id: "s1", action: "status" })).runs).toEqual([]); - Object.assign(f.cdp, { getFrameGraph: undefined }); + f.cdp.discoverChildEventSources = prevDiscover; expect((await f.manager.start("s1", 7)).state).toBe("capturing"); }); @@ -222,7 +235,7 @@ describe("task-scoped debug lifecycle", () => { const f = await fixture(); active.push(f.manager); let resolve!: (value: object) => void; - f.sendAttached.mockImplementation(async (_target, method) => + f.sendToEventSource.mockImplementation(async (_target, method) => method === "Page.getFrameTree" ? new Promise((done) => { resolve = done; @@ -238,10 +251,10 @@ describe("task-scoped debug lifecycle", () => { await starting; resolve({ frameTree: { frame: { id: "root" } } }); await Promise.resolve(); - expect(f.sendAttached.mock.calls.map(([, method]) => method)).not.toContain( + expect(f.sendToEventSource.mock.calls.map(([, method]) => method)).not.toContain( "Runtime.addBinding", ); - expect(f.sendAttached.mock.calls.map(([, method]) => method)).not.toContain( + expect(f.sendToEventSource.mock.calls.map(([, method]) => method)).not.toContain( "Page.addScriptToEvaluateOnNewDocument", ); }); @@ -250,7 +263,7 @@ describe("task-scoped debug lifecycle", () => { const f = await fixture(); active.push(f.manager); let resolve!: (value: object) => void; - f.sendAttached.mockImplementation(async (_target, method) => { + f.sendToEventSource.mockImplementation(async (_target, method) => { if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "root" } } }; if (method === "Page.addScriptToEvaluateOnNewDocument") return new Promise((done) => { @@ -267,13 +280,13 @@ describe("task-scoped debug lifecycle", () => { await starting; resolve({ identifier: "late-script" }); await vi.waitFor(() => - expect(f.sendAttached).toHaveBeenCalledWith( - { tabId: 7 }, + expect(f.sendToEventSource).toHaveBeenCalledWith( + rootEventSource(7), "Page.removeScriptToEvaluateOnNewDocument", { identifier: "late-script" }, ), ); - expect(f.sendAttached.mock.calls.map(([, method]) => method)).not.toContain( + expect(f.sendToEventSource.mock.calls.map(([, method]) => method)).not.toContain( "Page.createIsolatedWorld", ); }); @@ -301,10 +314,10 @@ describe("task-scoped debug lifecycle", () => { vi.useFakeTimers(); const f = await fixture(); active.push(f.manager); - const normal = f.sendAttached.getMockImplementation()!; + const normal = f.sendToEventSource.getMockImplementation()!; const expressions: string[] = []; - f.sendAttached.mockImplementation((async ( - target: CdpDebuggee, + f.sendToEventSource.mockImplementation((async ( + target: TabEventSource, method: string, params?: { expression?: string }, ) => { @@ -383,7 +396,7 @@ describe("task-scoped debug lifecycle", () => { }), ).toBeUndefined(); expect(f.cdp.onEvent).not.toHaveBeenCalled(); - expect(f.cdp.sendAttached).not.toHaveBeenCalled(); + expect(f.cdp.sendToEventSource).not.toHaveBeenCalled(); expect(f.tabs.query).not.toHaveBeenCalled(); }); it("requires task ownership even in local mode and isolates evidence between sessions", async () => { @@ -446,9 +459,9 @@ describe("task-scoped debug lifecycle", () => { expect(result.console).toHaveLength(1); expect(result.operation?.before?.text).toBe("Save failed"); expect(result.operation?.after?.state).toBe("available"); - const calls = f.sendAttached.mock.calls.length; + const calls = f.sendToEventSource.mock.calls.length; await f.manager.read({ action: "operation", session_id: "s1", id: list.operations![0].id }); - expect(f.cdp.sendAttached).toHaveBeenCalledTimes(calls); + expect(f.cdp.sendToEventSource).toHaveBeenCalledTimes(calls); }); it("closes the previous observation window when the next action begins", async () => { vi.useFakeTimers(); @@ -525,7 +538,7 @@ describe("task-scoped debug lifecycle", () => { f.manager.stopTab(7, "requested"); resolve(); expect((await starting).state).toBe("stopped"); - expect(f.cdp.sendAttached).not.toHaveBeenCalled(); + expect(f.cdp.sendToEventSource).not.toHaveBeenCalled(); }); it("cleans up failed starts and ignores evidence from unrelated tabs and replayed console messages", async () => { const f = await fixture(); @@ -964,14 +977,14 @@ describe("persistent debugging records", () => { expect((await f.manager.read({ action: "status", session_id: "s1" })).runs).toEqual([]); const restored = new DebugManager(f.sessions, f.cdp, f.tabs, () => 11000, archive); active.push(restored); - const calls = f.sendAttached.mock.calls.length; + const calls = f.sendToEventSource.mock.calls.length; const result = await restored.readHistory({ action: "export", run_id: run.id, session_id: "" }); expect(result.recording?.run.stop_reason).toBe("session_ended"); expect(result.recording?.requests[0].state).toBe("interrupted"); expect(result.recording?.console[0].text).toBe("Save failed"); expect(result.recording?.pages[0].title).toBe("App"); expect(JSON.stringify(result)).not.toContain("private"); - expect(f.sendAttached).toHaveBeenCalledTimes(calls); + expect(f.sendToEventSource).toHaveBeenCalledTimes(calls); const request = await restored.readHistory({ action: "request", run_id: run.id, diff --git a/apps/extension/src/debug/__tests__/network-control.browser.test.ts b/apps/extension/src/debug/__tests__/network-control.browser.test.ts index cd50f8ffa..8326e6215 100644 --- a/apps/extension/src/debug/__tests__/network-control.browser.test.ts +++ b/apps/extension/src/debug/__tests__/network-control.browser.test.ts @@ -4,7 +4,7 @@ import { writeFile } from "node:fs/promises"; import { createServer } from "node:http"; import { describe, expect, it, vi } from "vitest"; -import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; import { DebugManager } from "../manager"; import type { DebugRuleSpec } from "../types"; @@ -15,7 +15,7 @@ type Send = >( sessionId?: string, ) => Promise; type Event = { sessionId?: string; method: string; params?: Record }; -type Listener = (source: CdpDebuggee, method: string, params: unknown) => void; +type Listener = Parameters[0]; describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser network controls", () => { it("modifies actual traffic, mocks/blocks without server hits, links replay once and cleans up pending rules", async () => { const hits: { url: string; body: string; header?: string }[] = []; diff --git a/apps/extension/src/debug/__tests__/network-control.test.ts b/apps/extension/src/debug/__tests__/network-control.test.ts index a35c76132..9ab14c1bf 100644 --- a/apps/extension/src/debug/__tests__/network-control.test.ts +++ b/apps/extension/src/debug/__tests__/network-control.test.ts @@ -1,19 +1,29 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpDebuggee } from "@/browser-driver/chromium-cdp"; +import { + rootEventSource, + type TabEventSource, + targetEventSource, +} from "@/browser-driver/frame-handles"; import type { DebugCdp } from "../manager"; import { DebugNetworkControl } from "../network-control"; import { DebugNetworkStore } from "../network-store"; import type { DebugRuleSpec } from "../types"; +const ROOT = rootEventSource(7); +const CHILD = targetEventSource(7, "oopif:child"); + function fixture() { let owned = true, sequence = 0; - const sendAttached = vi.fn(async (_target: CdpDebuggee, method: string) => { + const sendToEventSource = vi.fn(async (_target: TabEventSource, method: string) => { if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "root" } } }; if (method === "Page.createIsolatedWorld") return { executionContextId: 1 }; return {}; }); - const cdp = { sendAttached, send: vi.fn(), ensureNetworkCapture: vi.fn() } as unknown as DebugCdp; + const cdp = { + sendToEventSource: sendToEventSource, + ensureNetworkCapture: vi.fn(), + } as unknown as DebugCdp; const network = new DebugNetworkStore("d1", cdp, () => ++sequence); const controls = new DebugNetworkControl( "d1", @@ -25,7 +35,7 @@ function fixture() { }, () => owned, ); - const event = (id: string, body = '{"name":"Alice"}', target = { tabId: 7 }) => { + const event = (id: string, body = '{"name":"Alice"}', target: TabEventSource = ROOT) => { network.onEvent(target, "Network.requestWillBeSent", { requestId: id, type: "Fetch", @@ -51,7 +61,7 @@ function fixture() { return { controls, network, - sendAttached, + sendToEventSource, event, release: () => { owned = false; @@ -66,7 +76,7 @@ afterEach(() => vi.useRealTimers()); describe("local request control lifecycle", () => { it("continues a large request unchanged when only adding a header", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); await f.controls.add({ match: mock.match, effect: { type: "modify", headers: { "x-debug": "1" } }, @@ -74,7 +84,7 @@ describe("local request control lifecycle", () => { f.event("large", "x".repeat(65537)); await vi.waitFor(() => expect(f.controls.list()[0].state).toBe("exhausted")); await vi.waitFor(() => - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.continueRequest", { + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.continueRequest", { requestId: "fetch-large", headers: [ { name: "content-type", value: "application/json" }, @@ -82,43 +92,39 @@ describe("local request control lifecycle", () => { ], }), ); - expect(f.sendAttached.mock.calls.some(([, method]) => method === "Fetch.failRequest")).toBe( - false, - ); + expect( + f.sendToEventSource.mock.calls.some(([, method]) => method === "Fetch.failRequest"), + ).toBe(false); await f.controls.stop(); }); it("has no Fetch subscription until a rule exists and attaches rules to child targets", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); - expect(f.sendAttached).not.toHaveBeenCalled(); + await f.controls.target(ROOT); + expect(f.sendToEventSource).not.toHaveBeenCalled(); await f.controls.add({ ...mock, effect: { type: "block" }, times: 0 }); - await f.controls.target({ tabId: 7, sessionId: "child" }); - expect(f.sendAttached).toHaveBeenCalledWith( - { tabId: 7, sessionId: "child" }, - "Fetch.enable", - expect.anything(), - ); + await f.controls.target(CHILD); + expect(f.sendToEventSource).toHaveBeenCalledWith(CHILD, "Fetch.enable", expect.anything()); await f.controls.stop(); - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.disable"); - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7, sessionId: "child" }, "Fetch.disable"); + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.disable"); + expect(f.sendToEventSource).toHaveBeenCalledWith(CHILD, "Fetch.disable"); }); it("does not widen default Fetch/XHR rules when another rule intercepts Documents", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); await f.controls.add({ ...mock, effect: { type: "mock", status: 200, body: "{}" }, times: 0 }); await f.controls.add({ match: { ...mock.match, resource_type: "Document" }, effect: { type: "block" }, }); - f.controls.onEvent({ tabId: 7 }, "Fetch.requestPaused", { + f.controls.onEvent(ROOT, "Fetch.requestPaused", { requestId: "doc", networkId: "doc", resourceType: "Document", request: { url: mock.match.url, method: "POST", headers: {} }, }); await vi.waitFor(() => - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.failRequest", { + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.failRequest", { requestId: "doc", errorReason: "BlockedByClient", }), @@ -128,17 +134,17 @@ describe("local request control lifecycle", () => { }); it("reserves a one-shot rule synchronously so concurrent requests do not both consume it", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); await f.controls.add({ ...mock, effect: { type: "block" } }); f.event("one"); f.event("two"); await vi.waitFor(() => - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.failRequest", { + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.failRequest", { requestId: "fetch-one", errorReason: "BlockedByClient", }), ); - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.continueRequest", { + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.continueRequest", { requestId: "fetch-two", }); expect(f.controls.list()[0]).toMatchObject({ state: "exhausted", hits: 1 }); @@ -146,17 +152,17 @@ describe("local request control lifecycle", () => { }); it("cancels a delayed mock on disable and does not fall through to the real server", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); await f.controls.add(mock); f.event("one"); await Promise.resolve(); await f.controls.update("d1:r1", "rule_disable"); - expect(f.sendAttached).not.toHaveBeenCalledWith( + expect(f.sendToEventSource).not.toHaveBeenCalledWith( expect.anything(), "Fetch.fulfillRequest", expect.anything(), ); - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.failRequest", { + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.failRequest", { requestId: "fetch-one", errorReason: "Aborted", }); @@ -165,14 +171,14 @@ describe("local request control lifecycle", () => { }); it("aborts on invalid JSON transformations instead of sending a partially edited request", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); await f.controls.add({ match: mock.match, effect: { type: "modify", json: { rename: { missing: "name" } } }, }); f.event("one"); await vi.waitFor(() => expect(f.controls.list()[0].failures).toBe(1)); - expect(f.sendAttached).not.toHaveBeenCalledWith( + expect(f.sendToEventSource).not.toHaveBeenCalledWith( expect.anything(), "Fetch.continueRequest", expect.anything(), @@ -182,26 +188,26 @@ describe("local request control lifecycle", () => { }); it("disables partially enabled interception after a failed configuration", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); - f.sendAttached.mockRejectedValueOnce(new Error("failed")); + await f.controls.target(ROOT); + f.sendToEventSource.mockRejectedValueOnce(new Error("failed")); await expect(f.controls.add(mock)).rejects.toThrow("configuration failed"); expect(f.controls.list()[0].state).toBe("disabled"); - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.disable"); + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.disable"); await f.controls.stop(); }); it("rolls back a cancelled rule setup and rejects writes after task ownership is lost", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); const ac = new AbortController(); let release: () => void = () => {}; - f.sendAttached.mockImplementationOnce( + f.sendToEventSource.mockImplementationOnce( () => new Promise((resolve) => { release = () => resolve({}); }), ); const adding = f.controls.add(mock, ac.signal); - await vi.waitFor(() => expect(f.sendAttached).toHaveBeenCalled()); + await vi.waitFor(() => expect(f.sendToEventSource).toHaveBeenCalled()); ac.abort(); release(); await expect(adding).rejects.toThrow("cancelled"); @@ -212,7 +218,7 @@ describe("local request control lifecycle", () => { }); it("keeps executable secrets out of request annotations and saved rule definitions", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); await f.controls.add({ match: mock.match, effect: { @@ -226,31 +232,33 @@ describe("local request control lifecycle", () => { const saved = JSON.stringify({ rules: f.controls.list(), requests: f.network.list() }); expect(saved).not.toContain("private-value"); expect(saved).not.toContain("hidden-value"); - const command = f.sendAttached.mock.calls.find((call) => call[1] === "Fetch.continueRequest"); + const command = f.sendToEventSource.mock.calls.find( + (call) => call[1] === "Fetch.continueRequest", + ); expect(JSON.stringify(command)).toContain("private-value"); await f.controls.stop(); }); it("rolls back re-enabling a rule if the agent action is cancelled during setup", async () => { const f = fixture(); - await f.controls.target({ tabId: 7 }); + await f.controls.target(ROOT); await f.controls.add(mock); await f.controls.update("d1:r1", "rule_disable"); - f.sendAttached.mockClear(); + f.sendToEventSource.mockClear(); const ac = new AbortController(); let release: () => void = () => {}; - f.sendAttached.mockImplementationOnce( + f.sendToEventSource.mockImplementationOnce( () => new Promise((resolve) => { release = () => resolve({}); }), ); const enabling = f.controls.update("d1:r1", "rule_enable", ac.signal); - await vi.waitFor(() => expect(f.sendAttached).toHaveBeenCalled()); + await vi.waitFor(() => expect(f.sendToEventSource).toHaveBeenCalled()); ac.abort(); release(); await expect(enabling).rejects.toThrow("cancelled"); expect(f.controls.list()[0].state).toBe("disabled"); - expect(f.sendAttached).toHaveBeenCalledWith({ tabId: 7 }, "Fetch.disable"); + expect(f.sendToEventSource).toHaveBeenCalledWith(ROOT, "Fetch.disable"); await f.controls.stop(); }); }); diff --git a/apps/extension/src/debug/__tests__/network-retention.test.ts b/apps/extension/src/debug/__tests__/network-retention.test.ts index 205dc50b2..70067f1da 100644 --- a/apps/extension/src/debug/__tests__/network-retention.test.ts +++ b/apps/extension/src/debug/__tests__/network-retention.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { mergeRequest } from "../journal"; import { DebugNetworkStore, MAX_INFLIGHT, MAX_REQUESTS } from "../network-store"; import type { DebugRequest } from "../types"; @@ -8,13 +9,13 @@ function fixture(send = vi.fn(async () => ({ body: '{"saved":false}' }))) { let sequence = 0; const store = new DebugNetworkStore( "d1", - { send: send as never }, + { sendToEventSource: send as never }, () => ++sequence, Date.now, (entry) => saved.set(entry.id, structuredClone(mergeRequest(saved.get(entry.id), entry))), ); const event = (method: string, data: object) => - store.onEvent({ tabId: 7 }, `Network.${method}`, data); + store.onEvent(rootEventSource(7), `Network.${method}`, data); const start = (id: string) => event("requestWillBeSent", { requestId: id, @@ -52,7 +53,9 @@ describe("in-flight evidence retention", () => { duration_ms: 19000, response_body: { state: "available", text: '{"saved":false}' }, }); - expect(f.send).toHaveBeenCalledWith(7, "Network.getResponseBody", { requestId: "slow" }); + expect(f.send).toHaveBeenCalledWith(rootEventSource(7), "Network.getResponseBody", { + requestId: "slow", + }); }); it("preserves an already running body job when its request leaves the recent cache", async () => { diff --git a/apps/extension/src/debug/__tests__/network-store.test.ts b/apps/extension/src/debug/__tests__/network-store.test.ts index 1467efa6c..7aab4178b 100644 --- a/apps/extension/src/debug/__tests__/network-store.test.ts +++ b/apps/extension/src/debug/__tests__/network-store.test.ts @@ -1,8 +1,11 @@ import { describe, expect, it, vi } from "vitest"; +import { rootEventSource, targetEventSource } from "@/browser-driver/frame-handles"; import { replayRequest } from "../control-model"; import { bodySlice, DebugNetworkStore, MAX_REQUESTS, requestProjection } from "../network-store"; import { BODY_CHARS, redactBody, redactHeaders, redactText, redactUrl } from "../redact"; +const ROOT = rootEventSource(7); + function fixture( send = vi.fn(async () => ({ body: '{"ok":false,"token":"secret","data":{"name":"Alice"}}' })), retain?: ConstructorParameters[4], @@ -11,17 +14,21 @@ function fixture( let now = 1000; const store = new DebugNetworkStore( "d1", - { send: send as never }, + { sendToEventSource: send as never }, () => ++sequence, () => now++, retain, ); - const event = (method: string, data: object, sessionId?: string) => - store.onEvent({ tabId: 7, ...(sessionId ? { sessionId } : {}) }, `Network.${method}`, { - requestId: "raw", - timestamp: now / 1000, - ...data, - }); + const event = (method: string, data: object, childKey?: string) => + store.onEvent( + childKey ? targetEventSource(7, `oopif:${childKey}`) : ROOT, + `Network.${method}`, + { + requestId: "raw", + timestamp: now / 1000, + ...data, + }, + ); const request = (data = {}, sessionId?: string) => event( "requestWillBeSent", @@ -79,7 +86,7 @@ describe("debug network evidence", () => { headers: { "content-type": "text/html" }, }; if (path === "annotation") - f.store.annotate({ tabId: 7 }, "raw", { + f.store.annotate(ROOT, "raw", { effective: { ...request, postData: html }, mock: { status: 200, headers: request.headers, body: html }, }); @@ -112,7 +119,7 @@ describe("debug network evidence", () => { const headers = { "content-security-policy": "x".repeat(3000) }; if (path === "extra-first") f.event("responseReceivedExtraInfo", { headers }); if (path === "mock") - f.store.annotate({ tabId: 7 }, "raw", { mock: { status: 200, headers, body: "{}" } }); + f.store.annotate(ROOT, "raw", { mock: { status: 200, headers, body: "{}" } }); f.request({ request: { url: "https://site.test/save", method: "GET", headers: { mode: "dry-run" } }, }); @@ -151,7 +158,7 @@ describe("debug network evidence", () => { ).toBe(4095); if (path === "extra-first") f.event("requestWillBeSentExtraInfo", { headers }); if (path === "annotation") - f.store.annotate({ tabId: 7 }, "raw", { + f.store.annotate(ROOT, "raw", { effective: { url: "https://site.test/save", method: "GET", headers }, }); f.request({ @@ -180,7 +187,7 @@ describe("debug network evidence", () => { request_body: { state: "empty", replay_safe: true }, }); const second = fixture(); - second.store.annotate({ tabId: 7 }, "raw", { effective: { url, method: "GET", headers: {} } }); + second.store.annotate(ROOT, "raw", { effective: { url, method: "GET", headers: {} } }); second.request(); expect(second.store.list()[0]).toMatchObject({ truncated: true, @@ -220,7 +227,7 @@ describe("debug network evidence", () => { values: "a".repeat(BODY_CHARS - base.length - 1), }); expect(body.length).toBeLessThan(BODY_CHARS); - f.store.annotate({ tabId: 7 }, "raw", { + f.store.annotate(ROOT, "raw", { intervention: { rule_id: "r1", type: "mock", state: "applied" }, effective: { url: "https://site.test/api", @@ -272,7 +279,7 @@ describe("debug network evidence", () => { '"Alice"', ); expect(requestProjection(entry, "response", 0, 5).response_body.next_offset).toBe(5); - expect(f.send).toHaveBeenCalledWith(7, "Network.getResponseBody", { requestId: "raw" }); + expect(f.send).toHaveBeenCalledWith(ROOT, "Network.getResponseBody", { requestId: "raw" }); }); it("correlates out-of-order ExtraInfo with redirect hops without mixing headers", () => { @@ -438,7 +445,7 @@ describe("redaction and projections", () => { const f = fixture(); f.request(); f.request({}, "child"); - f.store.detachTarget("child"); + f.store.detachTarget("oopif:child"); expect(f.store.list().map((entry) => entry.state)).toEqual(["pending", "interrupted"]); }); diff --git a/apps/extension/src/debug/__tests__/observer.browser.test.ts b/apps/extension/src/debug/__tests__/observer.browser.test.ts index 9952a9077..75819ef44 100644 --- a/apps/extension/src/debug/__tests__/observer.browser.test.ts +++ b/apps/extension/src/debug/__tests__/observer.browser.test.ts @@ -2,7 +2,9 @@ // Owns an isolated browser and fixture; never freezes a user's tab. import { createServer } from "node:http"; import { expect, it } from "vitest"; +import { rootEventSource, type TabEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; +import { createStubCdpRunner } from "@/tools/shared"; import { type DebugCdp, DebugManager } from "../manager"; it.skipIf(!process.env.BSK_TEST_CHROME)( @@ -30,16 +32,19 @@ it.skipIf(!process.env.BSK_TEST_CHROME)( zoom: 1, startupTimeout: 30000, onEvent: (event: { method: string; params?: object }) => - listener?.({ tabId: 7 }, event.method, event.params), + listener?.(rootEventSource(7), event.method, event.params), }, - async (send: (method: string, params?: object, sessionId?: string) => Promise) => { + async (send: (method: string, params?: object, targetSession?: string) => Promise) => { const { targetId } = await send("Target.createTarget", { url: "about:blank" }); - const { sessionId } = await send("Target.attachToTarget", { targetId, flatten: true }); + const { sessionId: targetSession } = await send("Target.attachToTarget", { + targetId, + flatten: true, + }); const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; - await send("Page.enable", {}, sessionId); - await send("Runtime.enable", {}, sessionId); - await send("Page.navigate", { url }, sessionId); - await send("Runtime.evaluate", { expression: "document.readyState" }, sessionId); + await send("Page.enable", {}, targetSession); + await send("Runtime.enable", {}, targetSession); + await send("Page.navigate", { url }, targetSession); + await send("Runtime.evaluate", { expression: "document.readyState" }, targetSession); const sessions = new SessionManager({ agentWindow: { create: async () => ({ windowId: 100, initialTabIds: [7] }), @@ -49,14 +54,20 @@ it.skipIf(!process.env.BSK_TEST_CHROME)( }); (await sessions.start("s1")).agentCreatedTabs.add(7); const hooks: object[] = []; - const cdp = { - sendAttached: (_target: unknown, method: string, params?: { expression?: string }) => { + const cdp: DebugCdp = createStubCdpRunner({ + send: ((_tabId: number, method: string, params?: object) => + send(method, params, targetSession)) as DebugCdp["send"], + sendToEventSource: (( + _target: TabEventSource, + method: string, + params?: { expression?: string }, + ) => { if (method === "Runtime.evaluate" && params?.expression?.includes(".agent(true,")) hooks.push(params); - return send(method, params, sessionId); - }, + return send(method, params, targetSession); + }) as DebugCdp["sendToEventSource"], ensureNetworkCapture: async () => { - await send("Network.enable", {}, sessionId); + await send("Network.enable", {}, targetSession); }, onEvent: (_tabId: number, fn: NonNullable) => { listener = fn; @@ -66,7 +77,7 @@ it.skipIf(!process.env.BSK_TEST_CHROME)( }, }; }, - } as unknown as DebugCdp; + }); const tabs = { get: async () => ({ id: 7, windowId: 100, url }) as chrome.tabs.Tab, query: async () => [{ id: 7 } as chrome.tabs.Tab], @@ -77,7 +88,7 @@ it.skipIf(!process.env.BSK_TEST_CHROME)( await send( "Runtime.evaluate", { expression: "setTimeout(()=>{while(true){}},50);true" }, - sessionId, + targetSession, ); await new Promise((resolve) => setTimeout(resolve, 150)); const before = Date.now(); @@ -91,25 +102,25 @@ it.skipIf(!process.env.BSK_TEST_CHROME)( manager.after(ticket); expect((await send("Browser.getVersion")).product).toContain("Chrome"); } finally { - await send("Runtime.terminateExecution", {}, sessionId); + await send("Runtime.terminateExecution", {}, targetSession); } try { // Re-deliver the obsolete enable after the disable and its admission deadline. // It must not suppress subsequent genuine user input. - await send("Runtime.evaluate", { expression: "true" }, sessionId); + await send("Runtime.evaluate", { expression: "true" }, targetSession); expect(hooks).toHaveLength(1); - await send("Runtime.evaluate", hooks[0], sessionId); + await send("Runtime.evaluate", hooks[0], targetSession); await send( "Input.dispatchMouseEvent", { type: "mousePressed", x: 40, y: 30, button: "left", clickCount: 1 }, - sessionId, + targetSession, ); await send( "Input.dispatchMouseEvent", { type: "mouseReleased", x: 40, y: 30, button: "left", clickCount: 1 }, - sessionId, + targetSession, ); - await send("Runtime.evaluate", { expression: "true" }, sessionId); + await send("Runtime.evaluate", { expression: "true" }, targetSession); await new Promise((resolve) => setTimeout(resolve, 50)); const result = await manager.read({ session_id: "s1", action: "operations" }); expect( diff --git a/apps/extension/src/debug/manager.ts b/apps/extension/src/debug/manager.ts index 875e896b0..c79c77425 100644 --- a/apps/extension/src/debug/manager.ts +++ b/apps/extension/src/debug/manager.ts @@ -1,9 +1,9 @@ import { - type CdpDebuggee, parseConsoleApiCalled, parseExceptionThrown, parseLogEntry, } from "@/browser-driver/chromium-cdp"; +import { rootEventSource, type TabEventSource } from "@/browser-driver/frame-handles"; import { isAgentControlledTab, type SessionContext, @@ -48,12 +48,12 @@ const ACTIONS = new Set( export interface DebugCdp extends CdpRunner { ensureNetworkCapture(tabId: number): Promise; - sendAttached( - target: CdpDebuggee & { tabId: number }, + sendToEventSource( + source: TabEventSource, method: string, params?: object, ): Promise; - getFrameGraph?: CdpRunner["getFrameGraph"]; + discoverChildEventSources?(tabId: number): Promise; } interface RunState { run: DebugRun; @@ -243,19 +243,10 @@ export class DebugManager { const network = new DebugNetworkStore( id, { - send: async (targetTab: number, method: string, params?: object) => { - if (!this.owned(sessionId, targetTab) || state.run.state !== "capturing") + sendToEventSource: async (source: TabEventSource, method: string, params?: object) => { + if (!this.owned(sessionId, source.tabId) || state.run.state !== "capturing") throw new Error("capture stopped"); - return deadline(this.cdp.sendAttached({ tabId: targetTab }, method, params), 2500); - }, - sendToTarget: async ( - target: CdpDebuggee & { tabId: number }, - method: string, - params?: object, - ) => { - if (!this.owned(sessionId, target.tabId) || state.run.state !== "capturing") - throw new Error("capture stopped"); - return deadline(this.cdp.sendAttached(target, method, params), 2500); + return deadline(this.cdp.sendToEventSource(source, method, params), 2500); }, }, () => this.change(state), @@ -301,7 +292,7 @@ export class DebugManager { ownedRuns.set(id, tabId); while (ownedRuns.size > HISTORY_LIMIT + MAX_RUNS) ownedRuns.delete(ownedRuns.keys().next().value!); - state.subscription = this.cdp.onEvent?.(tabId, (source, method, params) => + state.subscription = this.cdp.onEvent(tabId, (source, method, params) => this.onEvent(source, method, params), ); try { @@ -312,27 +303,28 @@ export class DebugManager { throw new Error("debug tab must remain in its Agent Window"); if (!this.runs.has(id) || state.run.state !== "capturing") throw new Error("capture stopped during debug start"); - this.cdp.trackSessionTab?.(sessionId, tabId); + this.cdp.trackSessionTab(sessionId, tabId); await wait(this.cdp.ensureNetworkCapture(tabId)); if (!this.owned(sessionId, tabId) || !this.runs.has(id)) throw new Error("task ended during debug start"); if (state.run.state !== "capturing") return this.summary(state); - await wait(this.enableTarget(state, { tabId }, startup.signal)); + await wait(this.enableTarget(state, rootEventSource(tabId), startup.signal)); if (state.run.state !== "capturing") return this.summary(state); - const graph = await wait( + const childSources = await wait( Promise.resolve( - this.cdp.getFrameGraph?.(tabId).catch(() => { + this.cdp.discoverChildEventSources?.(tabId).catch(() => { this.coverage(state, "child_capture_partial"); return undefined; }), ), ); - if (graph) { - // Frame graph discovery already belongs to the driver. Debug adds only + if (childSources) { + // Child target discovery already belongs to the driver. Debug adds only // bounded Network/Runtime listeners on its existing child attachments. - const targets = new Map(); - for (const frame of graph.frames) - if (frame.target.sessionId) targets.set(frame.target.sessionId, frame.target); + const targets = new Map(); + for (const childSource of childSources) { + if (!childSource.isRootTarget) targets.set(childSource.targetKey, childSource); + } await wait( Promise.all( [...targets.values()] @@ -379,10 +371,10 @@ export class DebugManager { private async enableTarget( state: RunState, - target: CdpDebuggee & { tabId: number }, + target: TabEventSource, signal?: AbortSignal, ): Promise { - const key = target.sessionId ?? "root"; + const key = target.targetKey; if ( state.targets.has(key) || state.run.state !== "capturing" || @@ -396,7 +388,7 @@ export class DebugManager { state.targets.add(key); try { await deadline( - this.cdp.sendAttached(target, "Network.enable", { + this.cdp.sendToEventSource(target, "Network.enable", { maxTotalBufferSize: 2 * 1024 * 1024, maxResourceBufferSize: 256 * 1024, maxPostDataSize: 64 * 1024, @@ -408,17 +400,17 @@ export class DebugManager { // the observer tracks its world through context events and bindings // only reach new documents while Runtime is on. if (state.run.state === "capturing" && this.owned(state.run.session_id, target.tabId)) - await deadline(this.cdp.sendAttached(target, "Runtime.enable"), 2500, signal); + await deadline(this.cdp.sendToEventSource(target, "Runtime.enable"), 2500, signal); // Root attachment no longer enables Log implicitly. Preserve browser // diagnostics in debug recordings without making Log a startup requirement. if ( - !target.sessionId && + target.isRootTarget && state.run.state === "capturing" && this.owned(state.run.session_id, target.tabId) && !signal?.aborted ) { try { - await deadline(this.cdp.sendAttached(target, "Log.enable"), 2500, signal); + await deadline(this.cdp.sendToEventSource(target, "Log.enable"), 2500, signal); } catch (error) { if (signal?.aborted) throw error; this.coverage(state, "browser_log_capture_unavailable"); @@ -439,20 +431,19 @@ export class DebugManager { } } - private onEvent(source: CdpDebuggee, method: string, params: unknown): void { + private onEvent(source: TabEventSource, method: string, params: unknown): void { if (typeof source.tabId !== "number") return; for (const state of this.runs.values()) { if (state.run.tab_id !== source.tabId) continue; if (state.run.state !== "capturing") { - if (state.controlCleanup) - state.controls?.onEvent({ ...source, tabId: source.tabId }, method, params); + if (state.controlCleanup) state.controls?.onEvent(source, method, params); continue; } if (!this.owned(state.run.session_id, source.tabId)) { this.stopState(state, "tab_released"); continue; } - if (!source.sessionId) { + if (source.isRootTarget) { state.observer?.contextEvent(method, params); if ( method === "Page.frameStartedLoading" && @@ -471,24 +462,22 @@ export class DebugManager { continue; } } - if (method === "Target.attachedToTarget") { - const child = params as { sessionId?: string; targetInfo?: { type?: string } }; - if (child.sessionId && child.targetInfo?.type === "iframe") - void this.enableTarget(state, { tabId: source.tabId, sessionId: child.sessionId }).catch( - () => this.coverage(state, "child_capture_partial"), - ); + if (method === "BskDriver.childTargetAttached" && !source.isRootTarget) { + void this.enableTarget(state, source).catch(() => + this.coverage(state, "child_capture_partial"), + ); + continue; } - if (method === "Target.detachedFromTarget") { - const child = params as { sessionId?: string }; - if (child.sessionId) { - state.targets.delete(child.sessionId); - state.network.detachTarget(child.sessionId); - state.controls?.detach(child.sessionId); - } + if (method === "BskDriver.childTargetDetached" && !source.isRootTarget) { + state.targets.delete(source.targetKey); + state.network.detachTarget(source.targetKey); + state.controls?.detach(source.targetKey); + continue; } - if (!source.sessionId && method === "Page.loadEventFired") void this.capturePage(state, true); - state.network.onEvent({ ...source, tabId: source.tabId }, method, params); - state.controls?.onEvent({ ...source, tabId: source.tabId }, method, params); + if (source.isRootTarget && method === "Page.loadEventFired") + void this.capturePage(state, true); + state.network.onEvent(source, method, params); + state.controls?.onEvent(source, method, params); if (method === "Network.loadingFinished") this.scheduleObservation(state); const parsed = method === "Runtime.consoleAPICalled" @@ -835,9 +824,9 @@ export class DebugManager { try { const [tree, tab, fieldValues] = await deadline( Promise.all([ - this.cdp.sendAttached<{ + this.cdp.sendToEventSource<{ nodes: { ignored?: boolean; role?: { value?: string }; name?: { value?: string } }[]; - }>({ tabId: state.run.tab_id }, "Accessibility.getFullAXTree"), + }>(rootEventSource(state.run.tab_id), "Accessibility.getFullAXTree"), this.tabs.get(state.run.tab_id), state.observer?.call("snapshot").catch(() => undefined), ]), diff --git a/apps/extension/src/debug/network-control.ts b/apps/extension/src/debug/network-control.ts index ceaaf0a4b..0cbed07c7 100644 --- a/apps/extension/src/debug/network-control.ts +++ b/apps/extension/src/debug/network-control.ts @@ -1,4 +1,4 @@ -import type { CdpDebuggee } from "@/browser-driver/chromium-cdp"; +import { rootEventSource, type TabEventSource } from "@/browser-driver/frame-handles"; import { editRequest, type LiveRequest, @@ -22,7 +22,7 @@ import type { DebugRuleSpec, } from "./types"; -type Target = CdpDebuggee & { tabId: number }; +type Target = TabEventSource; interface Rule { spec?: DebugRuleSpec; public: DebugRule; @@ -153,14 +153,14 @@ export class DebugNetworkControl { } async target(target: Target): Promise { if (!this.alive) return; - this.targets.set(target.sessionId ?? "root", target); + this.targets.set(target.targetKey, target); if (this.rules.size) await this.refresh(); } - detach(sessionId: string): void { - this.targets.delete(sessionId); - this.configured.delete(sessionId); + detach(targetKey: string): void { + this.targets.delete(targetKey); + this.configured.delete(targetKey); for (const item of this.pending.values()) - if (item.target.sessionId === sessionId) item.abort.abort(); + if (item.target.targetKey === targetKey) item.abort.abort(); } async add(input: DebugRuleSpec, signal?: AbortSignal): Promise { this.assertActive(); @@ -274,7 +274,7 @@ export class DebugNetworkControl { if (patterns.length && this.alive && this.allowed()) { this.configured.add(key); await bounded( - this.cdp.sendAttached(target, "Fetch.enable", { + this.cdp.sendToEventSource(target, "Fetch.enable", { patterns, handleAuthRequests: false, }), @@ -283,10 +283,10 @@ export class DebugNetworkControl { // Drain already-paused requests before disabling the domain. await Promise.allSettled( [...this.pending.values()] - .filter((item) => (item.target.sessionId ?? "root") === key) + .filter((item) => item.target.targetKey === key) .map((item) => item.done), ); - await bounded(this.cdp.sendAttached(target, "Fetch.disable")); + await bounded(this.cdp.sendToEventSource(target, "Fetch.disable")); this.configured.delete(key); } } catch (error) { @@ -301,7 +301,7 @@ export class DebugNetworkControl { return work; } onEvent(source: Target, method: string, raw: unknown): void { - if (source.tabId !== this.tabId || !this.targets.has(source.sessionId ?? "root")) return; + if (source.tabId !== this.tabId || !this.targets.has(source.targetKey)) return; if (method === "Network.requestWillBeSent") { const event = raw as { requestId?: string; @@ -327,11 +327,11 @@ export class DebugNetworkControl { if (method !== "Fetch.requestPaused") return; const event = raw as Paused; if (!event?.requestId || !event.request) return; - const key = `${source.sessionId ?? "root"}:${event.requestId}`; + const key = `${source.targetKey}:${event.requestId}`; if (this.pending.has(key)) return; if (!this.alive || !this.allowed()) { void bounded( - this.cdp.sendAttached(source, "Fetch.failRequest", { + this.cdp.sendToEventSource(source, "Fetch.failRequest", { requestId: event.requestId, errorReason: "Aborted", }), @@ -350,10 +350,10 @@ export class DebugNetworkControl { ); if (!rule?.spec) { void bounded( - this.cdp.sendAttached(source, "Fetch.continueRequest", { requestId: event.requestId }), + this.cdp.sendToEventSource(source, "Fetch.continueRequest", { requestId: event.requestId }), ).catch(() => bounded( - this.cdp.sendAttached(source, "Fetch.failRequest", { + this.cdp.sendToEventSource(source, "Fetch.failRequest", { requestId: event.requestId, errorReason: "Aborted", }), @@ -402,7 +402,7 @@ export class DebugNetworkControl { if (effect.type === "block") { annotate(); await bounded( - this.cdp.sendAttached(target, "Fetch.failRequest", { + this.cdp.sendToEventSource(target, "Fetch.failRequest", { requestId: event.requestId, errorReason: "BlockedByClient", }), @@ -422,7 +422,7 @@ export class DebugNetworkControl { .map((value) => redactText(value, 160)); annotate({ effective }); await bounded( - this.cdp.sendAttached(target, "Fetch.continueRequest", { + this.cdp.sendToEventSource(target, "Fetch.continueRequest", { requestId: event.requestId, ...(effect.url === undefined ? {} : { url: effective.url }), ...(effect.method === undefined ? {} : { method: effective.method }), @@ -449,7 +449,7 @@ export class DebugNetworkControl { const responseBody = event.request.method === "HEAD" ? "" : effect.body; annotate({ mock: { status: effect.status, headers, body: responseBody } }); await bounded( - this.cdp.sendAttached(target, "Fetch.fulfillRequest", { + this.cdp.sendToEventSource(target, "Fetch.fulfillRequest", { requestId: event.requestId, responseCode: effect.status, responseHeaders: Object.entries(headers).map(([name, value]) => ({ name, value })), @@ -468,7 +468,7 @@ export class DebugNetworkControl { rule.public.last_error = mark.error; annotate(); await bounded( - this.cdp.sendAttached(target, "Fetch.failRequest", { + this.cdp.sendToEventSource(target, "Fetch.failRequest", { requestId: event.requestId, errorReason: "Aborted", }), @@ -513,7 +513,7 @@ export class DebugNetworkControl { result.error = "replay cancelled; the request may have been sent"; if (record.context !== undefined) void bounded( - this.cdp.sendAttached({ tabId: this.tabId }, "Runtime.evaluate", { + this.cdp.sendToEventSource(rootEventSource(this.tabId), "Runtime.evaluate", { contextId: record.context, expression: `globalThis[${JSON.stringify(result.id)}]?.abort()`, }), @@ -528,15 +528,15 @@ export class DebugNetworkControl { record.promise = (async () => { try { const { frameTree } = await bounded( - this.cdp.sendAttached<{ frameTree: { frame: { id: string } } }>( - { tabId: this.tabId }, + this.cdp.sendToEventSource<{ frameTree: { frame: { id: string } } }>( + rootEventSource(this.tabId), "Page.getFrameTree", ), ); check(); const { executionContextId } = await bounded( - this.cdp.sendAttached<{ executionContextId: number }>( - { tabId: this.tabId }, + this.cdp.sendToEventSource<{ executionContextId: number }>( + rootEventSource(this.tabId), "Page.createIsolatedWorld", { frameId: frameTree.frame.id, worldName: `bsk-network-${this.runId}` }, ), @@ -555,8 +555,8 @@ export class DebugNetworkControl { // origin check executes in that same context, closing a navigation race. const expression = `(${replayInPage.toString()})(${JSON.stringify({ origin: new URL(pageUrl).origin, id: result.id, url: request.url, options })})\n//# sourceURL=bsk-replay://${result.id}`; const response = await bounded( - this.cdp.sendAttached<{ exceptionDetails?: unknown }>( - { tabId: this.tabId }, + this.cdp.sendToEventSource<{ exceptionDetails?: unknown }>( + rootEventSource(this.tabId), "Runtime.evaluate", { expression, contextId: executionContextId, awaitPromise: true, returnByValue: true }, ), @@ -596,7 +596,7 @@ export class DebugNetworkControl { if (context !== undefined) aborts.push( bounded( - this.cdp.sendAttached({ tabId: this.tabId }, "Runtime.evaluate", { + this.cdp.sendToEventSource(rootEventSource(this.tabId), "Runtime.evaluate", { contextId: context, expression: `globalThis[${JSON.stringify(result.id)}]?.abort()`, }), @@ -611,7 +611,7 @@ export class DebugNetworkControl { } catch (error) { // A broken Fetch domain must not outlive the capture and freeze the tab. // Detach only as a cleanup fallback; normal stop preserves shared domains. - if (this.cdp.detach) await bounded(this.cdp.detach(this.tabId)).catch(() => {}); + await bounded(this.cdp.detach(this.tabId)).catch(() => {}); throw error; } })(); diff --git a/apps/extension/src/debug/network-store.ts b/apps/extension/src/debug/network-store.ts index f606cd264..41b33754e 100644 --- a/apps/extension/src/debug/network-store.ts +++ b/apps/extension/src/debug/network-store.ts @@ -1,6 +1,4 @@ -import type { CdpDebuggee } from "@/browser-driver/chromium-cdp"; -import type { CdpRunner } from "@/tools/shared"; -import { sendToCdpTarget } from "@/tools/shared"; +import type { TabEventSource } from "@/browser-driver/frame-handles"; import { requestMetadata } from "./journal"; import { jsonPointer } from "./json-source"; import { @@ -54,10 +52,18 @@ interface Event { stack?: { callFrames?: { url?: string; lineNumber?: number; functionName?: string }[] }; }; } +export interface DebugNetworkCdp { + sendToEventSource( + source: TabEventSource, + method: string, + params?: object, + ): Promise; +} + interface RequestRecord { entry: DebugRequest; rawId: string; - target: CdpDebuggee & { tabId: number }; + target: TabEventSource; timestamp?: number; expectsExtra?: boolean; } @@ -125,14 +131,14 @@ export class DebugNetworkStore { constructor( private readonly runId: string, - private readonly cdp: CdpRunner, + private readonly cdp: DebugNetworkCdp, private readonly changed: () => number, private readonly now: () => number = Date.now, private readonly retain?: (entry: DebugRequest) => void, ) {} - private key(source: CdpDebuggee, requestId: string): string { - return `${source.sessionId ?? "root"}:${requestId}`; + private key(source: TabEventSource, requestId: string): string { + return `${source.targetKey}:${requestId}`; } private tracked(id: string): boolean { @@ -149,7 +155,7 @@ export class DebugNetworkStore { return this.entries.size + this.inflight.size; } - onEvent(source: CdpDebuggee & { tabId: number }, method: string, raw: unknown): void { + onEvent(source: TabEventSource, method: string, raw: unknown): void { if (!this.accepting || !method.startsWith("Network.")) return; const event = raw as Event; if (!event?.requestId) return; @@ -401,12 +407,12 @@ export class DebugNetworkStore { this.jobs += 1; // Never call send(), which can reattach a returned tab. The production // runner supplies a direct command guarded by current task ownership. - void sendToCdpTarget<{ body: string; base64Encoded?: boolean }>( - this.cdp, - record.target, - "Network.getResponseBody", - { requestId: record.rawId }, - ) + void this.cdp + .sendToEventSource<{ body: string; base64Encoded?: boolean }>( + record.target, + "Network.getResponseBody", + { requestId: record.rawId }, + ) .then((result) => { if (!this.alive || !this.tracked(record.entry.id)) return; let body = result.body; @@ -527,7 +533,7 @@ export class DebugNetworkStore { } /** Called on the CDP event path; annotations can precede requestWillBeSent. */ - annotate(source: CdpDebuggee, rawId: string, annotation: ControlAnnotation): void { + annotate(source: TabEventSource, rawId: string, annotation: ControlAnnotation): void { const key = this.key(source, rawId); let requestBody: ReturnType | undefined; let responseBody: ReturnType | undefined; @@ -630,10 +636,10 @@ export class DebugNetworkStore { return (this.entries.get(id) ?? this.inflight.get(id))?.entry; } - detachTarget(sessionId: string): void { + detachTarget(targetKey: string): void { for (const record of this.records()) { const { entry, target } = record; - if (target.sessionId !== sessionId || entry.state !== "pending") continue; + if (target.targetKey !== targetKey || entry.state !== "pending") continue; entry.state = "interrupted"; entry.error = "frame_detached"; entry.finished_at = this.now(); diff --git a/apps/extension/src/debug/observer.ts b/apps/extension/src/debug/observer.ts index b4bc3249f..dcba79506 100644 --- a/apps/extension/src/debug/observer.ts +++ b/apps/extension/src/debug/observer.ts @@ -1,3 +1,4 @@ +import { rootEventSource } from "@/browser-driver/frame-handles"; import type { DebugCdp } from "./manager"; import { installPerformance } from "./performance-observer"; import { redactText } from "./redact"; @@ -342,7 +343,7 @@ export class DebugObserver { return !!frameId && frameId === this.rootFrame; } private send(method: string, params?: object): Promise { - return this.cdp.sendAttached({ tabId: this.tabId }, method, params); + return this.cdp.sendToEventSource(rootEventSource(this.tabId), method, params); } async start(): Promise { const tree = await this.send<{ frameTree: { frame: { id: string } } }>("Page.getFrameTree"); diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index 693ca0df7..69dd20477 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -58,6 +58,10 @@ import { } from "@/tools/borrow-confirmation"; import { ToolDispatcher } from "@/tools/dispatcher"; import { clearHelpForSession, hasActiveHelp } from "@/tools/human-loop"; +import { + clearObserveIdentityStreaksForSession, + clearObserveIdentityStreaksForTab, +} from "@/tools/observation"; import { activeRecordingCount, attachRecordFinishListener, @@ -132,8 +136,9 @@ export default defineBackground(() => { cooperation.release(ctx.sessionId); debug.releaseSession(ctx.sessionId); void video.stopSession(ctx.sessionId).catch(() => {}); - clearRecordingForSession(ctx.sessionId); + clearRecordingForSession(ctx.sessionId, recordDeps); clearHelpForSession(ctx.sessionId); + clearObserveIdentityStreaksForSession(ctx.sessionId); }, onSessionsChanged: onOverlaySessionStateChanged, }); @@ -451,22 +456,13 @@ export default defineBackground(() => { debug.stopTab(tabId, "tab_closed"); sessions.forgetClosedTab(tabId, { isWindowClosing: removeInfo.isWindowClosing }); cdp.forgetClosedTab?.(tabId); + dispatcher.forgetClosedTab(tabId); + clearObserveIdentityStreaksForTab(tabId); debug.sync(); }); // Reconcile dynamic Tier B content-script registrations on SW startup so // idle browser startups always begin with zero registered content scripts. void runtimeScripts.sync(); - const cleanupAfterDisconnect = createDisconnectCleanup({ - manager: sessions, - // Same contract as the dispatcher's `tool.session_stop`: without these - // deps the agent-tab cleanup and the window-release path are dead code. - sessionStopDeps: { - cdp, - tabManagement: { tabs: chromeTabMutationApi }, - tabsQuery: chromeTabsApi, - }, - onSessionsChanged: onOverlaySessionStateChanged, - }); const recordDeps = { tabsApi: chrome.tabs, cdp, @@ -487,6 +483,18 @@ export default defineBackground(() => { } }, } satisfies RecordRuntimeDeps; + const cleanupAfterDisconnect = createDisconnectCleanup({ + manager: sessions, + // Same contract as the dispatcher's `tool.session_stop`: without these + // deps the agent-tab cleanup and the window-release path are dead code. + sessionStopDeps: { + cdp, + recording: recordDeps, + tabManagement: { tabs: chromeTabMutationApi }, + tabsQuery: chromeTabsApi, + }, + onSessionsChanged: onOverlaySessionStateChanged, + }); recordFrameCoordinator.attach(); attachRecordStepListener(recordDeps); attachRecordFinishListener(recordDeps); diff --git a/apps/extension/src/lib/__tests__/document-settle.test.ts b/apps/extension/src/lib/__tests__/document-settle.test.ts index 84a02af1b..fea1435d0 100644 --- a/apps/extension/src/lib/__tests__/document-settle.test.ts +++ b/apps/extension/src/lib/__tests__/document-settle.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { waitForDocumentSettled } from "../recording/document-settle"; function quietCdp(): { cdp: CdpRunner; send: ReturnType } { @@ -7,7 +7,7 @@ function quietCdp(): { cdp: CdpRunner; send: ReturnType } { result: { value: { idleMs: 1_000, readyState: "complete" } }, })); return { - cdp: { send: send as unknown as CdpRunner["send"] }, + cdp: createStubCdpRunner({ send: send as unknown as CdpRunner["send"] }), send, }; } @@ -53,7 +53,7 @@ describe("waitForDocumentSettled", () => { if (method === "Page.createIsolatedWorld") return { executionContextId: 91 }; return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; }); - const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] }; + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as CdpRunner["send"] }); const settled = waitForDocumentSettled(cdp, { tabId: 7, @@ -81,11 +81,11 @@ describe("waitForDocumentSettled", () => { if (method === "Page.createIsolatedWorld") return { executionContextId: 27 }; return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as CdpRunner["send"], - sendToFrame: sendToFrame as unknown as NonNullable, + sendToFrame: sendToFrame as unknown as CdpRunner["sendToFrame"], hasOwnFrameTarget: (_tabId, frameId) => frameId === "oopif-frame", - }; + }); const handle = { tabId: 7, frameId: "oopif-frame", loaderId: "oopif-loader" }; const settled = waitForDocumentSettled(cdp, { tabId: 7, handle }); @@ -97,7 +97,6 @@ describe("waitForDocumentSettled", () => { handle, "Runtime.evaluate", expect.objectContaining({ contextId: 27 }), - undefined, ); }); }); diff --git a/apps/extension/src/lib/__tests__/recording-runtime.test.ts b/apps/extension/src/lib/__tests__/recording-runtime.test.ts index 2c3d43d94..168c156e6 100644 --- a/apps/extension/src/lib/__tests__/recording-runtime.test.ts +++ b/apps/extension/src/lib/__tests__/recording-runtime.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { type CdpRunner, type ChromeTabsApi, createStubCdpRunner } from "@/tools/shared"; const captureRecordingObservation = vi.hoisted(() => vi.fn()); const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const)); @@ -31,7 +31,7 @@ function observation(url = "https://example.com/") { function runtime(): RecordingObservationRuntime { return new RecordingObservationRuntime({ - cdp: { send: vi.fn() as unknown as CdpRunner["send"] }, + cdp: createStubCdpRunner({ send: vi.fn() as unknown as CdpRunner["send"] }), tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi, }); } diff --git a/apps/extension/src/lib/__tests__/settle-controller.test.ts b/apps/extension/src/lib/__tests__/settle-controller.test.ts index 4b209c06e..6263a6982 100644 --- a/apps/extension/src/lib/__tests__/settle-controller.test.ts +++ b/apps/extension/src/lib/__tests__/settle-controller.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { type CdpRunner, type ChromeTabsApi, createStubCdpRunner } from "@/tools/shared"; import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture"; import { RecordingObservationSession } from "../recording/observation-session"; import { SettleController } from "../recording/settle-controller"; @@ -17,11 +17,11 @@ describe("SettleController", () => { it("aborts superseded capture work and lands the prior action on the next origin", async () => { vi.useFakeTimers(); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({ result: { value: { idleMs: 1_000, readyState: "complete" } }, })) as unknown as CdpRunner["send"], - }; + }); const tabsApi: ChromeTabsApi = { get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab), query: vi.fn(async () => []), @@ -64,11 +64,11 @@ describe("SettleController", () => { it("does not consume a newer redirect while reading the prior landing URL", async () => { vi.useFakeTimers(); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({ result: { value: { idleMs: 1_000, readyState: "complete" } }, })) as unknown as CdpRunner["send"], - }; + }); let resolveFirstTab!: (tab: chrome.tabs.Tab) => void; const firstTab = new Promise((resolve) => { resolveFirstTab = resolve; @@ -105,11 +105,11 @@ describe("SettleController", () => { if (method === "Page.createIsolatedWorld") return { executionContextId: 39 }; return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as CdpRunner["send"], - sendToFrame: sendToFrame as unknown as NonNullable, + sendToFrame: sendToFrame as unknown as CdpRunner["sendToFrame"], hasOwnFrameTarget: (_tabId, frameId) => frameId === "oopif-frame", - }; + }); const tabsApi: ChromeTabsApi = { get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab), query: vi.fn(async () => []), @@ -131,7 +131,6 @@ describe("SettleController", () => { handle, "Runtime.evaluate", expect.objectContaining({ contextId: 39 }), - undefined, ); expect(drafts[0]?.postStateId).toBe("s-next"); }); diff --git a/apps/extension/src/lib/dom-predicates.ts b/apps/extension/src/lib/dom-predicates.ts new file mode 100644 index 000000000..368f011ec --- /dev/null +++ b/apps/extension/src/lib/dom-predicates.ts @@ -0,0 +1,41 @@ +import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "./overlay-bridge"; +import { + RECORD_PASSWORD_ATTRIBUTE, + SENSITIVE_AUTOCOMPLETE_SELECTOR, + sensitiveAutocomplete, +} from "./sensitive-autocomplete"; + +export { RECORD_PASSWORD_ATTRIBUTE, SENSITIVE_AUTOCOMPLETE_SELECTOR, sensitiveAutocomplete }; + +/** Selector matching BrowserSkill's injected overlay hosts and Shadow DOM roots. */ +export const BSK_OVERLAY_SELECTOR = `[data-bsk-overlay],[${OVERLAY_HOST_MARKER_ATTR}],${OVERLAY_HOST_NAME}`; + +/** Non-rendered or BrowserSkill-internal ancestor subtrees excluded from passive DOM reads. */ +export const READ_REDACTED_ANCESTOR_SELECTOR = `script,style,template,noscript,${BSK_OVERLAY_SELECTOR}`; + +/** Sensitive input controls whose values must never be emitted by `extract`. */ +export const SENSITIVE_EXTRACT_INPUT_SELECTOR = `input[type="password"],input[type="hidden"],[${RECORD_PASSWORD_ATTRIBUTE}],${SENSITIVE_AUTOCOMPLETE_SELECTOR}`; + +/** Sensitive input controls whose values/attributes are redacted in `read` and `expect` checks. */ +export const SENSITIVE_READ_INPUT_SELECTOR = `input[type="password"],input[type="file"],input[type="hidden"],[${RECORD_PASSWORD_ATTRIBUTE}],${SENSITIVE_AUTOCOMPLETE_SELECTOR}`; + +/** Canonical WAI-ARIA interactive widget roles shared across VOM fact capture and semantic resolution. */ +export const INTERACTIVE_ARIA_ROLES: ReadonlySet = new Set([ + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +]); diff --git a/apps/extension/src/lib/recording/document-settle.ts b/apps/extension/src/lib/recording/document-settle.ts index 4f8ad0311..266d7529b 100644 --- a/apps/extension/src/lib/recording/document-settle.ts +++ b/apps/extension/src/lib/recording/document-settle.ts @@ -1,4 +1,4 @@ -import type { FrameDocumentHandle } from "@/browser-driver/frame-graph"; +import type { FrameDocumentHandle } from "@/browser-driver/frame-handles"; import { type CdpRunner, sendToFrameHandle } from "@/tools/shared"; const SETTLE_MIN_MS = 150; diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts index c903c9e84..779a3d77d 100644 --- a/apps/extension/src/lib/recording/observation-capture.ts +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -1,5 +1,5 @@ import type { RenderedRef } from "@browser-skill/vom"; -import type { FrameDocumentHandle } from "@/browser-driver/frame-graph"; +import type { FrameDocumentHandle } from "@/browser-driver/frame-handles"; import { type CaptureVomMatchNode, type CaptureVomObservationResult, diff --git a/apps/extension/src/lib/recording/recording-runtime.ts b/apps/extension/src/lib/recording/recording-runtime.ts index bdf0b8b3e..149f52b91 100644 --- a/apps/extension/src/lib/recording/recording-runtime.ts +++ b/apps/extension/src/lib/recording/recording-runtime.ts @@ -47,40 +47,26 @@ export class RecordingObservationRuntime { // the original CDP connection, so cancellation cannot prevent detaching. this.#cdp = input.isTabAllowed ? delegateCdpRunner(input.cdp, { - send: async (tabId, method, params) => { + send: async (tabId, method, params, signal) => { check(tabId); - return input.cdp.send(tabId, method, params); + return input.cdp.send(tabId, method, params, signal); + }, + sendToFrame: async (handle, method, params, options) => { + check(handle.tabId); + return input.cdp.sendToFrame(handle, method, params, options); + }, + getFrameDocuments: async (tabId, signal) => { + check(tabId); + return input.cdp.getFrameDocuments(tabId, signal); + }, + getMainFrameHandle: async (tabId, signal) => { + check(tabId); + return input.cdp.getMainFrameHandle(tabId, signal); + }, + ensureAttachedToUrl: async (tabId, url) => { + check(tabId); + return input.cdp.ensureAttachedToUrl(tabId, url); }, - sendToTarget: input.cdp.sendToTarget - ? async (target, method, params) => { - check(target.tabId); - return input.cdp.sendToTarget!(target, method, params); - } - : undefined, - sendToFrame: input.cdp.sendToFrame - ? async (handle, method, params, options) => { - check(handle.tabId); - return input.cdp.sendToFrame!(handle, method, params, options); - } - : undefined, - getFrameGraph: input.cdp.getFrameGraph - ? async (tabId) => { - check(tabId); - return input.cdp.getFrameGraph!(tabId); - } - : undefined, - getMainFrameHandle: input.cdp.getMainFrameHandle - ? async (tabId) => { - check(tabId); - return input.cdp.getMainFrameHandle!(tabId); - } - : undefined, - ensureAttachedToUrl: input.cdp.ensureAttachedToUrl - ? async (tabId, url) => { - check(tabId); - return input.cdp.ensureAttachedToUrl!(tabId, url); - } - : undefined, }) : input.cdp; this.#tabsApi = input.tabsApi; diff --git a/apps/extension/src/session-manager/ref-store.ts b/apps/extension/src/session-manager/ref-store.ts index 3c750b630..981b5dd88 100644 --- a/apps/extension/src/session-manager/ref-store.ts +++ b/apps/extension/src/session-manager/ref-store.ts @@ -1,4 +1,4 @@ -import type { FrameNodeHandle } from "@/browser-driver/frame-graph"; +import type { FrameNodeHandle } from "@/browser-driver/frame-handles"; import type { VisualCandidate } from "@/tools/vom/visual-discovery"; export type BackendNodeId = number; diff --git a/apps/extension/src/tools/__tests__/background-execution.test.ts b/apps/extension/src/tools/__tests__/background-execution.test.ts index 38273f3d8..d9e2479ea 100644 --- a/apps/extension/src/tools/__tests__/background-execution.test.ts +++ b/apps/extension/src/tools/__tests__/background-execution.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { RequestFrame } from "@/transport/types"; import { prepareBackgroundExecution } from "../background-execution"; +import { createStubCdpRunner } from "../shared"; async function fixture() { const manager = new SessionManager({ @@ -22,11 +23,17 @@ async function fixture() { [{ id: 1, windowId: 100, url: "https://fixture.test", active: true }] as chrome.tabs.Tab[], ), }; - const cdp = { - send: vi.fn(), - acquireBackgroundExecution: vi.fn(async () => {}), - releaseSessionTab: vi.fn(async () => {}), - }; + const acquireBackgroundExecution = vi.fn(async () => {}); + const releaseSessionTab = vi.fn(async () => {}); + const cdp = Object.assign( + createStubCdpRunner({ + send: vi.fn(), + }), + { + acquireBackgroundExecution, + releaseSessionTab, + }, + ); return { manager, ctx, tabs, cdp }; } diff --git a/apps/extension/src/tools/__tests__/click-listener-observation.browser.test.ts b/apps/extension/src/tools/__tests__/click-listener-observation.browser.test.ts index 8efa59a06..784e6bf47 100644 --- a/apps/extension/src/tools/__tests__/click-listener-observation.browser.test.ts +++ b/apps/extension/src/tools/__tests__/click-listener-observation.browser.test.ts @@ -2,10 +2,11 @@ // Opt in with BSK_TEST_CHROME; the harness owns an isolated browser/profile. import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleClick, handleHover } from "../interaction"; import { handleObserve, handleSnapshot } from "../observation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, @@ -43,20 +44,29 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("click listener observations", () expect(reply.exceptionDetails).toBeUndefined(); return reply.result.value; }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId, method, params) => local(method, params), - sendToTarget: (_target, method, params) => local(method, params), + sendToFrame: (_handle, method, params) => local(method, params), getAttachmentId: () => sessionId, - getFrameGraph: async () => { - const { frameTree } = await local<{ frameTree: { frame: { id: string } } }>( - "Page.getFrameTree", - ); + getFrameDocuments: async () => { + const { frameTree } = await local<{ + frameTree: { frame: { id: string; loaderId?: string } }; + }>("Page.getFrameTree"); + const loaderId = frameTree.frame.loaderId ?? ""; return { rootFrameId: frameTree.frame.id, - frames: [{ frameId: frameTree.frame.id, target: { tabId: 4 } }], + frames: [ + { + handle: toFrameDocumentHandle(4, { frameId: frameTree.frame.id, loaderId }), + frameId: frameTree.frame.id, + loaderId, + processKey: "root", + isOutOfProcess: false, + }, + ], }; }, - }; + }); const manager = new SessionManager({ agentWindow: { create: async () => ({ windowId: 100, initialTabIds: [4] }), @@ -223,20 +233,29 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("click listener observations", () }; await local("Page.enable"); await local("Page.bringToFront"); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId, method, params) => local(method, params), - sendToTarget: (_target, method, params) => local(method, params), + sendToFrame: (_handle, method, params) => local(method, params), getAttachmentId: () => sessionId, - getFrameGraph: async () => { - const { frameTree } = await local<{ frameTree: { frame: { id: string } } }>( - "Page.getFrameTree", - ); + getFrameDocuments: async () => { + const { frameTree } = await local<{ + frameTree: { frame: { id: string; loaderId?: string } }; + }>("Page.getFrameTree"); + const loaderId = frameTree.frame.loaderId ?? ""; return { rootFrameId: frameTree.frame.id, - frames: [{ frameId: frameTree.frame.id, target: { tabId: 4 } }], + frames: [ + { + handle: toFrameDocumentHandle(4, { frameId: frameTree.frame.id, loaderId }), + frameId: frameTree.frame.id, + loaderId, + processKey: "root", + isOutOfProcess: false, + }, + ], }; }, - }; + }); const manager = new SessionManager({ agentWindow: { create: async () => ({ windowId: 100, initialTabIds: [4] }), @@ -253,7 +272,9 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("click listener observations", () .split("\n") .map((line) => line.trim()) .find((line) => { - const match = line.match(/^(@e\d+)\s+menuitem\s+"([^"]+)"$/); + const match = line.match( + /^(@e\d+)\s+menuitem\s+"([^"]+)"(?:\s+\[(?:has-submenu|expanded)\])*$/, + ); if (!match) return false; const cleaned = match[2].replace(/ \[(?:has-submenu|expanded)\]$/, ""); return cleaned === exactName; diff --git a/apps/extension/src/tools/__tests__/click.browser.test.ts b/apps/extension/src/tools/__tests__/click.browser.test.ts index 52a2ec139..2ab86957f 100644 --- a/apps/extension/src/tools/__tests__/click.browser.test.ts +++ b/apps/extension/src/tools/__tests__/click.browser.test.ts @@ -8,7 +8,7 @@ import { createElement } from "react"; import { renderToStaticMarkup } from "react-dom/server"; import ts from "typescript"; import { describe, expect, it, vi } from "vitest"; -import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { ControlOverlay } from "@/content/ControlOverlay"; import { INPUT_PASSTHROUGH, @@ -19,7 +19,7 @@ import { SessionManager } from "@/session-manager/manager"; import { prepareBackgroundExecution } from "../background-execution"; import { handleClick, handlePress } from "../interaction"; import { handleObserve } from "../observation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { handleWheel } from "../wheel"; type Send = >( @@ -34,7 +34,7 @@ interface CdpEvent { params?: object; } -type CdpEventListener = (source: CdpDebuggee, method: string, params: unknown) => void; +type CdpEventListener = Parameters[0]; describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser click readiness", () => { it("delivers hidden native input, preserves disabled semantics and restores visibility", async () => { @@ -122,13 +122,13 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser click readiness", () hidden ? "hidden" : "visible", ); const commands: { method: string; params?: object }[] = []; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: async (_tabId, method, params) => { commands.push({ method, params }); return (await send(method, params, target.sessionId)) as never; }, getAttachmentId: () => target.sessionId, - }; + }); const tab = { id: 4, windowId: 100, active: !hidden } as chrome.tabs.Tab; const tabsApi = { get: async () => tab, query: async () => [tab] }; let action: { ref?: string; selector?: string } = { @@ -578,7 +578,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser click readiness", () const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab; const tabsApi = { get: async () => tab, query: async () => [tab] }; const mouse: string[] = []; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: async (_tabId, method, params) => { if (method === "Input.dispatchMouseEvent") { mouse.push((params as { type: string }).type); @@ -592,7 +592,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser click readiness", () } return (await local(method, params)) as never; }, - }; + }); const phases: string[] = []; const sendInputPassthrough: InputPassthroughSendToTab = async (_tabId, message) => { phases.push(message.phase); diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 07b78ad75..c08dcccd7 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { CooperationController } from "@/cooperation/controller"; import { INTERACTION_STORAGE_KEY, InteractionPreferenceStore } from "@/lib/interaction-preferences"; +import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { ConnectionStateHandler, FrameHandler, Transport } from "@/transport/transport"; import type { @@ -17,9 +18,16 @@ import { resetBrowserObservationForTests, setBrowserObservationAttachForTests, } from "../record"; +import { type CdpRunnerStubOverrides, createStubCdpRunner } from "../shared"; type TestDispatcherCdp = NonNullable[0]["cdp"]>; +function stubDispatcherCdp>(overrides: T): TestDispatcherCdp & T { + return createStubCdpRunner( + overrides as unknown as CdpRunnerStubOverrides, + ) as unknown as TestDispatcherCdp & T; +} + function fakeTransport() { const handlers = new Set(); const stateHandlers = new Set(); @@ -52,16 +60,20 @@ function makeRequest(method: string, params: unknown): RequestFrame { } function makeRecordTestCdp(tabId: number): TestDispatcherCdp { - type EventListener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => void; + type EventListener = ( + source: { tabId: number; targetKey: string; isRootTarget: boolean }, + method: string, + params: unknown, + ) => void; const listeners: EventListener[] = []; - return { + return stubDispatcherCdp({ send: vi.fn(async (_tabId: number, method: string) => { if (method === "Page.getFrameTree") { return { frameTree: { frame: { id: "frame-1", loaderId: "loader-before" } } }; } if (method === "Page.navigate") { for (const listener of [...listeners]) { - listener({ tabId }, "Page.lifecycleEvent", { + listener({ tabId, targetKey: "root", isRootTarget: true }, "Page.lifecycleEvent", { name: "load", frameId: "frame-1", loaderId: "loader-after", @@ -92,7 +104,7 @@ function makeRecordTestCdp(tabId: number): TestDispatcherCdp { }, }; }, - } as unknown as TestDispatcherCdp; + }); } describe("ToolDispatcher", () => { @@ -118,13 +130,13 @@ describe("ToolDispatcher", () => { sequence: 1, decision_deadline: 60000, }; - const cdp = { + const cdp = stubDispatcherCdp({ send: vi.fn(() => { throw new Error("Renderer blocked"); }), pendingDialogs: () => [dialog], - resolveDialog: vi.fn(async () => ({ ...dialog, handled: "accepted" })), - } as unknown as TestDispatcherCdp; + resolveDialog: vi.fn(async () => ({ ...dialog, handled: "accepted" as const })), + }); const debug = { before: vi.fn(() => { throw new Error("Screenshot blocked"); @@ -198,10 +210,10 @@ describe("ToolDispatcher", () => { }, }); await sessions.start("owner"); - const cdp = { + const cdp = stubDispatcherCdp({ send: vi.fn(), - pendingDialogs: () => [{ id: "async-dialog", tab_id: 7 }], - } as unknown as TestDispatcherCdp; + pendingDialogs: () => [{ id: "async-dialog", tab_id: 7 } as PendingJavaScriptDialog], + }) as TestDispatcherCdp; const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); dispatcher.start(); deliver(makeRequest("tool.evaluate", { session_id: "owner", expression: "dangerousAction()" })); @@ -226,7 +238,10 @@ describe("ToolDispatcher", () => { }, }); await sessions.start("owner"); - const cdp = { send: vi.fn(), pendingDialogs: () => [] } as unknown as TestDispatcherCdp; + const cdp = stubDispatcherCdp({ + send: vi.fn(), + pendingDialogs: () => [], + }) as TestDispatcherCdp; // The first operation (for example a cancelled screenshot) has not settled. const debug = { before: vi.fn(() => new Promise(() => {})), after: vi.fn(), dispose: vi.fn() }; const dispatcher = new ToolDispatcher({ transport, sessions, cdp, debug: debug as never }); @@ -291,11 +306,11 @@ describe("ToolDispatcher", () => { }); await sessions.start("test"); const error = new Error("Cannot access a chrome-extension:// URL of different extension"); - const cdp = { + const cdp = stubDispatcherCdp({ send: vi.fn().mockRejectedValue(error), ensureConsoleCapture: vi.fn().mockRejectedValue(error), consoleEntriesSince: vi.fn(), - } as unknown as TestDispatcherCdp; + }) as TestDispatcherCdp; const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); dispatcher.start(); deliver( @@ -330,7 +345,7 @@ describe("ToolDispatcher", () => { }, }); await sessions.start("remote"); - const cdp = { send: vi.fn() } as unknown as TestDispatcherCdp; + const cdp = stubDispatcherCdp({ send: vi.fn() }); const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); dispatcher.start(); deliver(makeRequest(method, { session_id: "remote" })); @@ -634,7 +649,7 @@ describe("ToolDispatcher", () => { let leased = false; let visibility = "visible"; let visibilityReads = 0; - const cdp = { + const cdp = stubDispatcherCdp({ acquireBackgroundExecution: vi.fn(async (sessionId: string, tabId: number) => { expect([sessionId, tabId]).toEqual(["aa11", 7]); commands.push("acquire"); @@ -670,7 +685,7 @@ describe("ToolDispatcher", () => { if (method === "Input.dispatchMouseEvent") return {}; throw new Error(`unexpected CDP call ${method}`); }), - } as unknown as TestDispatcherCdp; + }) as TestDispatcherCdp; const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); dispatcher.start(); @@ -897,7 +912,11 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp: stubDispatcherCdp(cdp), + }); dispatcher.start(); deliver(makeRequest("tool.console", { session_id: "aa11" })); @@ -986,7 +1005,11 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp: stubDispatcherCdp(cdp), + }); dispatcher.start(); deliver( @@ -1028,7 +1051,11 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp: stubDispatcherCdp(cdp), + }); dispatcher.start(); deliver(makeRequest("tool.session_stop", { session_id: "aa11" })); @@ -1098,7 +1125,7 @@ describe("ToolDispatcher", () => { const dispatcher = new ToolDispatcher({ transport, sessions, - cdp: cdp as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), cooperation, }); dispatcher.start(); @@ -1248,7 +1275,7 @@ describe("ToolDispatcher", () => { const { transport, sent, deliver } = fakeTransport(); const onBrowserControlResumed = vi.fn(); let resolveNode: ((value: object) => void) | undefined; - const cdp = { + const cdp = stubDispatcherCdp({ send: vi.fn(async (_tabId: number, method: string) => { if (method === "Runtime.evaluate") return { result: { value: "visible" } }; if (method === "DOM.resolveNode") @@ -1261,7 +1288,7 @@ describe("ToolDispatcher", () => { if (method === "Runtime.callFunctionOn") return { result: { value: true } }; return {}; }), - } as unknown as TestDispatcherCdp; + }) as TestDispatcherCdp; const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed }); const hover = dispatcher as unknown as { rememberHover: (sessionId: string, result: object) => void; @@ -1403,7 +1430,11 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp: stubDispatcherCdp(cdp), + }); ( dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown } @@ -1481,7 +1512,11 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp: stubDispatcherCdp(cdp), + }); const helpers = dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown; setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; @@ -1542,7 +1577,7 @@ describe("ToolDispatcher", () => { const dispatcher = new ToolDispatcher({ transport, sessions, - cdp: cdp as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), }); const helpers = dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown; @@ -1598,7 +1633,7 @@ describe("ToolDispatcher", () => { const dispatcher = new ToolDispatcher({ transport, sessions, - cdp: cdp as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), }); const helpers = dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown; @@ -1709,7 +1744,11 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp: stubDispatcherCdp(cdp), + }); const helpers = dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown; withHoverReassert: ( @@ -1892,7 +1931,7 @@ describe("ToolDispatcher", () => { const dispatcher = new ToolDispatcher({ transport, sessions, - cdp: { detachSession } as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp({ send: vi.fn(), detachSession }), }); dispatcher.start(); deliver({ id: "one-stop", method: "tool.session_stop", params: { session_id: "one" } }); @@ -1988,12 +2027,12 @@ describe("background execution dispatch integration", () => { get: async () => ({ id: 7, windowId: 100, active: false, url: "https://fixture.test" }), }, }); - const cdp = { + const cdp = stubDispatcherCdp({ send: vi.fn(), acquireBackgroundExecution: vi.fn(async () => { throw new Error("simulation unavailable"); }), - } as unknown as TestDispatcherCdp; + }) as TestDispatcherCdp; const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); dispatcher.start(); try { @@ -2246,7 +2285,7 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { transport, sessions, cooperation, - cdp: cdp as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), interactionPreferences: makePreferenceStore(false), onAgentTabClaimed, }); @@ -2340,7 +2379,7 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { transport, sessions, cooperation, - cdp: cdp as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), interactionPreferences: makePreferenceStore(true), }); dispatcher.start(); @@ -2362,7 +2401,7 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { transport: t2, sessions, cooperation, - cdp: cdp as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), interactionPreferences: makePreferenceStore(false), }); dispatcherNoConfirm.start(); @@ -2476,7 +2515,7 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { transport, sessions, cooperation, - cdp: cdp as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), interactionPreferences: makePreferenceStore(false), }); dispatcher.start(); @@ -2751,7 +2790,7 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { changed: vi.fn(), }); cooperation.startSession("aa11"); - const cdp = { + const cdp = stubDispatcherCdp({ ensureConsoleCapture: vi.fn(async () => {}), consoleEntriesSince: vi.fn(() => ({ tab_id: 7, @@ -2760,7 +2799,7 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { truncated: false, })), send: vi.fn(async () => ({})), - } as unknown as TestDispatcherCdp; + }) as TestDispatcherCdp; const dispatcher = new ToolDispatcher({ transport, sessions, cdp, cooperation }); dispatcher.start(); try { @@ -2869,15 +2908,15 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { } return {} as T; }), - sendGuarded: vi.fn( + sendToFrame: vi.fn( async ( - target: unknown, + handle: { tabId: number }, method: string, - _params: Record | undefined, - guard: { onDispatch?: () => void }, + params: Record | undefined, + guard?: { onDispatch?: () => void }, ) => { - guard.onDispatch?.(); - return (await cdp.send(target, method)) as T; + guard?.onDispatch?.(); + return (await cdp.send(handle.tabId, method)) as T; }, ), detachSession: vi.fn(async () => {}), @@ -2890,7 +2929,7 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { transport, sessions, cooperation, - cdp: cdp as unknown as TestDispatcherCdp, + cdp: stubDispatcherCdp(cdp), interactionPreferences: makePreferenceStore(false), debug: { before: async (req: { id: string }) => { @@ -2987,4 +3026,397 @@ describe("implicit auto-borrow when confirmTabBorrow is false", () => { vi.unstubAllGlobals(); } }); + + it("awaits compositor paint on tool.wheel when ownsBackgroundExecution is true on an unfocused/minimized window", async () => { + const { transport, sent, deliver } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [42] }), + remove: async () => {}, + ensureActiveTab: async () => 42, + }, + }); + const ctx = await sessions.start("wheel-bg"); + ctx.refStore.set("e1", 701, { tabId: 42 }); + + const tab = { + id: 42, + windowId: 100, + active: true, + url: "https://wheel.example/", + } as chrome.tabs.Tab; + vi.stubGlobal("chrome", { + tabs: { + get: vi.fn(async () => tab), + query: vi.fn(async () => [tab]), + }, + }); + + let windowState: { id: number; focused: boolean; state: chrome.windows.Window["state"] } = { + id: 100, + focused: false, + state: "minimized", + }; + const evaluatedExpressions: string[] = []; + const dispatchedMethods: string[] = []; + const cdp = stubDispatcherCdp({ + ownsBackgroundExecution: vi.fn(() => true), + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + dispatchedMethods.push(method); + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 300, 100, 300] } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[100, 100, 300, 100, 300, 300, 100, 300]] }; + } + if (method === "Runtime.evaluate") { + const expr = (params as { expression?: string } | undefined)?.expression ?? ""; + evaluatedExpressions.push(expr); + if (expr === "document.visibilityState") { + return { result: { value: "visible" } }; + } + if (expr.includes("requestAnimationFrame")) { + return { result: { value: true } }; + } + return { result: { value: { x: 0, y: 120 } } }; + } + if (method === "Runtime.callFunctionOn") { + return { result: { value: { scrollTop: 120, scrollLeft: 0 } } }; + } + if (method === "DOM.resolveNode") { + return { object: { objectId: "scroller-1" } }; + } + return {}; + }), + }); + + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp, + windowsApi: { get: vi.fn(async () => windowState) }, + }); + dispatcher.start(); + try { + deliver({ + id: "wheel-minimized", + method: "tool.wheel", + params: { session_id: "wheel-bg", tab_id: 42, delta_y: 120 }, + }); + await vi.waitFor(() => expect(sent).toHaveLength(1)); + expect(sent[0]).toMatchObject({ + id: "wheel-minimized", + result: { tab_id: 42, delta_y: 120 }, + }); + const wheelIdx = dispatchedMethods.indexOf("Input.dispatchMouseEvent"); + expect(wheelIdx).toBeGreaterThanOrEqual(0); + expect(evaluatedExpressions.some((expr) => expr.includes("requestAnimationFrame"))).toBe( + true, + ); + + // Foreground focused window does not wait on extra hidden-surface paint frames. + evaluatedExpressions.length = 0; + windowState = { id: 100, focused: true, state: "normal" }; + deliver({ + id: "wheel-focused", + method: "tool.wheel", + params: { session_id: "wheel-bg", tab_id: 42, delta_y: 120 }, + }); + await vi.waitFor(() => expect(sent).toHaveLength(2)); + expect(sent[1]).toMatchObject({ + id: "wheel-focused", + result: { tab_id: 42, delta_y: 120 }, + }); + expect(evaluatedExpressions.some((expr) => expr.includes("requestAnimationFrame"))).toBe( + false, + ); + } finally { + dispatcher.stop(); + vi.unstubAllGlobals(); + } + }); + + it("cleans up hover latches and overlay bypass refcounts when tabsOnRemoved fires", async () => { + const { transport, sent, deliver } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [42, 43] }), + remove: async () => {}, + ensureActiveTab: async () => 42, + }, + }); + const ctx = await sessions.start("hover-cleanup"); + ctx.refStore.set("e1", 501, { tabId: 42 }); + + const tabs = new Map([ + [42, { id: 42, windowId: 100, active: true, url: "https://a.example/" } as chrome.tabs.Tab], + [43, { id: 43, windowId: 100, active: false, url: "https://b.example/" } as chrome.tabs.Tab], + ]); + const sendMessage = vi.fn(async () => ({ ok: true })); + vi.stubGlobal("chrome", { + tabs: { + get: vi.fn(async (id: number) => tabs.get(id)!), + query: vi.fn(async () => [...tabs.values()]), + sendMessage, + }, + }); + + const removedListeners = new Set<(tabId: number) => void>(); + const tabsOnRemoved = { + addListener: (cb: (tabId: number) => void) => removedListeners.add(cb), + removeListener: (cb: (tabId: number) => void) => removedListeners.delete(cb), + }; + + const mouseMovedTabs: number[] = []; + const cdp = stubDispatcherCdp({ + send: vi.fn(async (tabId: number, method: string, params?: object) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[10, 10, 50, 10, 50, 50, 10, 50]] }; + } + if (method === "Runtime.evaluate") { + const expr = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expr.includes("overlayHostPresent")) { + return { + result: { + value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, + }, + }; + } + return { result: { value: "visible" } }; + } + if (method === "Input.dispatchMouseEvent") { + mouseMovedTabs.push(tabId); + return {}; + } + return {}; + }), + }); + + const dispatcher = new ToolDispatcher({ + transport, + sessions, + cdp, + tabsOnRemoved, + }); + dispatcher.start(); + try { + deliver({ + id: "hover-42", + method: "tool.hover", + params: { session_id: "hover-cleanup", tab_id: 42, ref: "e1", settle_ms: 0 }, + }); + await vi.waitFor(() => expect(sent).toHaveLength(1)); + expect(sent[0]).toMatchObject({ id: "hover-42", result: { tab_id: 42 } }); + expect(sendMessage).toHaveBeenCalledWith(42, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled: true, + }); + + // Closing tab 42 forgets both the hover latch and the per-tab bypass ownership entry. + sendMessage.mockClear(); + mouseMovedTabs.length = 0; + tabs.delete(42); + for (const listener of removedListeners) listener(42); + + // Subsequent tool.click on tab 43 (or session-wide) does not reassert hover on closed tab 42, + // and session_stop does not send redundant bypass messages to closed tab 42. + ctx.refStore.set("e2", 502, { tabId: 43 }); + deliver({ + id: "click-43", + method: "tool.click", + params: { session_id: "hover-cleanup", ref: "e2", tab_id: 43 }, + }); + await vi.waitFor(() => expect(sent).toHaveLength(2)); + expect(mouseMovedTabs.every((id) => id === 43)).toBe(true); + expect(sendMessage).not.toHaveBeenCalledWith(42, expect.anything()); + + // If Chrome later reuses tab ID 42, hover re-acquires overlay bypass instead of hitting stale owner cache. + tabs.set(42, { + id: 42, + windowId: 100, + active: true, + url: "https://reused.example/", + } as chrome.tabs.Tab); + ctx.refStore.set("e3", 503, { tabId: 42 }); + sendMessage.mockClear(); + deliver({ + id: "hover-42-reused", + method: "tool.hover", + params: { session_id: "hover-cleanup", tab_id: 42, ref: "e3", settle_ms: 0 }, + }); + await vi.waitFor(() => expect(sent).toHaveLength(3)); + expect(sendMessage).toHaveBeenCalledWith(42, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled: true, + }); + } finally { + dispatcher.stop(); + vi.unstubAllGlobals(); + } + }); + + it("returns structured session_busy on concurrent tool.download without sharing module-global lock across dispatchers", async () => { + const t1 = fakeTransport(); + const t2 = fakeTransport(); + const makeSessions = async (sessionId: string, windowId: number, tabId: number) => { + const sm = new SessionManager({ + agentWindow: { + create: async () => ({ windowId, initialTabIds: [tabId] }), + remove: async () => {}, + ensureActiveTab: async () => tabId, + }, + }); + const ctx = await sm.start(sessionId); + ctx.refStore.set("e1", 901, { tabId }); + return sm; + }; + const sessions1 = await makeSessions("dl-1", 101, 51); + const sessions2 = await makeSessions("dl-2", 102, 52); + + let releasePress1!: (val: object) => void; + const press1Started = new Promise((resolve) => { + vi.stubGlobal("chrome", { + tabs: { + get: vi.fn(async (id: number) => ({ + id, + windowId: id === 51 ? 101 : 102, + active: true, + url: "https://files.example/", + })), + query: vi.fn(async () => []), + sendMessage: vi.fn(async () => ({})), + }, + downloads: { + onDeterminingFilename: { addListener: vi.fn(), removeListener: vi.fn() }, + onCreated: { addListener: vi.fn(), removeListener: vi.fn() }, + onChanged: { addListener: vi.fn(), removeListener: vi.fn() }, + search: vi.fn(async () => []), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }, + webNavigation: { + onCreatedNavigationTarget: { addListener: vi.fn(), removeListener: vi.fn() }, + }, + }); + void resolve; + }); + void press1Started; + + let enteredFirstDownload!: () => void; + const firstEntered = new Promise((resolve) => { + enteredFirstDownload = resolve; + }); + let secondDispatcherEntered = false; + + const makeDownloadCdp = (isFirst: boolean) => + stubDispatcherCdp({ + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[10, 10, 40, 10, 40, 40, 10, 40]] }; + } + if (method === "Runtime.evaluate") { + const expr = (params as { expression?: string } | undefined)?.expression ?? ""; + return { + result: { value: expr === "document.visibilityState" ? "visible" : "absent" }, + }; + } + if ( + method === "Input.dispatchMouseEvent" && + (params as { type?: string } | undefined)?.type === "mousePressed" + ) { + if (isFirst) { + enteredFirstDownload(); + return new Promise((resolve) => { + releasePress1 = resolve; + }); + } + secondDispatcherEntered = true; + } + return {}; + }), + }); + + const d1 = new ToolDispatcher({ + transport: t1.transport, + sessions: sessions1, + cdp: makeDownloadCdp(true), + }); + const d2 = new ToolDispatcher({ + transport: t2.transport, + sessions: sessions2, + cdp: makeDownloadCdp(false), + }); + d1.start(); + d2.start(); + try { + t1.deliver({ + id: "dl-first", + method: "tool.download", + params: { + session_id: "dl-1", + tab_id: 51, + ref: "e1", + browser_relative_dir: "BrowserSkill/d1", + timeout_ms: 5_000, + }, + }); + await firstEntered; + + // Second concurrent download on the same dispatcher receives structured session_busy with effect_state: "none". + t1.deliver({ + id: "dl-second-same-dispatcher", + method: "tool.download", + params: { + session_id: "dl-1", + tab_id: 51, + ref: "e1", + browser_relative_dir: "BrowserSkill/d1", + timeout_ms: 5_000, + }, + }); + await vi.waitFor(() => expect(t1.sent).toHaveLength(1)); + expect(t1.sent[0]).toMatchObject({ + id: "dl-second-same-dispatcher", + error: { + code: "timeout", + data: { + reason: "session_busy", + effect_state: "none", + dispatched: false, + }, + }, + }); + + // Separate ToolDispatcher instance has its own DownloadConcurrencyGate (no module-global lock leak). + t2.deliver({ + id: "dl-other-dispatcher", + method: "tool.download", + params: { + session_id: "dl-2", + tab_id: 52, + ref: "e1", + browser_relative_dir: "BrowserSkill/d2", + timeout_ms: 50, + }, + }); + await vi.waitFor(() => expect(secondDispatcherEntered).toBe(true)); + + t1.deliver({ id: "cancel-dl-first", method: "cancel", params: { rpc_id: "dl-first" } }); + releasePress1?.({}); + await vi.waitFor(() => expect(t1.sent.length).toBeGreaterThanOrEqual(3)); + } finally { + d1.stop(); + d2.stop(); + vi.unstubAllGlobals(); + } + }); }); diff --git a/apps/extension/src/tools/__tests__/download-deadline.browser.test.ts b/apps/extension/src/tools/__tests__/download-deadline.browser.test.ts index feb86c130..4628fce15 100644 --- a/apps/extension/src/tools/__tests__/download-deadline.browser.test.ts +++ b/apps/extension/src/tools/__tests__/download-deadline.browser.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest"; import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; import { handleDownload } from "../download"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, delegateCdpRunner } from "../shared"; type Send = >( method: string, @@ -119,15 +119,13 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("download trigger retirement", () } return value; }; - const wrapped: CdpRunner = { - send: async (tabId, method, params) => - holdReply(method, params, await cdp.send(tabId, method, params)), - sendGuarded: async (target, method, params, guard) => - holdReply(method, params, await cdp.sendGuarded(target, method, params, guard)), - detach: cdp.detach.bind(cdp), - getAttachmentId: cdp.getAttachmentId.bind(cdp), + const wrapped: CdpRunner = delegateCdpRunner(cdp, { + send: async (tabId, method, params, signal) => + holdReply(method, params, await cdp.send(tabId, method, params, signal)), + sendToFrame: async (handle, method, params, guard) => + holdReply(method, params, await cdp.sendToFrame(handle, method, params, guard)), onEvent: () => ({ dispose() {} }), - }; + }); const event = () => ({ addListener() {}, removeListener() {} }); const downloads = { onCreated: event(), @@ -218,9 +216,8 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("download trigger retirement", () return sendCommand(target, method, params); }; let interceptMove = true; - const wrapped: CdpRunner = { - send: cdp.send.bind(cdp), - sendGuarded: async (target, method, params, guard) => { + const wrapped: CdpRunner = delegateCdpRunner(cdp, { + sendToFrame: async (handle, method, params, guard) => { if (method === "Input.dispatchMouseEvent" && interceptMove) { interceptMove = false; await cdp.detach(7); @@ -230,17 +227,15 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("download trigger retirement", () await held; }; try { - return await cdp.sendGuarded(target, method, params, guard); + return await cdp.sendToFrame(handle, method, params, guard); } finally { settled(); } } - return cdp.sendGuarded(target, method, params, guard); + return cdp.sendToFrame(handle, method, params, guard); }, - detach: cdp.detach.bind(cdp), - getAttachmentId: cdp.getAttachmentId.bind(cdp), onEvent: () => ({ dispose() {} }), - }; + }); const event = () => ({ addListener() {}, removeListener() {} }); const pending = handleDownload( sessions, diff --git a/apps/extension/src/tools/__tests__/download-deadline.test.ts b/apps/extension/src/tools/__tests__/download-deadline.test.ts index 1b354b9d4..cc907a8c0 100644 --- a/apps/extension/src/tools/__tests__/download-deadline.test.ts +++ b/apps/extension/src/tools/__tests__/download-deadline.test.ts @@ -1,9 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import type { ClickResult } from "@/transport/types"; import { handleDownload } from "../download"; import { captureBrowserDownload, type DownloadsApi } from "../download-capture"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; function deferred() { let resolve!: (value: T) => void; @@ -50,13 +51,13 @@ function fixture() { }; let listener!: Parameters>[1]; const dispose = vi.fn(); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], onEvent: (_tabId, callback) => { listener = callback; return { dispose }; }, - }; + }); const item = { id: 9, url: "https://example.test/report", @@ -67,7 +68,7 @@ function fixture() { totalBytes: 4, } as chrome.downloads.DownloadItem; const offer = () => { - listener({ tabId: 7 }, "Page.downloadWillBegin", { + listener(rootEventSource(7), "Page.downloadWillBegin", { url: item.url, suggestedFilename: item.filename, }); @@ -83,7 +84,7 @@ function fixture() { onChanged, onDeterminingFilename, intent: (url: string, filename: string) => - listener({ tabId: 7 }, "Page.downloadWillBegin", { url, suggestedFilename: filename }), + listener(rootEventSource(7), "Page.downloadWillBegin", { url, suggestedFilename: filename }), }; } diff --git a/apps/extension/src/tools/__tests__/download-effect-state.browser.test.ts b/apps/extension/src/tools/__tests__/download-effect-state.browser.test.ts index a41fd7efd..06cc1eec9 100644 --- a/apps/extension/src/tools/__tests__/download-effect-state.browser.test.ts +++ b/apps/extension/src/tools/__tests__/download-effect-state.browser.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest"; import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; import { handleDownload } from "../download"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, delegateCdpRunner } from "../shared"; type Send = >( method: string, @@ -87,7 +87,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("download effect metadata", () => `, }); const ac = new AbortController(); - const wrapped: CdpRunner = { + const wrapped: CdpRunner = delegateCdpRunner(cdp, { send: (async (tabId: number, method: string, params?: object) => { const reply = await cdp.send(tabId, method, params); if ( @@ -98,7 +98,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("download effect metadata", () => return reply; }) as CdpRunner["send"], onEvent: () => ({ dispose() {} }), - }; + }); const event = () => ({ addListener() {}, removeListener() {} }); const downloads = { onCreated: event(), diff --git a/apps/extension/src/tools/__tests__/download-effect-state.test.ts b/apps/extension/src/tools/__tests__/download-effect-state.test.ts index 2c0367736..c2d5c175d 100644 --- a/apps/extension/src/tools/__tests__/download-effect-state.test.ts +++ b/apps/extension/src/tools/__tests__/download-effect-state.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { captureBrowserDownload, type DownloadsApi } from "../download-capture"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; describe("download trigger effect state", () => { it.each(["returned", "thrown"])("distinguishes pre/post-dispatch %s failures", async (kind) => { @@ -14,7 +14,10 @@ describe("download trigger effect state", () => { cancel: vi.fn(async () => {}), removeFile: vi.fn(async () => {}), }; - const cdp: CdpRunner = { send: vi.fn(), onEvent: () => ({ dispose: vi.fn() }) }; + const cdp: CdpRunner = createStubCdpRunner({ + send: vi.fn(), + onEvent: () => ({ dispose: vi.fn() }), + }); const result = await captureBrowserDownload({ cdp, downloads, diff --git a/apps/extension/src/tools/__tests__/download-lifecycle.test.ts b/apps/extension/src/tools/__tests__/download-lifecycle.test.ts index 9d7dca703..a3e4d333c 100644 --- a/apps/extension/src/tools/__tests__/download-lifecycle.test.ts +++ b/apps/extension/src/tools/__tests__/download-lifecycle.test.ts @@ -1,10 +1,10 @@ import { afterEach, expect, it, vi } from "vitest"; import { - type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp, DEFAULT_IDLE_DETACH_MS, } from "@/browser-driver/chromium-cdp"; +import { toFrameDocumentHandle, toRootFrameHandle } from "@/browser-driver/frame-handles"; import type { InputPassthroughMessage } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { handleDownload } from "../download"; @@ -31,8 +31,10 @@ function event() { }; } +type RawDebuggerEventArgs = Parameters[0]>; + async function fixture(options: { idleDetachMs?: number } = {}) { - const onEvent = event<[CdpDebuggee, string, unknown]>(); + const onEvent = event(); const onDetach = event<[chrome.debugger.Debuggee, string]>(); const inputs: string[] = []; const api: CdpDebuggerApi = { @@ -208,9 +210,9 @@ it.each([ }); it.each([ - undefined, - "child-session", -])("checks cancellation at native dispatch for target %s", async (sessionId) => { + "", + "child-frame", +])("checks cancellation at native dispatch for frame %s", async (frameId) => { const f = await fixture(); const abort = new AbortController(); const attach = deferred(); @@ -220,8 +222,8 @@ it.each([ return attach.promise; }); const scope = pointerTransactionDeps(f.deps, abort.signal); - const pending = scope.deps.cdp.sendToTarget!( - { tabId: 7, sessionId }, + const pending = scope.deps.cdp.sendToFrame( + toFrameDocumentHandle(7, { frameId }), "Input.dispatchMouseEvent", { type: "mousePressed" }, ); @@ -249,8 +251,8 @@ it("refuses guarded cleanup sends and detaches for a replaced attachment without const attachmentId = f.cdp.getAttachmentId(7)!; await f.cdp.detach(7); const cleanup = () => - f.cdp.sendGuarded( - { tabId: 7 }, + f.cdp.sendToFrame( + toRootFrameHandle(7), "Input.dispatchMouseEvent", { type: "mouseReleased" }, { attachmentId }, diff --git a/apps/extension/src/tools/__tests__/drag.browser.test.ts b/apps/extension/src/tools/__tests__/drag.browser.test.ts index 8487ee680..281f19eb0 100644 --- a/apps/extension/src/tools/__tests__/drag.browser.test.ts +++ b/apps/extension/src/tools/__tests__/drag.browser.test.ts @@ -4,14 +4,19 @@ import { readFile } from "node:fs/promises"; import { createRequire } from "node:module"; import { build } from "esbuild"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentHandle, + type FrameDocumentInfo, + type FrameNodeHandle, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { pointerTargetValidator } from "@/cooperation/pointer-target"; import { SessionManager } from "@/session-manager/manager"; import type { DragParams } from "@/transport/types"; import { handleDrag } from "../drag"; import { handleClick, handleHover } from "../interaction"; import { handleObserve } from "../observation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, @@ -101,20 +106,20 @@ async function harness(targetId: string, background: boolean) { const ctx = await manager.start("aa11"); let hook: Hook = () => {}; const calls: { method: string; params?: Record }[] = []; - const forward = async (target: CdpTarget, method: string, params?: object): Promise => { + const forward = async ( + _handle: FrameDocumentHandle | FrameNodeHandle | { tabId: number }, + method: string, + params?: object, + ): Promise => { calls.push({ method, params: params as Record }); - const result = await send(method, params, target.sessionId ?? sessionId); + const result = await send(method, params, sessionId); await hook(method, params as Record); return result; }; - const cdp: CdpRunner = { - send: (_tab, method, params) => forward({ tabId: 4 }, method, params), - sendToTarget: forward, - }; const frames = async () => { - type Tree = { frame: { id: string; name?: string }; childFrames?: Tree[] }; + type Tree = { frame: { id: string; loaderId?: string; name?: string }; childFrames?: Tree[] }; const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, sessionId); - const result: CdpFrame[] = []; + const result: FrameDocumentInfo[] = []; const visit = async (tree: Tree, parentFrameId?: string) => { const owner = parentFrameId ? await send<{ backendNodeId: number }>( @@ -123,18 +128,26 @@ async function harness(targetId: string, background: boolean) { sessionId, ) : undefined; + const loaderId = tree.frame.loaderId ?? ""; result.push({ + handle: toFrameDocumentHandle(4, { frameId: tree.frame.id, loaderId }), frameId: tree.frame.id, + loaderId, parentFrameId, - target: { tabId: 4 }, ownerBackendNodeId: owner?.backendNodeId, + processKey: "root", + isOutOfProcess: false, }); for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); }; await visit(frameTree); return { rootFrameId: frameTree.frame.id, frames: result }; }; - cdp.getFrameGraph = frames; + const cdp: CdpRunner = createStubCdpRunner({ + send: (_tab, method, params) => forward({ tabId: 4 }, method, params), + sendToFrame: forward, + getFrameDocuments: frames, + }); cdp.verifyPointerTarget = pointerTargetValidator(cdp); const deps = { cdp, diff --git a/apps/extension/src/tools/__tests__/drag.test.ts b/apps/extension/src/tools/__tests__/drag.test.ts index 9dfd8bc37..42a338655 100644 --- a/apps/extension/src/tools/__tests__/drag.test.ts +++ b/apps/extension/src/tools/__tests__/drag.test.ts @@ -5,7 +5,7 @@ import type { DragParams } from "@/transport/types"; import { withClickOverlay } from "../click-overlay"; import { validateDrag } from "../drag"; import { pointerTransactionDeps } from "../pointer-transaction"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; it.each([ { source: { selector: "" } }, @@ -33,12 +33,12 @@ it.each([ it("cancels native drag and releases the last position after control is revoked", async () => { let revoked = false; const sent: { method: string; params?: object }[] = []; - const raw: CdpRunner = { + const raw: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tab, method, params) => { sent.push({ method, params }); return {}; }) as CdpRunner["send"], - }; + }); const cdp = guardedRunner(raw, { prepare: async () => { if (revoked) throw new Error("revoked"); @@ -80,7 +80,7 @@ it("cancels native drag and releases the last position after control is revoked" ).rejects.toThrow(); }); it("does not allow an unrelated request to cancel dragging after revocation", async () => { - const raw: CdpRunner = { send: vi.fn() as CdpRunner["send"] }; + const raw: CdpRunner = createStubCdpRunner({ send: vi.fn() as CdpRunner["send"] }); const cdp = guardedRunner(raw, { prepare: async () => { throw new Error("revoked"); @@ -92,13 +92,13 @@ it("does not allow an unrelated request to cancel dragging after revocation", as it("does not release a replacement attachment after native cancellation settles", async () => { let attachment = "original"; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ getAttachmentId: () => attachment, send: vi.fn(async (_tab, method) => { if (method === "Input.cancelDragging") attachment = "replacement"; return {}; }) as CdpRunner["send"], - }; + }); const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, undefined, @@ -125,13 +125,13 @@ it("does not release a replacement attachment after native cancellation settles" }); it("does not complete a drop when native cancellation cannot be confirmed", async () => { - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tab, method) => { if (method === "Input.cancelDragging") throw new Error("disconnected"); return {}; }) as CdpRunner["send"], detach: vi.fn(async () => {}), - }; + }); const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, undefined, @@ -160,11 +160,11 @@ it("does not complete a drop when native cancellation cannot be confirmed", asyn it("refuses a long gesture before input when overlay passthrough cannot cover it", async () => { let passthrough = false; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({ result: { value: passthrough ? "clear" : "covered" }, })) as CdpRunner["send"], - }; + }); const action = vi.fn(async () => ({ done: true })); const result = await withClickOverlay( 4, diff --git a/apps/extension/src/tools/__tests__/element-geometry.test.ts b/apps/extension/src/tools/__tests__/element-geometry.test.ts index 2a05905e0..8ad1ebc64 100644 --- a/apps/extension/src/tools/__tests__/element-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/element-geometry.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { nodeContentRegion } from "../element-geometry"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; describe("nodeContentRegion", () => { it("rejects invalid content quads before using the box model fallback", async () => { @@ -12,7 +12,9 @@ describe("nodeContentRegion", () => { throw new Error(`unexpected CDP call ${method}`); }; - await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([ + await expect( + nodeContentRegion(createStubCdpRunner({ send: send as CdpRunner["send"] }), 7, 555), + ).resolves.toEqual([ [ { x: 5, y: 10 }, { x: 25, y: 10 }, @@ -29,9 +31,9 @@ describe("nodeContentRegion", () => { [15, 50, 90, 50, 90, 80, 15, 80], ], }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as CdpRunner["send"], - }; + }); await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([ [ @@ -66,7 +68,7 @@ describe("nodeContentRegion", () => { throw new Error(`unexpected CDP call ${method}`); } }; - const cdp = { send: send as CdpRunner["send"] }; + const cdp = createStubCdpRunner({ send: send as CdpRunner["send"] }); await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([ [ diff --git a/apps/extension/src/tools/__tests__/evaluate.test.ts b/apps/extension/src/tools/__tests__/evaluate.test.ts index 2888a7f1a..57d93b02a 100644 --- a/apps/extension/src/tools/__tests__/evaluate.test.ts +++ b/apps/extension/src/tools/__tests__/evaluate.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; +import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { handleEvaluate } from "../evaluate"; function fakeAgentWindow(ids: number[]) { @@ -54,10 +55,13 @@ function makeFakeCdp(opts: FakeCdpOptions = {}) { if (opts.evaluate) return opts.evaluate(params); return { result: { type: "number", value: 2 } }; } + if (method === "Runtime.releaseObject") { + return {}; + } throw new Error(`unexpected CDP call ${method}`); }; const send = vi.fn(sendImpl); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as ( tabId: number, method: string, @@ -68,7 +72,7 @@ function makeFakeCdp(opts: FakeCdpOptions = {}) { dialogsSince: vi.fn((_tabId: number, cursor: number) => dialogRecords.filter((d) => d.sequence > cursor), ), - }; + }); const windowId = opts.windowId ?? 100; const tabsApi = { get: vi.fn(async (tabId: number) => ({ id: tabId, windowId, active: true }) as chrome.tabs.Tab), @@ -311,4 +315,103 @@ describe("handleEvaluate", () => { data: { reason: "agent_window_scope" }, }); }); + + it("deterministically releases V8 RemoteObjects via Runtime.releaseObject when return_by_value=false", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const abort = new AbortController(); + const fake = makeFakeCdp({ + evaluate: () => { + abort.abort(); + return { + result: { + type: "object", + className: "HTMLDivElement", + objectId: "leaked-node-handle-9", + description: "div#app", + }, + }; + }, + }); + const res = await handleEvaluate( + sm, + { + session_id: "aa11", + expression: "document.getElementById('app')", + return_by_value: false, + }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, + ); + expect(res).toMatchObject({ + code: "cancelled", + data: { effect_state: "unknown" }, + }); + expect(fake.sent).toContainEqual({ + tabId: 4, + method: "Runtime.releaseObject", + params: { objectId: "leaked-node-handle-9" }, + }); + }); + + it("does not resurrect a detached tab or mask the result when releasing RemoteObjects on exit", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const detachListeners = new Set<(source: chrome.debugger.Debuggee, reason: string) => void>(); + const api: CdpDebuggerApi = { + attach: vi.fn(async () => {}), + detach: vi.fn(async () => {}), + sendCommand: vi.fn(async (target: CdpDebuggee, method: string) => { + if (method === "Runtime.evaluate") { + for (const listener of detachListeners) { + listener({ tabId: target.tabId }, "target_closed"); + } + return { + result: { + type: "object", + className: "HTMLDivElement", + objectId: "detached-node-1", + description: "div#gone", + }, + }; + } + return {}; + }), + // biome-ignore lint/suspicious/noExplicitAny: minimal chrome.events.Event shim + onEvent: { addListener: vi.fn(), removeListener: vi.fn() } as any, + onDetach: { + addListener: vi.fn((cb) => detachListeners.add(cb)), + removeListener: vi.fn((cb) => detachListeners.delete(cb)), + // biome-ignore lint/suspicious/noExplicitAny: minimal chrome.events.Event shim + } as any, + }; + const realCdp = new ChromiumCdp(api); + const tabsApi = { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + + const res = await handleEvaluate( + sm, + { + session_id: "aa11", + expression: "document.getElementById('gone')", + return_by_value: false, + }, + { cdp: realCdp, tabsApi }, + ); + expect(res).toMatchObject({ + ok: true, + tab_id: 4, + value: { + type: "object", + className: "HTMLDivElement", + objectId: "detached-node-1", + description: "div#gone", + }, + }); + expect(api.attach).toHaveBeenCalledTimes(1); + expect(realCdp.getAttachmentId(4)).toBeUndefined(); + }); }); diff --git a/apps/extension/src/tools/__tests__/file-transfer-readiness.test.ts b/apps/extension/src/tools/__tests__/file-transfer-readiness.test.ts index b30005da1..212f656dd 100644 --- a/apps/extension/src/tools/__tests__/file-transfer-readiness.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer-readiness.test.ts @@ -1,7 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleDownload } from "../download"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { handleUpload } from "../upload"; function deferred() { @@ -71,10 +72,10 @@ async function fixture(kind: "upload" | "download", hidden = true) { const detach = vi.fn(async (_tabId: number, expected?: string) => { if (attachment === expected) attachment = undefined; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ detach, getAttachmentId: () => attachment, - sendGuarded: async (target, method, params, guard) => { + sendToFrame: async (handle, method, params, guard) => { if ( holdEnable && method === "Emulation.setFocusEmulationEnabled" && @@ -83,9 +84,9 @@ async function fixture(kind: "upload" | "download", hidden = true) { focusQueued = true; await enableFocus.promise; } - if (guard.signal?.aborted) throw new DOMException("aborted", "AbortError"); - guard.onDispatch?.(); - return cdp.send(target.tabId, method, params); + if (guard?.signal?.aborted) throw new DOMException("aborted", "AbortError"); + guard?.onDispatch?.(); + return cdp.send(handle.tabId, method, params); }, onEvent: (_tabId, callback) => { listener = callback; @@ -145,7 +146,7 @@ async function fixture(kind: "upload" | "download", hidden = true) { } return {}; }) as CdpRunner["send"], - }; + }); const tab = { id: 7, windowId: 100, active: true } as chrome.tabs.Tab; const deps = { cdp, @@ -153,7 +154,7 @@ async function fixture(kind: "upload" | "download", hidden = true) { tabsApi: { get: async () => tab, query: async () => [tab] }, }; const finishDownload = () => { - listener?.({ tabId: 7 }, "Page.downloadWillBegin", { + listener?.(rootEventSource(7), "Page.downloadWillBegin", { url: item.url, suggestedFilename: "sample.bin", }); diff --git a/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts b/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts index 78559349c..edc90a0cf 100644 --- a/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer-trigger.test.ts @@ -1,16 +1,16 @@ import { afterEach, expect, it, vi } from "vitest"; import { pointerTransactionDeps } from "../pointer-transaction"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; afterEach(() => vi.useRealTimers()); it("does not send cleanup input to a replacement attachment", async () => { const abort = new AbortController(); let attachment = "first"; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], getAttachmentId: () => attachment, - }; + }); const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, abort.signal, @@ -31,10 +31,10 @@ it("does not send cleanup input to a replacement attachment", async () => { it("bounds a lost button-release reply and detaches without replaying input", async () => { vi.useFakeTimers(); const abort = new AbortController(); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], detach: vi.fn(async () => {}), - }; + }); const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, abort.signal, @@ -52,10 +52,10 @@ it("bounds a lost button-release reply and detaches without replaying input", as it("fences an already pending release rather than sending another one", async () => { const abort = new AbortController(); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], detach: vi.fn(async () => {}), - }; + }); const scope = pointerTransactionDeps( { cdp, tabsApi: { get: vi.fn(), query: vi.fn() } }, abort.signal, @@ -65,6 +65,6 @@ it("fences an already pending release rather than sending another one", async () void scope.deps.cdp.send(7, "Input.dispatchMouseEvent", { type: "mouseReleased" }); abort.abort(); await scope.cleanup(Date.now() + 1_000); - expect(cdp.detach).toHaveBeenCalledWith(7); + expect(cdp.detach).toHaveBeenCalledWith(7, "stub-attachment"); expect(cdp.send).toHaveBeenCalledTimes(2); }); diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index c9c11a71c..ad6c520a4 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -1,4 +1,9 @@ import { describe, expect, it, vi } from "vitest"; +import { + createFrameDocumentInfo, + rootEventSource, + targetEventSource, +} from "@/browser-driver/frame-handles"; import type { InputPassthroughMessage } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; @@ -6,7 +11,7 @@ import { captureBrowserDownload } from "../download-capture"; import { uploadThroughFileDrop } from "../file-drop-transaction"; import { uploadThroughActivatedFileInput } from "../file-input-transaction"; import type { ResolvedActionTarget } from "../interaction"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { handleUpload } from "../upload"; function sessions() { @@ -118,11 +123,13 @@ function popupDownloadFakes() { }; } -function silentCdp(): CdpRunner { - return { - send: vi.fn(async () => ({})) as CdpRunner["send"], +function silentCdp( + send: CdpRunner["send"] = vi.fn(async () => ({})) as CdpRunner["send"], +): CdpRunner { + return createStubCdpRunner({ + send, onEvent: () => ({ dispose: vi.fn() }), - }; + }); } function actionTarget(frameId = "", loaderId = ""): ResolvedActionTarget { @@ -195,21 +202,21 @@ function uploadCdp( (params as { type?: string }).type === "mousePressed" && options.chooser ) { - cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser); + cdpEvent?.(rootEventSource(4), "Page.fileChooserOpened", options.chooser); } return {}; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as CdpRunner["send"], onEvent: (_tabId, handler) => { cdpEvent = handler; return { dispose: vi.fn() }; }, - }; + }); return { calls, emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) => - cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event), + cdpEvent?.(rootEventSource(4), "Page.fileChooserOpened", event), cdp, }; } @@ -252,10 +259,10 @@ function dropCdp( if (method === "Runtime.releaseObjectGroup") return {}; throw new Error(`unexpected drop CDP command ${method}`); }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as CdpRunner["send"], detach, - }; + }); return { calls, cdp, detach }; } @@ -325,14 +332,14 @@ describe("file transfer tools", () => { it("fails before clicking when chooser interception is unavailable", async () => { const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 })); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tabId: number, method: string) => { if (method === "Page.setInterceptFileChooserDialog") { throw new Error("method unavailable"); } return {}; }) as CdpRunner["send"], - }; + }); const result = await uploadThroughActivatedFileInput({ cdp, @@ -530,19 +537,23 @@ describe("file transfer tools", () => { it("dispatches an OOPIF file drop in the child target's local coordinates", async () => { const rootCalls: Array<{ method: string; params?: object }> = []; const childCalls: Array<{ method: string; params?: object }> = []; - const cdp: CdpRunner = { - send: vi.fn(async (_tabId, method, params) => { - rootCalls.push({ method, params }); - if (method === "DOM.scrollIntoViewIfNeeded") return {}; - if (method === "DOM.getBoxModel") { - return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; - } - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + const rootSend = vi.fn(async (_tabId: number, method: string, params?: object) => { + rootCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + throw new Error(`unexpected root command ${method}`); + }); + const cdp: CdpRunner = createStubCdpRunner({ + send: rootSend as CdpRunner["send"], + sendToFrame: vi.fn(async (handle, method, params) => { + if (handle.frameId !== "child") { + return rootSend(handle.tabId, method, params); } - throw new Error(`unexpected root command ${method}`); - }) as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method, params) => { childCalls.push({ method, params }); if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "DOM.getContentQuads") { @@ -565,21 +576,23 @@ describe("file transfer tools", () => { return {}; } throw new Error(`unexpected child command ${method}`); - }) as CdpRunner["sendToTarget"], - getFrameGraph: vi.fn(async () => ({ + }) as CdpRunner["sendToFrame"], + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child-session" }, + processKey: "oopif:child", + isOutOfProcess: true, loaderId: "child-loader", - }, + }), ], })), - }; + }); const result = await uploadThroughFileDrop({ cdp, @@ -760,7 +773,7 @@ describe("file transfer tools", () => { method === "Input.dispatchMouseEvent" && (params as { type?: string }).type === "mousePressed" ) { - cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + cdpEvent?.(rootEventSource(4), "Page.downloadWillBegin", { url: initial.url, suggestedFilename: "result.zip", }); @@ -774,13 +787,13 @@ describe("file transfer tools", () => { } return {}; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as CdpRunner["send"], onEvent: (_tabId, handler) => { cdpEvent = handler; return { dispose: vi.fn() }; }, - }; + }); const result = await handleDownload( manager, @@ -829,15 +842,15 @@ describe("file transfer tools", () => { removeFile: vi.fn(async () => {}), }; let cdpEvent: Parameters>[1] | undefined; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as unknown as CdpRunner["send"], isEventFromFrame: (source, handle) => - source.tabId === handle.tabId && source.sessionId === "expected-child", + source.tabId === handle.tabId && source.targetKey === "expected-child", onEvent: (_tabId, handler) => { cdpEvent = handler; return { dispose: vi.fn() }; }, - }; + }); const unrelated = { id: 17, url: "https://example.test/unrelated.zip", @@ -854,7 +867,7 @@ describe("file transfer tools", () => { browserRelativeDir: "BrowserSkill/tr_1", timeoutMs: 5, trigger: async () => { - cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", { + cdpEvent?.(targetEventSource(4, "other-child"), "Page.downloadWillBegin", { url: unrelated.url, suggestedFilename: unrelated.filename, }); @@ -908,13 +921,13 @@ describe("file transfer tools", () => { removeFile: vi.fn(async () => {}), }; let cdpEvent: Parameters>[1] | undefined; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], onEvent: (_tabId, handler) => { cdpEvent = handler; return { dispose: vi.fn() }; }, - }; + }); let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined; const result = await captureBrowserDownload({ @@ -930,7 +943,7 @@ describe("file transfer tools", () => { resolve(); }); }); - cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + cdpEvent?.(rootEventSource(4), "Page.downloadWillBegin", { url: initial.url, suggestedFilename: initial.filename, }); @@ -984,13 +997,13 @@ describe("file transfer tools", () => { removeFile: vi.fn(async () => {}), }; let cdpEvent: Parameters>[1] | undefined; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], onEvent: (_tabId, handler) => { cdpEvent = handler; return { dispose: vi.fn() }; }, - }; + }); const result = await captureBrowserDownload({ cdp, @@ -1000,7 +1013,7 @@ describe("file transfer tools", () => { maxByteSize: 4, timeoutMs: 1_000, trigger: async () => { - cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + cdpEvent?.(rootEventSource(4), "Page.downloadWillBegin", { url: initial.url, suggestedFilename: initial.filename, }); @@ -1060,13 +1073,13 @@ describe("file transfer tools", () => { removeFile: vi.fn(async () => {}), }; let cdpEvent: Parameters>[1] | undefined; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], onEvent: (_tabId, handler) => { cdpEvent = handler; return { dispose: vi.fn() }; }, - }; + }); const result = await captureBrowserDownload({ cdp, @@ -1075,7 +1088,7 @@ describe("file transfer tools", () => { browserRelativeDir: "BrowserSkill/tr_23", timeoutMs: 80, trigger: async () => { - cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + cdpEvent?.(rootEventSource(4), "Page.downloadWillBegin", { url: initial.url, suggestedFilename: initial.filename, }); @@ -1115,13 +1128,13 @@ describe("file transfer tools", () => { removeFile: vi.fn(async () => {}), }; let cdpEvent: Parameters>[1] | undefined; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], onEvent: (_tabId, handler) => { cdpEvent = handler; return { dispose: vi.fn() }; }, - }; + }); const defaults: number[] = []; const candidate = (id: number) => ({ @@ -1139,7 +1152,7 @@ describe("file transfer tools", () => { browserRelativeDir: "BrowserSkill/tr_ambiguous", timeoutMs: 100, trigger: async () => { - cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + cdpEvent?.(rootEventSource(4), "Page.downloadWillBegin", { url: "https://example.test/same.bin", suggestedFilename: "same.bin", }); @@ -1196,7 +1209,7 @@ describe("file transfer tools", () => { timeout_ms: 1_000, }, { - cdp: { ...silentCdp(), send: send as unknown as CdpRunner["send"] }, + cdp: silentCdp(send as unknown as CdpRunner["send"]), tabsApi: tabsApi(), downloads: fakes.downloads, navigationTargets: fakes.navigationTargets, diff --git a/apps/extension/src/tools/__tests__/fill.browser.test.ts b/apps/extension/src/tools/__tests__/fill.browser.test.ts index 64cf66b52..450a8c628 100644 --- a/apps/extension/src/tools/__tests__/fill.browser.test.ts +++ b/apps/extension/src/tools/__tests__/fill.browser.test.ts @@ -1,11 +1,11 @@ // @vitest-environment node // Opt in with BSK_TEST_CHROME=/path/to/chrome; the suite owns its browser/profile and each test gets a fresh tab. import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import type { CdpFrame } from "@/browser-driver/frame-graph"; +import { type FrameDocumentInfo, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleFill } from "../interaction"; import { handleObserve } from "../observation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, @@ -105,43 +105,47 @@ async function withFillBrowser( const ctx = await manager.start("aa11"); const calls: string[] = []; let afterCommand: ((method: string, params?: object) => void | Promise) | undefined; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: async (_tabId, method, params) => { calls.push(method); const result = await send(method, params, sessionId); await afterCommand?.(method, params); return result as never; }, - }; - cdp.sendToTarget = (_target, method, params) => cdp.send(4, method, params); - cdp.getAttachmentId = () => sessionId; - cdp.getFrameGraph = async () => { - type FrameTree = { frame: { id: string }; childFrames?: FrameTree[] }; - const { frameTree } = await send<{ frameTree: FrameTree }>( - "Page.getFrameTree", - {}, - sessionId, - ); - const frames: CdpFrame[] = []; - const visit = async (tree: FrameTree, parentFrameId?: string) => { - const owner = parentFrameId - ? await send<{ backendNodeId: number }>( - "DOM.getFrameOwner", - { frameId: tree.frame.id }, - sessionId, - ) - : undefined; - frames.push({ - frameId: tree.frame.id, - parentFrameId, - target: { tabId: 4 }, - ownerBackendNodeId: owner?.backendNodeId, - }); - for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); - }; - await visit(frameTree); - return { rootFrameId: frameTree.frame.id, frames }; - }; + sendToFrame: (_handle, method, params) => cdp.send(4, method, params), + getAttachmentId: () => sessionId, + getFrameDocuments: async () => { + type FrameTree = { frame: { id: string; loaderId?: string }; childFrames?: FrameTree[] }; + const { frameTree } = await send<{ frameTree: FrameTree }>( + "Page.getFrameTree", + {}, + sessionId, + ); + const frames: FrameDocumentInfo[] = []; + const visit = async (tree: FrameTree, parentFrameId?: string) => { + const owner = parentFrameId + ? await send<{ backendNodeId: number }>( + "DOM.getFrameOwner", + { frameId: tree.frame.id }, + sessionId, + ) + : undefined; + const loaderId = tree.frame.loaderId ?? ""; + frames.push({ + handle: toFrameDocumentHandle(4, { frameId: tree.frame.id, loaderId }), + frameId: tree.frame.id, + loaderId, + parentFrameId, + ownerBackendNodeId: owner?.backendNodeId, + processKey: "root", + isOutOfProcess: false, + }); + for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); + }; + await visit(frameTree); + return { rootFrameId: frameTree.frame.id, frames }; + }, + }); const deps = { cdp, tabsApi: { diff --git a/apps/extension/src/tools/__tests__/fill.test.ts b/apps/extension/src/tools/__tests__/fill.test.ts index d8061883c..64164d8c2 100644 --- a/apps/extension/src/tools/__tests__/fill.test.ts +++ b/apps/extension/src/tools/__tests__/fill.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { RECORD_PASSWORD_ATTRIBUTE } from "@/lib/sensitive-autocomplete"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { handleFill } from "../interaction"; interface ScriptParams { @@ -135,7 +135,7 @@ async function setup(markup = '') { manager, { session_id: "aa11", ref: "e1", value, clear_before: clearBefore }, { - cdp: { send: send as CdpRunner["send"] }, + cdp: createStubCdpRunner({ send: send as CdpRunner["send"] }), tabsApi: { get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab, query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab], @@ -421,7 +421,7 @@ describe("fill result verification", () => { manager, { session_id: "aa11", ref: "e1", value: "2026-10-14", clear_before: true }, { - cdp: { send: send as CdpRunner["send"] }, + cdp: createStubCdpRunner({ send: send as CdpRunner["send"] }), tabsApi: { get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab, query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab], diff --git a/apps/extension/src/tools/__tests__/focus.browser.test.ts b/apps/extension/src/tools/__tests__/focus.browser.test.ts index 36a1fb411..8ec8f4326 100644 --- a/apps/extension/src/tools/__tests__/focus.browser.test.ts +++ b/apps/extension/src/tools/__tests__/focus.browser.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { handleBlur, handleFocus } from "../interaction"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, @@ -61,14 +61,14 @@ async function withFocusBrowser( const ctx = await manager.start("aa11"); const calls: string[] = []; const afterCommand = { current: (_method: string) => {} }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: async (_tabId, method, params) => { calls.push(method); const result = await send(method, params, sessionId); afterCommand.current(method); return result as never; }, - }; + }); const deps = { cdp, tabsApi: { diff --git a/apps/extension/src/tools/__tests__/focus.test.ts b/apps/extension/src/tools/__tests__/focus.test.ts index b8aa5e484..2f41c8535 100644 --- a/apps/extension/src/tools/__tests__/focus.test.ts +++ b/apps/extension/src/tools/__tests__/focus.test.ts @@ -1,14 +1,14 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleBlur, handleFocus } from "../interaction"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; interface ScriptParams { functionDeclaration: string; } interface Call { - target: CdpTarget; + target: { tabId: number; sessionId?: string }; method: string; } @@ -42,7 +42,11 @@ async function setup(element?: HTMLElement, childSession = false) { }); const focus = vi.fn(() => targetElement.focus()); const release = vi.fn(async () => ({})); - const send = async (target: CdpTarget, method: string, params?: object) => { + const send = async ( + target: { tabId: number; sessionId?: string }, + method: string, + params?: object, + ) => { const call = { target, method }; calls.push(call); try { @@ -71,9 +75,8 @@ async function setup(element?: HTMLElement, childSession = false) { afterCommand(call); } }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (tabId, method, params) => send({ tabId }, method, params) as never, - sendToTarget: send as CdpRunner["sendToTarget"], sendToFrame: (handle, method, params) => send( { @@ -83,23 +86,32 @@ async function setup(element?: HTMLElement, childSession = false) { method, params, ) as never, - getFrameGraph: vi.fn(async () => ({ + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, + { + handle: toFrameDocumentHandle(4, { frameId: "main", loaderId: "" }), + frameId: "main", + loaderId: "", + processKey: "root", + isOutOfProcess: false, + }, ...(childSession ? [ { + handle: toFrameDocumentHandle(4, { frameId: "child", loaderId: "child-loader" }), frameId: "child", + loaderId: "child-loader", parentFrameId: "main", ownerBackendNodeId: 99, - target: { tabId: 4, sessionId: "child-session" }, + processKey: "oopif:child", + isOutOfProcess: true, }, ] : []), ], })), - }; + }); const tabsApi = { get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index 7ab697071..17e067c72 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -1,4 +1,10 @@ import { describe, expect, it, vi } from "vitest"; +import { + createFrameDocumentInfo, + type FrameDocumentHandle, + type FrameNodeHandle, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import { resolveNodeGeometry } from "../frame-geometry"; import { clipPolygon, @@ -11,25 +17,27 @@ import { regionBounds, } from "../geometry"; import { GeometryContext } from "../geometry/frame-context"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; function scrollbarDriver( visible = { width: 185, height: 89 }, frameSize: unknown = { width: 200, height: 100 }, ) { - const send = vi.fn(async (target: { sessionId?: string }, method: string) => { + const isChildHandle = (handle: FrameDocumentHandle | FrameNodeHandle) => + handle.frameId === "child" || handle.frameId === "same-child"; + const send = vi.fn(async (handle: FrameDocumentHandle | FrameNodeHandle, method: string) => { if (method === "Runtime.evaluate") return { result: { value: frameSize } }; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { - clientWidth: target.sessionId ? visible.width : 800, - clientHeight: target.sessionId ? visible.height : 600, + clientWidth: isChildHandle(handle) ? visible.width : 800, + clientHeight: isChildHandle(handle) ? visible.height : 600, }, }; if (method === "DOM.getBoxModel") return { model: { - content: target.sessionId + content: isChildHandle(handle) ? [0, 0, 200, 0, 200, 100, 0, 100] : [100, 100, 500, 100, 500, 300, 100, 300], }, @@ -38,30 +46,35 @@ function scrollbarDriver( return { quads: [[180, 80, 230, 80, 230, 120, 180, 120]] }; throw new Error(`unexpected ${method}`); }); - const cdp: CdpRunner = { - send: (tabId, method) => send({ tabId } as never, method) as never, - sendToTarget: send as CdpRunner["sendToTarget"], - getFrameGraph: async () => ({ + const cdp = createStubCdpRunner({ + send: ((tabId: number, method: string) => + send(toRootFrameHandle(tabId), method)) as CdpRunner["send"], + sendToFrame: send as CdpRunner["sendToFrame"], + getFrameDocuments: async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 }, loaderId: "main-loader" }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main", loaderId: "main-loader" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child" }, + processKey: "oopif:child", + isOutOfProcess: true, loaderId: "child-loader", - }, - { + }), + createFrameDocumentInfo({ + tabId: 4, frameId: "same-child", parentFrameId: "child", ownerBackendNodeId: 20, - target: { tabId: 4, sessionId: "child" }, + processKey: "oopif:child", + isOutOfProcess: true, loaderId: "same-child-loader", - }, + }), ], }), - }; + }); return { cdp, send }; } @@ -116,11 +129,11 @@ describe("frame geometry projection", () => { it("clips an inner OOPIF against an intermediate OOPIF's occupied scrollbar strips", async () => { const { cdp } = scrollbarDriver(); - const outer = cdp.sendToTarget!; - cdp.sendToTarget = async (target, method, params) => { - if (method === "DOM.getBoxModel") + const outer = cdp.sendToFrame; + cdp.sendToFrame = (async (handle, method, params, guard) => { + if (method === "DOM.getBoxModel" && handle.frameId === "child") return { model: { content: [150, 60, 250, 60, 250, 120, 150, 120] } } as never; - if (target.sessionId !== "inner") return outer(target, method, params); + if (handle.frameId !== "inner") return outer(handle, method, params, guard); if (method === "Runtime.evaluate") return { result: { value: { width: 100, height: 60 } } } as never; if (method === "Page.getLayoutMetrics") @@ -128,16 +141,18 @@ describe("frame geometry projection", () => { if (method === "DOM.getContentQuads") return { quads: [[0, 0, 100, 0, 100, 60, 0, 60]] } as never; throw new Error(`unexpected ${method}`); - }; - const graph = await cdp.getFrameGraph!(4); - graph.frames[2] = { + }) as CdpRunner["sendToFrame"]; + const graph = await cdp.getFrameDocuments(4); + graph.frames[2] = createFrameDocumentInfo({ + tabId: 4, frameId: "inner", parentFrameId: "child", ownerBackendNodeId: 20, - target: { tabId: 4, sessionId: "inner" }, + processKey: "oopif:inner", + isOutOfProcess: true, loaderId: "inner-loader", - }; - cdp.getFrameGraph = async () => graph; + }); + cdp.getFrameDocuments = async () => graph; expect( await resolveNodeGeometry(cdp, 4, { tabId: 4, @@ -198,7 +213,7 @@ describe("frame geometry projection", () => { }); it("does not reapply same-target frame transforms around an OOPIF", async () => { - const send = vi.fn(async (_tabId, method, params) => { + const rootSend = vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; } @@ -210,33 +225,38 @@ describe("frame geometry projection", () => { } throw new Error(`unexpected root command ${method}`); }); - const cdp: CdpRunner = { - send: send as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method) => { + const cdp: CdpRunner = createStubCdpRunner({ + send: rootSend as CdpRunner["send"], + sendToFrame: vi.fn(async (handle, method, params) => { + if (handle.frameId !== "oopif") { + return rootSend(handle.tabId, method, params); + } if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } }; if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; } throw new Error(`unexpected child command ${method}`); - }) as CdpRunner["sendToTarget"], - }; + }) as CdpRunner["sendToFrame"], + }); const projection = await new GeometryContext(cdp, 4, { rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "same-process-parent", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4 }, - }, - { + }), + createFrameDocumentInfo({ + tabId: 4, frameId: "oopif", parentFrameId: "same-process-parent", ownerBackendNodeId: 20, - target: { tabId: 4, sessionId: "oopif-session" }, - }, + processKey: "oopif:oopif", + isOutOfProcess: true, + }), ], }).targetProjection("oopif"); @@ -262,17 +282,21 @@ describe("frame geometry projection", () => { }); it("clips live OOPIF geometry at every frame boundary", async () => { - const cdp: CdpRunner = { - send: vi.fn(async (_tabId, method) => { - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; - } - if (method === "DOM.getBoxModel") { - return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + const rootSend = vi.fn(async (_tabId: number, method: string) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + throw new Error(`unexpected root command ${method}`); + }); + const cdp: CdpRunner = createStubCdpRunner({ + send: rootSend as CdpRunner["send"], + sendToFrame: vi.fn(async (handle, method) => { + if (handle.frameId !== "child") { + return rootSend(handle.tabId, method); } - throw new Error(`unexpected root command ${method}`); - }) as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method) => { if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } }; if (method === "Page.getLayoutMetrics") { @@ -282,20 +306,22 @@ describe("frame geometry projection", () => { return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] }; } throw new Error(`unexpected child command ${method}`); - }) as CdpRunner["sendToTarget"], - getFrameGraph: vi.fn(async () => ({ + }) as CdpRunner["sendToFrame"], + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child-session" }, - }, + processKey: "oopif:child", + isOutOfProcess: true, + }), ], })), - }; + }); const geometry = await resolveNodeGeometry(cdp, 4, { tabId: 4, @@ -312,30 +338,36 @@ describe("frame geometry projection", () => { }); it("reuses one frame graph while scrolling and projecting live geometry", async () => { - const getFrameGraph = vi.fn(async () => ({ + const getFrameDocuments = vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child-session" }, - }, + processKey: "oopif:child", + isOutOfProcess: true, + }), ], })); - const cdp: CdpRunner = { - send: vi.fn(async (_tabId, method) => { - if (method === "DOM.scrollIntoViewIfNeeded") return {}; - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; - } - if (method === "DOM.getBoxModel") { - return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + const rootSend = vi.fn(async (_tabId: number, method: string) => { + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + throw new Error(`unexpected root command ${method}`); + }); + const cdp: CdpRunner = createStubCdpRunner({ + send: rootSend as CdpRunner["send"], + sendToFrame: vi.fn(async (handle, method) => { + if (handle.frameId !== "child") { + return rootSend(handle.tabId, method); } - throw new Error(`unexpected root command ${method}`); - }) as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method) => { if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } }; @@ -346,9 +378,9 @@ describe("frame geometry projection", () => { return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; } throw new Error(`unexpected child command ${method}`); - }) as CdpRunner["sendToTarget"], - getFrameGraph, - }; + }) as CdpRunner["sendToFrame"], + getFrameDocuments, + }); await expect( resolveNodeGeometry( @@ -363,11 +395,11 @@ describe("frame geometry projection", () => { { scrollIntoView: true }, ), ).resolves.not.toMatchObject({ code: expect.any(String) }); - expect(getFrameGraph).toHaveBeenCalledOnce(); + expect(getFrameDocuments).toHaveBeenCalledOnce(); }); it("clips live geometry to same-target iframe ancestors", async () => { - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tabId, method) => { if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; @@ -380,19 +412,19 @@ describe("frame geometry projection", () => { } throw new Error(`unexpected ${method}`); }) as CdpRunner["send"], - getFrameGraph: vi.fn(async () => ({ + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4 }, - }, + }), ], })), - }; + }); const geometry = await resolveNodeGeometry(cdp, 4, { tabId: 4, @@ -409,15 +441,18 @@ describe("frame geometry projection", () => { }); it("fails closed when an OOPIF frame graph is unavailable", async () => { - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(), - sendToTarget: vi.fn(async (_target, method) => { + getFrameDocuments: vi.fn(async () => { + throw new Error("frame graph unavailable"); + }), + sendToFrame: vi.fn(async (_handle, method) => { if (method === "DOM.getContentQuads") { return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; } throw new Error(`unexpected ${method}`); - }) as CdpRunner["sendToTarget"], - }; + }) as CdpRunner["sendToFrame"], + }); await expect( resolveNodeGeometry(cdp, 4, { diff --git a/apps/extension/src/tools/__tests__/geometry-context.test.ts b/apps/extension/src/tools/__tests__/geometry-context.test.ts index 3e273f298..f6d713a96 100644 --- a/apps/extension/src/tools/__tests__/geometry-context.test.ts +++ b/apps/extension/src/tools/__tests__/geometry-context.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it, vi } from "vitest"; -import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { + createFrameDocumentInfo, + type FrameDocumentGraph, + rootProcessBoundary, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { resolveNodeGeometry } from "../frame-geometry"; import { rectPolygon } from "../geometry"; import { @@ -9,20 +14,20 @@ import { snapshotViewportRect, } from "../geometry/coordinate-types"; import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; -const target = { tabId: 4 }; -const graph: CdpFrameGraph = { +const process = rootProcessBoundary(4); +const graph: FrameDocumentGraph = { rootFrameId: "main", frames: [ - { frameId: "main", target, loaderId: "main-loader" }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main", loaderId: "main-loader" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target, loaderId: "child-loader", - }, + }), ], }; @@ -42,7 +47,11 @@ function driver() { if (method === "Runtime.releaseObject") return {}; throw new Error(method); }); - return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send }; + return createStubCdpRunner({ + send: send as CdpRunner["send"], + getFrameDocuments: vi.fn(async () => graph), + calls: send, + }); } describe("measurement geometry context", () => { @@ -63,7 +72,7 @@ describe("measurement geometry context", () => { expect( snapshotViewportRect( [160 * scale, 320 * scale, 120 * scale, 40 * scale], - { target }, + { process }, coordinates, )?.rect, ).toEqual({ x: 80, y: 120, width: 120, height: 40 }); @@ -71,7 +80,7 @@ describe("measurement geometry context", () => { it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => { const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 }); - expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({ + expect(snapshotViewportRect([200, 800, 240, 80], { process }, coordinates)?.rect).toEqual({ x: 60, y: 300, width: 120, @@ -100,7 +109,7 @@ describe("measurement geometry context", () => { }), ).toBe(2); expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull(); - expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull(); + expect(snapshotViewportRect([10, 20, 120, 40], { process }, null)).toBeNull(); }); it("shares in-flight frame measurements, but never shares them with the next context", async () => { @@ -110,20 +119,20 @@ describe("measurement geometry context", () => { Array.from({ length: 20 }, () => context.targetProjection("child")), ); expect(results.every((value) => value === results[0])).toBe(true); - expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(cdp.getFrameDocuments).toHaveBeenCalledTimes(1); expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([ "Page.getLayoutMetrics", "DOM.getBoxModel", ]); await new GeometryContext(cdp, 4).targetProjection("child"); - expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2); + expect(cdp.getFrameDocuments).toHaveBeenCalledTimes(2); expect(cdp.calls.mock.calls).toHaveLength(4); }); it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => { const cdp = driver(); const context = new GeometryContext(cdp, 4); - const source = { target, frameId: "child" }; + const source = { process, frameId: "child" }; const projection = await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000, @@ -187,10 +196,10 @@ describe("measurement geometry context", () => { return original(tabId, method, params); }; const context = new GeometryContext(cdp, 4); - context.registerSnapshotOwners(target, new Set([10, 11])); + context.registerSnapshotOwners(process, new Set([10, 11])); const values = await Promise.all( [10, 11].map((id) => - context.snapshotProjection({ target, frameId: String(id) }, id, [], { + context.snapshotProjection({ process, frameId: String(id) }, id, [], { width: 1000, height: 1000, }), @@ -199,7 +208,7 @@ describe("measurement geometry context", () => { expect(values.every((value) => value.status === "available")).toBe(true); expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1); expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1); - await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], { + await new GeometryContext(cdp, 4).snapshotProjection({ process, frameId: "child" }, 10, [], { width: 1000, height: 1000, }); @@ -227,10 +236,10 @@ describe("measurement geometry context", () => { return original(tabId, method, params); }; const context = new GeometryContext(cdp, 4, undefined, controller.signal); - context.registerSnapshotOwners(target, new Set([10, 11])); + context.registerSnapshotOwners(process, new Set([10, 11])); let settled = false; const operation = context - .snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 }) + .snapshotProjection({ process, frameId: "child" }, 10, [], { width: 1000, height: 1000 }) .finally(() => { settled = true; }); @@ -246,11 +255,11 @@ describe("measurement geometry context", () => { it("rejects owner mismatches and invalid numeric bounds", () => { const input = snapshotViewportRect( [30, 50, 120, 40], - { target, frameId: "main" }, + { process, frameId: "main" }, { layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } }, )!; const projection = { - source: { target, frameId: "main" }, + source: { process, frameId: "main" }, geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } }, }; expect(projectSnapshotRect(input, projection)).toEqual({ @@ -260,44 +269,44 @@ describe("measurement geometry context", () => { height: 40, }); expect( - projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }), + projectSnapshotRect(input, { ...projection, source: { process, frameId: "other" } }), ).toBeNull(); expect( projectSnapshotRect(input, { ...projection, - source: { target: { tabId: 5 }, frameId: "main" }, + source: { process: rootProcessBoundary(5), frameId: "main" }, }), ).toBeNull(); expect( snapshotViewportRect( [0, 0, NaN, 40], - { target }, + { process }, { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } }, ), ).toBeNull(); expect( snapshotViewportRect( [0, 0, 10, 40], - { target }, + { process }, { layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } }, ), ).toBeNull(); }); it("does not turn a border quad or a failed owner read into a content projection", async () => { - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({ model: { border: [0, 0, 100, 0, 100, 100, 0, 100] }, })) as CdpRunner["send"], - }; + }); expect( - await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], { + await new GeometryContext(cdp, 4).snapshotProjection({ process, frameId: "child" }, 10, [], { width: 100, height: 100, }), ).toEqual({ status: "unavailable", - source: { target, frameId: "child" }, + source: { process, frameId: "child" }, ownerBackendNodeId: 10, }); }); @@ -305,7 +314,7 @@ describe("measurement geometry context", () => { it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => { const cdp = driver(); const projection = await new GeometryContext(cdp, 4).snapshotProjection( - { target, frameId: "nested" }, + { process, frameId: "nested" }, 10, [rectPolygon({ x: 60, y: 620, w: 100, h: 100 })], { width: 1000, height: 1000 }, @@ -313,7 +322,7 @@ describe("measurement geometry context", () => { if (projection.status !== "available") throw new Error("expected available projection"); const input = snapshotViewportRect( [0, 0, 300, 200], - { target, frameId: "nested" }, + { process, frameId: "nested" }, { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } }, ); expect(projectSnapshotRect(input!, projection.projection)).toEqual({ @@ -326,18 +335,18 @@ describe("measurement geometry context", () => { it("rejects malformed frame ancestry without looping", async () => { const cdp = driver(); - const cycle: CdpFrameGraph = { + const cycle: FrameDocumentGraph = { rootFrameId: "a", frames: [ - { frameId: "a", parentFrameId: "b", target }, - { frameId: "b", parentFrameId: "a", target }, + createFrameDocumentInfo({ tabId: 4, frameId: "a", parentFrameId: "b" }), + createFrameDocumentInfo({ tabId: 4, frameId: "b", parentFrameId: "a" }), ], }; expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull(); expect(cdp.calls).not.toHaveBeenCalled(); - const orphan: CdpFrameGraph = { + const orphan: FrameDocumentGraph = { rootFrameId: "a", - frames: [{ frameId: "a", parentFrameId: "missing", target }], + frames: [createFrameDocumentInfo({ tabId: 4, frameId: "a", parentFrameId: "missing" })], }; expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull(); }); @@ -372,13 +381,13 @@ describe("measurement geometry context", () => { }); const controller = new AbortController(); const context = new GeometryContext( - { send: send as CdpRunner["send"] }, + createStubCdpRunner({ send: send as CdpRunner["send"] }), 4, undefined, controller.signal, ); const tasks = Array.from({ length: 8 }, (_, index) => - context.viewport({ tabId: 4, sessionId: String(index) }), + context.viewport(toFrameDocumentHandle(4, { frameId: `frame-${index}` })), ); const settled = Promise.allSettled(tasks); expect(send).toHaveBeenCalledTimes(4); @@ -401,7 +410,7 @@ describe("measurement geometry context", () => { }); const context = new GeometryContext(cdp, 4, undefined, controller.signal); await expect( - context.snapshotProjection({ target, frameId: "child" }, 10, [], { + context.snapshotProjection({ process, frameId: "child" }, 10, [], { width: 1000, height: 1000, }), @@ -437,7 +446,7 @@ describe("measurement geometry context", () => { { scrollIntoView: true }, ), ).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } }); - expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(cdp.getFrameDocuments).toHaveBeenCalledTimes(1); expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength( 2, // Target layout flush plus the iframe owner, both measured after scrolling. ); @@ -459,10 +468,10 @@ describe("measurement geometry context", () => { const cdp = driver(); cdp.calls.mockRejectedValueOnce(new Error("unavailable")); const context = new GeometryContext(cdp, 4); - await expect(context.viewport(target)).rejects.toThrow("unavailable"); - await expect(context.viewport(target)).rejects.toThrow("unavailable"); + await expect(context.viewport(process)).rejects.toThrow("unavailable"); + await expect(context.viewport(process)).rejects.toThrow("unavailable"); expect(cdp.calls).toHaveBeenCalledTimes(1); - expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({ + expect(await new GeometryContext(cdp, 4).viewport(process)).toEqual({ width: 1000, height: 1000, }); diff --git a/apps/extension/src/tools/__tests__/human-loop.test.ts b/apps/extension/src/tools/__tests__/human-loop.test.ts index 4283b47d8..8678cf7f4 100644 --- a/apps/extension/src/tools/__tests__/human-loop.test.ts +++ b/apps/extension/src/tools/__tests__/human-loop.test.ts @@ -3,11 +3,13 @@ import { createRequire } from "node:module"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { InteractionPreferenceStore } from "@/lib/interaction-preferences"; import { SessionManager } from "@/session-manager/manager"; import { RefStore } from "@/session-manager/ref-store"; import type { RequestHelpParams } from "@/transport/types"; import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; function chromeEvent unknown>() { const listeners = new Set(); @@ -1014,7 +1016,7 @@ describe("handleRequestHelp", () => { findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null), } as unknown as SessionManager; const deps = baseDeps({ - cdp: { + cdp: createStubCdpRunner({ send: vi.fn(async (_tabId, method) => { if (method === "DOM.getDocument") return { root: { nodeId: 1 } }; if (method === "DOM.scrollIntoViewIfNeeded") return {}; @@ -1026,7 +1028,7 @@ describe("handleRequestHelp", () => { } throw new Error(`unexpected ${method}`); }), - } as unknown as RequestHelpDeps["cdp"], + }), }); const res = await handleRequestHelp( mgr, @@ -1086,34 +1088,48 @@ describe("handleRequestHelp", () => { } throw new Error(`unexpected root ${method}`); }); - const sendToTarget = vi.fn(async (_target, method) => { - if (method === "DOM.scrollIntoViewIfNeeded") return {}; - if (method === "DOM.getContentQuads") { - return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; - } - if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } }; - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; - } - throw new Error(`unexpected child ${method}`); - }); + const sendToFrame = vi.fn( + async (handle: { tabId: number; frameId: string }, method: string) => { + if (handle.frameId !== "child") return send(handle.tabId, method); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; + } + if (method === "Runtime.evaluate") + return { result: { value: { width: 200, height: 100 } } }; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + throw new Error(`unexpected child ${method}`); + }, + ); + const childHandle = toFrameDocumentHandle(5, { frameId: "child", loaderId: "child-loader" }); const deps = baseDeps({ - cdp: { + cdp: createStubCdpRunner({ send, - sendToTarget, - getFrameGraph: vi.fn(async () => ({ + sendToFrame: sendToFrame as CdpRunner["sendToFrame"], + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 5 } }, { + handle: toFrameDocumentHandle(5, { frameId: "main", loaderId: "main-loader" }), + frameId: "main", + loaderId: "main-loader", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: childHandle, frameId: "child", + loaderId: "child-loader", parentFrameId: "main", ownerBackendNodeId: 99, - target: { tabId: 5, sessionId: "child-session" }, + processKey: "oopif:child", + isOutOfProcess: true, }, ], })), - } as unknown as NonNullable, + }), }); const res = await handleRequestHelp( @@ -1125,10 +1141,12 @@ describe("handleRequestHelp", () => { const sentMsg = (deps.sendToTab as ReturnType).mock.calls[0][1]; expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]); expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] }); - expect(sendToTarget).toHaveBeenCalledWith( - { tabId: 5, sessionId: "child-session" }, + expect(sendToFrame).toHaveBeenCalledWith( + expect.objectContaining(childHandle), "DOM.getContentQuads", - { backendNodeId: 42 }, + { + backendNodeId: 42, + }, ); }); diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index a92aa1d88..d126a22f4 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -1,11 +1,12 @@ import { describe, expect, it, vi } from "vitest"; +import { rootEventSource, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { INPUT_PASSTHROUGH, INPUT_PASSTHROUGH_TTL_MS, type InputPassthroughMessage, } from "@/lib/input-passthrough-bridge"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { withInputReady } from "../input-readiness"; import { handleBlur, @@ -67,14 +68,14 @@ function makeFakeCdp( return h(params); }; const send = vi.fn(sendImpl); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as ( tabId: number, method: string, params?: object, ) => Promise, trackSessionTab: vi.fn(), - }; + }); const tabsApi = { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, @@ -250,22 +251,33 @@ describe("handleClick", () => { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] }, }), }); - fake.cdp.getFrameGraph = vi.fn(async () => ({ + fake.cdp.getFrameDocuments = vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 }, loaderId: "main-loader" }, { + handle: toFrameDocumentHandle(4, { frameId: "main", loaderId: "main-loader" }), + frameId: "main", + loaderId: "main-loader", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: toFrameDocumentHandle(4, { frameId: "child-frame", loaderId: "child-loader" }), frameId: "child-frame", parentFrameId: "main", ownerBackendNodeId: 99, - target: { tabId: 4, sessionId: "child-session" }, loaderId: "child-loader", + processKey: "oopif:child-frame", + isOutOfProcess: true, }, ], })); - const targetCalls: Array<{ sessionId?: string; method: string }> = []; - fake.cdp.sendToTarget = vi.fn(async (target, method) => { - targetCalls.push({ sessionId: target.sessionId, method }); + const targetCalls: Array<{ frameId?: string; method: string }> = []; + fake.cdp.sendToFrame = vi.fn(async (handle, method, params) => { + if (handle.frameId !== "child-frame") { + return fake.cdp.send(handle.tabId, method, params); + } + targetCalls.push({ frameId: handle.frameId, method }); if (method === "Accessibility.getPartialAXTree") return { nodes: [] }; if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "DOM.getContentQuads") { @@ -276,7 +288,7 @@ describe("handleClick", () => { return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; } throw new Error(`unexpected child CDP call ${method}`); - }) as CdpRunner["sendToTarget"]; + }) as CdpRunner["sendToFrame"]; const res = await handleClick( sm, @@ -288,11 +300,11 @@ describe("handleClick", () => { // The iframe content box starts at (204, 306) after its border and is scaled 2x. expect(res).toMatchObject({ x: 324, y: 386 }); expect(targetCalls).toEqual([ - { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, - { sessionId: "child-session", method: "DOM.getBoxModel" }, - { sessionId: "child-session", method: "DOM.getContentQuads" }, - { sessionId: "child-session", method: "Page.getLayoutMetrics" }, - { sessionId: "child-session", method: "Runtime.evaluate" }, + { frameId: "child-frame", method: "DOM.scrollIntoViewIfNeeded" }, + { frameId: "child-frame", method: "DOM.getBoxModel" }, + { frameId: "child-frame", method: "DOM.getContentQuads" }, + { frameId: "child-frame", method: "Page.getLayoutMetrics" }, + { frameId: "child-frame", method: "Runtime.evaluate" }, ]); expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3); }); @@ -1437,7 +1449,7 @@ describe("click input readiness", () => { if ((params as { type?: string }).type === "mousePressed") { unselectedDuringPress.push(`${currentTool}:${activeTabId}`); if (currentTool === "upload") { - cdpListener?.({ tabId: 4 }, "Page.fileChooserOpened", { + cdpListener?.(rootEventSource(4), "Page.fileChooserOpened", { mode: "selectSingle", backendNodeId: 1234, }); @@ -1461,7 +1473,7 @@ describe("click input readiness", () => { canResume: false, startTime: new Date().toISOString(), } as chrome.downloads.DownloadItem; - cdpListener?.({ tabId: 4 }, "Page.downloadWillBegin", { + cdpListener?.(rootEventSource(4), "Page.downloadWillBegin", { url: "https://app.test/report.csv", suggestedFilename: "report.csv", }); @@ -1477,9 +1489,7 @@ describe("click input readiness", () => { // and only turns "hidden" once unselected to tab 1! () => (activeTabId === 4 || leaseOverrideIntact ? "visible" : "hidden"), ); - let cdpListener: - | ((source: { tabId?: number }, method: string, params: unknown) => void) - | undefined; + let cdpListener: Parameters[1] | undefined; fake.cdp.onEvent = (_tabId, fn) => { cdpListener = fn; return { dispose: () => {} }; @@ -1701,28 +1711,39 @@ describe("handleFocus and handleBlur", () => { const fake = makeFakeCdp({ "DOM.scrollIntoViewIfNeeded": () => ({}), }); - fake.cdp.getFrameGraph = vi.fn(async () => ({ + fake.cdp.getFrameDocuments = vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 }, loaderId: "main-loader" }, { + handle: toFrameDocumentHandle(4, { frameId: "main", loaderId: "main-loader" }), + frameId: "main", + loaderId: "main-loader", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: toFrameDocumentHandle(4, { frameId: "child-frame", loaderId: "child-loader" }), frameId: "child-frame", parentFrameId: "main", ownerBackendNodeId: 99, - target: { tabId: 4, sessionId: "child-session" }, loaderId: "child-loader", + processKey: "oopif:child-frame", + isOutOfProcess: true, }, ], })); - const targetCalls: Array<{ sessionId?: string; method: string }> = []; - fake.cdp.sendToTarget = vi.fn(async (target, method) => { - targetCalls.push({ sessionId: target.sessionId, method }); + const targetCalls: Array<{ frameId?: string; method: string }> = []; + fake.cdp.sendToFrame = vi.fn(async (handle, method, params) => { + if (handle.frameId !== "child-frame") { + return fake.cdp.send(handle.tabId, method, params); + } + targetCalls.push({ frameId: handle.frameId, method }); if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "DOM.focus" || method === "Runtime.releaseObject") return {}; if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } }; if (method === "Runtime.callFunctionOn") return { result: { value: true } }; throw new Error(`unexpected child CDP call ${method}`); - }) as CdpRunner["sendToTarget"]; + }) as CdpRunner["sendToFrame"]; const res = await handleFocus( sm, @@ -1733,11 +1754,11 @@ describe("handleFocus and handleBlur", () => { if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.focused).toBe(true); expect(targetCalls).toEqual([ - { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, - { sessionId: "child-session", method: "DOM.resolveNode" }, - { sessionId: "child-session", method: "DOM.focus" }, - { sessionId: "child-session", method: "Runtime.callFunctionOn" }, - { sessionId: "child-session", method: "Runtime.releaseObject" }, + { frameId: "child-frame", method: "DOM.scrollIntoViewIfNeeded" }, + { frameId: "child-frame", method: "DOM.resolveNode" }, + { frameId: "child-frame", method: "DOM.focus" }, + { frameId: "child-frame", method: "Runtime.callFunctionOn" }, + { frameId: "child-frame", method: "Runtime.releaseObject" }, ]); }); @@ -2014,15 +2035,24 @@ describe("handleFill", () => { "Runtime.callFunctionOn": successfulFillScript, "Runtime.releaseObject": () => ({}), }); - fake.cdp.getFrameGraph = async () => ({ + fake.cdp.getFrameDocuments = async () => ({ rootFrameId: "root", frames: [ - { frameId: "root", target: { tabId: 4 } }, { + handle: toFrameDocumentHandle(4, { frameId: "root", loaderId: "" }), + frameId: "root", + loaderId: "", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: toFrameDocumentHandle(4, { frameId: "subframe-1", loaderId: "" }), frameId: "subframe-1", + loaderId: "", parentFrameId: "root", ownerBackendNodeId: 900, - target: { tabId: 4 }, + processKey: "root", + isOutOfProcess: false, }, ], }); diff --git a/apps/extension/src/tools/__tests__/navigation-history.browser.test.ts b/apps/extension/src/tools/__tests__/navigation-history.browser.test.ts index e4184a931..47ce0c1bd 100644 --- a/apps/extension/src/tools/__tests__/navigation-history.browser.test.ts +++ b/apps/extension/src/tools/__tests__/navigation-history.browser.test.ts @@ -2,9 +2,10 @@ // Opt in with BSK_TEST_CHROME. The shared harness owns its browser and profile. import { createServer } from "node:http"; import { describe, expect, it } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleNavigate, handleNavigateBack, handleNavigateForward } from "../navigation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, @@ -49,7 +50,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser history navigation", if (event.method === "Page.frameNavigated" && params.type === "BackForwardCacheRestore") restores.push(new URL(params.frame?.url ?? origin).pathname); for (const listener of [...listeners]) - listener({ tabId: 4 }, event.method, event.params); + listener(rootEventSource(4), event.method, event.params); }, }, async (send: Send) => { @@ -76,14 +77,14 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser history navigation", }, }); await manager.start("history"); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId: number, method: string, params?: object) => send(method, params, sessionId), onEvent: (_tabId, listener) => { listeners.add(listener); return { dispose: () => void listeners.delete(listener) }; }, - }; + }); const tab = async () => ({ id: 4, diff --git a/apps/extension/src/tools/__tests__/navigation-recovery.test.ts b/apps/extension/src/tools/__tests__/navigation-recovery.test.ts index 591ce70bb..4a57244be 100644 --- a/apps/extension/src/tools/__tests__/navigation-recovery.test.ts +++ b/apps/extension/src/tools/__tests__/navigation-recovery.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { type BrowserNavigationApi, navigateWithBrowserApi } from "../browser-navigation"; import { @@ -7,7 +8,7 @@ import { handleNavigateForward, handleReload, } from "../navigation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; const denied = "Cannot access a chrome-extension:// URL of different extension"; const url = "https://example.test/recovered"; @@ -52,7 +53,7 @@ async function fixture() { } if (method === "Runtime.evaluate") return { result: { value: "complete" } }; if (method === "Page.setLifecycleEventsEnabled" && state.emitIdle) { - cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", { + cdpEvents.fire(rootEventSource(4), "Page.lifecycleEvent", { frameId: "main", loaderId: "new-loader", name: "networkIdle", @@ -61,13 +62,13 @@ async function fixture() { if (method === "Page.navigate") return { frameId: "main", loaderId: "new-loader" }; return {}; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as CdpRunner["send"], onEvent: (_tabId, listener) => { cdpEvents.addListener(listener); return { dispose: () => cdpEvents.removeListener(listener) }; }, - }; + }); const main = { tabId: 4, frameId: 0, documentId: "new-document", url }; const navigate = async () => { state.blocked = state.destinationBlocked; @@ -656,14 +657,14 @@ it("keeps native succession active throughout network-idle waiting", async () => f.send.mock.calls.filter(([, method]) => method === "Page.setLifecycleEventsEnabled"), ).toHaveLength(2), ); - f.cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", { + f.cdpEvents.fire(rootEventSource(4), "Page.lifecycleEvent", { frameId: "main", loaderId: "first", name: "networkIdle", }); await Promise.resolve(); expect(settled).toBe(false); - f.cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", { + f.cdpEvents.fire(rootEventSource(4), "Page.lifecycleEvent", { frameId: "main", loaderId: "second", name: "networkIdle", @@ -848,7 +849,7 @@ it("keeps waiting for load when a recovered same-document page is still loading" }); await vi.waitFor(() => expect(f.cdpEvents.listeners.size).toBe(1)); expect(settled).toBe(false); - f.cdpEvents.fire({ tabId: 4 }, "Page.lifecycleEvent", { + f.cdpEvents.fire(rootEventSource(4), "Page.lifecycleEvent", { frameId: "main", loaderId: "new-loader", name: "load", diff --git a/apps/extension/src/tools/__tests__/navigation-same-document.browser.test.ts b/apps/extension/src/tools/__tests__/navigation-same-document.browser.test.ts index 9ec5d103f..af4d53f0d 100644 --- a/apps/extension/src/tools/__tests__/navigation-same-document.browser.test.ts +++ b/apps/extension/src/tools/__tests__/navigation-same-document.browser.test.ts @@ -3,10 +3,11 @@ import { createServer } from "node:http"; import { describe, expect, it, vi } from "vitest"; import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleNavigate, handleNavigateBack, handleNavigateForward } from "../navigation"; import { handleRead } from "../read"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, @@ -125,7 +126,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser same-document naviga }, { onDocumentChanged: (tabId) => context.refStore.invalidateTab(tabId) }, ); - const graph = vi.spyOn(cdp, "getFrameGraph"); + const graph = vi.spyOn(cdp, "getFrameDocuments"); const tab = async () => { const { targetInfo } = await send<{ targetInfo: { url: string } }>( "Target.getTargetInfo", @@ -214,7 +215,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser same-document naviga onEvent: (event: CdpEvent) => { if (sessionId && event.sessionId === sessionId) for (const listener of [...listeners]) - listener({ tabId: 4 }, event.method, event.params); + listener(rootEventSource(4), event.method, event.params); }, }, async (send: Send) => { @@ -248,7 +249,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser same-document naviga }, }); await manager.start("navigation"); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId: number, method: string, params?: object) => send(method, params, sessionId), onEvent: (_tabId, listener) => { @@ -259,7 +260,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser same-document naviga }, }; }, - }; + }); const tab = async () => ({ id: 4, diff --git a/apps/extension/src/tools/__tests__/navigation-same-document.test.ts b/apps/extension/src/tools/__tests__/navigation-same-document.test.ts index 59f79325b..880e43be9 100644 --- a/apps/extension/src/tools/__tests__/navigation-same-document.test.ts +++ b/apps/extension/src/tools/__tests__/navigation-same-document.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import type { WaitUntil } from "@/transport/types"; import { @@ -7,7 +8,7 @@ import { handleNavigateForward, handleReload, } from "../navigation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; const destination = "https://example.test/page#two"; type Listener = Parameters>[1]; @@ -31,7 +32,7 @@ async function fixture() { action: () => {}, }; const fire = (method: string, params: object, tabId = 4) => { - for (const listener of [...listeners]) listener({ tabId }, method, params); + for (const listener of [...listeners]) listener(rootEventSource(tabId), method, params); }; const sameDocument = (url = destination, frameId = "main") => fire("Page.navigatedWithinDocument", { frameId, url, navigationType: "fragment" }); @@ -58,13 +59,13 @@ async function fixture() { for (const name of state.replay) lifecycle(name); return {}; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as CdpRunner["send"], onEvent: (_tabId, listener) => { listeners.add(listener); return { dispose: () => listeners.delete(listener) }; }, - }; + }); const tab = { id: 4, windowId: 100, active: true, url: destination } as chrome.tabs.Tab; const run = (operation: Operation, wait_until: WaitUntil, signal?: AbortSignal) => { const deps = { diff --git a/apps/extension/src/tools/__tests__/navigation-wait.browser.test.ts b/apps/extension/src/tools/__tests__/navigation-wait.browser.test.ts index 958cb4c39..a871c1bab 100644 --- a/apps/extension/src/tools/__tests__/navigation-wait.browser.test.ts +++ b/apps/extension/src/tools/__tests__/navigation-wait.browser.test.ts @@ -2,8 +2,9 @@ // Opt in with BSK_TEST_CHROME. The shared harness owns its browser and profile. import { createServer } from "node:http"; import { describe, expect, it } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { handleWaitForNavigation } from "../waits"; type Send = >( @@ -81,7 +82,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser wait_for_navigation" pendingUrl = undefined; } for (const listener of [...listeners]) - listener({ tabId: 4 }, event.method, event.params); + listener(rootEventSource(4), event.method, event.params); }, }, async (send: Send) => { @@ -120,14 +121,14 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser wait_for_navigation" }, }); await manager.start("wait-nav"); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId: number, method: string, params?: object) => send(method, params, sessionId), onEvent: (_tabId, listener) => { listeners.add(listener); return { dispose: () => void listeners.delete(listener) }; }, - }; + }); const tab = async () => ({ id: 4, diff --git a/apps/extension/src/tools/__tests__/navigation.test.ts b/apps/extension/src/tools/__tests__/navigation.test.ts index 8838f4c4c..9e6c59264 100644 --- a/apps/extension/src/tools/__tests__/navigation.test.ts +++ b/apps/extension/src/tools/__tests__/navigation.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; +import { rootEventSource, type TabEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { eventLoaderIsRelevant, handleNavigate, @@ -27,7 +28,7 @@ function fakeAgentWindow(ids: number[]) { } interface EventListener { - (source: chrome.debugger.Debuggee, method: string, params: unknown): void; + (source: TabEventSource, method: string, params: unknown): void; } /** @@ -76,7 +77,7 @@ function makeFakeCdp(opts?: { opts.fireLifecycleDuringNavigateLoaderId ?? opts.navigateLoaderId ?? "loader-after", }; for (const listener of [...events]) { - listener({ tabId: 4 }, "Page.lifecycleEvent", payload); + listener(rootEventSource(4), "Page.lifecycleEvent", payload); } } return { @@ -92,7 +93,7 @@ function makeFakeCdp(opts?: { loaderId: opts.navigateLoaderId ?? "loader-after", }; for (const listener of [...events]) { - listener({ tabId: 4 }, "Page.lifecycleEvent", payload); + listener(rootEventSource(4), "Page.lifecycleEvent", payload); } } return {}; @@ -126,7 +127,7 @@ function makeFakeCdp(opts?: { throw new Error(`unexpected CDP call ${method}`); }; const send = vi.fn(sendImpl); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as ( tabId: number, method: string, @@ -142,7 +143,7 @@ function makeFakeCdp(opts?: { }, }; }), - }; + }); const tabsApi = { get: vi.fn( async (tabId: number) => @@ -172,7 +173,8 @@ function makeFakeCdp(opts?: { loaderId = opts?.navigateLoaderId ?? "loader-after", ) { const payload = { name, frameId, loaderId }; - for (const listener of [...events]) listener({ tabId: 4 }, "Page.lifecycleEvent", payload); + for (const listener of [...events]) + listener(rootEventSource(4), "Page.lifecycleEvent", payload); }, fireFrameNavigated( frameId = opts?.navigateFrameId ?? "frame-1", @@ -183,7 +185,8 @@ function makeFakeCdp(opts?: { frame: { id: frameId, loaderId, url: opts?.finalUrl ?? "https://example.com/" }, ...(type ? { type } : {}), }; - for (const listener of [...events]) listener({ tabId: 4 }, "Page.frameNavigated", payload); + for (const listener of [...events]) + listener(rootEventSource(4), "Page.frameNavigated", payload); }, listeners: events, }; @@ -270,7 +273,9 @@ describe("handleNavigate", () => { fake.cdp.send = vi.fn(async (tabId, method, params) => { if (method === "Page.navigate") { for (const listener of [...fake.listeners]) - listener({ tabId }, "Page.javascriptDialogOpening", { type: "beforeunload" }); + listener(rootEventSource(tabId), "Page.javascriptDialogOpening", { + type: "beforeunload", + }); return { frameId: "frame-1", errorText: "net::ERR_ABORTED" }; } return original(tabId, method, params); @@ -289,7 +294,7 @@ describe("handleNavigate", () => { ); expect(finished).toBe(false); for (const listener of [...fake.listeners]) - listener({ tabId: 4 }, "Page.javascriptDialogClosed", { result: false }); + listener(rootEventSource(4), "Page.javascriptDialogClosed", { result: false }); expect(await navigation).toMatchObject({ code: "cancelled", message: expect.stringContaining("page was kept"), @@ -770,7 +775,7 @@ it.each([ await vi.waitFor(() => expect(fake.sent.some((s) => s.method === "Page.navigate")).toBe(true)); fake.fireFrameNavigated(); for (const listener of [...fake.listeners]) - listener({ tabId: 4 }, "Page.frameRequestedNavigation", { + listener(rootEventSource(4), "Page.frameRequestedNavigation", { frameId: "frame-1", disposition: "currentTab", }); @@ -796,7 +801,7 @@ it("does not overwrite a successor observed before Page.navigate resolves with i if (method === "Page.navigate") { fake.fireFrameNavigated(); for (const listener of [...fake.listeners]) - listener({ tabId: 4 }, "Page.frameRequestedNavigation", { + listener(rootEventSource(4), "Page.frameRequestedNavigation", { frameId: "frame-1", disposition: "currentTab", }); @@ -832,7 +837,7 @@ it.each([ return send(tabId, method, params); }) as CdpRunner["send"]; const fire = (method: string, params: Record) => { - for (const listener of [...fake.listeners]) listener({ tabId: 4 }, method, params); + for (const listener of [...fake.listeners]) listener(rootEventSource(4), method, params); }; let settled = false; const work = handleNavigate( @@ -884,7 +889,7 @@ it.each([ return send(tabId, method, params); }) as CdpRunner["send"]; const fire = (method: string, params: Record) => { - for (const listener of [...fake.listeners]) listener({ tabId: 4 }, method, params); + for (const listener of [...fake.listeners]) listener(rootEventSource(4), method, params); }; let settled = false; const work = handleNavigate( @@ -937,7 +942,7 @@ it.each([ return send(tabId, method, params); }) as CdpRunner["send"]; const fire = (method: string, params: Record) => { - for (const listener of [...fake.listeners]) listener({ tabId: 4 }, method, params); + for (const listener of [...fake.listeners]) listener(rootEventSource(4), method, params); }; const begin = (loaderId: string) => { fire("Page.frameRequestedNavigation", { frameId: "frame-1", disposition: "currentTab" }); diff --git a/apps/extension/src/tools/__tests__/observation-layers.browser.test.ts b/apps/extension/src/tools/__tests__/observation-layers.browser.test.ts index 1eb3be7b4..a3a301e12 100644 --- a/apps/extension/src/tools/__tests__/observation-layers.browser.test.ts +++ b/apps/extension/src/tools/__tests__/observation-layers.browser.test.ts @@ -2,18 +2,22 @@ // Reuses the CI browser opt-in; every run owns its Chrome profile. import { createServer } from "node:http"; import { describe, expect, it } from "vitest"; -import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentGraph, + type FrameDocumentInfo, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleClick } from "../interaction"; import { handleObserve } from "../observation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, params?: object, sessionId?: string, ) => Promise; -type FrameTree = { frame: { id: string }; childFrames?: FrameTree[] }; +type FrameTree = { frame: { id: string; loaderId?: string }; childFrames?: FrameTree[] }; const cases = [ { name: "sidebar 64%", width: 64 }, @@ -109,32 +113,36 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser observation layers", ); expect(metrics.cssVisualViewport.zoom).toBeCloseTo(zoom, 4); expect(await evaluate("devicePixelRatio")).toBeCloseTo(deviceScale * zoom, 4); - const graph = async (): Promise => { + const graph = async (): Promise => { const { frameTree } = await local<{ frameTree: FrameTree }>("Page.getFrameTree"); - const frames: CdpFrame[] = []; + const frames: FrameDocumentInfo[] = []; const visit = async (tree: FrameTree, parentFrameId?: string) => { const owner = parentFrameId ? await local<{ backendNodeId: number }>("DOM.getFrameOwner", { frameId: tree.frame.id, }) : undefined; + const loaderId = tree.frame.loaderId ?? ""; frames.push({ + handle: toFrameDocumentHandle(4, { frameId: tree.frame.id, loaderId }), frameId: tree.frame.id, + loaderId, parentFrameId, - target: { tabId: 4 }, ownerBackendNodeId: owner?.backendNodeId, + processKey: "root", + isOutOfProcess: false, }); for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); }; await visit(frameTree); return { rootFrameId: frameTree.frame.id, frames }; }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId, method, params) => local(method, params), - sendToTarget: (_target, method, params) => local(method, params), + sendToFrame: (_handle, method, params) => local(method, params), getAttachmentId: () => sessionId, - getFrameGraph: graph, - }; + getFrameDocuments: graph, + }); const manager = new SessionManager({ agentWindow: { create: async () => ({ windowId: 100, initialTabIds: [] }), diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index af804e0a9..683262a18 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1,9 +1,10 @@ import { renderVom } from "@browser-skill/vom"; import { describe, expect, it, vi } from "vitest"; +import { createFrameDocumentInfo, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-suppress-bridge"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import type { BuildVomSceneOptions, VomFrameDocument } from "../observation"; import { buildFrameVomScene, @@ -23,6 +24,30 @@ interface SceneFixture extends CapturedSceneInput { documents?: VomFrameDocument[]; } +function frameDoc( + input: Omit & { + tabId?: number; + loaderId?: string; + processKey?: string; + isOutOfProcess?: boolean; + }, +): VomFrameDocument { + const { + tabId = 7, + loaderId = "", + processKey = "root", + isOutOfProcess = processKey !== "root", + ...rest + } = input; + return { + ...rest, + loaderId, + processKey, + isOutOfProcess, + handle: toFrameDocumentHandle(tabId, { frameId: rest.frameId, loaderId }), + }; +} + // Only supplies the common root fixture; child documents declare their own ownership. function buildTestScene( axNodes: CdpAxNode[], @@ -32,14 +57,13 @@ function buildTestScene( const rootFrameId = captured.rootFrameId ?? "root"; return buildFrameVomScene( [ - { + frameDoc({ frameId: rootFrameId, contextScopeId: rootFrameId, - target: { tabId: 7 }, url: options.pageUrl, domNodes: captured.nodes, axNodes, - }, + }), ...(captured.documents ?? []), ], captured, @@ -102,7 +126,13 @@ function makeFakeCdp(handlers: Record unknown>) { if (!handler) throw new Error(`unexpected CDP call ${method}`); return handler(params); }); - return { cdp: { send, trackSessionTab: vi.fn() } as unknown as CdpRunner, sent }; + return { + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], + trackSessionTab: vi.fn(), + }), + sent, + }; } describe("stripDataUrlPrefix", () => { @@ -617,54 +647,53 @@ describe("buildFrameVomScene", () => { }); const result = buildFrameVomScene( [ - { + frameDoc({ frameId: "main", contextScopeId: "main", - target: { tabId: 7 }, domNodes: mainNodes, axNodes: [ axNode("root", 1, "RootWebArea"), axNode("frame-1", 10, "Iframe", "root", "First"), axNode("frame-2", 20, "Iframe", "root", "Second"), ], - }, - { + }), + frameDoc({ frameId: "first", contextScopeId: "first", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 7, sessionId: "session-first" }, + processKey: "oopif:first", domNodes: firstNodes, axNodes: [ axNode("root", 100, "RootWebArea"), axNode("button", 101, "button", "root", "First action"), ], - }, - { + }), + frameDoc({ frameId: "second", contextScopeId: "second", parentFrameId: "main", ownerBackendNodeId: 20, - target: { tabId: 7, sessionId: "session-second" }, + processKey: "oopif:second", domNodes: secondNodes, axNodes: [ axNode("root", 200, "RootWebArea"), axNode("heading", 201, "heading", "root", "Second heading"), axNode("nested-frame", 210, "Iframe", "root", "Nested"), ], - }, - { + }), + frameDoc({ frameId: "nested", contextScopeId: "nested", parentFrameId: "second", ownerBackendNodeId: 210, - target: { tabId: 7, sessionId: "session-nested" }, + processKey: "oopif:nested", domNodes: nestedNodes, axNodes: [ axNode("root", 300, "RootWebArea"), axNode("input", 301, "textbox", "root", "Nested field"), ], - }, + }), ], captured, ); @@ -700,31 +729,32 @@ describe("buildFrameVomScene", () => { const owners = [owner(10), owner(20)]; const result = buildFrameVomScene( [ - { + frameDoc({ frameId: "main", contextScopeId: "main", - target: { tabId: 7 }, domNodes: owners, axNodes: [], - }, - ...["left", "right"].map((frameId, i) => ({ - frameId, - contextScopeId: frameId, - parentFrameId: "main", - ownerBackendNodeId: owners[i].backendNodeId, - target: { tabId: 7, sessionId: frameId }, - excludedBackendNodeIds: new Set(i === 0 ? [7] : []), - domNodes: [], - axNodes: [ - { - nodeId: "button", - frameId, - backendDOMNodeId: 7, - role: { type: "role", value: "button" }, - name: { type: "computedString", value: frameId + " action" }, - }, - ], - })), + }), + ...["left", "right"].map((frameId, i) => + frameDoc({ + frameId, + contextScopeId: frameId, + parentFrameId: "main", + ownerBackendNodeId: owners[i].backendNodeId, + processKey: `oopif:${frameId}`, + excludedBackendNodeIds: new Set(i === 0 ? [7] : []), + domNodes: [], + axNodes: [ + { + nodeId: "button", + frameId, + backendDOMNodeId: 7, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: frameId + " action" }, + }, + ], + }), + ), ], { nodes: owners, @@ -760,18 +790,16 @@ describe("buildFrameVomScene", () => { const result = buildFrameVomScene( [ - { + frameDoc({ frameId: "main", contextScopeId: "main", - target: { tabId: 7 }, domNodes: [], axNodes: [], - }, - { + }), + frameDoc({ frameId: "child", contextScopeId: "child", parentFrameId: "main", - target: { tabId: 7 }, domNodes: [childNode], axNodes: [ { @@ -782,7 +810,7 @@ describe("buildFrameVomScene", () => { name: { type: "computedString", value: "Child action" }, }, ], - }, + }), ], captured, ); @@ -825,10 +853,9 @@ describe("buildFrameVomScene", () => { const result = buildFrameVomScene( [ - { + frameDoc({ frameId: "main", contextScopeId: "main", - target: { tabId: 7 }, domNodes: mainNodes, axNodes: [ { @@ -844,13 +871,13 @@ describe("buildFrameVomScene", () => { name: { type: "computedString", value: "Remote" }, }, ], - }, - { + }), + frameDoc({ frameId: "child", contextScopeId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 7, sessionId: "child-session" }, + processKey: "oopif:child", domNodes: [], axNodes: [ { @@ -866,7 +893,7 @@ describe("buildFrameVomScene", () => { name: { type: "computedString", value: "AX fallback action" }, }, ], - }, + }), ], captured, ); @@ -1113,12 +1140,11 @@ describe("buildFrameVomScene", () => { }, ], documents: [ - { + frameDoc({ frameId: "child", contextScopeId: "child", parentFrameId: "root", ownerBackendNodeId: 20, - target: { tabId: 7 }, axNodes: [], domNodes: [ { @@ -1142,7 +1168,7 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, + }), ], excludedBackendNodeIds: new Set(), }; @@ -1216,12 +1242,11 @@ describe("buildFrameVomScene", () => { }, ], documents: [ - { + frameDoc({ frameId: "child", contextScopeId: "child", parentFrameId: "root", ownerBackendNodeId: 20, - target: { tabId: 7 }, axNodes: [axNodes[2]], domNodes: [ { @@ -1235,7 +1260,7 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, + }), ], excludedBackendNodeIds: new Set(), }; @@ -1295,12 +1320,11 @@ describe("buildFrameVomScene", () => { }, ], documents: [ - { + frameDoc({ frameId: "child", contextScopeId: "child", parentFrameId: "root", ownerBackendNodeId: 20, - target: { tabId: 7 }, axNodes: [], domNodes: [ { @@ -1324,7 +1348,7 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, + }), ], excludedBackendNodeIds: new Set(), }, @@ -1376,12 +1400,11 @@ describe("buildFrameVomScene", () => { }, ], documents: [ - { + frameDoc({ frameId: "child", contextScopeId: "child", parentFrameId: "root", ownerBackendNodeId: 20, - target: { tabId: 7 }, axNodes: [], domNodes: [ { @@ -1405,7 +1428,7 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, + }), ], excludedBackendNodeIds: new Set(), }, @@ -2411,7 +2434,9 @@ describe("buildFrameVomScene", () => { expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual( expect.objectContaining({ - name: "Products [expanded]", + name: "Products", + hasPopup: true, + expanded: true, attrs: expect.objectContaining({ "aria-expanded": "true", "aria-controls": "products-panel", @@ -2421,10 +2446,11 @@ describe("buildFrameVomScene", () => { expect(scene.activeScopeBlocks).toEqual([ { triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, - label: "Products [expanded]", + label: "Products", lines: ["Shoes Bags"], }, ]); + expect(renderVom(scene).text).toContain('@e1 button "Products" [expanded]'); expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true); }); @@ -2605,14 +2631,10 @@ describe("handleSnapshot", () => { }; const send = vi.fn(sendImpl); const trackSessionTab = vi.fn(); - const cdp = { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, + const cdp = createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab, - }; + }); return { cdp, tabsApi: { @@ -2728,14 +2750,10 @@ describe("handleSnapshot", () => { sm, { session_id: "aa11" }, { - cdp: { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab: vi.fn(), - }, + }), tabsApi: { get: vi.fn( async (tabId: number) => @@ -2841,14 +2859,10 @@ describe("handleSnapshot", () => { sm, { session_id: "aa11", probe_hover: true }, { - cdp: { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab: vi.fn(), - }, + }), tabsApi: { get: vi.fn( async (tabId: number) => @@ -2935,14 +2949,10 @@ describe("handleSnapshot", () => { throw new Error(`unexpected CDP method ${method}`); }); const deps = { - cdp: { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab: vi.fn(), - }, + }), tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, @@ -3132,14 +3142,10 @@ describe("handleSnapshot", () => { throw new Error(`unexpected CDP method ${method}`); }); const deps = { - cdp: { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab: vi.fn(), - }, + }), tabsApi: { get: vi.fn(), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), @@ -3148,7 +3154,12 @@ describe("handleSnapshot", () => { const pending = handleSnapshot(sm, { session_id: "aa11" }, deps, controller.signal); await vi.waitFor(() => - expect(send).toHaveBeenCalledWith(4, "DOMSnapshot.captureSnapshot", expect.any(Object)), + expect(send).toHaveBeenCalledWith( + 4, + "DOMSnapshot.captureSnapshot", + expect.any(Object), + controller.signal, + ), ); controller.abort(); resolveCapture({ strings: [], documents: [] }); @@ -3166,13 +3177,9 @@ describe("handleSnapshot", () => { }; const send = vi.fn(sendImpl); const deps = { - cdp: { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, - }, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], + }), tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, @@ -3194,10 +3201,10 @@ describe("handleSnapshot", () => { throw new Error(`unexpected CDP method ${method}`); }); return { - cdp: { - send: send as unknown as (t: number, m: string, p?: object) => Promise, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab: vi.fn(), - }, + }), tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, @@ -3447,7 +3454,7 @@ describe("handleSnapshot", () => { if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; throw new Error(`unexpected root CDP method ${method}`); }); - const sendToTarget = vi.fn(async (_target, method: string) => { + const sendToChildFrame = vi.fn(async (method: string) => { if (method === "Runtime.evaluate") return { result: { value: { width: 400, height: 300 } } }; if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable") return { @@ -3461,23 +3468,32 @@ describe("handleSnapshot", () => { if (method === "Accessibility.getFullAXTree") return { nodes: childAx }; throw new Error(`unexpected child CDP method ${method}`); }); - const cdp = { + const cdp = createStubCdpRunner({ send: send as unknown as CdpRunner["send"], - sendToTarget: sendToTarget as unknown as NonNullable, - getFrameGraph: vi.fn(async () => ({ + getAttachmentId: () => undefined, + hasOwnFrameTarget: (_tabId, frameId) => frameId === "child", + sendToFrame: (async (handle, method, params, guard) => { + if (handle.frameId === "child") { + return sendToChildFrame(method); + } + return cdp.send(handle.tabId, method, params, guard?.signal); + }) as CdpRunner["sendToFrame"], + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 12, - target: { tabId: 4, sessionId: "child-session" }, - }, + processKey: "oopif:child", + isOutOfProcess: true, + }), ], })), trackSessionTab: vi.fn(), - } satisfies CdpRunner; + }); return { cdp, tabsApi: { @@ -3533,12 +3549,12 @@ describe("handleSnapshot", () => { }, ]; const deps = makeOverlayDeps(ax, snapshot, VP_METRICS); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ ...deps.cdp, - getFrameGraph: async () => { + getFrameDocuments: async () => { throw new Error("graph unavailable"); }, - }; + }); const result = await captureVomObservation( cdp, 4, @@ -3786,11 +3802,13 @@ describe("handleSnapshot", () => { ["Accessibility.getFullAXTree", "ax"], ])("reports partial capture when child %s fails", async (failedMethod, stage) => { const { cdp } = makeFrameAwareDeps("available"); - const send = cdp.sendToTarget; - cdp.sendToTarget = async (target, method, params) => { - if (method === failedMethod) throw new Error("unavailable child data"); - return send(target, method, params); - }; + const send = cdp.sendToFrame; + cdp.sendToFrame = (async (handle, method, params, guard) => { + if (handle.frameId === "child" && method === failedMethod) { + throw new Error("unavailable child data"); + } + return send(handle, method, params, guard); + }) as CdpRunner["sendToFrame"]; for (const maxTokens of [undefined, 100, 0]) { const result = await captureVomObservation(cdp, 4, "https://example.com", { maxTokens }); expect(result.text).toContain(`@warning observation incomplete: some ${stage} data`); @@ -3812,7 +3830,11 @@ describe("handleSnapshot", () => { cdp.send = async (tabId, method, params) => { const result = await send(tabId, method, params); if (method === "DOMSnapshot.captureSnapshot") { - const child = await cdp.sendToTarget({ tabId, sessionId: "child-session" }, method, params); + const child = await cdp.sendToFrame( + toFrameDocumentHandle(tabId, { frameId: "child" }), + method, + params, + ); (result as { documents: unknown[] }).documents.push( ...(child as { documents: unknown[] }).documents, ); @@ -4116,10 +4138,10 @@ describe("handleSnapshot", () => { throw new Error(`unexpected ${method}`); }); const deps = { - cdp: { - send: send as unknown as (t: number, m: string, p?: object) => Promise, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab: vi.fn(), - }, + }), tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, @@ -4325,10 +4347,10 @@ describe("handleSnapshot", () => { throw new Error(`unexpected ${method}`); }); const deps = { - cdp: { - send: send as unknown as (t: number, m: string, p?: object) => Promise, + cdp: createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab: vi.fn(), - }, + }), tabsApi: { get: vi.fn( async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, @@ -4358,14 +4380,10 @@ describe("handleGetHtml", () => { const send = vi.fn(sendImpl); const trackSessionTab = vi.fn(); // Cast to the generic CdpRunner.send signature for handleGetHtml. - const cdp = { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, + const cdp = createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], trackSessionTab, - }; + }); return { cdp, tabsApi: { diff --git a/apps/extension/src/tools/__tests__/record-steps.test.ts b/apps/extension/src/tools/__tests__/record-steps.test.ts index 481cd5683..19e78651a 100644 --- a/apps/extension/src/tools/__tests__/record-steps.test.ts +++ b/apps/extension/src/tools/__tests__/record-steps.test.ts @@ -1,7 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { RECORD_FINISH, RECORD_START, RECORD_STEP, RECORD_STOP } from "@/lib/record-bridge"; -import type { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { rootEventSource, type TabEventSource } from "@/browser-driver/frame-handles"; +import { + RECORD_CANCEL, + RECORD_FINISH, + RECORD_START, + RECORD_STEP, + RECORD_STOP, +} from "@/lib/record-bridge"; +import { SessionManager } from "@/session-manager/manager"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { EXTENSION_VERSION } from "@/transport/handshake"; import type { RecordStopResult, TraceV3 } from "@/transport/types"; import { @@ -11,6 +18,7 @@ import { handleRecordStop as rawHandleRecordStop, resetBrowserObservationForTests, } from "../record"; +import { handleSessionStop } from "../session"; const AGENT_WINDOW_ID = 100; const TAB_ID = 4; @@ -126,7 +134,7 @@ function makeFakeCdp( trees?: unknown[], options?: { failCaptures?: boolean; treesByTab?: Record }, ): FakeCdp { - type EventListener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => void; + type EventListener = (source: TabEventSource, method: string, params: unknown) => void; const events: EventListener[] = []; const script = [...(trees ?? [])]; let failing = options?.failCaptures ?? false; @@ -140,7 +148,7 @@ function makeFakeCdp( }), "Page.navigate": () => { for (const listener of [...events]) { - listener({ tabId: TAB_ID }, "Page.lifecycleEvent", { + listener(rootEventSource(TAB_ID), "Page.lifecycleEvent", { name: "load", frameId: "frame-1", loaderId: "loader-after", @@ -167,7 +175,7 @@ function makeFakeCdp( }, }; - return { + return createStubCdpRunner({ send: (async (_tabId: number, method: string, params: unknown) => { const handler = handlers[method]; if (!handler) throw new Error(`unsupported CDP call ${method}`); @@ -189,7 +197,7 @@ function makeFakeCdp( }, }; }, - }; + }); } function makeTabsApi() { @@ -1815,4 +1823,89 @@ describe("recorded user steps reach the exported trace", () => { const trace = asTraceV3((stopped as RecordStopResult).trace); expect(trace.steps.map((step) => step.op)).toEqual(["switch_tab"]); }); + + it("cancels active recording and resets automation-bypass refcounts on borrowed tabs during handleSessionStop using injected deps", async () => { + installChrome(); + const realManager = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: AGENT_WINDOW_ID, initialTabIds: [] }), + remove: async () => {}, + ensureActiveTab: async () => 99, + }, + }); + const ctx = await realManager.start("borrowed-rec"); + ctx.borrowedTabs.set(99, { tabId: 99, originalWindowId: 200, originalIndex: 0 }); + + const tabsApi = makeMultiTabsApi([ + { + id: 99, + windowId: AGENT_WINDOW_ID, + active: true, + status: "complete", + url: START_URL, + } as chrome.tabs.Tab, + ]); + const sentMessages: Array<{ tabId: number; message: unknown }> = []; + const sendToTab = vi.fn(async (tabId: number, message: unknown) => { + sentMessages.push({ tabId, message }); + return { ok: true }; + }); + const bypassOverlay = vi.fn(async () => {}); + const cdp = makeFakeCdp(); + + await handleRecordStart( + realManager, + { session_id: "borrowed-rec", tab_id: 99 }, + { tabsApi, sendToTab, bypassOverlay, cdp }, + ); + expect(bypassOverlay).toHaveBeenCalledWith(99, true); + + sentMessages.length = 0; + bypassOverlay.mockClear(); + + const stopRes = await handleSessionStop( + realManager, + { session_id: "borrowed-rec" }, + { + cdp: { + detachSession: vi.fn(async () => {}), + releaseSessionTab: vi.fn(async () => {}), + }, + tabsQuery: tabsApi, + tabManagement: { + tabs: { + get: tabsApi.get, + create: vi.fn(async () => ({ id: 1 }) as chrome.tabs.Tab), + remove: vi.fn(async () => {}), + update: vi.fn(async (id) => tabsApi.get(id)), + move: vi.fn(async (id, props) => { + const tab = await tabsApi.get(id); + tab.windowId = props.windowId ?? tab.windowId; + return tab; + }), + query: tabsApi.query, + group: vi.fn(async () => 1), + }, + windows: { + get: vi.fn(async (id) => ({ id, type: "normal" }) as chrome.windows.Window), + getLastFocused: vi.fn( + async () => ({ id: 200, type: "normal" }) as chrome.windows.Window, + ), + create: vi.fn(async () => ({ id: 300, type: "normal" }) as chrome.windows.Window), + remove: vi.fn(async () => {}), + }, + agentOverlayReset: { resetAgentOverlays: vi.fn(async () => {}) }, + }, + recording: { tabsApi, sendToTab, bypassOverlay, cdp }, + }, + ); + expect(stopRes).toEqual({ returned_tab_ids: [99] }); + expect( + sentMessages.some( + (entry) => + entry.tabId === 99 && (entry.message as { type?: string })?.type === RECORD_CANCEL, + ), + ).toBe(true); + expect(bypassOverlay).toHaveBeenCalledWith(99, false); + }); }); diff --git a/apps/extension/src/tools/__tests__/screenshot-full-page.test.ts b/apps/extension/src/tools/__tests__/screenshot-full-page.test.ts index ac4718f99..fc91d92c7 100644 --- a/apps/extension/src/tools/__tests__/screenshot-full-page.test.ts +++ b/apps/extension/src/tools/__tests__/screenshot-full-page.test.ts @@ -1,10 +1,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { rootEventSource, targetEventSource } from "@/browser-driver/frame-handles"; import { capturePage } from "@/long-screenshot/capture"; import { ScreenshotExports } from "@/long-screenshot/exports"; import { exportPng } from "@/long-screenshot/png"; import { SessionManager } from "@/session-manager/manager"; import { handleFullPageScreenshot } from "../screenshot-full-page"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; vi.mock("@/long-screenshot/page-client", () => ({ createPageClient: () => ({ documentId: "document-one", prepare: async () => {}, page: vi.fn() }), @@ -45,25 +46,35 @@ async function setup() { const disposeEvents = vi.fn(() => { onEvent = undefined; }); - const cdp = { - onEvent: vi.fn((_tabId: number, handler: NonNullable) => { - onEvent = handler; - return { dispose: disposeEvents }; - }), - getAttachmentId: vi.fn(() => "attachment-one" as string | undefined), - send: vi.fn(async (_id: number, method: string, _params?: object): Promise => { - if (method === "Page.startScreencast") { - onEvent?.({ tabId: 7 }, "Page.screencastFrame", { sessionId: 1 }); - return {} as T; - } - if (method === "Page.stopScreencast" || method === "Page.screencastFrameAck") return {} as T; - if (method === "Page.getFrameTree") return {} as T; - if (method === "Page.captureScreenshot") return { data: "cG5n" } as T; - throw new Error(`Unexpected CDP call: ${method}`); + const onEventMock = vi.fn((_tabId: number, handler: NonNullable) => { + onEvent = handler; + return { dispose: disposeEvents }; + }); + const getAttachmentIdMock = vi.fn(() => "attachment-one" as string | undefined); + const sendMock = vi.fn(async (_id: number, method: string, _params?: object): Promise => { + if (method === "Page.startScreencast") { + onEvent?.(rootEventSource(7), "Page.screencastFrame", { sessionId: 1 }); + return {} as T; + } + if (method === "Page.stopScreencast" || method === "Page.screencastFrameAck") return {} as T; + if (method === "Page.getFrameTree") return {} as T; + if (method === "Page.captureScreenshot") return { data: "cG5n" } as T; + throw new Error(`Unexpected CDP call: ${method}`); + }); + const cdp = Object.assign( + createStubCdpRunner({ + send: sendMock as CdpRunner["send"], + onEvent: onEventMock, + getAttachmentId: getAttachmentIdMock, }), - }; + { + send: sendMock, + onEvent: onEventMock, + getAttachmentId: getAttachmentIdMock, + }, + ); const deps = { - cdp: cdp as CdpRunner & typeof cdp, + cdp, tabsApi: { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) }, exports: new ScreenshotExports((id) => manager.has(id)), }; @@ -420,12 +431,12 @@ describe("Windows full-page rendering lifetime", () => { vi.mocked(capturePage).mockImplementationOnce(async (options) => { await options.screenshot(); const frame = deps.cdp.onEvent.mock.calls[0][1]; - frame({ tabId: 99 }, "Page.screencastFrame", { sessionId: 2 }); - frame({ tabId: 7, sessionId: "child" }, "Page.screencastFrame", { sessionId: 3 }); - frame({ tabId: 7 }, "Page.screencastVisibilityChanged", {}); - frame({ tabId: 7 }, "Page.screencastFrame", { sessionId: 4 }); + frame(rootEventSource(99), "Page.screencastFrame", { sessionId: 2 }); + frame(targetEventSource(7, "child"), "Page.screencastFrame", { sessionId: 3 }); + frame(rootEventSource(7), "Page.screencastVisibilityChanged", {}); + frame(rootEventSource(7), "Page.screencastFrame", { sessionId: 4 }); deps.cdp.getAttachmentId.mockReturnValue("replacement"); - frame({ tabId: 7 }, "Page.screencastFrame", { sessionId: 5 }); + frame(rootEventSource(7), "Page.screencastFrame", { sessionId: 5 }); return { width: 64, height: 256 }; }); expect(await handleFullPageScreenshot(manager, { session_id: "one" }, deps)).toMatchObject({ diff --git a/apps/extension/src/tools/__tests__/scroll.browser.test.ts b/apps/extension/src/tools/__tests__/scroll.browser.test.ts index 2aef57d56..4dc1f6217 100644 --- a/apps/extension/src/tools/__tests__/scroll.browser.test.ts +++ b/apps/extension/src/tools/__tests__/scroll.browser.test.ts @@ -1,14 +1,19 @@ // @vitest-environment node // Opt in with BSK_TEST_CHROME=/path/to/chrome. Uses an isolated browser/profile. import { describe, expect, it } from "vitest"; -import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentGraph, + type FrameDocumentHandle, + type FrameDocumentInfo, + type FrameNodeHandle, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { pointerTargetValidator } from "@/cooperation/pointer-target"; import { SessionManager } from "@/session-manager/manager"; import { resolveNodeGeometry } from "../frame-geometry"; import type { ViewportRect } from "../geometry"; import { handleScrollTo } from "../scroll"; -import type { CdpRunner } from "../shared"; -import { isRpcError } from "../shared"; +import { type CdpRunner, createStubCdpRunner, isRpcError } from "../shared"; import { pointBelongsToTarget } from "../target-hit"; type Send = >( @@ -54,19 +59,30 @@ async function harness(send: Send) { const ctx = await manager.start("aa11"); const calls: { method: string; sessionId?: string }[] = []; const after = { command: (_method: string) => {} }; - const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; - const forward = async (target: CdpTarget, method: string, params?: object): Promise => { - calls.push({ method, sessionId: target.sessionId }); - const result = await send(method, params, sessionFor(target)); + type FrameRecord = FrameDocumentInfo & { name: string; targetSession?: string }; + let currentFrames: FrameRecord[] = []; + const sessionForFrame = (frameId?: string) => + (frameId ? currentFrames.find((f) => f.frameId === frameId)?.targetSession : undefined) ?? + rootSession; + const forward = async ( + handle: FrameDocumentHandle | FrameNodeHandle | { tabId: number; frameId?: string }, + method: string, + params?: object, + ): Promise => { + const targetSession = handle.frameId + ? currentFrames.find((f) => f.frameId === handle.frameId)?.targetSession + : undefined; + calls.push({ method, sessionId: targetSession }); + const result = await send(method, params, targetSession ?? rootSession); after.command(method); return result; }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tab, method, params) => forward({ tabId: 4 }, method, params), - sendToTarget: forward, - }; - const evaluate = async (expression: string, frame?: CdpFrame): Promise => { - const session = frame ? sessionFor(frame.target) : rootSession; + sendToFrame: forward, + }); + const evaluate = async (expression: string, frame?: FrameRecord): Promise => { + const session = frame ? sessionForFrame(frame.frameId) : rootSession; const world = frame ? await send<{ executionContextId: number }>( "Page.createIsolatedWorld", @@ -90,20 +106,21 @@ async function harness(send: Send) { expect(reply.exceptionDetails).toBeUndefined(); return reply.result.value; }; - const rememberRef = (frame: CdpFrame, backendNodeId: number) => + const rememberRef = (frame: FrameRecord, backendNodeId: number) => ctx.refStore.set("e1", backendNodeId, { tabId: 4, frameId: frame.frameId, loaderId: frame.loaderId, }); - const ref = async (frame: CdpFrame, selector = "#probe") => { + const ref = async (frame: FrameRecord, selector = "#probe") => { + const frameSession = sessionForFrame(frame.frameId); const { executionContextId } = await send<{ executionContextId: number }>( "Page.createIsolatedWorld", { frameId: frame.frameId, worldName: "scroll-oracle", }, - sessionFor(frame.target), + frameSession, ); const { result } = await send<{ result: { objectId: string } }>( "Runtime.evaluate", @@ -111,14 +128,14 @@ async function harness(send: Send) { expression: `document.querySelector(${JSON.stringify(selector)})`, contextId: executionContextId, }, - sessionFor(frame.target), + frameSession, ); const { node } = await send<{ node: { backendNodeId: number } }>( "DOM.describeNode", { objectId: result.objectId }, - sessionFor(frame.target), + frameSession, ); - await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target)); + await send("Runtime.releaseObject", { objectId: result.objectId }, frameSession); rememberRef(frame, node.backendNodeId); return node.backendNodeId; }; @@ -147,12 +164,21 @@ async function harness(send: Send) { }); sessions.push(sessionId); } - const frames: (CdpFrame & { name: string })[] = []; + const frames: FrameRecord[] = []; for (const session of sessions) { const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session); - const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) }; + const targetSession = session !== rootSession ? session : undefined; const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => { - frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" }); + frames.push({ + handle: toFrameDocumentHandle(4, { frameId: tree.frame.id, loaderId: "" }), + frameId: tree.frame.id, + loaderId: "", + parentFrameId, + processKey: targetSession ? `oopif:${targetSession}` : "root", + isOutOfProcess: Boolean(targetSession), + ...(targetSession ? { targetSession } : {}), + name: tree.frame.name ?? "", + }); for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); }; visit(frameTree); @@ -163,12 +189,13 @@ async function harness(send: Send) { const owner = await send<{ backendNodeId: number }>( "DOM.getFrameOwner", { frameId: frame.frameId }, - sessionFor(parent.target), + parent.targetSession ?? rootSession, ); - frame.ownerBackendNodeId = owner.backendNodeId; + (frame as { ownerBackendNodeId?: number }).ownerBackendNodeId = owner.backendNodeId; } - const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames }; - cdp.getFrameGraph = async () => graph; + currentFrames = frames; + const graph: FrameDocumentGraph = { rootFrameId: frames[0].frameId, frames }; + cdp.getFrameDocuments = async () => graph; return frames; }; return { cdp, send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after }; @@ -336,7 +363,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser scroll-to", () => { expect(isRpcError(geometry)).toBe(false); if (isRpcError(geometry)) throw new Error(geometry.message); await pointerTargetValidator(h.cdp)({ handle }, geometry.actionPoint); - if (frame.target.sessionId) { + if (frame.targetSession) { await h.evaluate( "window.frameCover=document.createElement('div');frameCover.style='position:fixed;inset:0;z-index:99999;background:white';document.body.append(frameCover)", ); diff --git a/apps/extension/src/tools/__tests__/scroll.test.ts b/apps/extension/src/tools/__tests__/scroll.test.ts index e75eee1d4..7c64ba281 100644 --- a/apps/extension/src/tools/__tests__/scroll.test.ts +++ b/apps/extension/src/tools/__tests__/scroll.test.ts @@ -1,13 +1,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { type FrameDocumentGraph, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types"; import type { ViewportRect } from "../geometry"; import { handleScrollTo } from "../scroll"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; interface Call { - target: CdpTarget; + target: { tabId: number; sessionId?: string }; method: string; params?: Record; aborted: boolean; @@ -27,15 +27,24 @@ async function fixture(frame: "top" | "same-target" | "oopif" = "top") { ...(frame !== "top" ? { frameId: "child", loaderId: "child-loader" } : {}), }); const abort = new AbortController(); - const graph: CdpFrameGraph = { + const graph: FrameDocumentGraph = { rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, { + handle: toFrameDocumentHandle(4, { frameId: "main", loaderId: "" }), + frameId: "main", + loaderId: "", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: toFrameDocumentHandle(4, { frameId: "child", loaderId: "child-loader" }), frameId: "child", + loaderId: "child-loader", parentFrameId: "main", ownerBackendNodeId: 99, - target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) }, + processKey: frame === "oopif" ? "oopif:child" : "root", + isOutOfProcess: frame === "oopif", }, ], }; @@ -48,7 +57,11 @@ async function fixture(frame: "top" | "same-target" | "oopif" = "top") { after: (_call: Call) => {}, reply: (_call: Call): object | undefined => undefined, }; - const send = async (target: CdpTarget, method: string, params?: object): Promise => { + const send = async ( + target: { tabId: number; sessionId?: string }, + method: string, + params?: object, + ): Promise => { const call = { target, method, @@ -94,21 +107,31 @@ async function fixture(frame: "top" | "same-target" | "oopif" = "top") { if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`); return replies[method] as T; }; - const cdp: CdpRunner = { - send: (tabId, method, params) => send({ tabId }, method, params), - sendToTarget: send, - sendToFrame: (handle, method, params) => { - const target = handle.frameId - ? graph.frames.find((entry) => entry.frameId === handle.frameId)?.target - : undefined; - return send(target ?? { tabId: handle.tabId }, method, params); - }, - getFrameGraph: vi.fn(async () => { - hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted }); - return graph; + const getFrameDocuments = vi.fn(async () => { + hooks.after({ + target: { tabId: 4 }, + method: "getFrameDocuments", + aborted: abort.signal.aborted, + }); + return graph; + }); + const trackSessionTab = vi.fn(); + const cdp = Object.assign( + createStubCdpRunner({ + send: (tabId, method, params) => send({ tabId }, method, params), + sendToFrame: (handle, method, params) => { + const isChildOopif = handle.frameId === "child" && frame === "oopif"; + return send( + { tabId: handle.tabId, ...(isChildOopif ? { sessionId: "child-session" } : {}) }, + method, + params, + ); + }, + getFrameDocuments, + trackSessionTab, }), - trackSessionTab: vi.fn(), - }; + { getFrameDocuments, trackSessionTab }, + ); const tabsApi = { get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), @@ -170,7 +193,7 @@ describe("handleScrollTo", () => { .filter((call) => call.method === "DOM.scrollIntoViewIfNeeded") .map((call) => call.params!.backendNodeId), ).toEqual([99, 1234]); - expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(f.cdp.getFrameDocuments).toHaveBeenCalledTimes(1); if (frame === "oopif") expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe( "child-session", @@ -226,7 +249,7 @@ describe("handleScrollTo", () => { }); it.each([ - "getFrameGraph", + "getFrameDocuments", "DOM.scrollIntoViewIfNeeded", "DOM.resolveNode", "Runtime.callFunctionOn", diff --git a/apps/extension/src/tools/__tests__/select.browser.test.ts b/apps/extension/src/tools/__tests__/select.browser.test.ts index 2ed370505..a74287cac 100644 --- a/apps/extension/src/tools/__tests__/select.browser.test.ts +++ b/apps/extension/src/tools/__tests__/select.browser.test.ts @@ -3,7 +3,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { handleSelect } from "../interaction"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, @@ -105,9 +105,9 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("select disabled state in Chrome", }, }); await manager.start("aa11"); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: async (_tabId, method, params) => send(method, params, sessionId) as never, - }; + }); const result = await handleSelect( manager, { diff --git a/apps/extension/src/tools/__tests__/snapshot-coordinates.browser.test.ts b/apps/extension/src/tools/__tests__/snapshot-coordinates.browser.test.ts index 32d37b8cf..4d23154e8 100644 --- a/apps/extension/src/tools/__tests__/snapshot-coordinates.browser.test.ts +++ b/apps/extension/src/tools/__tests__/snapshot-coordinates.browser.test.ts @@ -1,8 +1,12 @@ // @vitest-environment node import { describe, expect, it } from "vitest"; -import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentGraph, + type FrameDocumentInfo, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { resolveNodeGeometry } from "../frame-geometry"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { captureObservationFacts } from "../vom/capture-coordinator"; type Send = >( @@ -120,21 +124,26 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real DOMSnapshot coordinate contr sessions.push(sessionId); } expect(sessions.length).toBe(configuration.fixture === "oopif-scrollbars" ? 3 : 2); - const frames: CdpFrame[] = []; + type FrameRecord = FrameDocumentInfo & { targetSession?: string }; + const frames: FrameRecord[] = []; const names = new Map(); - const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; for (const sessionId of sessions) { const { frameTree } = await send<{ frameTree: Tree }>( "Page.getFrameTree", {}, sessionId, ); - const target: CdpTarget = { - tabId: 1, - ...(sessionId === rootSession ? {} : { sessionId }), - }; + const targetSession = sessionId === rootSession ? undefined : sessionId; const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => { - frames.push({ frameId: tree.frame.id, parentFrameId, target }); + frames.push({ + handle: toFrameDocumentHandle(1, { frameId: tree.frame.id, loaderId: "" }), + frameId: tree.frame.id, + loaderId: "", + parentFrameId, + processKey: targetSession ? `oopif:${targetSession}` : "root", + isOutOfProcess: Boolean(targetSession), + ...(targetSession ? { targetSession } : {}), + }); names.set(tree.frame.id, tree.frame.name ?? ""); for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); }; @@ -147,32 +156,38 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real DOMSnapshot coordinate contr const owner = await send<{ backendNodeId: number }>( "DOM.getFrameOwner", { frameId: frame.frameId }, - sessionFor(parent.target), + parent.targetSession ?? rootSession, ); - frame.ownerBackendNodeId = owner.backendNodeId; + (frame as { ownerBackendNodeId?: number }).ownerBackendNodeId = owner.backendNodeId; } - const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames }; + const graph: FrameDocumentGraph = { rootFrameId: frames[0].frameId, frames }; const calls: string[] = []; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (tabId, method, params) => { calls.push(`${tabId}:${method}`); return send(method, params, rootSession); }, - sendToTarget: (target, method, params) => { - calls.push(`${target.sessionId ?? target.tabId}:${method}`); + sendToFrame: (handle, method, params) => { + const targetSession = handle.frameId + ? frames.find((item) => item.frameId === handle.frameId)?.targetSession + : undefined; + const prefix = targetSession ?? handle.tabId; + calls.push(`${prefix}:${method}`); if ( method === "Runtime.evaluate" && (params as { expression?: string })?.expression?.includes("window.innerWidth") ) - calls.push(`${target.sessionId ?? target.tabId}:viewport-size`); - return send(method, params, sessionFor(target)); + calls.push(`${prefix}:viewport-size`); + return send(method, params, targetSession ?? rootSession); }, - getFrameGraph: async () => graph, + hasOwnFrameTarget: (_tabId, frameId) => + Boolean(frames.find((item) => item.frameId === frameId)?.isOutOfProcess), + getFrameDocuments: async () => graph, getAttachmentId: () => rootSession, - }; + }); const oracles = new Map(); for (const frame of frames) { - const session = sessionFor(frame.target); + const session = frame.targetSession ?? rootSession; const { executionContextId } = await send<{ executionContextId: number }>( "Page.createIsolatedWorld", { frameId: frame.frameId, worldName: "coordinate-oracle" }, diff --git a/apps/extension/src/tools/__tests__/soft-detach-click.browser.test.ts b/apps/extension/src/tools/__tests__/soft-detach-click.browser.test.ts index 5ac24ed89..3da09aa57 100644 --- a/apps/extension/src/tools/__tests__/soft-detach-click.browser.test.ts +++ b/apps/extension/src/tools/__tests__/soft-detach-click.browser.test.ts @@ -7,7 +7,7 @@ import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { CooperationController } from "@/cooperation/controller"; import { pointerTargetValidator } from "@/cooperation/pointer-target"; import type { CooperationFrames, FrameState } from "@/cooperation/types"; @@ -34,7 +34,7 @@ interface CdpEvent { params?: object; } -type CdpEventListener = (source: CdpDebuggee, method: string, params: unknown) => void; +type CdpEventListener = Parameters[0]; const stubInputClock = (cdp: ChromiumCdp) => { vi.spyOn(cdp, "prepareInputTimestamp").mockImplementation(async () => ({ diff --git a/apps/extension/src/tools/__tests__/stealth-stripper.browser.test.ts b/apps/extension/src/tools/__tests__/stealth-stripper.browser.test.ts index 767454858..b3e1433af 100644 --- a/apps/extension/src/tools/__tests__/stealth-stripper.browser.test.ts +++ b/apps/extension/src/tools/__tests__/stealth-stripper.browser.test.ts @@ -2,17 +2,21 @@ // End-to-end real-browser regression for computed-style stealth prompt-injection stripping. import { createServer } from "node:http"; import { describe, expect, it } from "vitest"; -import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentGraph, + type FrameDocumentInfo, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleObserve, handleSnapshot } from "../observation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, params?: object, sessionId?: string, ) => Promise; -type FrameTree = { frame: { id: string }; childFrames?: FrameTree[] }; +type FrameTree = { frame: { id: string; loaderId?: string }; childFrames?: FrameTree[] }; const IFRAME_HTML = ` @@ -265,32 +269,36 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)( })()`); expect(await evaluate("devicePixelRatio")).toBeCloseTo(2, 4); - const graph = async (): Promise => { + const graph = async (): Promise => { const { frameTree } = await local<{ frameTree: FrameTree }>("Page.getFrameTree"); - const frames: CdpFrame[] = []; + const frames: FrameDocumentInfo[] = []; const visit = async (tree: FrameTree, parentFrameId?: string) => { const owner = parentFrameId ? await local<{ backendNodeId: number }>("DOM.getFrameOwner", { frameId: tree.frame.id, }) : undefined; + const loaderId = tree.frame.loaderId ?? ""; frames.push({ + handle: toFrameDocumentHandle(7, { frameId: tree.frame.id, loaderId }), frameId: tree.frame.id, + loaderId, parentFrameId, - target: { tabId: 7 }, ownerBackendNodeId: owner?.backendNodeId, + processKey: "root", + isOutOfProcess: false, }); for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); }; await visit(frameTree); return { rootFrameId: frameTree.frame.id, frames }; }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId, method, params) => local(method, params), - sendToTarget: (_target, method, params) => local(method, params), + sendToFrame: (_handle, method, params) => local(method, params), getAttachmentId: () => sessionId, - getFrameGraph: graph, - }; + getFrameDocuments: graph, + }); const manager = new SessionManager({ agentWindow: { create: async () => ({ windowId: 100, initialTabIds: [] }), diff --git a/apps/extension/src/tools/__tests__/tabs.test.ts b/apps/extension/src/tools/__tests__/tabs.test.ts index a4b52e8d1..9bb54380c 100644 --- a/apps/extension/src/tools/__tests__/tabs.test.ts +++ b/apps/extension/src/tools/__tests__/tabs.test.ts @@ -1461,7 +1461,11 @@ describe("handleTabReturn", () => { throw new Error("simulated move failure"); }); const { api: windowsApi } = makeWindowsApi(state); - const res = await returnBorrowedTab(ctx, 7, { tabs: api, windows: windowsApi }); + const res = await returnBorrowedTab(ctx, 7, { + tabs: api, + windows: windowsApi, + isAgentWindowId: (id) => sm.findByWindowId(id) !== null, + }); expect(res).toMatchObject({ code: "cdp_failed" }); }); @@ -1483,7 +1487,11 @@ describe("handleTabReturn", () => { createWindowId: 777, }); - const res = await returnBorrowedTab(ctx, 7, { tabs: api, windows: windowsApi }); + const res = await returnBorrowedTab(ctx, 7, { + tabs: api, + windows: windowsApi, + isAgentWindowId: (id) => sm.findByWindowId(id) !== null, + }); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); expect(res.toWindowId).toBe(777); @@ -2107,3 +2115,39 @@ it("claims an unremovable tab into agentCreatedTabs when chromeTabMutationApi.cr vi.stubGlobal("chrome", prevChrome); } }); + +it("never returns a borrowed user tab into another live session's Agent Window when original window closed", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100, 101]) }); + const ctxA = await sm.start("sess-a"); + await sm.start("sess-b"); // owns Agent Window 101 + + const state: FakeTabState = { + tabs: new Map([[7, { id: 7, windowId: 100, index: 0 } as chrome.tabs.Tab]]), + nextTabId: 50, + windowsClosed: new Set([200]), // original user window 200 is closed + }; + ctxA.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 0 }); + + const { api, spies } = makeTabMutationApi(state); + // Last focused window is session B's Agent Window (101), so returnBorrowedTab must skip 101 and create a fresh user window. + const windowsApi = { + get: vi.fn(async (windowId: number) => { + if (state.windowsClosed.has(windowId)) throw new Error("window closed"); + return { id: windowId, type: "normal" } as chrome.windows.Window; + }), + getLastFocused: vi.fn(async () => ({ id: 101, type: "normal" }) as chrome.windows.Window), + create: vi.fn(async () => ({ id: 999, type: "normal" }) as chrome.windows.Window), + remove: vi.fn(async () => {}), + }; + + const res = await returnBorrowedTab(ctxA, 7, { + tabs: api, + windows: windowsApi, + isAgentWindowId: (id) => sm.findByWindowId(id) !== null, + }); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.toWindowId).toBe(999); + expect(res.toWindowId).not.toBe(101); + expect(spies.move).toHaveBeenCalledWith(7, { windowId: 999, index: 0 }); +}); diff --git a/apps/extension/src/tools/__tests__/target-hit.browser.test.ts b/apps/extension/src/tools/__tests__/target-hit.browser.test.ts index b820cabbf..0526572e0 100644 --- a/apps/extension/src/tools/__tests__/target-hit.browser.test.ts +++ b/apps/extension/src/tools/__tests__/target-hit.browser.test.ts @@ -2,17 +2,22 @@ // Opt in with BSK_TEST_CHROME; the fixture owns its browser, profile and server. import { createServer } from "node:http"; import { expect, it } from "vitest"; -import type { CdpFrameGraph, CdpFrameTreeNode } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentGraph, + toFrameDocumentHandle, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import { pointerTargetValidator } from "@/cooperation/pointer-target"; import { SessionManager } from "@/session-manager/manager"; import { handleClick } from "../interaction"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; type Send = >( method: string, params?: object, sessionId?: string, ) => Promise; +type TreeNode = { frame: { id: string; url?: string }; childFrames?: TreeNode[] }; it.skipIf(!process.env.BSK_TEST_CHROME)( "keeps selector hits inside their document and clicks frame refs through checked ancestors", @@ -91,47 +96,61 @@ it.skipIf(!process.env.BSK_TEST_CHROME)( { targetId: crossTarget!.targetId, flatten: true }, ); await expect.poll(() => evaluate("typeof clicks", crossSession)).toBe("number"); - const { frameTree } = await local<{ frameTree: CdpFrameTreeNode }>("Page.getFrameTree"); - const root = { tabId: 4 }; - const graph: CdpFrameGraph = { + const { frameTree } = await local<{ frameTree: TreeNode }>("Page.getFrameTree"); + const graph: FrameDocumentGraph = { rootFrameId: frameTree.frame.id, - frames: [{ frameId: frameTree.frame.id, target: root }], + frames: [ + { + handle: toFrameDocumentHandle(4, { frameId: frameTree.frame.id, loaderId: "" }), + frameId: frameTree.frame.id, + loaderId: "", + processKey: "root", + isOutOfProcess: false, + }, + ], }; const sameFrame = frameTree.childFrames!.find( (child) => child.frame.url === `http://127.0.0.1:${port}/child`, )!; - const { frameTree: crossTree } = await send<{ frameTree: CdpFrameTreeNode }>( + const { frameTree: crossTree } = await send<{ frameTree: TreeNode }>( "Page.getFrameTree", {}, crossSession, ); + const childSessions = new Map(); for (const [frameId, childSession] of [ [sameFrame.frame.id, undefined], [crossTree.frame.id, crossSession], ] as const) { const owner = await local<{ backendNodeId: number }>("DOM.getFrameOwner", { frameId }); + if (childSession) childSessions.set(frameId, childSession); graph.frames.push({ + handle: toFrameDocumentHandle(4, { frameId, loaderId: "" }), frameId, + loaderId: "", parentFrameId: graph.rootFrameId, ownerBackendNodeId: owner.backendNodeId, - target: { tabId: 4, ...(childSession ? { sessionId: childSession } : {}) }, + processKey: childSession ? `oopif:${frameId}` : "root", + isOutOfProcess: Boolean(childSession), }); } let presses = 0; - const dispatch: CdpRunner["sendToTarget"] = async (target, method, params) => { + const dispatch: CdpRunner["sendToFrame"] = async (handle, method, params) => { if ( method === "Input.dispatchMouseEvent" && (params as { type?: string }).type === "mousePressed" ) presses++; - return send(method, params, target.sessionId ?? sessionId) as never; + const targetSession = handle.frameId ? childSessions.get(handle.frameId) : undefined; + return send(method, params, targetSession ?? sessionId) as never; }; - const cdp: CdpRunner = { - send: (tab, method, params) => dispatch({ tabId: tab }, method, params), - sendToTarget: dispatch, - getFrameGraph: async () => graph, + const cdp: CdpRunner = createStubCdpRunner({ + send: (tab, method, params) => dispatch(toRootFrameHandle(tab), method, params), + sendToFrame: dispatch, + hasOwnFrameTarget: (_tab, frameId) => childSessions.has(frameId), + getFrameDocuments: async () => graph, getAttachmentId: () => sessionId, - }; + }); cdp.verifyPointerTarget = pointerTargetValidator(cdp); const manager = new SessionManager({ agentWindow: { diff --git a/apps/extension/src/tools/__tests__/target-hit.test.ts b/apps/extension/src/tools/__tests__/target-hit.test.ts index 9c5f19619..ebb06a2c8 100644 --- a/apps/extension/src/tools/__tests__/target-hit.test.ts +++ b/apps/extension/src/tools/__tests__/target-hit.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it, vi } from "vitest"; import { - type CdpFrameGraph, - type CdpTarget, + type FrameDocumentGraph, type FrameNodeHandle, StaleFrameHandleError, -} from "@/browser-driver/frame-graph"; + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { projectPolygon, type Quad, unprojectPoint } from "../geometry"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { pointBelongsToTarget, pointReachesTarget } from "../target-hit"; describe("hit-test document identity", () => { @@ -27,7 +27,7 @@ describe("hit-test document identity", () => { throw new Error(`Unexpected method ${method}`); } }); - const cdp = { send } as unknown as CdpRunner; + const cdp = createStubCdpRunner({ send }); const check = (frameId?: string) => pointBelongsToTarget( cdp, @@ -133,29 +133,48 @@ function nestedFrames(blocked?: string) { const root = { tabId: 4 }; const parent = { tabId: 4, sessionId: "parent" }; const child = { tabId: 4, sessionId: "child" }; - const graph: CdpFrameGraph = { + const targetByFrame = new Map([ + ["root-frame", root], + ["parent-frame", parent], + ["child-frame", child], + ]); + const graph: FrameDocumentGraph = { rootFrameId: "root-frame", frames: [ - { frameId: "root-frame", loaderId: "root-loader", target: root }, { + handle: toFrameDocumentHandle(4, { frameId: "root-frame", loaderId: "root-loader" }), + frameId: "root-frame", + loaderId: "root-loader", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: toFrameDocumentHandle(4, { frameId: "parent-frame", loaderId: "parent-loader" }), frameId: "parent-frame", loaderId: "parent-loader", parentFrameId: "root-frame", - target: parent, ownerBackendNodeId: 100, + processKey: "oopif:parent", + isOutOfProcess: true, }, { + handle: toFrameDocumentHandle(4, { frameId: "child-frame", loaderId: "child-loader" }), frameId: "child-frame", loaderId: "child-loader", parentFrameId: "parent-frame", - target: child, ownerBackendNodeId: 200, + processKey: "oopif:child", + isOutOfProcess: true, }, ], }; const hits: { target: string; x: number; y: number }[] = []; const released: string[] = []; - const request = async (target: CdpTarget, method: string, params?: object): Promise => { + const request = async ( + target: { tabId: number; sessionId?: string }, + method: string, + params?: object, + ): Promise => { const key = target.sessionId ?? "root"; const p = params as Record; const sizes = { root: [1000, 800], parent: [400, 300], child: [200, 100] }; @@ -195,19 +214,19 @@ function nestedFrames(blocked?: string) { throw new Error(`Unexpected method ${method}`); } }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: ((_tab, method, params) => request(root, method, params)) as CdpRunner["send"], - sendToTarget: request as CdpRunner["sendToTarget"], sendToFrame: ((handle, method, params) => { + if (!handle.frameId) return request(root, method, params); const frame = graph.frames.find((entry) => entry.frameId === handle.frameId); if (!frame || (handle.loaderId && frame.loaderId !== handle.loaderId)) { throw new StaleFrameHandleError(handle); } - return request(frame.target, method, params); + return request(targetByFrame.get(frame.frameId) ?? root, method, params); }) as CdpRunner["sendToFrame"], hasOwnFrameTarget: (_tab, frameId) => frameId !== "root-frame", - getFrameGraph: async () => graph, - }; + getFrameDocuments: async () => graph, + }); const handle: FrameNodeHandle = { tabId: 4, frameId: "child-frame", diff --git a/apps/extension/src/tools/__tests__/viewport-screenshot.test.ts b/apps/extension/src/tools/__tests__/viewport-screenshot.test.ts index f5cd8225c..c491a231c 100644 --- a/apps/extension/src/tools/__tests__/viewport-screenshot.test.ts +++ b/apps/extension/src/tools/__tests__/viewport-screenshot.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import type { CaptureSuppressMessage } from "@/lib/capture-suppress-bridge"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { captureControlledViewport } from "../viewport-screenshot"; const PNG = @@ -19,10 +19,10 @@ function fixture() { if (method === "Page.captureScreenshot") return { data: state.png }; return {}; }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as CdpRunner["send"], getAttachmentId: () => state.attachment, - }; + }); const bridge = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => { await hook.bridge(message.phase); }); diff --git a/apps/extension/src/tools/__tests__/visual-geometry.browser.test.ts b/apps/extension/src/tools/__tests__/visual-geometry.browser.test.ts index 4014307b6..a7dafe20b 100644 --- a/apps/extension/src/tools/__tests__/visual-geometry.browser.test.ts +++ b/apps/extension/src/tools/__tests__/visual-geometry.browser.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import { describe, expect, it } from "vitest"; -import type { CdpFrame } from "@/browser-driver/frame-graph"; -import type { CdpRunner } from "../shared"; +import { type FrameDocumentInfo, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { captureVisualScreenshot } from "../visual-screenshot"; import { resolveVisualRegionNow, verifyVisualHit } from "../visual-target"; import { captureObservationFacts } from "../vom/capture-coordinator"; @@ -43,12 +43,12 @@ async function withPage( await send("Target.activateTarget", { targetId }); const local: Send = (method, params) => send(method, params, sessionId); const { frameTree } = await local<{ frameTree: Tree }>("Page.getFrameTree"); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tab, method, params) => local(method, params), getAttachmentId: () => "visual-geometry-browser-test", - getFrameGraph: async () => { + getFrameDocuments: async () => { const { frameTree } = await local<{ frameTree: Tree }>("Page.getFrameTree"); - const frames: CdpFrame[] = []; + const frames: FrameDocumentInfo[] = []; const visit = async (tree: Tree, parentFrameId?: string) => { const owner = parentFrameId ? await local<{ backendNodeId: number }>("DOM.getFrameOwner", { @@ -56,17 +56,20 @@ async function withPage( }) : undefined; frames.push({ + handle: toFrameDocumentHandle(1, { frameId: tree.frame.id, loaderId: "" }), frameId: tree.frame.id, - target: { tabId: 1 }, + loaderId: "", parentFrameId, ownerBackendNodeId: owner?.backendNodeId, + processKey: "root", + isOutOfProcess: false, }); for (const child of tree.childFrames ?? []) await visit(child, tree.frame.id); }; await visit(frameTree); return { rootFrameId: frameTree.frame.id, frames }; }, - }; + }); await run({ cdp, send: local, diff --git a/apps/extension/src/tools/__tests__/visual-ref.test.ts b/apps/extension/src/tools/__tests__/visual-ref.test.ts index c4fb8ffd4..3c54a8977 100644 --- a/apps/extension/src/tools/__tests__/visual-ref.test.ts +++ b/apps/extension/src/tools/__tests__/visual-ref.test.ts @@ -47,7 +47,7 @@ async function setup() { const send = vi.fn(async () => { throw new Error("unexpected target operation"); }); - const cdp = { send, sendToTarget: send, trackSessionTab: vi.fn() } as unknown as CdpRunner; + const cdp = { send, sendToFrame: send, trackSessionTab: vi.fn() } as unknown as CdpRunner; const tab = { id: 4, windowId: 100, active: true, url: "https://example.com" } as chrome.tabs.Tab; const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) }; return { manager, ctx, send, cdp, tabsApi }; diff --git a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts index d18c528e0..b209c8e6e 100644 --- a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts +++ b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts @@ -1,10 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import { CdpReadGate, READ_TIMEOUT_MS } from "@/browser-driver/command-deadline"; -import type { CdpTarget } from "@/browser-driver/frame-graph"; import { SessionManager } from "@/session-manager/manager"; import { handleClick } from "../interaction"; import { handleScreenshot } from "../observation"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { consumeVisualCapture, issueVisualCapture } from "../visual-capture"; import { captureVisualScreenshot, visualScreenshotScale } from "../visual-screenshot"; import { resolveVisualRegionNow, verifyVisualHit } from "../visual-target"; @@ -106,7 +105,7 @@ function fixture(child = false, oopif = false) { const controller = new AbortController(); const send = vi.fn( async ( - target: CdpTarget, + target: { tabId: number; sessionId?: string }, method: string, params: Record = {}, ): Promise> => { @@ -165,24 +164,27 @@ function fixture(child = false, oopif = false) { throw new Error(`unexpected ${method} ${target.sessionId}`); }, ); - const cdp: CdpRunner = { - send: ((tabId, method, params) => - send({ tabId }, method, params as Record)) as CdpRunner["send"], - sendToTarget: send as unknown as CdpRunner["sendToTarget"], - sendToFrame: ((handle, method, params) => - send( - oopif && handle.frameId !== "top" - ? { tabId: handle.tabId, sessionId: "oopif" } - : { tabId: handle.tabId }, - method, - params as Record, - )) as CdpRunner["sendToFrame"], - hasOwnFrameTarget: (_tabId, frameId) => oopif && frameId !== "top", - getAttachmentId: () => control.attachmentId, - getFrameGraph: vi.fn(async () => { - throw new Error("whole page discovery forbidden"); + const getFrameDocuments = vi.fn(async () => { + throw new Error("whole page discovery forbidden"); + }); + const cdp = Object.assign( + createStubCdpRunner({ + send: ((tabId, method, params) => + send({ tabId }, method, params as Record)) as CdpRunner["send"], + sendToFrame: ((handle, method, params) => + send( + oopif && handle.frameId && handle.frameId !== "top" + ? { tabId: handle.tabId, sessionId: "oopif" } + : { tabId: handle.tabId }, + method, + params as Record, + )) as CdpRunner["sendToFrame"], + hasOwnFrameTarget: (_tabId, frameId) => oopif && frameId !== "top", + getAttachmentId: () => control.attachmentId, + getFrameDocuments, }), - }; + { getFrameDocuments }, + ); return { candidate, cdp, send, control, controller, topRows, childRows }; } @@ -238,7 +240,7 @@ describe("visual screenshot", () => { const f = fixture(child, oopif); const result = await captureVisualScreenshot(f.cdp, f.candidate); expect(result).toMatchObject({ width: child ? 200 : 100, height: child ? 80 : 40 }); - expect(f.cdp.getFrameGraph).not.toHaveBeenCalled(); + expect(f.cdp.getFrameDocuments).not.toHaveBeenCalled(); const names = f.send.mock.calls.map((c) => c[1]); expect(names.filter((n) => n === "DOM.getFrameOwner")).toHaveLength(child ? 2 : 0); expect(names.filter((n) => n === "DOM.resolveNode")).toHaveLength(child ? 4 : 2); @@ -310,7 +312,7 @@ describe("visual screenshot", () => { expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({ clip: crop, }); - expect(f.cdp.getFrameGraph).not.toHaveBeenCalled(); + expect(f.cdp.getFrameDocuments).not.toHaveBeenCalled(); }); it("runs the live reader on only connected local ancestry", async () => { @@ -619,7 +621,7 @@ it.each([ expect(result).toMatchObject({ x: child ? 220 : 60, y: child ? 280 : 40 }); expect(f.hitPoints.at(-1)).toEqual([60, 40]); expect(f.input.map((e) => e.type)).toEqual(["mouseMoved", "mousePressed", "mouseReleased"]); - expect(f.cdp.getFrameGraph).not.toHaveBeenCalled(); + expect(f.cdp.getFrameDocuments).not.toHaveBeenCalled(); const count = f.input.length; expect(await handleClick(f.manager, f.params, f.deps)).toHaveProperty( "data.reason", @@ -893,7 +895,7 @@ describe("visual hit shadow boundaries", () => { } target.remove(); expect(await verifyVisualHit(f.cdp, state, { x: 20, y: 30 })).toBe(false); - expect(f.cdp.getFrameGraph).not.toHaveBeenCalled(); + expect(f.cdp.getFrameDocuments).not.toHaveBeenCalled(); } finally { for (const { root, original } of hits) { if (original) Object.defineProperty(root, "elementFromPoint", original); @@ -978,7 +980,7 @@ it.each([ expect(result).toHaveProperty("data.reason", "visual_capture_stale"); expect(f.input).toHaveLength(0); } - expect(f.cdp.getFrameGraph).not.toHaveBeenCalled(); + expect(f.cdp.getFrameDocuments).not.toHaveBeenCalled(); } finally { if (hitDescriptor) Object.defineProperty(document, "elementFromPoint", hitDescriptor); else Reflect.deleteProperty(document, "elementFromPoint"); diff --git a/apps/extension/src/tools/__tests__/visual-target.test.ts b/apps/extension/src/tools/__tests__/visual-target.test.ts index af83c7dda..a8c7fcb20 100644 --- a/apps/extension/src/tools/__tests__/visual-target.test.ts +++ b/apps/extension/src/tools/__tests__/visual-target.test.ts @@ -1,6 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { - type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp, DEFAULT_IDLE_DETACH_MS, @@ -81,7 +80,8 @@ afterEach(() => { }); async function softDetachVisualFixture(oopif = false) { - const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>(); + const onEvent = + fakeChromeEvent[0]>>(); const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>(); let attachGeneration = 0; let rootState = { diff --git a/apps/extension/src/tools/__tests__/waits.browser.test.ts b/apps/extension/src/tools/__tests__/waits.browser.test.ts index 30d4a2b21..b0c622215 100644 --- a/apps/extension/src/tools/__tests__/waits.browser.test.ts +++ b/apps/extension/src/tools/__tests__/waits.browser.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { WaitForElementResult } from "@/transport/types"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { handleWaitForElement } from "../waits"; type Send = >( @@ -41,9 +41,9 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser element waits", () = targetId, flatten: true, }); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tabId, method, params) => send(method, params, sessionId), - }; + }); const tab = { id: 4, windowId: 100, active: !background } as chrome.tabs.Tab; const deps = { cdp, tabsApi: { get: async () => tab, query: async () => [tab] } }; const evaluate = async (expression: string) => { @@ -145,14 +145,14 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser element waits", () = await evaluate( `document.body.insertAdjacentHTML('beforeend', '
Loading
')`, ); - const removalCdp: CdpRunner = { + const removalCdp: CdpRunner = createStubCdpRunner({ async send(tabId: number, method: string, params?: object): Promise { if (method === "DOM.describeNode") { await evaluate(`document.querySelector('#race-mask')?.remove()`); } return cdp.send(tabId, method, params); }, - }; + }); expect( await handleWaitForElement( manager, diff --git a/apps/extension/src/tools/__tests__/waits.test.ts b/apps/extension/src/tools/__tests__/waits.test.ts index 6f18dd1da..7c6d940be 100644 --- a/apps/extension/src/tools/__tests__/waits.test.ts +++ b/apps/extension/src/tools/__tests__/waits.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { rootEventSource, type TabEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { handleWaitForElement, handleWaitForNavigation, satisfiesState } from "../waits"; function fakeAgentWindow(ids: number[]) { @@ -17,7 +18,7 @@ function fakeAgentWindow(ids: number[]) { } interface EventListener { - (source: chrome.debugger.Debuggee, method: string, params: unknown): void; + (source: TabEventSource, method: string, params: unknown): void; } function makeFakeCdp( @@ -57,7 +58,7 @@ function makeFakeCdp( throw new Error(`unexpected CDP call ${method}`); }; const send = vi.fn(sendImpl); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: send as unknown as ( tabId: number, method: string, @@ -73,7 +74,7 @@ function makeFakeCdp( }, }; }), - }; + }); const windowId = opts.windowId ?? 100; const tabsApi = { get: vi.fn(async (tabId: number) => ({ id: tabId, windowId, active: true }) as chrome.tabs.Tab), @@ -90,10 +91,11 @@ function makeFakeCdp( loaderId = "loader-1", ) { const payload = { name, frameId, loaderId }; - for (const listener of [...events]) listener({ tabId }, "Page.lifecycleEvent", payload); + for (const listener of [...events]) + listener(rootEventSource(tabId), "Page.lifecycleEvent", payload); }, fire(method: string, params: unknown, tabId = 4) { - for (const listener of [...events]) listener({ tabId }, method, params); + for (const listener of [...events]) listener(rootEventSource(tabId), method, params); }, /** Simulate the browser committing a new main document. */ commit(loaderId: string, readyState: string, type = "Navigation") { @@ -101,7 +103,7 @@ function makeFakeCdp( opts.readyState = readyState; const frame = { id: opts.mainFrameId ?? "frame-1", loaderId }; for (const listener of [...events]) - listener({ tabId: 4 }, "Page.frameNavigated", { frame, type }); + listener(rootEventSource(4), "Page.frameNavigated", { frame, type }); }, listeners: events, }; @@ -995,9 +997,9 @@ describe("wait-for-element regression coverage", () => { const manager = await sessionWithRef(); const fake = makeFakeCdp(); const dispatched = vi.fn(); - fake.cdp.sendGuarded = async (_target, _method, _params, guard) => { + fake.cdp.sendToFrame = async (_handle, _method, _params, guard) => { await new Promise((resolve) => setTimeout(resolve, 500)); - guard.onDispatch?.(); + guard?.onDispatch?.(); dispatched(); return {} as never; }; diff --git a/apps/extension/src/tools/__tests__/wheel.browser.test.ts b/apps/extension/src/tools/__tests__/wheel.browser.test.ts index b46901049..4434f7136 100644 --- a/apps/extension/src/tools/__tests__/wheel.browser.test.ts +++ b/apps/extension/src/tools/__tests__/wheel.browser.test.ts @@ -1,10 +1,16 @@ // @vitest-environment node // Opt in with BSK_TEST_CHROME=/path/to/chrome. Uses an isolated browser/profile. import { describe, expect, it } from "vitest"; -import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentGraph, + type FrameDocumentHandle, + type FrameDocumentInfo, + type FrameNodeHandle, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import type { WheelParams } from "@/transport/types"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { handleWheel } from "../wheel"; type Send = >( @@ -49,19 +55,30 @@ async function harness(send: Send) { const ctx = await manager.start("aa11"); const calls: { method: string; sessionId?: string }[] = []; const after = { command: (_method: string) => {} }; - const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; - const forward = async (target: CdpTarget, method: string, params?: object): Promise => { - calls.push({ method, sessionId: target.sessionId }); - const result = await send(method, params, sessionFor(target)); + type FrameRecord = FrameDocumentInfo & { name: string; targetSession?: string }; + let currentFrames: FrameRecord[] = []; + const sessionForFrame = (frameId?: string) => + (frameId ? currentFrames.find((f) => f.frameId === frameId)?.targetSession : undefined) ?? + rootSession; + const forward = async ( + handle: FrameDocumentHandle | FrameNodeHandle | { tabId: number; frameId?: string }, + method: string, + params?: object, + ): Promise => { + const targetSession = handle.frameId + ? currentFrames.find((f) => f.frameId === handle.frameId)?.targetSession + : undefined; + calls.push({ method, sessionId: targetSession }); + const result = await send(method, params, targetSession ?? rootSession); after.command(method); return result; }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tab, method, params) => forward({ tabId: 4 }, method, params), - sendToTarget: forward, - }; - const evaluate = async (expression: string, frame?: CdpFrame): Promise => { - const session = frame ? sessionFor(frame.target) : rootSession; + sendToFrame: forward, + }); + const evaluate = async (expression: string, frame?: FrameRecord): Promise => { + const session = frame ? sessionForFrame(frame.frameId) : rootSession; const world = frame ? await send<{ executionContextId: number }>( "Page.createIsolatedWorld", @@ -85,20 +102,21 @@ async function harness(send: Send) { expect(reply.exceptionDetails).toBeUndefined(); return reply.result.value; }; - const rememberRef = (frame: CdpFrame, backendNodeId: number) => + const rememberRef = (frame: FrameRecord, backendNodeId: number) => ctx.refStore.set("e1", backendNodeId, { tabId: 4, frameId: frame.frameId, loaderId: frame.loaderId, }); - const ref = async (frame: CdpFrame, selector = "#probe") => { + const ref = async (frame: FrameRecord, selector = "#probe") => { + const frameSession = sessionForFrame(frame.frameId); const { executionContextId } = await send<{ executionContextId: number }>( "Page.createIsolatedWorld", { frameId: frame.frameId, worldName: "wheel-oracle", }, - sessionFor(frame.target), + frameSession, ); const { result } = await send<{ result: { objectId: string } }>( "Runtime.evaluate", @@ -106,14 +124,14 @@ async function harness(send: Send) { expression: `document.querySelector(${JSON.stringify(selector)})`, contextId: executionContextId, }, - sessionFor(frame.target), + frameSession, ); const { node } = await send<{ node: { backendNodeId: number } }>( "DOM.describeNode", { objectId: result.objectId }, - sessionFor(frame.target), + frameSession, ); - await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target)); + await send("Runtime.releaseObject", { objectId: result.objectId }, frameSession); rememberRef(frame, node.backendNodeId); }; const wheel = (params: Partial, signal?: AbortSignal) => @@ -141,12 +159,21 @@ async function harness(send: Send) { }); sessions.push(sessionId); } - const frames: (CdpFrame & { name: string })[] = []; + const frames: FrameRecord[] = []; for (const session of sessions) { const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session); - const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) }; + const targetSession = session !== rootSession ? session : undefined; const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => { - frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" }); + frames.push({ + handle: toFrameDocumentHandle(4, { frameId: tree.frame.id, loaderId: "" }), + frameId: tree.frame.id, + loaderId: "", + parentFrameId, + processKey: targetSession ? `oopif:${targetSession}` : "root", + isOutOfProcess: Boolean(targetSession), + ...(targetSession ? { targetSession } : {}), + name: tree.frame.name ?? "", + }); for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); }; visit(frameTree); @@ -157,12 +184,13 @@ async function harness(send: Send) { const owner = await send<{ backendNodeId: number }>( "DOM.getFrameOwner", { frameId: frame.frameId }, - sessionFor(parent.target), + parent.targetSession ?? rootSession, ); - frame.ownerBackendNodeId = owner.backendNodeId; + (frame as { ownerBackendNodeId?: number }).ownerBackendNodeId = owner.backendNodeId; } - const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames }; - cdp.getFrameGraph = async () => graph; + currentFrames = frames; + const graph: FrameDocumentGraph = { rootFrameId: frames[0].frameId, frames }; + cdp.getFrameDocuments = async () => graph; return frames; }; return { send, rootSession, evaluate, ref, rememberRef, wheel, loadFrames, calls, after }; @@ -275,7 +303,7 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser wheel", () => { .toBe(true); const frames = await h.loadFrames(); expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3); - expect(frames.some((frame) => frame.target.sessionId)).toBe(true); + expect(frames.some((frame) => frame.isOutOfProcess)).toBe(true); for (const frame of frames) await h.evaluate(`${LISTEN} window.blockWheel=true`, frame); for (const frame of frames) { for (const other of frames) await h.evaluate("window.wheels=[]", other); diff --git a/apps/extension/src/tools/__tests__/wheel.test.ts b/apps/extension/src/tools/__tests__/wheel.test.ts index 4ec92930e..dead9b0a1 100644 --- a/apps/extension/src/tools/__tests__/wheel.test.ts +++ b/apps/extension/src/tools/__tests__/wheel.test.ts @@ -1,13 +1,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { type FrameDocumentGraph, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import type { JavaScriptDialogInfo, WheelParams } from "@/transport/types"; import type { ViewportRect } from "../geometry"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { handleWheel } from "../wheel"; interface Call { - target: CdpTarget; + target: { tabId: number; sessionId?: string }; method: string; params?: Record; aborted: boolean; @@ -27,15 +27,24 @@ async function fixture(frame: "top" | "same-target" | "oopif" = "top") { ...(frame !== "top" ? { frameId: "child", loaderId: "child-loader" } : {}), }); const abort = new AbortController(); - const graph: CdpFrameGraph = { + const graph: FrameDocumentGraph = { rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, { + handle: toFrameDocumentHandle(4, { frameId: "main", loaderId: "" }), + frameId: "main", + loaderId: "", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: toFrameDocumentHandle(4, { frameId: "child", loaderId: "child-loader" }), frameId: "child", + loaderId: "child-loader", parentFrameId: "main", ownerBackendNodeId: 99, - target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) }, + processKey: frame === "oopif" ? "oopif:child" : "root", + isOutOfProcess: frame === "oopif", }, ], }; @@ -48,7 +57,11 @@ async function fixture(frame: "top" | "same-target" | "oopif" = "top") { after: (_call: Call) => {}, reply: (_call: Call): object | undefined => undefined, }; - const send = async (target: CdpTarget, method: string, params?: object): Promise => { + const send = async ( + target: { tabId: number; sessionId?: string }, + method: string, + params?: object, + ): Promise => { const call = { target, method, @@ -97,21 +110,31 @@ async function fixture(frame: "top" | "same-target" | "oopif" = "top") { if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`); return replies[method] as T; }; - const cdp: CdpRunner = { - send: (tabId, method, params) => send({ tabId }, method, params), - sendToTarget: send, - sendToFrame: (handle, method, params) => { - const target = handle.frameId - ? graph.frames.find((entry) => entry.frameId === handle.frameId)?.target - : undefined; - return send(target ?? { tabId: handle.tabId }, method, params); - }, - getFrameGraph: vi.fn(async () => { - hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted }); - return graph; + const getFrameDocuments = vi.fn(async () => { + hooks.after({ + target: { tabId: 4 }, + method: "getFrameDocuments", + aborted: abort.signal.aborted, + }); + return graph; + }); + const trackSessionTab = vi.fn(); + const cdp = Object.assign( + createStubCdpRunner({ + send: (tabId, method, params) => send({ tabId }, method, params), + sendToFrame: (handle, method, params) => { + const isChildOopif = handle.frameId === "child" && frame === "oopif"; + return send( + { tabId: handle.tabId, ...(isChildOopif ? { sessionId: "child-session" } : {}) }, + method, + params, + ); + }, + getFrameDocuments, + trackSessionTab, }), - trackSessionTab: vi.fn(), - }; + { getFrameDocuments, trackSessionTab }, + ); const tabsApi = { get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), @@ -292,7 +315,7 @@ describe("handleWheel", () => { .filter((c) => c.method === "DOM.scrollIntoViewIfNeeded") .map((c) => c.params!.backendNodeId), ).toEqual([99, 1234]); - expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1); + expect(f.cdp.getFrameDocuments).toHaveBeenCalledTimes(1); expect( f.calls .filter((c) => c.method === "Input.dispatchMouseEvent") @@ -393,7 +416,7 @@ describe("handleWheel", () => { }); it.each([ - "getFrameGraph", + "getFrameDocuments", "DOM.scrollIntoViewIfNeeded", "DOM.getContentQuads", "DOM.resolveNode", diff --git a/apps/extension/src/tools/__tests__/window.browser.test.ts b/apps/extension/src/tools/__tests__/window.browser.test.ts index 1035f30e1..c7e8bdcbd 100644 --- a/apps/extension/src/tools/__tests__/window.browser.test.ts +++ b/apps/extension/src/tools/__tests__/window.browser.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner, ChromeTabsApi } from "../shared"; +import { type CdpRunner, ChromeTabsApi, createStubCdpRunner } from "../shared"; import { handleWindowResize, type WindowResizeApi } from "../window"; type Send = >( @@ -123,12 +123,12 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("window resize in a real browser", get: (id) => evaluate(`chrome.tabs.get(${id})`), query: (query) => evaluate(`chrome.tabs.query(${JSON.stringify(query)})`), }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (id, method, params) => evaluate( `chrome.debugger.sendCommand({tabId:${id}},${JSON.stringify(method)},${JSON.stringify(params ?? {})})`, ), - }; + }); const manager = new SessionManager({ agentWindow: { create: async () => ({ windowId, initialTabIds: [tabId] }), diff --git a/apps/extension/src/tools/__tests__/window.test.ts b/apps/extension/src/tools/__tests__/window.test.ts index 1b1e29055..a04926cd5 100755 --- a/apps/extension/src/tools/__tests__/window.test.ts +++ b/apps/extension/src/tools/__tests__/window.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { INTERACTION_STORAGE_KEY, InteractionPreferenceStore } from "@/lib/interaction-preferences"; import { SessionManager } from "@/session-manager/manager"; import { handleSessionStart } from "../session"; -import type { CdpRunner, ChromeTabsApi } from "../shared"; +import { type CdpRunner, type ChromeTabsApi, createStubCdpRunner } from "../shared"; import { handleWindowResize, type WindowResizeDeps } from "../window"; function fakeAgentWindow(ids: number[]) { @@ -56,12 +56,12 @@ function resizeFixture( state.viewport = { width: state.bounds.width, height: state.bounds.height - 143 }; }), }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({ cssLayoutViewport: { clientWidth: state.viewport.width, clientHeight: state.viewport.height }, cssVisualViewport: { zoom: opts.zoom ?? 1 }, })) as CdpRunner["send"], - }; + }); const wait = vi.fn(async () => { state.waits++; if (opts.delayViewport && state.waits === 2) diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index c5514fdca..2512ab672 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -36,13 +36,14 @@ import { prepareBackgroundExecution } from "./background-execution"; import { handleConsole } from "./console"; import { handleDebug } from "./debug"; import { handleDialog } from "./dialog-control"; -import { handleDownload } from "./download"; +import { DownloadConcurrencyGate, handleDownload } from "./download"; import { handleDrag } from "./drag"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { classifyCdpError, rpcError } from "./errors"; import { handleEvaluate } from "./evaluate"; import { handleExtract } from "./extract"; import { handleRequestHelp } from "./human-loop"; +import type { InputWindowsApi } from "./input-readiness"; import { handleBlur, handleClick, @@ -62,6 +63,8 @@ import { handleNetwork, type NetworkCdpRunner } from "./network"; import { type CdpRunner, chromeTabsCaptureApi, + clearObserveIdentityStreaksForSession, + clearObserveIdentityStreaksForTab, handleGetHtml, handleObserve, handleScreenshot, @@ -127,6 +130,11 @@ interface HoverLatchScope { tab_id?: number; } +export interface TabRemovedEvent { + addListener(callback: (tabId: number) => void): void; + removeListener(callback: (tabId: number) => void): void; +} + export interface DispatcherDeps { cooperation?: CooperationController; video?: VideoManager; @@ -135,6 +143,8 @@ export interface DispatcherDeps { sessions: SessionManager; cdp?: DispatcherCdpRunner; recording?: RecordRuntimeDeps; + tabsOnRemoved?: TabRemovedEvent; + windowsApi?: InputWindowsApi; /** * Invoked whenever a dispatched RPC may have changed the live * session set (currently `tool.session_start` and @@ -188,9 +198,13 @@ export class ToolDispatcher { private readonly approveBorrow?: BorrowConfirmationApprover; private readonly interactionPreferences?: InteractionPreferenceStore; private readonly helpNotificationCopy?: () => { title: string; body: string }; + private readonly tabsOnRemoved?: TabRemovedEvent; + private readonly windowsApi?: InputWindowsApi; private subscription: { dispose(): void } | null = null; + private tabRemovedSubscription: { dispose(): void } | null = null; private readonly hoverBypassTabs = new Map(); private readonly hoverLatches = new Map(); + private readonly downloadGate = new DownloadConcurrencyGate(); private readonly activeDialogOperations = new Map< string, { request: RequestFrame; operationId: string } @@ -222,10 +236,22 @@ export class ToolDispatcher { this.approveBorrow = deps.approveBorrow; this.interactionPreferences = deps.interactionPreferences; this.helpNotificationCopy = deps.helpNotificationCopy; + this.tabsOnRemoved = + deps.tabsOnRemoved ?? (typeof chrome === "undefined" ? undefined : chrome.tabs?.onRemoved); + this.windowsApi = deps.windowsApi; } start(): void { if (this.subscription) return; + if (this.tabsOnRemoved && !this.tabRemovedSubscription) { + const onTabRemoved = (tabId: number) => { + this.forgetClosedTab(tabId); + }; + this.tabsOnRemoved.addListener(onTabRemoved); + this.tabRemovedSubscription = { + dispose: () => this.tabsOnRemoved?.removeListener(onTabRemoved), + }; + } this.dialogSubscription = this.cdp?.onDialogChanged?.((tabId, dialog, reason) => { const sessionId = this.sessions.findControllingSession(tabId); @@ -285,6 +311,8 @@ export class ToolDispatcher { stop(): void { this.debug?.dispose(); + this.tabRemovedSubscription?.dispose(); + this.tabRemovedSubscription = null; this.subscription?.dispose(); this.subscription = null; // Trip every outstanding controller so dependent waits unblock @@ -308,6 +336,13 @@ export class ToolDispatcher { void exports.dispose(); } + /** Drop per-tab hover latches, overlay-bypass bookkeeping, and observe streaks for a closed tab. */ + forgetClosedTab(tabId: number): void { + this.hoverBypassTabs.delete(tabId); + this.hoverLatches.delete(tabId); + clearObserveIdentityStreaksForTab(tabId); + } + /** Revoke in-flight work locally; do not wait for the daemon's round trip. */ cancelSession(sessionId: string): void { for (const [rpcId, owner] of this.requestSessions) { @@ -675,12 +710,16 @@ export class ToolDispatcher { await this.releaseHoverLatch(params.session_id); const result = await handleSessionStop(this.sessions, params, { cdp, + ...(this.recording ? { recording: this.recording } : {}), ...(typeof chrome === "undefined" ? {} : { tabManagement: { tabs: chromeTabMutationApi }, tabsQuery: chromeTabsApi }), signal, }); - if (!this.sessions.has(params.session_id)) this.cooperation?.release(params.session_id); + if (!this.sessions.has(params.session_id)) { + this.cooperation?.release(params.session_id); + clearObserveIdentityStreaksForSession(params.session_id); + } return result; }, "tool.tab_list": async (params, { signal }) => { @@ -723,7 +762,7 @@ export class ToolDispatcher { cdp, beforeReturn: async (sessionId, tabId) => { this.debug?.stopTab(tabId); - clearRecordingForSession(sessionId); + clearRecordingForSession(sessionId, this.recording); await this.releaseHoverLatch(sessionId, tabId); }, }); @@ -925,6 +964,7 @@ export class ToolDispatcher { handleWheel(this.sessions, params, { cdp, tabsApi: chromeTabsApi, + ...(this.windowsApi ? { windowsApi: this.windowsApi } : {}), signal, bypassOverlay, }), @@ -1015,6 +1055,7 @@ export class ToolDispatcher { signal, completionSignal, bypassOverlay, + downloadGate: this.downloadGate, }) : Promise.resolve({ code: "unsupported", diff --git a/apps/extension/src/tools/dom-read-runtime.ts b/apps/extension/src/tools/dom-read-runtime.ts index 983dda578..b5f72b4c3 100644 --- a/apps/extension/src/tools/dom-read-runtime.ts +++ b/apps/extension/src/tools/dom-read-runtime.ts @@ -1,5 +1,5 @@ -import { type CdpFrame, toFrameDocumentHandle } from "@/browser-driver/frame-graph"; -import { type CdpRunner, sendToFrameHandle } from "./shared"; +import type { FrameDocumentHandle } from "@/browser-driver/frame-handles"; +import { type CdpRunner, sendToFrameHandle, withObjectGroup } from "./shared"; import type { DocumentIdentity } from "./vom/facts"; export const SERIALIZATION = { serialization: "deep", @@ -39,24 +39,22 @@ export function unwrap(reply: RuntimeReply): unknown { } export async function documentIdentity( cdp: CdpRunner, - frame: CdpFrame, + frameHandle: FrameDocumentHandle, attachmentId: string, unavailable: () => never = () => { throw new Error("Could not establish DOM document identity"); }, ): Promise<{ identity: DocumentIdentity; contextId: number }> { - const frameHandle = toFrameDocumentHandle(frame.target.tabId, frame); const world = await sendToFrameHandle<{ executionContextId: number }>( cdp, frameHandle, "Page.createIsolatedWorld", { - frameId: frame.frameId, + frameId: frameHandle.frameId, worldName: "bsk-extract", }, ); - const group = `bsk-extract-root-${crypto.randomUUID()}`; - try { + return withObjectGroup(cdp, frameHandle, "bsk-extract-root", async (group) => { const root = unwrap( await sendToFrameHandle(cdp, frameHandle, "Runtime.evaluate", { expression: "document.documentElement", @@ -66,7 +64,7 @@ export async function documentIdentity( }), ) as { backendNodeId?: number } | null; if (!root?.backendNodeId) unavailable(); - cdp.recordFrameDocumentElement?.( + cdp.recordFrameDocumentElement( frameHandle.tabId, frameHandle.frameId, frameHandle.loaderId, @@ -82,9 +80,5 @@ export async function documentIdentity( }, contextId: world.executionContextId, }; - } finally { - await sendToFrameHandle(cdp, frameHandle, "Runtime.releaseObjectGroup", { - objectGroup: group, - }).catch(() => {}); - } + }); } diff --git a/apps/extension/src/tools/download-capture.ts b/apps/extension/src/tools/download-capture.ts index b731c7c32..3435f1ade 100644 --- a/apps/extension/src/tools/download-capture.ts +++ b/apps/extension/src/tools/download-capture.ts @@ -5,7 +5,7 @@ // no CDP intent on the clicked target, so a navigation target that the clicked // tab opens after mouse press supplies the intent URL instead. -import type { FrameDocumentHandle } from "@/browser-driver/frame-graph"; +import type { FrameDocumentHandle } from "@/browser-driver/frame-handles"; import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types"; import { transferError } from "./errors"; import { type CdpRunner, isRpcError } from "./shared"; @@ -329,15 +329,9 @@ export async function captureBrowserDownload( popupUrls.add(details.url); reconcile(); }; - let mockFrameKnown = !options.handle.frameId; - let mockFrameSessionId: string | undefined; - const cdpSubscription = options.cdp.onEvent?.(options.handle.tabId, (source, method, raw) => { + const cdpSubscription = options.cdp.onEvent(options.handle.tabId, (source, method, raw) => { if (settled || method !== "Page.downloadWillBegin") return; - const matchesSource = options.cdp.isEventFromFrame - ? options.cdp.isEventFromFrame(source, options.handle) - : source.tabId === options.handle.tabId && - (!mockFrameKnown || (source.sessionId ?? undefined) === mockFrameSessionId); - if (!matchesSource) return; + if (!options.cdp.isEventFromFrame(source, options.handle)) return; if (capturedId !== undefined && options.inputSignal?.aborted) return; const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown }; if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return; @@ -356,24 +350,10 @@ export async function captureBrowserDownload( }; reconcile(); }); - if (!cdpSubscription) { - return captureError("CDP download intent subscription unavailable", "none", "arm"); - } let releaseTabAttachment: (() => void) | undefined; try { - releaseTabAttachment = options.cdp.retainTabAttachment?.(options.handle.tabId); - if (!options.cdp.isEventFromFrame && options.handle.frameId && options.cdp.getFrameGraph) { - const graph = await waitBounded( - options.cdp.getFrameGraph(options.handle.tabId), - deadline, - options.signal, - "resolving download frame target timed out", - ); - mockFrameSessionId = graph.frames.find((entry) => entry.frameId === options.handle.frameId) - ?.target.sessionId; - mockFrameKnown = true; - } + releaseTabAttachment = options.cdp.retainTabAttachment(options.handle.tabId); options.downloads.onDeterminingFilename.addListener(determiningListener); options.downloads.onCreated.addListener(createdListener); options.downloads.onChanged.addListener(changedListener); diff --git a/apps/extension/src/tools/download.ts b/apps/extension/src/tools/download.ts index 6cf39090b..c6b4a8cb2 100644 --- a/apps/extension/src/tools/download.ts +++ b/apps/extension/src/tools/download.ts @@ -10,20 +10,60 @@ import { type DownloadsApi, type NavigationTargetsApi, } from "./download-capture"; +import { rpcError } from "./errors"; import { withInputReady } from "./input-readiness"; import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; import { pointerTransactionDeps } from "./pointer-transaction"; import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared"; -let downloadActive = false; - export type { DownloadsApi, NavigationTargetsApi } from "./download-capture"; +/** + * Instance-scoped concurrency gate for `chrome.downloads` attribution. + * `ToolDispatcher` owns one gate per browser runtime; direct callers/tests + * without an explicit gate derive an isolated gate per `DownloadsApi` instance. + */ +export class DownloadConcurrencyGate { + private active = false; + + isBusy(): boolean { + return this.active; + } + + tryAcquire(): (() => void) | null { + if (this.active) return null; + this.active = true; + let released = false; + return () => { + if (!released) { + released = true; + this.active = false; + } + }; + } +} + +const gatesByDownloadsApi = new WeakMap(); + +function resolveDownloadGate( + downloads: DownloadsApi, + explicitGate?: DownloadConcurrencyGate, +): DownloadConcurrencyGate { + if (explicitGate) return explicitGate; + let gate = gatesByDownloadsApi.get(downloads); + if (!gate) { + gate = new DownloadConcurrencyGate(); + gatesByDownloadsApi.set(downloads, gate); + } + return gate; +} + export interface DownloadDeps extends InteractionDeps { /** Human input revokes the trigger; explicit cancellation also stops receipt collection. */ completionSignal?: AbortSignal; downloads?: DownloadsApi; navigationTargets?: NavigationTargetsApi; + downloadGate?: DownloadConcurrencyGate; } export async function handleDownload( @@ -31,18 +71,25 @@ export async function handleDownload( params: DownloadParams, deps: DownloadDeps, ): Promise { - if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" }; - downloadActive = true; + const ctx = lookupSession(manager, params, "download"); + if (isRpcError(ctx)) return ctx; + if (!params.browser_relative_dir) { + return { code: "invalid_params", message: "download requires a daemon capability directory" }; + } + const downloads = deps.downloads ?? chromeDownloadsApi; + const releaseGate = resolveDownloadGate(downloads, deps.downloadGate).tryAcquire(); + if (!releaseGate) { + return rpcError("timeout", "session_busy", "another bsk download is active", { + effect_state: "none", + dispatched: false, + phase: "before_trigger", + }); + } try { - const ctx = lookupSession(manager, params, "download"); - if (isRpcError(ctx)) return ctx; const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "download"); if (denied) return denied; - if (!params.browser_relative_dir) { - return { code: "invalid_params", message: "download requires a daemon capability directory" }; - } const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download"); if (isRpcError(address)) return address; @@ -52,7 +99,7 @@ export async function handleDownload( const capture = await captureBrowserDownload({ cdp: deps.cdp, handle: address.handle, - downloads: deps.downloads ?? chromeDownloadsApi, + downloads, navigationTargets: deps.navigationTargets ?? chromeNavigationTargetsApi, browserRelativeDir: params.browser_relative_dir, maxByteSize: params.max_byte_size, @@ -97,6 +144,6 @@ export async function handleDownload( browser_path: item.filename, }; } finally { - downloadActive = false; + releaseGate(); } } diff --git a/apps/extension/src/tools/drag.ts b/apps/extension/src/tools/drag.ts index 5d08781a5..799ba8fc7 100644 --- a/apps/extension/src/tools/drag.ts +++ b/apps/extension/src/tools/drag.ts @@ -1,4 +1,4 @@ -import { type FrameNodeHandle, isStaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { type FrameNodeHandle, isStaleFrameHandleError } from "@/browser-driver/frame-handles"; import type { SessionManager } from "@/session-manager/manager"; import type { DragParams, DragPoint, DragResult, DragTarget, RpcError } from "@/transport/types"; import { withClickOverlay } from "./click-overlay"; @@ -10,6 +10,7 @@ import { type InteractionDeps, modifiersBitfield, resolveActionTarget } from "./ import { pointerTransactionDeps } from "./pointer-transaction"; import { type CdpRunner, + delegateCdpRunner, enforceAgentWindow, isRpcError, lookupSession, @@ -19,21 +20,9 @@ import { import { pointReachesTarget } from "./target-hit"; import { BoundedWaitError, waitBounded } from "./transfer-transaction"; -async function isOutOfProcessNode( - cdp: CdpRunner, - tabId: number, - handle: FrameNodeHandle, -): Promise { +function isOutOfProcessNode(cdp: CdpRunner, tabId: number, handle: FrameNodeHandle): boolean { if (!handle.frameId) return false; - if (cdp.hasOwnFrameTarget) { - return cdp.hasOwnFrameTarget(tabId, handle.frameId); - } - if (cdp.getFrameGraph) { - const graph = await cdp.getFrameGraph(tabId); - const frame = graph.frames.find((entry) => entry.frameId === handle.frameId); - return Boolean(frame?.target.sessionId); - } - return false; + return cdp.hasOwnFrameTarget(tabId, handle.frameId); } const validPoint = (p: DragPoint | undefined) => @@ -126,18 +115,19 @@ export async function handleDrag( }; // Check identity even between frames of a gesture. The transaction also fences // late CDP continuations after the outer time boundary has retired the operation. - const cdp = new Proxy(inputDeps.cdp, { - get(target, key) { - if (key === "resourceCleanup") return deps.cdp.resourceCleanup ?? deps.cdp; - const value = Reflect.get(target, key); - return typeof value === "function" - ? (...args: unknown[]) => { - check(); - return value.apply(target, args); - } - : value; + const cdp = delegateCdpRunner( + inputDeps.cdp, + { resourceCleanup: deps.cdp.resourceCleanup ?? deps.cdp }, + { + intercept: (target, _key, value) => + typeof value === "function" + ? (...args: unknown[]) => { + check(); + return value.apply(target, args); + } + : undefined, }, - }) as CdpRunner; + ); let outcome: DragResult | RpcError; try { outcome = await waitBounded( @@ -184,11 +174,11 @@ export async function handleDrag( y: p.y + (o?.y ?? 0), }); const sourceOopif = params.source.offset - ? await isOutOfProcessNode(cdp, tab.tabId, source.handle) + ? isOutOfProcessNode(cdp, tab.tabId, source.handle) : false; const targetOopif = target && params.target?.offset - ? await isOutOfProcessNode(cdp, tab.tabId, target.handle) + ? isOutOfProcessNode(cdp, tab.tabId, target.handle) : false; if (sourceOopif || targetOopif) return { @@ -272,9 +262,13 @@ export async function handleDrag( } finally { const cleanup = deps.cdp.resourceCleanup ?? deps.cdp; for (const node of target ? [source, target] : [source]) - await sendToFrameHandle(cleanup, node.handle, "Runtime.releaseObjectGroup", { - objectGroup: group, - }).catch(() => {}); + await sendToFrameHandle( + cleanup, + node.handle, + "Runtime.releaseObjectGroup", + { objectGroup: group }, + { allowReattach: false }, + ).catch(() => {}); } await cdp.verifyPointerTarget?.( { handle: source.handle }, @@ -403,6 +397,7 @@ export async function handleDrag( target.handle, "Runtime.releaseObjectGroup", { objectGroup: group }, + { allowReattach: false }, ).catch(() => {}); } await cdp.verifyPointerTarget?.( diff --git a/apps/extension/src/tools/element-geometry.ts b/apps/extension/src/tools/element-geometry.ts index 7869f24a6..9cde37c28 100644 --- a/apps/extension/src/tools/element-geometry.ts +++ b/apps/extension/src/tools/element-geometry.ts @@ -1,7 +1,7 @@ // Resolve and scroll nodes inside one CDP target. Cross-frame projection // belongs to frame-geometry.ts. -import { isStaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { isStaleFrameHandleError } from "@/browser-driver/frame-handles"; import type { RpcError } from "@/transport/types"; import { rpcError } from "./errors"; import { diff --git a/apps/extension/src/tools/errors.ts b/apps/extension/src/tools/errors.ts index c7584f068..d97621ea2 100644 --- a/apps/extension/src/tools/errors.ts +++ b/apps/extension/src/tools/errors.ts @@ -2,7 +2,7 @@ // rendering. Extension handlers attach reasons here; human-facing copy // lives in bsk-cli `render_error.rs`. -import { isStaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { isStaleFrameHandleError } from "@/browser-driver/frame-handles"; import type { ErrorCode, RpcError, diff --git a/apps/extension/src/tools/evaluate.ts b/apps/extension/src/tools/evaluate.ts index 54da43f95..067d70d0c 100644 --- a/apps/extension/src/tools/evaluate.ts +++ b/apps/extension/src/tools/evaluate.ts @@ -30,6 +30,7 @@ import { isRpcError, lookupSession, resolveTargetTab, + withRemoteObject, } from "./shared"; export interface EvaluateDeps { @@ -147,7 +148,7 @@ export async function handleEvaluate( return { code: "cancelled", message: "evaluate aborted" }; } const dialogCursor = markDialogCursor(deps.cdp, target.tabId); - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + deps.cdp.trackSessionTab(ctx.sessionId, target.tabId); try { await deps.cdp.beginDomMutation?.(); const reply = await deps.cdp.send(target.tabId, "Runtime.evaluate", { @@ -156,28 +157,43 @@ export async function handleEvaluate( returnByValue: params.return_by_value ?? true, throwOnSideEffect: false, }); - if (deps.signal?.aborted) - return { - code: "cancelled", - message: "evaluate cancelled after dispatch; page effects may have occurred", - data: { effect_state: "unknown" }, - }; - // The receipt says so when nothing about the page was verified. - const unverified = params.unverified === true ? { verified: false as const } : {}; - if (reply.exceptionDetails) { + const releaseAndFormat = async (): Promise => { + if (deps.signal?.aborted) + return { + code: "cancelled", + message: "evaluate cancelled after dispatch; page effects may have occurred", + data: { effect_state: "unknown" }, + }; + // The receipt says so when nothing about the page was verified. + const unverified = params.unverified === true ? { verified: false as const } : {}; + if (reply.exceptionDetails) { + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + ok: false, + tab_id: target.tabId, + error: buildEvaluateError(reply.exceptionDetails), + ...unverified, + }); + } return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - ok: false, + ok: true, tab_id: target.tabId, - error: buildEvaluateError(reply.exceptionDetails), + value: extractValue(reply.result), ...unverified, }); + }; + const objectIds = [ + ...new Set( + [reply.result?.objectId, reply.exceptionDetails?.exception?.objectId].filter( + (id): id is string => typeof id === "string" && id.length > 0, + ), + ), + ]; + let run = releaseAndFormat; + for (const objectId of objectIds) { + const next = run; + run = () => withRemoteObject(deps.cdp, target.tabId, objectId, next); } - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - ok: true, - tab_id: target.tabId, - value: extractValue(reply.result), - ...unverified, - }); + return await run(); } catch (err) { return { code: "cdp_failed", diff --git a/apps/extension/src/tools/extract.ts b/apps/extension/src/tools/extract.ts index 65711fe1f..a97e01885 100644 --- a/apps/extension/src/tools/extract.ts +++ b/apps/extension/src/tools/extract.ts @@ -1,9 +1,11 @@ -import type { - CdpFrame, - CdpFrameGraph, - FrameDocumentHandle, - FrameNodeHandle, -} from "@/browser-driver/frame-graph"; +import { + type FrameDocumentGraph, + type FrameDocumentHandle, + type FrameDocumentInfo, + type FrameNodeHandle, + isStaleFrameHandleError, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { RpcError } from "@/transport/types"; import { documentIdentity, type RuntimeReply, SERIALIZATION, unwrap } from "./dom-read-runtime"; @@ -12,13 +14,13 @@ import { fitExtractBudget, normalizeExtract } from "./extract/normalize"; import type { CollectorOptions, ExtractParams, ExtractResult, RawCapture } from "./extract/types"; import { ExtractionFailure, extractionFailure, validateExtract } from "./extract/validation"; import { + type CdpDispatchGuard, type CdpRunner, type ChromeTabsApi, delegateCdpRunner, isRpcError, lookupSession, resolveCdpAccessibleTargetTab, - sendToCdpTarget, sendToFrameHandle, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; @@ -122,8 +124,8 @@ function frameOwnerVisible(this: Element): boolean { async function frameVisible( cdp: CdpRunner, - graph: CdpFrameGraph, - frame: CdpFrame, + graph: FrameDocumentGraph, + frame: FrameDocumentInfo, attachmentId: string, deadline: number, signal?: AbortSignal, @@ -136,7 +138,7 @@ async function frameVisible( throw new ExtractionFailure("extract_limit", "Extraction deadline exceeded"); const parent = graph.frames.find((candidate) => candidate.frameId === frame.parentFrameId); if (!parent || frame.ownerBackendNodeId === undefined) stale(); - const { identity } = await documentIdentity(cdp, parent, attachmentId, stale); + const { identity } = await documentIdentity(cdp, parent.handle, attachmentId, stale); const owner = await resolveVerifiedNode(cdp, identity, frame.ownerBackendNodeId, signal); if (owner.status !== "current") stale(); try { @@ -188,67 +190,63 @@ export async function handleExtract( "extract", ); if (isRpcError(tab)) return tab; - deps.cdp.trackSessionTab?.(ctx.sessionId, tab.tabId); - await deps.cdp.ensureAttachedToUrl?.(tab.tabId, tab.url); + deps.cdp.trackSessionTab(ctx.sessionId, tab.tabId); + await deps.cdp.ensureAttachedToUrl(tab.tabId, tab.url); throwIfAborted(signal); - const attachmentId = deps.cdp.getAttachmentId?.(tab.tabId); + const attachmentId = deps.cdp.getAttachmentId(tab.tabId); if (!attachmentId) return { code: "unsupported", message: "extract requires document identity support" }; const check = () => { throwIfAborted(signal); if ( manager.get(ctx.sessionId) !== ctx || - deps.cdp.getAttachmentId?.(tab.tabId) !== attachmentId + deps.cdp.getAttachmentId(tab.tabId) !== attachmentId ) stale(); }; const guardFor = (method: string) => { if (method === "Runtime.releaseObjectGroup") { - if (deps.cdp.getAttachmentId?.(tab.tabId) !== attachmentId) return null; - return { attachmentId }; + if (deps.cdp.getAttachmentId(tab.tabId) !== attachmentId) return null; + return { attachmentId, allowReattach: false }; } check(); return { signal, attachmentId }; }; + const sendFrame = async ( + handle: FrameDocumentHandle | FrameNodeHandle, + method: string, + args?: object, + extraGuard?: CdpDispatchGuard, + ): Promise => { + const guard = guardFor(method); + if (!guard) return {} as T; + return deps.cdp.sendToFrame(handle, method, args, { + ...extraGuard, + ...guard, + signal: extraGuard?.signal ?? guard.signal, + }); + }; // Guard each native dispatch, including those in shared identity helpers. const cdp = delegateCdpRunner(deps.cdp, { - send: async (tabId: number, method: string, args?: object): Promise => { - const guard = guardFor(method); - if (!guard) return undefined as T; - return deps.cdp.sendGuarded - ? deps.cdp.sendGuarded({ tabId }, method, args, guard) - : deps.cdp.send(tabId, method, args); - }, - sendToTarget: async ( - target: Parameters>[0], - method: string, - args?: object, - ): Promise => { - const guard = guardFor(method); - if (!guard) return undefined as T; - return deps.cdp.sendGuarded - ? deps.cdp.sendGuarded(target, method, args, guard) - : sendToCdpTarget(deps.cdp, target, method, args); - }, - sendToFrame: async ( + resourceCleanup: undefined, + send: (tabId: number, method: string, args?: object, sig?: AbortSignal): Promise => + sendFrame(toRootFrameHandle(tabId), method, args, sig ? { signal: sig } : undefined), + sendToFrame: ( handle: FrameDocumentHandle | FrameNodeHandle, method: string, args?: object, - ): Promise => { - const guard = guardFor(method); - if (!guard) return undefined as T; - return sendToFrameHandle(deps.cdp, handle, method, args, guard); - }, + extraGuard?: CdpDispatchGuard, + ): Promise => sendFrame(handle, method, args, extraGuard), }); - const graph = await deps.cdp.getFrameGraph?.(tab.tabId); + const graph = await deps.cdp.getFrameDocuments(tab.tabId, signal); check(); - const root = graph?.frames.find((frame) => frame.frameId === graph.rootFrameId); + const root = graph.frames.find((frame) => frame.frameId === graph.rootFrameId); if (!root) return { code: "unsupported", message: "Could not establish the target document's frame identity", }; - const rootDoc = await documentIdentity(cdp, root, attachmentId, stale); + const rootDoc = await documentIdentity(cdp, root.handle, attachmentId, stale); let selectedFrame = root; let backendNodeId: number | undefined; let expectedIdentity: DocumentIdentity | undefined; @@ -256,7 +254,7 @@ export async function handleExtract( if (!saved || saved.identity.tabId !== tab.tabId) stale(); expectedIdentity = saved.identity; backendNodeId = saved.backendNodeId; - const frame = graph!.frames.find( + const frame = graph.frames.find( (f) => f.frameId === saved.identity.frameId && (!saved.identity.loaderId || !f.loaderId || f.loaderId === saved.identity.loaderId), @@ -266,9 +264,9 @@ export async function handleExtract( } else if (params.ref) { const ref = resolveSnapshotRef(ctx, params.ref, tab.tabId); if (isRpcError(ref)) return ref; - const frame = graph!.frames.find( + const frame = graph.frames.find( (f) => - f.frameId === (ref.frameId || graph!.rootFrameId) && + f.frameId === (ref.frameId || graph.rootFrameId) && (!ref.loaderId || !f.loaderId || f.loaderId === ref.loaderId), ); if (!frame) stale(); @@ -277,18 +275,26 @@ export async function handleExtract( attachmentId, tabId: tab.tabId, frameId: frame.frameId, - loaderId: frame.loaderId ?? ref.loaderId ?? "", + loaderId: frame.loaderId || ref.loaderId || "", documentElementBackendNodeId: ref.documentElementBackendNodeId, }; } selectedFrame = frame; backendNodeId = ref.backendNodeId; } - const frames = - options.action === "discover" && !options.selector - ? [root, ...graph!.frames.filter((frame) => frame.frameId !== root.frameId)].slice(0, 16) - : [selectedFrame]; + const multiFrameSearch = + !params.ref && + !params.target_id && + (options.action === "discover" || Boolean(options.selector)); + const frames = multiFrameSearch + ? [root, ...graph.frames.filter((frame) => frame.frameId !== root.frameId)].slice(0, 16) + : [selectedFrame]; let output: ExtractResult | undefined; + let outputIdentity: DocumentIdentity | undefined; + let lastNotFoundError: ExtractionFailure | undefined; + let hiddenFrameError: ExtractionFailure | undefined; + let unavailableChildError: unknown; + const frameWarnings: string[] = []; const pendingTargets: [string, RetainedTarget][] = []; const started = performance.now(); for (const frame of frames) { @@ -299,14 +305,14 @@ export async function handleExtract( if ( !(await frameVisible( cdp, - graph!, + graph, frame, attachmentId, started + options.timeout_ms, signal, )) ) { - if (options.action === "discover") continue; + if (multiFrameSearch) continue; throw new ExtractionFailure("extract_target_hidden", "The extraction frame is hidden"); } const remaining = options.timeout_ms - (performance.now() - started); @@ -315,7 +321,7 @@ export async function handleExtract( const current = frame.frameId === root.frameId ? rootDoc - : await documentIdentity(cdp, frame, attachmentId, stale); + : await documentIdentity(cdp, frame.handle, attachmentId, stale); if ( expectedIdentity && (await verifyDocumentIdentity(cdp, expectedIdentity, signal)) !== "current" @@ -341,8 +347,16 @@ export async function handleExtract( }; const normalized = normalizeExtract(raw, options, tab.tabId, source); if (options.action !== "discover") { + if (output) { + throw new ExtractionFailure( + "extract_target_ambiguous", + "The selector matched containers in multiple frames; use discover or a ref", + ); + } output = normalized; - break; + outputIdentity = current.identity; + if (!multiFrameSearch || frames.length === 1) break; + continue; } if (!output) output = { ...normalized, targets: [] }; output.warnings.push(...raw.warnings); @@ -372,6 +386,25 @@ export async function handleExtract( }); } } catch (error) { + if ( + multiFrameSearch && + !signal?.aborted && + error instanceof ExtractionFailure && + error.reason === "selector_not_found" + ) { + lastNotFoundError = error; + continue; + } + if ( + multiFrameSearch && + frames.length > 1 && + !signal?.aborted && + error instanceof ExtractionFailure && + error.reason === "extract_target_hidden" + ) { + hiddenFrameError ??= error; + continue; + } if ( options.action === "discover" && output && @@ -386,19 +419,27 @@ export async function handleExtract( break; } if ( - options.action !== "discover" || + !multiFrameSearch || frame.frameId === root.frameId || + error instanceof ExtractionFailure || isCaptureTerminalError(error) || signal?.aborted ) throw error; - if (output) { - output.warnings.push(`frame_unavailable:${frame.frameId}`); - output.coverage.dataset_complete = "incomplete"; - } + unavailableChildError ??= error; + frameWarnings.push(`frame_unavailable:${frame.frameId}`); } } - if (!output) throw new Error("No extraction result"); + if (!output) { + if (hiddenFrameError) throw hiddenFrameError; + if (unavailableChildError && options.action !== "discover") throw unavailableChildError; + if (lastNotFoundError) throw lastNotFoundError; + throw new Error("No extraction result"); + } + if (frameWarnings.length > 0) { + output.warnings.push(...frameWarnings); + output.coverage.dataset_complete = "incomplete"; + } if (options.action === "discover") { output.targets!.sort((a, b) => Number(a.kind === "list") - Number(b.kind === "list")); if (output.targets!.length > 32) { @@ -406,13 +447,13 @@ export async function handleExtract( output.coverage.truncated = true; output.coverage.stop_reason ??= "target_limit"; } - if (graph!.frames.length > frames.length && !options.selector) { + if (graph.frames.length > frames.length) { output.coverage.truncated = true; output.coverage.stop_reason = "frame_limit"; } - for (const frame of graph!.unavailableFrames ?? []) + for (const frame of graph.unavailableFrames ?? []) output.warnings.push(`frame_unavailable:${frame.frameId}`); - if (output.coverage.truncated || graph!.unavailableFrames?.length) + if (output.coverage.truncated || graph.unavailableFrames?.length) output.coverage.dataset_complete = "incomplete"; while ( output.targets!.length > 0 && @@ -425,6 +466,12 @@ export async function handleExtract( } } if ((await verifyDocumentIdentity(cdp, rootDoc.identity, signal)) !== "current") stale(); + if ( + outputIdentity && + outputIdentity.frameId !== rootDoc.identity.frameId && + (await verifyDocumentIdentity(cdp, outputIdentity, signal)) !== "current" + ) + stale(); check(); output.warnings = [...new Set(output.warnings)]; output = fitExtractBudget(output, options.max_bytes); @@ -439,6 +486,13 @@ export async function handleExtract( if (signal?.aborted || (error instanceof Error && error.name === "AbortError")) return { code: "cancelled", message: "extract aborted" }; if (error instanceof ExtractionFailure) return extractionFailure(error); + if (isStaleFrameHandleError(error)) + return extractionFailure( + new ExtractionFailure( + "extract_target_stale", + "The extraction document or target changed; inspect it again", + ), + ); return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) }; } } diff --git a/apps/extension/src/tools/extract/collector.ts b/apps/extension/src/tools/extract/collector.ts index 6b4508bde..88865a402 100644 --- a/apps/extension/src/tools/extract/collector.ts +++ b/apps/extension/src/tools/extract/collector.ts @@ -25,7 +25,10 @@ export function collectExtractDom(this: Element, options: CollectorOptions): Raw const tableSelector = 'table,[role="table"],[role="grid"],[role="treegrid"]'; const listSelector = 'ul,ol,[role="list"]'; const sensitiveInputSelector = + options.sensitive ?? 'input[type="password"],input[type="hidden"],[data-bsk-record-password],[autocomplete~="current-password" i],[autocomplete~="new-password" i],[autocomplete~="one-time-code" i],[autocomplete^="cc-" i],[autocomplete*=" cc-" i]'; + const overlaySelector = + options.overlaySelector ?? "[data-bsk-overlay],[data-bsk-overlay-host],browser-skill-overlay"; const hiddenSubtrees = new WeakMap(); const styles = new WeakMap(); const styleOf = (element: Element) => { @@ -354,6 +357,37 @@ export function collectExtractDom(this: Element, options: CollectorOptions): Raw return fail("extract_selector_invalid", `Invalid CSS selector: ${selector}`); } }; + const isOverlayElement = (element: Element): boolean => + element.hasAttribute("data-bsk-overlay") || + element.hasAttribute("data-bsk-overlay-host") || + element.tagName.toLowerCase() === "browser-skill-overlay" || + Boolean(element.closest(overlaySelector)); + const selectAcrossOpenShadowRoots = (root: ParentNode, selector: string): Element[] => { + const matches: Element[] = []; + for (const node of select(root, selector)) { + if (!isOverlayElement(node)) matches.push(node); + } + const roots: ParentNode[] = [root]; + while (roots.length) { + const currentRoot = roots.pop()!; + const walker = document.createTreeWalker(currentRoot, NodeFilter.SHOW_ELEMENT); + for ( + let el = walker.nextNode() as Element | null; + el; + el = walker.nextNode() as Element | null + ) { + tick(); + if (isOverlayElement(el)) continue; + if (el.shadowRoot) { + for (const node of select(el.shadowRoot, selector)) { + if (!isOverlayElement(node)) matches.push(node); + } + roots.push(el.shadowRoot); + } + } + } + return matches; + }; // Invalid ARIA positions fall back to DOM order with a warning, like spans // and counts; only conflicting cells remain structural errors. const ariaIndex = ( @@ -413,7 +447,7 @@ export function collectExtractDom(this: Element, options: CollectorOptions): Raw fail("extract_target_stale", "The extraction target no longer belongs to this document"); let root = options.anchored ? this : document.documentElement; if (options.selector) { - const matches = select(document, options.selector); + const matches = selectAcrossOpenShadowRoots(document, options.selector); if (matches.length === 0) fail("selector_not_found", "No extraction target matches the selector"); if (matches.length !== 1) @@ -484,7 +518,10 @@ export function collectExtractDom(this: Element, options: CollectorOptions): Raw return result; } if (!options.anchored && !options.selector) { - const matches = select(root, options.action === "table" ? tableSelector : listSelector); + const matches = selectAcrossOpenShadowRoots( + root, + options.action === "table" ? tableSelector : listSelector, + ); let selected: Element | undefined; for (const node of matches) { tick(); diff --git a/apps/extension/src/tools/extract/extract.browser.test.ts b/apps/extension/src/tools/extract/extract.browser.test.ts index acf40bc76..43a44cbf5 100644 --- a/apps/extension/src/tools/extract/extract.browser.test.ts +++ b/apps/extension/src/tools/extract/extract.browser.test.ts @@ -4,8 +4,9 @@ // isolated world, followed by the same normalizer and byte budget. import { createServer } from "node:http"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { handleExtract } from "../extract"; import { collectExtractDom } from "./collector"; import { fitExtractBudget, normalizeExtract } from "./normalize"; @@ -281,15 +282,24 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("real browser structured extractio active: true, url: `${origin}/${fixture}`, } as chrome.tabs.Tab; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: (_tab, method, args) => local(method, args), getAttachmentId: () => "a1", ensureAttachedToUrl: async () => {}, - getFrameGraph: async () => ({ + getFrameDocuments: async () => ({ rootFrameId: frameTree.frame.id, - frames: [{ frameId: frameTree.frame.id, target: { tabId: 1 }, url: tab.url }], + frames: [ + { + handle: toFrameDocumentHandle(1, { frameId: frameTree.frame.id, loaderId: "" }), + frameId: frameTree.frame.id, + loaderId: "", + url: tab.url, + processKey: "root", + isOutOfProcess: false, + }, + ], }), - }; + }); const tabsApi = { get: async () => tab, query: async () => [tab] }; const discovered = await handleExtract( manager, diff --git a/apps/extension/src/tools/extract/handler.test.ts b/apps/extension/src/tools/extract/handler.test.ts index dbef7cc48..178f623b7 100644 --- a/apps/extension/src/tools/extract/handler.test.ts +++ b/apps/extension/src/tools/extract/handler.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { handleExtract } from "../extract"; import type { RawCapture } from "./types"; @@ -51,15 +52,29 @@ async function setup() { } throw new Error("Unexpected CDP call: " + method); }); - const cdp: CdpRunner = { - send: send as CdpRunner["send"], - getAttachmentId: () => state.attachment, - ensureAttachedToUrl: vi.fn(async () => {}), - getFrameGraph: vi.fn(async () => ({ - rootFrameId: "root", - frames: [{ frameId: "root", target: { tabId: 7 }, url: tab.url }], - })), - }; + const ensureAttachedToUrl = vi.fn(async () => {}); + const getFrameDocuments = vi.fn(async () => ({ + rootFrameId: "root", + frames: [ + { + handle: toFrameDocumentHandle(7, { frameId: "root", loaderId: "" }), + frameId: "root", + loaderId: "", + url: tab.url, + processKey: "root", + isOutOfProcess: false, + }, + ], + })); + const cdp = Object.assign( + createStubCdpRunner({ + send: send as CdpRunner["send"], + getAttachmentId: () => state.attachment, + ensureAttachedToUrl, + getFrameDocuments, + }), + { ensureAttachedToUrl, getFrameDocuments }, + ); const tabsApi = { get: vi.fn(async () => tab), query: vi.fn(async () => [tab]) }; const run = (params: object = {}, signal?: AbortSignal) => handleExtract( @@ -153,4 +168,123 @@ describe("extraction lifecycle", () => { expect(await h.run()).toMatchObject({ data: { reason: "extract_target_stale" } }); expect(h.send.mock.calls.at(-1)?.[1]).toBe("Runtime.releaseObjectGroup"); }); + + it("searches child frames for --selector and rejects cross-frame selector ambiguity", async () => { + const h = await setup(); + const rootHandle = toFrameDocumentHandle(7, { frameId: "root", loaderId: "L-root" }); + const childHandle = toFrameDocumentHandle(7, { frameId: "child", loaderId: "L-child" }); + h.cdp.getFrameDocuments.mockResolvedValue({ + rootFrameId: "root", + frames: [ + { + handle: rootHandle, + frameId: "root", + loaderId: "L-root", + url: "https://example.test/", + processKey: "root", + isOutOfProcess: false, + }, + { + handle: childHandle, + frameId: "child", + parentFrameId: "root", + loaderId: "L-child", + ownerBackendNodeId: 55, + url: "https://child.example.test/", + processKey: "oopif:child", + isOutOfProcess: true, + }, + ], + }); + + let rootMatches = false; + h.cdp.sendToFrame = vi.fn(async (handle, method, args) => { + if (method === "Page.createIsolatedWorld") { + return { executionContextId: handle.frameId === "child" ? 9 : 3 }; + } + if (method === "Runtime.releaseObjectGroup") return {}; + if (method === "Runtime.evaluate") { + return { + result: { + deepSerializedValue: { + type: "node", + value: { backendNodeId: handle.frameId === "child" ? 200 : 1 }, + }, + }, + }; + } + if (method === "DOM.resolveNode") return { object: { objectId: "owner-obj" } }; + if (method === "Runtime.callFunctionOn") { + const params = args as { arguments?: unknown[]; functionDeclaration?: string }; + if (params.functionDeclaration?.includes("checkVisibility")) { + return { result: { value: true } }; + } + if (!params.arguments) { + return { + result: { + deepSerializedValue: { + type: "node", + value: { backendNodeId: handle.frameId === "child" ? 200 : 1 }, + }, + }, + }; + } + if (handle.frameId === "root" && !rootMatches) { + return { + result: { + value: { + frame_url: "https://example.test/", + title: "root", + captured_at: "2026-10-01T00:00:00Z", + rows: [], + items: [], + item_sources: [], + warnings: [], + truncated: false, + targets: [], + error: { reason: "selector_not_found", message: "not in root" }, + }, + }, + }; + } + return { + result: { + value: { + frame_url: + handle.frameId === "child" + ? "https://child.example.test/" + : "https://example.test/", + title: handle.frameId, + captured_at: "2026-10-01T00:00:00Z", + columns: [{ name: "Order", index: 0 }], + rows: [{ cells: ["#1001"] }], + items: [], + item_sources: [], + warnings: [], + truncated: false, + targets: [], + }, + }, + }; + } + throw new Error(`Unexpected sendToFrame: ${method}`); + }) as CdpRunner["sendToFrame"]; + + const childOnly = await h.run({ action: "table", selector: "#orders-shadow-table" }); + expect(childOnly).toMatchObject({ + kind: "table", + source: { + frame_id: "child", + frame_url: "https://child.example.test/", + selector: "#orders-shadow-table", + }, + }); + + // When both root and child frames match the same selector, extract rejects ambiguity cleanly. + rootMatches = true; + const ambiguous = await h.run({ action: "table", selector: "#orders-shadow-table" }); + expect(ambiguous).toMatchObject({ + data: { reason: "extract_target_ambiguous" }, + }); + }); }); diff --git a/apps/extension/src/tools/extract/types.ts b/apps/extension/src/tools/extract/types.ts index bec9a6874..99b61f13f 100644 --- a/apps/extension/src/tools/extract/types.ts +++ b/apps/extension/src/tools/extract/types.ts @@ -91,6 +91,8 @@ export interface CollectorOptions { max_columns: number; max_bytes: number; timeout_ms: number; + sensitive?: string; + overlaySelector?: string; } export interface RawCell { text: string | null; diff --git a/apps/extension/src/tools/extract/validation.ts b/apps/extension/src/tools/extract/validation.ts index 8963f993a..6c624ea12 100644 --- a/apps/extension/src/tools/extract/validation.ts +++ b/apps/extension/src/tools/extract/validation.ts @@ -1,3 +1,4 @@ +import { BSK_OVERLAY_SELECTOR, SENSITIVE_EXTRACT_INPUT_SELECTOR } from "@/lib/dom-predicates"; import type { RpcError, RpcErrorReason } from "@/transport/types"; import type { CollectorOptions, ExtractParams } from "./types"; @@ -115,5 +116,7 @@ export function validateExtract(params: ExtractParams): CollectorOptions { max_columns: maxColumns, max_bytes: bound(params.max_bytes, 1_048_576, 4_194_304, "max_bytes", 1_024), timeout_ms: bound(params.timeout_ms, 5_000, 15_000, "timeout_ms", 100), + sensitive: SENSITIVE_EXTRACT_INPUT_SELECTOR, + overlaySelector: BSK_OVERLAY_SELECTOR, }; } diff --git a/apps/extension/src/tools/file-drop-transaction.ts b/apps/extension/src/tools/file-drop-transaction.ts index 3caec11b0..da97a45c1 100644 --- a/apps/extension/src/tools/file-drop-transaction.ts +++ b/apps/extension/src/tools/file-drop-transaction.ts @@ -2,7 +2,7 @@ // events are sent to the target that owns the node: root targets use top-level // coordinates, while OOPIF targets require coordinates local to their session. -import { isStaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { isStaleFrameHandleError } from "@/browser-driver/frame-handles"; import type { RpcError, TransferEffectState } from "@/transport/types"; import { rpcError, transferError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; @@ -99,6 +99,7 @@ export async function uploadThroughFileDrop( let dragStarted = false; let dropSent = false; let dragPoint = { x: 0, y: 0 }; + let objectGroupRequested = false; let cleanupFailed = false; let releaseTabAttachment: (() => void) | undefined; let outcome: FileDropTransactionResult | RpcError = failure( @@ -129,6 +130,7 @@ export async function uploadThroughFileDrop( outcome = await options.withOverlayHidden(async () => { try { try { + objectGroupRequested = true; const belongs = await waitBounded( pointBelongsToTarget(options.cdp, { handle }, point.x, point.y, objectGroup), deadline, @@ -264,20 +266,32 @@ export async function uploadThroughFileDrop( return outcome; } finally { try { - try { - await waitBounded( - sendToFrameHandle( - options.cdp.resourceCleanup ?? options.cdp, - handle, - "Runtime.releaseObjectGroup", - { objectGroup }, - ), - Date.now() + CLEANUP_TIMEOUT_MS, - undefined, - "releasing file-drop object group timed out", - ); - } catch { - cleanupFailed = true; + if (objectGroupRequested) { + try { + await waitBounded( + sendToFrameHandle( + options.cdp.resourceCleanup ?? options.cdp, + handle, + "Runtime.releaseObjectGroup", + { objectGroup }, + { allowReattach: false }, + ), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing file-drop object group timed out", + ); + } catch (err) { + if ( + !( + err instanceof Error && + (err.message === "No resources owned on this tab" || + err.message === "debugger detached") + ) && + !isStaleFrameHandleError(err) + ) { + cleanupFailed = true; + } + } } const effect = isRpcError(outcome) diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts index 01fd09ffd..3d5688a3b 100644 --- a/apps/extension/src/tools/file-input-transaction.ts +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -3,11 +3,11 @@ // chooser events and a frame-scoped DOM probe are independent input-location // signals, so event delivery is not required for standard file inputs. -import { isStaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { isStaleFrameHandleError } from "@/browser-driver/frame-handles"; import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types"; import { rpcError, transferError } from "./errors"; import type { ResolvedActionTarget } from "./interaction"; -import { type CdpRunner, isRpcError, sendToFrameHandle } from "./shared"; +import { type CdpRunner, isRpcError, sendToFrameHandle, withRemoteObject } from "./shared"; import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction"; const CLEANUP_TIMEOUT_MS = 1_000; @@ -110,11 +110,10 @@ export async function uploadThroughActivatedFileInput( const objectGroup = `bsk-upload-${crypto.randomUUID()}`; const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`; const chooserEvents: ChooserEvent[] = []; - let mockFrameKnown = !handle.frameId; - let mockFrameSessionId: string | undefined; let interceptionArmed = false; let probeArmed = false; let triggerObjectId: string | undefined; + let objectGroupRequested = false; let releaseTabAttachment: (() => void) | undefined; let chooserSubscription: { dispose(): void } | undefined; let outcome: FileInputTransactionResult | RpcError = transferFailure( @@ -126,30 +125,15 @@ export async function uploadThroughActivatedFileInput( ); try { - releaseTabAttachment = options.cdp.retainTabAttachment?.(handle.tabId); - chooserSubscription = options.cdp.onEvent?.(handle.tabId, (source, method, raw) => { + releaseTabAttachment = options.cdp.retainTabAttachment(handle.tabId); + chooserSubscription = options.cdp.onEvent(handle.tabId, (source, method, raw) => { if (method !== "Page.fileChooserOpened") return; - const matchesSource = options.cdp.isEventFromFrame - ? options.cdp.isEventFromFrame(source, handle) - : source.tabId === handle.tabId && - (!mockFrameKnown || (source.sessionId ?? undefined) === mockFrameSessionId); - if (!matchesSource) return; + if (!options.cdp.isEventFromFrame(source, handle)) return; const event = raw as ChooserEvent; chooserEvents.push(event); }); try { - if (!options.cdp.isEventFromFrame && handle.frameId && options.cdp.getFrameGraph) { - const graph = await waitBounded( - options.cdp.getFrameGraph(handle.tabId), - deadline, - options.signal, - "resolving upload frame target timed out", - ); - mockFrameSessionId = graph.frames.find((entry) => entry.frameId === handle.frameId)?.target - .sessionId; - mockFrameKnown = true; - } await waitBounded( sendToFrameHandle(options.cdp, handle, "Page.setInterceptFileChooserDialog", { enabled: true, @@ -177,6 +161,7 @@ export async function uploadThroughActivatedFileInput( } try { + objectGroupRequested = true; const resolved = await waitBounded( sendToFrameHandle<{ object?: { objectId?: string } }>( options.cdp, @@ -359,16 +344,24 @@ export async function uploadThroughActivatedFileInput( const inputError = runtimeError(input, "failed to resolve activated file input object"); if (inputError) throw inputError; if (!input.result?.objectId) throw new Error("activated file input returned no objectId"); - const described = await sendToFrameHandle<{ node?: { backendNodeId?: number } }>( + backendNodeId = await withRemoteObject( options.cdp, handle, - "DOM.describeNode", - { objectId: input.result.objectId }, + input.result.objectId, + async (inputObjectId) => { + const described = await sendToFrameHandle<{ node?: { backendNodeId?: number } }>( + options.cdp, + handle, + "DOM.describeNode", + { objectId: inputObjectId }, + ); + const nodeId = described.node?.backendNodeId; + if (typeof nodeId !== "number") { + throw new Error("DOM.describeNode returned no file input backendNodeId"); + } + return nodeId; + }, ); - backendNodeId = described.node?.backendNodeId; - if (typeof backendNodeId !== "number") { - throw new Error("DOM.describeNode returned no file input backendNodeId"); - } } if (!multiple && options.files.length !== 1) { @@ -476,20 +469,32 @@ export async function uploadThroughActivatedFileInput( cleanupFailed = true; } } - try { - await waitBounded( - sendToFrameHandle( - options.cdp.resourceCleanup ?? options.cdp, - handle, - "Runtime.releaseObjectGroup", - { objectGroup }, - ), - Date.now() + CLEANUP_TIMEOUT_MS, - undefined, - "releasing upload object group timed out", - ); - } catch { - cleanupFailed = true; + if (objectGroupRequested) { + try { + await waitBounded( + sendToFrameHandle( + options.cdp.resourceCleanup ?? options.cdp, + handle, + "Runtime.releaseObjectGroup", + { objectGroup }, + { allowReattach: false }, + ), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing upload object group timed out", + ); + } catch (err) { + if ( + !( + err instanceof Error && + (err.message === "No resources owned on this tab" || + err.message === "debugger detached") + ) && + !isStaleFrameHandleError(err) + ) { + cleanupFailed = true; + } + } } const effect = isRpcError(outcome) diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index 63b70d017..dbb5ae46f 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -1,11 +1,9 @@ import { - type CdpFrame, - type FrameDocumentHandle, type FrameNodeHandle, isStaleFrameHandleError, + rootProcessBoundary, StaleFrameHandleError, - toFrameDocumentHandle, -} from "@/browser-driver/frame-graph"; +} from "@/browser-driver/frame-handles"; import type { RpcError } from "@/transport/types"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { rpcError } from "./errors"; @@ -23,7 +21,7 @@ import { } from "./geometry"; import type { CssViewport } from "./geometry/coordinate-types"; import { cssViewport, GeometryContext } from "./geometry/frame-context"; -import { type CdpRunner, cdpRunnerForFrame, cdpRunnerForTarget, isRpcError } from "./shared"; +import { type CdpRunner, cdpRunnerForFrame, isRpcError } from "./shared"; export type NodeAddress = FrameNodeHandle; @@ -42,15 +40,6 @@ function geometryError(message: string): RpcError { return { code: "cdp_failed", message }; } -function frameRunner( - cdp: CdpRunner, - handle: FrameDocumentHandle | FrameNodeHandle, - resolvedFrame?: CdpFrame, -): CdpRunner { - if (cdp.sendToFrame || !resolvedFrame) return cdpRunnerForFrame(cdp, handle); - return cdpRunnerForTarget(cdp, resolvedFrame.target); -} - async function scrollFrameOwners( cdp: CdpRunner, tabId: number, @@ -65,7 +54,7 @@ async function scrollFrameOwners( return geometryError(`could not resolve frame owner for ${child.frameId}`); } const error = await scrollNodeIntoView( - frameRunner(cdp, toFrameDocumentHandle(tabId, parent), parent), + cdpRunnerForFrame(cdp, parent.handle), tabId, child.ownerBackendNodeId, ); @@ -94,7 +83,7 @@ async function scrollElementWithContext( const error = await scrollFrameOwners(cdp, tabId, context, handle.frameId); if (error) return error; } - return scrollNodeIntoView(frameRunner(cdp, handle, frame), tabId, handle.backendNodeId); + return scrollNodeIntoView(cdpRunnerForFrame(cdp, handle), tabId, handle.backendNodeId); } export async function scrollElementAndFramesIntoView( @@ -165,13 +154,13 @@ export async function resolveNodeGeometry( // Only topology was read above. Begin live measurements after scrolling. const localRegion = await nodeContentRegion( - frameRunner(cdp, address, frame), + cdpRunnerForFrame(cdp, address), tabId, address.backendNodeId, ); if (isRpcError(localRegion)) return localRegion; - const isOopifTarget = Boolean(frame?.target.sessionId); + const isOopifTarget = Boolean(frame?.isOutOfProcess); let topVisibleRegions: Region; let targetActionPoint: Point | null = null; if (address.frameId && graph && frame) { @@ -180,7 +169,7 @@ export async function resolveNodeGeometry( return geometryError(`could not resolve frame geometry for ${address.frameId}`); topVisibleRegions = projectRegionToViewport(localRegion, projection); if (isOopifTarget) { - const localViewport = await context.viewport(frame.target); + const localViewport = await context.targetViewport(address.frameId); if (!localViewport) return geometryError("could not resolve target viewport geometry"); const localVisibleRegions = localRegion .map((polygon) => @@ -199,7 +188,7 @@ export async function resolveNodeGeometry( targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; } } else { - const viewport = await context.viewport({ tabId }); + const viewport = await context.viewport(rootProcessBoundary(tabId)); if (!viewport) return geometryError("could not resolve top viewport geometry"); topVisibleRegions = localRegion .map((polygon) => @@ -223,7 +212,7 @@ export async function resolveNodeGeometry( topBounds, actionPoint, targetActionPoint, - topViewport: cssViewport(await context.layoutMetrics({ tabId })), + topViewport: cssViewport(await context.topLayoutMetrics(rootProcessBoundary(tabId))), }; } catch (error) { if (isStaleFrameHandleError(error)) { diff --git a/apps/extension/src/tools/geometry/coordinate-types.ts b/apps/extension/src/tools/geometry/coordinate-types.ts index 5b2c48eee..e51cc59f7 100644 --- a/apps/extension/src/tools/geometry/coordinate-types.ts +++ b/apps/extension/src/tools/geometry/coordinate-types.ts @@ -1,9 +1,9 @@ -import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import { type FrameProcessBoundary, frameProcessIdentityKey } from "@/browser-driver/frame-handles"; import type { GeometryProjection, ViewportRect } from "../geometry"; import { projectRectToViewport } from "../geometry"; export interface CoordinateOwner { - target: CdpTarget; + process: FrameProcessBoundary; frameId?: string; } @@ -115,7 +115,8 @@ export function projectSnapshotRect( ): ViewportRect | null { if ( input.owner.frameId !== projection.source.frameId || - cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target) + frameProcessIdentityKey(input.owner.process) !== + frameProcessIdentityKey(projection.source.process) ) { return null; } diff --git a/apps/extension/src/tools/geometry/frame-context.ts b/apps/extension/src/tools/geometry/frame-context.ts index ab77bb8fc..0292f5863 100644 --- a/apps/extension/src/tools/geometry/frame-context.ts +++ b/apps/extension/src/tools/geometry/frame-context.ts @@ -1,11 +1,10 @@ import { - type CdpFrame, - type CdpFrameGraph, - type CdpTarget, - cdpTargetKey, + type FrameDocumentGraph, type FrameDocumentHandle, - StaleFrameHandleError, -} from "@/browser-driver/frame-graph"; + type FrameDocumentInfo, + type FrameProcessBoundary, + frameProcessIdentityKey, +} from "@/browser-driver/frame-handles"; import { type GeometryProjection, type Polygon, @@ -16,21 +15,21 @@ import { type Size, viewportPolygon, } from "../geometry"; -import { type CdpRunner, sendToCdpTarget } from "../shared"; +import type { CdpRunner } from "../shared"; import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types"; import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes"; -export type GeometryEndpoint = CdpTarget | FrameDocumentHandle; +export type GeometryEndpoint = FrameDocumentHandle | FrameProcessBoundary; -function isFrameEndpoint(endpoint: GeometryEndpoint): endpoint is FrameDocumentHandle { - return "loaderId" in endpoint; +function isProcessEndpoint(endpoint: GeometryEndpoint): endpoint is FrameProcessBoundary { + return "processKey" in endpoint; } function geometryEndpointKey(endpoint: GeometryEndpoint): string { - return isFrameEndpoint(endpoint) - ? `${endpoint.tabId}:handle:${endpoint.frameId}:${endpoint.loaderId}` - : cdpTargetKey(endpoint); + return isProcessEndpoint(endpoint) + ? `${endpoint.tabId}:process:${endpoint.processKey}` + : `${endpoint.tabId}:handle:${endpoint.frameId}:${endpoint.loaderId}`; } export interface LayoutMetrics { @@ -95,21 +94,21 @@ export class GeometryContext { private readonly sizeBatches = new Map Promise>>(); private active = 0; private readonly waiting: Array<() => void> = []; - private graphPromise?: Promise; - private framesPromise?: Promise>; + private graphPromise?: Promise; + private framesPromise?: Promise>; constructor( private readonly cdp: CdpRunner, private readonly tabId: number, - graph?: CdpFrameGraph, + graph?: FrameDocumentGraph, private readonly signal?: AbortSignal, private readonly frameHandles?: ReadonlyMap, ) { if (graph) this.graphPromise = Promise.resolve(graph); } - private endpointForFrame(frame: CdpFrame): GeometryEndpoint { - return this.frameHandles?.get(frame.frameId) ?? frame.target; + private endpointForFrame(frame: FrameDocumentInfo): GeometryEndpoint { + return this.frameHandles?.get(frame.frameId) ?? frame.handle; } private async request( @@ -124,31 +123,18 @@ export class GeometryContext { try { if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError"); - if (isFrameEndpoint(endpoint)) { - if (this.cdp.sendToFrame) { - return await this.cdp.sendToFrame(endpoint, method, params); - } - const frame = endpoint.frameId ? await this.frame(endpoint.frameId) : undefined; - if (endpoint.frameId && !frame) { - throw new StaleFrameHandleError( - endpoint, - `frame ${endpoint.frameId} is no longer attached`, - ); - } - if (endpoint.loaderId && frame?.loaderId && frame.loaderId !== endpoint.loaderId) { - throw new StaleFrameHandleError( - endpoint, - `frame ${endpoint.frameId} navigated (${endpoint.loaderId} -> ${frame.loaderId})`, - ); - } - return await sendToCdpTarget( - this.cdp, - frame?.target ?? { tabId: endpoint.tabId }, - method, - params, - ); + if (isProcessEndpoint(endpoint)) { + return await this.cdp.sendToFrame(endpoint.rootHandle, method, params, { + ignoreFrameNavigation: true, + ...(cleanup ? { allowReattach: false } : {}), + }); } - return await sendToCdpTarget(this.cdp, endpoint, method, params); + return await this.cdp.sendToFrame( + endpoint, + method, + params, + cleanup ? { allowReattach: false, ignoreFrameNavigation: true } : undefined, + ); } finally { const next = this.waiting.shift(); if (next) next(); @@ -156,27 +142,26 @@ export class GeometryContext { } } - graph(): Promise { - this.graphPromise ??= - this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null); + graph(): Promise { + this.graphPromise ??= this.cdp.getFrameDocuments(this.tabId).catch(() => null); return this.graphPromise; } - private frames(): Promise> { + private frames(): Promise> { this.framesPromise ??= this.graph().then( (graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []), ); return this.framesPromise; } - async frame(frameId: string): Promise { + async frame(frameId: string): Promise { return (await this.frames()).get(frameId); } /** Topology only: safe to resolve before the caller scrolls frame owners. */ - async ancestry(frameId: string): Promise { + async ancestry(frameId: string): Promise { const frames = await this.frames(); - const path: CdpFrame[] = []; + const path: FrameDocumentInfo[] = []; const seen = new Set(); let current = frames.get(frameId); if (!current) return null; @@ -258,7 +243,7 @@ export class GeometryContext { contentSize?: Size, sourceEndpoint?: GeometryEndpoint, ): Promise { - const endpoint = sourceEndpoint ?? source.target; + const endpoint = sourceEndpoint ?? source.process; const key = `${geometryEndpointKey(endpoint)}:${ownerBackendNodeId}`; let promise = this.snapshotOwners.get(key); if (!promise) { @@ -301,13 +286,13 @@ export class GeometryContext { /** Register one target's snapshot owners; the batch starts only when a * projection actually needs a size. Single owners keep the cheaper old path. */ - registerSnapshotOwners(target: CdpTarget, owners: ReadonlySet): void { - const key = cdpTargetKey(target); + registerSnapshotOwners(process: FrameProcessBoundary, owners: ReadonlySet): void { + const key = geometryEndpointKey(process); if (owners.size < 2 || this.sizeBatches.has(key)) return; let pending: Promise> | undefined; this.sizeBatches.set(key, () => { pending ??= readSnapshotOwnerSizes( - (method, params, cleanup) => this.request(target, method, params, cleanup), + (method, params, cleanup) => this.request(process, method, params, cleanup), owners, ).catch((error) => { if (error && typeof error === "object" && "name" in error && error.name === "AbortError") @@ -368,16 +353,29 @@ export class GeometryContext { } } - private isDifferentTargetBoundary(current: CdpFrame, parent: CdpFrame): boolean { + private isDifferentTargetBoundary( + current: FrameDocumentInfo, + parent: FrameDocumentInfo, + ): boolean { if (this.frameHandles?.has(current.frameId)) { - return this.cdp.hasOwnFrameTarget - ? this.cdp.hasOwnFrameTarget(this.tabId, current.frameId, parent.frameId) - : false; + return this.cdp.hasOwnFrameTarget(this.tabId, current.frameId, parent.frameId); } - return cdpTargetKey(parent.target) !== cdpTargetKey(current.target); + return ( + frameProcessIdentityKey({ + tabId: this.tabId, + processKey: parent.processKey, + }) !== + frameProcessIdentityKey({ + tabId: this.tabId, + processKey: current.processKey, + }) + ); } - private targetRoot(frames: Map, frame: CdpFrame): CdpFrame | null { + private targetRoot( + frames: Map, + frame: FrameDocumentInfo, + ): FrameDocumentInfo | null { const seen = new Set(); let current = frame; while (current.parentFrameId) { @@ -392,9 +390,9 @@ export class GeometryContext { } private async clips( - frames: Map, - frame: CdpFrame, - root: CdpFrame, + frames: Map, + frame: FrameDocumentInfo, + root: FrameDocumentInfo, ): Promise { const clips: Polygon[] = []; let current = frame; @@ -425,6 +423,21 @@ export class GeometryContext { return promise; } + async targetViewport(frameId: string): Promise { + const frames = await this.frames(); + const frame = frames.get(frameId); + if (!frame) return null; + const root = this.targetRoot(frames, frame); + return root ? this.viewport(this.endpointForFrame(root)) : null; + } + + async topLayoutMetrics(fallback: FrameProcessBoundary): Promise { + if (!this.graphPromise) return this.layoutMetrics(fallback); + const graph = await this.graph(); + const root = graph?.frames.find((frame) => frame.frameId === graph.rootFrameId); + return this.layoutMetrics(root ? this.endpointForFrame(root) : fallback); + } + private async buildTargetProjection(frameId: string): Promise { const frames = await this.frames(); const frame = frames.get(frameId); diff --git a/apps/extension/src/tools/human-loop.ts b/apps/extension/src/tools/human-loop.ts index ff8f2ef8c..1875f2a75 100644 --- a/apps/extension/src/tools/human-loop.ts +++ b/apps/extension/src/tools/human-loop.ts @@ -30,6 +30,7 @@ import type { RpcError, } from "@/transport/types"; import { resolveNodeGeometry } from "./frame-geometry"; +import { selectorExistsInMainDocument } from "./selector-resolution"; import { type CdpRunner, type ChromeTabsApi, @@ -200,23 +201,6 @@ async function sendHelpRequestWithAck( throw lastError ?? new Error("failed to show help overlay"); } -async function selectorExists( - cdp: CdpRunner, - tabId: number, - rootNodeId: number, - selector: string, -): Promise { - try { - const res = await cdp.send<{ nodeId?: number }>(tabId, "DOM.querySelector", { - nodeId: rootNodeId, - selector, - }); - return typeof res.nodeId === "number" && res.nodeId !== 0; - } catch { - return false; - } -} - async function resolveHelpTargets( ctx: ReturnType, tabId: number, @@ -249,7 +233,7 @@ async function resolveHelpTargets( selectors.push(tgt.selector); const matched = deps.cdp && rootNodeId !== undefined - ? await selectorExists(deps.cdp, tabId, rootNodeId, tgt.selector) + ? await selectorExistsInMainDocument(deps.cdp, tabId, tgt.selector, rootNodeId) : deps.cdp ? false : true; diff --git a/apps/extension/src/tools/input-readiness.ts b/apps/extension/src/tools/input-readiness.ts index 7200defdf..9ddd55e7c 100644 --- a/apps/extension/src/tools/input-readiness.ts +++ b/apps/extension/src/tools/input-readiness.ts @@ -1,12 +1,19 @@ +import { toRootFrameHandle } from "@/browser-driver/frame-handles"; import { AGENT_WINDOW_HOME } from "@/session-manager/agent-window"; import type { SessionContext } from "@/session-manager/manager"; import { beginTabCreation } from "@/session-manager/tab-creation"; import type { RpcError } from "@/transport/types"; -import { type CdpRunner, isRpcError } from "./shared"; +import { type CdpRunner, type ChromeTabsApi, isRpcError } from "./shared"; import { isAbortError } from "./vom/capture-abort"; +export interface InputWindowsApi { + get(windowId: number): Promise>; +} + export interface InputReadinessDeps { cdp: CdpRunner; + tabsApi?: Pick; + windowsApi?: InputWindowsApi; signal?: AbortSignal; deadline?: number; /** Keep native pointer input from activating an unfocused Agent Window on Linux. */ @@ -172,6 +179,21 @@ async function restoreLeasedInput( } } +async function lacksForegroundPresentation( + ctx: SessionContext, + tabId: number, + deps: InputReadinessDeps, +): Promise { + const tabsApi = deps.tabsApi ?? globalThis.chrome?.tabs; + const windowsApi = deps.windowsApi ?? globalThis.chrome?.windows; + const tab = await tabsApi?.get?.(tabId).catch(() => undefined); + if (tab && tab.active === false) return true; + const windowId = typeof tab?.windowId === "number" ? tab.windowId : ctx.agentWindowId; + const win = await windowsApi?.get?.(windowId).catch(() => undefined); + if (!win) return false; + return win.state === "minimized" || win.focused === false; +} + async function needsInactiveInputTab(ctx: SessionContext, tabId: number): Promise { // Headless Chrome reports windows as unfocused without a desktop activation path. if (globalThis.navigator?.userAgent.includes("HeadlessChrome/")) return false; @@ -310,6 +332,7 @@ export async function withInputReady( if (aborted) return aborted; const documentRevision = ctx.refStore.documentRevision(tabId); let restoreFocus = false; + let hiddenSurface = false; let releaseTab: (() => Promise) | undefined; let attachmentId: string | undefined; let inputSent = false; @@ -371,6 +394,13 @@ export async function withInputReady( } else if (visibility !== "visible") { throw new Error("Could not determine input target visibility"); } + hiddenSurface = + unselect || + visibility === "hidden" || + (await lacksForegroundPresentation(ctx, tabId, deps)); + const cancelledAfterSurface = abortError(deps.signal); + if (cancelledAfterSurface) return cancelledAfterSurface; + checkActive(); } else { const visibility = await sampleVisibility(tabId, deps); const cancelledAfterSample = abortError(deps.signal); @@ -383,16 +413,15 @@ export async function withInputReady( attachmentId = deps.cdp.getAttachmentId?.(tabId); // Mark ownership before awaiting: a failed reply may still have enabled it. restoreFocus = true; + hiddenSurface = true; deps.registerCleanup?.(cleanupFocus); await waitForInputReply( - deps.cdp.sendGuarded - ? deps.cdp.sendGuarded( - { tabId }, - "Emulation.setFocusEmulationEnabled", - { enabled: true }, - { signal: deps.signal, onDispatch: checkActive }, - ) - : deps.cdp.send(tabId, "Emulation.setFocusEmulationEnabled", { enabled: true }), + deps.cdp.sendToFrame( + toRootFrameHandle(tabId), + "Emulation.setFocusEmulationEnabled", + { enabled: true }, + { signal: deps.signal, onDispatch: checkActive }, + ), deps.signal, 5000, deps.deadline, @@ -435,7 +464,7 @@ export async function withInputReady( // Recompute geometry after waking: the background viewport may have changed. ready = true; result = await action({ - hidden: restoreFocus, + hidden: hiddenSurface, markSent: () => { checkActive(); inputSent = true; diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 291bae90c..02b92eb76 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -17,7 +17,7 @@ import { type FrameDocumentHandle, type FrameNodeHandle, isStaleFrameHandleError, -} from "@/browser-driver/frame-graph"; +} from "@/browser-driver/frame-handles"; import { RECORD_PASSWORD_ATTRIBUTE, SENSITIVE_AUTOCOMPLETE_SELECTOR, @@ -48,12 +48,12 @@ import { backendNodeToObject } from "./element-geometry"; import { rpcError } from "./errors"; import { FOCUS_CHECK } from "./focus-state"; import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry"; +import { resolveMainDocumentSelector } from "./selector-resolution"; import { type CdpDispatchGuard, type CdpRunner, type ChromeTabsApi, cdpRunnerForFrame, - cdpRunnerForTarget, delegateCdpRunner, enforceAgentWindow, isRpcError, @@ -61,6 +61,7 @@ import { type ResolvedTargetTab, resolveTargetTab, sendToFrameHandle, + withRemoteObject, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; @@ -192,72 +193,11 @@ export async function resolveBackendNode( }; } // selector path - try { - // Selector lookup itself attaches CDP, even when no element is found. - cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const doc = await cdp.send<{ root?: { nodeId?: number } }>(target.tabId, "DOM.getDocument", { - depth: 0, - }); - const rootNodeId = doc.root?.nodeId; - if (typeof rootNodeId !== "number") { - return { - code: "cdp_failed", - message: "DOM.getDocument returned no root nodeId", - }; - } - const query = { nodeId: rootNodeId, selector: params.selector }; - let nodeId: number | undefined; - if (strictSelector) { - const found = await cdp.send<{ nodeIds: number[] }>( - target.tabId, - "DOM.querySelectorAll", - query, - ); - if (found.nodeIds.length > 1) - return { - code: "invalid_params", - message: `${toolName}: selector matched ${found.nodeIds.length} elements; use a VOM ref or a narrower selector`, - }; - nodeId = found.nodeIds[0]; - } else { - nodeId = (await cdp.send<{ nodeId?: number }>(target.tabId, "DOM.querySelector", query)) - .nodeId; - } - if (typeof nodeId !== "number" || nodeId === 0) { - return rpcError( - "not_found", - "selector_not_found", - `selector ${params.selector} did not match any element`, - ); - } - const described = await cdp.send<{ node?: { backendNodeId?: number } }>( - target.tabId, - "DOM.describeNode", - { nodeId }, - ); - const backendNodeId = described.node?.backendNodeId; - if (typeof backendNodeId !== "number") { - return { - code: "cdp_failed", - message: "DOM.describeNode returned no backendNodeId", - }; - } - return { - handle: { - tabId: target.tabId, - frameId: "", - loaderId: "", - backendNodeId, - }, - backendNodeId, - usedSelector: params.selector, - }; - } catch (err) { - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; - } + cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + return resolveMainDocumentSelector(cdp, target.tabId, params.selector as string, { + toolName, + multiMatch: strictSelector ? "reject" : "first", + }); } const BLUR_TARGET = `function() { @@ -281,17 +221,9 @@ function checkFocusAbort(signal: AbortSignal | undefined): void { // inside selector resolution and frame scrolling, without changing other tools. function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner { return delegateCdpRunner(cdp, { - send: (tabId: number, method: string, params?: object) => { + send: (tabId: number, method: string, params?: object, sig?: AbortSignal) => { checkFocusAbort(signal); - return cdp.send(tabId, method, params); - }, - sendToTarget: ( - target: Parameters>[0], - method: string, - params?: object, - ) => { - checkFocusAbort(signal); - return cdpRunnerForTarget(cdp, target).send(target.tabId, method, params); + return cdp.send(tabId, method, params, sig ?? signal); }, sendToFrame: ( handle: FrameDocumentHandle | FrameNodeHandle, @@ -300,20 +232,22 @@ function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner { guard?: CdpDispatchGuard, ) => { checkFocusAbort(signal); - return sendToFrameHandle(cdp, handle, method, params, guard); + return sendToFrameHandle( + cdp, + handle, + method, + params, + guard?.signal || !signal ? guard : { ...guard, signal }, + ); + }, + getFrameDocuments: (tabId, sig) => { + checkFocusAbort(signal); + return cdp.getFrameDocuments(tabId, sig ?? signal); + }, + getMainFrameHandle: (tabId, sig) => { + checkFocusAbort(signal); + return cdp.getMainFrameHandle(tabId, sig ?? signal); }, - getFrameGraph: cdp.getFrameGraph - ? (tabId) => { - checkFocusAbort(signal); - return cdp.getFrameGraph!(tabId); - } - : undefined, - getMainFrameHandle: cdp.getMainFrameHandle - ? (tabId) => { - checkFocusAbort(signal); - return cdp.getMainFrameHandle!(tabId); - } - : undefined, }); } @@ -358,9 +292,6 @@ async function changeFocus( const ctx = lookupSession(manager, params, action); if (isRpcError(ctx)) return ctx; const cdp = focusCdp(deps.cdp, deps.signal); - let objectId: string | undefined; - let cleanupCdp: CdpRunner | undefined; - let tabId: number | undefined; try { checkFocusAbort(deps.signal); const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); @@ -368,13 +299,12 @@ async function changeFocus( if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, action); if (denied) return denied; - tabId = target.tabId; + const tabId = target.tabId; const dialogCursor = markDialogCursor(deps.cdp, tabId); const node = await resolveBackendNode(cdp, ctx, target, params, action); checkFocusAbort(deps.signal); if (isRpcError(node)) return node; const nodeCdp = cdpRunnerForFrame(cdp, node.handle); - cleanupCdp = cdpRunnerForFrame(deps.cdp, node.handle); deps.cdp.trackSessionTab?.(ctx.sessionId, tabId); if (action === "focus") { @@ -383,64 +313,67 @@ async function changeFocus( if (scrollErr) return scrollErr; } const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId); - // Keep the handle before checking cancellation so finally can release it. - if (!isRpcError(resolved)) objectId = resolved; - checkFocusAbort(deps.signal); - if (isRpcError(resolved)) return resolved; + if (isRpcError(resolved)) { + checkFocusAbort(deps.signal); + return resolved; + } - const runScript = async (functionDeclaration: string): Promise => { - const reply = await nodeCdp.send>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId, - functionDeclaration, - returnByValue: true, - }, - ); + return await withRemoteObject(deps.cdp, node.handle, resolved, async (objectId) => { checkFocusAbort(deps.signal); - if (reply.exceptionDetails) { - const details = reply.exceptionDetails; - throw new Error( - `${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`, + const runScript = async (functionDeclaration: string): Promise => { + const reply = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration, + returnByValue: true, + }, ); - } - return reply.result?.value; - }; + checkFocusAbort(deps.signal); + if (reply.exceptionDetails) { + const details = reply.exceptionDetails; + throw new Error( + `${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`, + ); + } + return reply.result?.value; + }; - let wasFocused: boolean | undefined; - if (action === "focus") { - await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); - checkFocusAbort(deps.signal); - } else { - const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET); - if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") { - throw new Error("blur script returned an unexpected result"); + let wasFocused: boolean | undefined; + if (action === "focus") { + await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + checkFocusAbort(deps.signal); + } else { + const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET); + if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") { + throw new Error("blur script returned an unexpected result"); + } + if (!mutation.ok) { + return { code: "invalid_params", message: "target element does not support blur()" }; + } + wasFocused = mutation.was_focused; } - if (!mutation.ok) { - return { code: "invalid_params", message: "target element does not support blur()" }; + // Use a separate call so microtasks from focus/blur handlers finish before + // we report success. In particular, a blur handler can restore focus. + const focused = await runScript(FOCUS_CHECK); + if (typeof focused !== "boolean") { + throw new Error(`${action} verification returned an unexpected result`); } - wasFocused = mutation.was_focused; - } - // Use a separate call so microtasks from focus/blur handlers finish before - // we report success. In particular, a blur handler can restore focus. - const focused = await runScript(FOCUS_CHECK); - if (typeof focused !== "boolean") { - throw new Error(`${action} verification returned an unexpected result`); - } - if (focused !== (action === "focus")) { - throw new Error( - action === "focus" - ? "target element did not become focused" - : "target element remained focused after blur()", - ); - } - return attachDialogs(deps.cdp, tabId, dialogCursor, { - tab_id: tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, - focused, - ...(action === "blur" ? { was_focused: wasFocused! } : {}), + if (focused !== (action === "focus")) { + throw new Error( + action === "focus" + ? "target element did not become focused" + : "target element remained focused after blur()", + ); + } + return attachDialogs(deps.cdp, tabId, dialogCursor, { + tab_id: tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + focused, + ...(action === "blur" ? { was_focused: wasFocused! } : {}), + }); }); } catch (err) { const aborted = throwIfAborted(deps.signal); @@ -450,16 +383,6 @@ async function changeFocus( code: "cdp_failed", message: err instanceof Error ? err.message : String(err), }; - } finally { - if (objectId !== undefined && cleanupCdp && tabId !== undefined) { - // Cleanup must also run after cancellation. Navigation may have already - // disposed the object, which must not replace the operation's result. - try { - await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId }); - } catch { - // The target or execution context may no longer exist. - } - } } } @@ -1346,58 +1269,59 @@ export async function handleFill( throw err; } if (isRpcError(objectIdOrErr)) return objectIdOrErr; - const objectId = objectIdOrErr; + const resolvedObjectId = objectIdOrErr; const clearBefore = params.clear_before ?? true; - try { - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "fill aborted" }; - } - await deps.cdp.beginDomMutation?.(); - - // Step 1: Merged validation, conditional scroll, focus, and clearing/caret setup. - const step1Reply = await nodeCdp.send>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId, - functionDeclaration: FILL_STEP1_FUNCTION, - arguments: [{ value: params.value }, { value: clearBefore }, { value: !node.frameId }], - awaitPromise: true, - returnByValue: true, - }, - ); - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - if (step1Reply.exceptionDetails || !step1Reply.result?.value) { - return fillError("fill_failed", "fill preparation script failed"); - } - const step1 = step1Reply.result.value; - if ("error" in step1) { - // Step 1 reports these validation errors only before clearing; its focus is - // preparation, so the requested write never started. `fill_target_changed` - // is reported after clearing and keeps an unknown effect. - const preWrite = - step1.reason === "target_not_fillable" || step1.reason === "fill_value_invalid"; - return fillError(step1.reason, step1.error, preWrite); - } - if ( - typeof step1.before !== "string" || - typeof step1.expected !== "string" || - (step1.write !== "insert" && step1.write !== "value") - ) { - return fillError("fill_failed", "fill preparation returned an unexpected result"); - } + return await withRemoteObject(deps.cdp, node.handle, resolvedObjectId, async (objectId) => { + try { + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "fill aborted" }; + } + await deps.cdp.beginDomMutation?.(); - const checkReady = async ( - placeCaret = false, - verifyCaret = false, - ): Promise => { - const reply = await nodeCdp.send>( + // Step 1: Merged validation, conditional scroll, focus, and clearing/caret setup. + const step1Reply = await nodeCdp.send>( target.tabId, "Runtime.callFunctionOn", { objectId, - functionDeclaration: `function(before, placeCaret, verifyCaret) { + functionDeclaration: FILL_STEP1_FUNCTION, + arguments: [{ value: params.value }, { value: clearBefore }, { value: !node.frameId }], + awaitPromise: true, + returnByValue: true, + }, + ); + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; + if (step1Reply.exceptionDetails || !step1Reply.result?.value) { + return fillError("fill_failed", "fill preparation script failed"); + } + const step1 = step1Reply.result.value; + if ("error" in step1) { + // Step 1 reports these validation errors only before clearing; its focus is + // preparation, so the requested write never started. `fill_target_changed` + // is reported after clearing and keeps an unknown effect. + const preWrite = + step1.reason === "target_not_fillable" || step1.reason === "fill_value_invalid"; + return fillError(step1.reason, step1.error, preWrite); + } + if ( + typeof step1.before !== "string" || + typeof step1.expected !== "string" || + (step1.write !== "insert" && step1.write !== "value") + ) { + return fillError("fill_failed", "fill preparation returned an unexpected result"); + } + + const checkReady = async ( + placeCaret = false, + verifyCaret = false, + ): Promise => { + const reply = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: `function(before, placeCaret, verifyCaret) { if (!(${FILL_EDITABLE_FUNCTION}).call(this) || (${FILL_VALUE_FUNCTION}).call(this) !== before) return 'fill_target_changed'; if (this.getRootNode().activeElement !== this) return 'fill_focus_lost'; @@ -1437,159 +1361,159 @@ export async function handleFill( } return this.ownerDocument.hasFocus() ? 'ready' : 'background'; }`, - arguments: [{ value: step1.before }, { value: placeCaret }, { value: verifyCaret }], - returnByValue: true, - }, - ); - const state = reply.result?.value; - if ( - reply.exceptionDetails || - (state !== "ready" && - state !== "background" && - state !== "fill_target_changed" && - state !== "fill_focus_lost") - ) { - return fillError( - "fill_failed", - "fill readiness script failed or returned an unexpected result", + arguments: [{ value: step1.before }, { value: placeCaret }, { value: verifyCaret }], + returnByValue: true, + }, ); - } - return state; - }; + const state = reply.result?.value; + if ( + reply.exceptionDetails || + (state !== "ready" && + state !== "background" && + state !== "fill_target_changed" && + state !== "fill_focus_lost") + ) { + return fillError( + "fill_failed", + "fill readiness script failed or returned an unexpected result", + ); + } + return state; + }; - const focusViaCdp = async (): Promise => { - try { - await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); - return null; - } catch (error) { - const editable = await nodeCdp.send>( + const focusViaCdp = async (): Promise => { + try { + await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + return null; + } catch (error) { + const editable = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { objectId, functionDeclaration: FILL_EDITABLE_FUNCTION, returnByValue: true }, + ); + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "fill aborted" }; + } + if (!editable.exceptionDetails && editable.result?.value === false) { + return fillError( + "target_not_fillable", + "fill target is not editable or its input type is unsupported", + ); + } + throw error; + } + }; + + // An unfocused page applies focus(), but runs its focus listeners only when + // the renderer receives focus with the next Input command. Deliver it before + // clearing or writing, so listeners that set defaults cannot run in between. + // On a visible page, a pointer move outside the viewport (sent to the tab, + // which also focuses OOPIF widgets) reaches no element and grants no user + // activation. A hidden page processes pointer moves only on its next frame, + // and a hovered page would see the pointer leave, so those, and a move that + // does not deliver focus, use a no-op key press, which pages can observe as + // an empty key. + let pointerFlush: boolean | undefined; + const canFlushWithPointer = async (): Promise => { + if (pointerFlush !== undefined) return pointerFlush; + pointerFlush = step1.visible && step1.hovered === false; + if (step1.visible && step1.hovered === null && node.frameId) { + // The target's frame cannot read a cross-origin parent; ask the tab. + const top = await deps.cdp + .send<{ result?: { value?: unknown } }>(target.tabId, "Runtime.evaluate", { + expression: "document.querySelector(':hover') === null", + returnByValue: true, + }) + .catch(() => undefined); + pointerFlush = top?.result?.value === true; + } + return pointerFlush; + }; + const focusDelivered = async (): Promise => { + const reply = await nodeCdp.send>( target.tabId, "Runtime.callFunctionOn", - { objectId, functionDeclaration: FILL_EDITABLE_FUNCTION, returnByValue: true }, + { + objectId, + functionDeclaration: "function() { return Boolean(this.ownerDocument?.hasFocus?.()); }", + returnByValue: true, + }, ); - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "fill aborted" }; + return !reply.exceptionDetails && reply.result?.value === true; + }; + const flushBackgroundFocus = async (): Promise => { + if (await canFlushWithPointer()) { + let timer: ReturnType | undefined; + const moved = await Promise.race([ + deps.cdp + .send(target.tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: -1, y: -1 }) + // A refused or late move falls back to the key press below. + .then( + () => true, + () => false, + ), + // The page became hidden after Step 1; its move waits for a frame. + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), FILL_POINTER_FLUSH_TIMEOUT_MS); + }), + ]).finally(() => clearTimeout(timer)); + if (throwIfAborted(deps.signal)) return; + if (moved && (await focusDelivered())) return; + pointerFlush = false; } - if (!editable.exceptionDetails && editable.result?.value === false) { - return fillError( - "target_not_fillable", - "fill target is not editable or its input type is unsupported", - ); + try { + await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { + type: "rawKeyDown", + commands: [], + }); + } finally { + await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" }); } - throw error; - } - }; - - // An unfocused page applies focus(), but runs its focus listeners only when - // the renderer receives focus with the next Input command. Deliver it before - // clearing or writing, so listeners that set defaults cannot run in between. - // On a visible page, a pointer move outside the viewport (sent to the tab, - // which also focuses OOPIF widgets) reaches no element and grants no user - // activation. A hidden page processes pointer moves only on its next frame, - // and a hovered page would see the pointer leave, so those, and a move that - // does not deliver focus, use a no-op key press, which pages can observe as - // an empty key. - let pointerFlush: boolean | undefined; - const canFlushWithPointer = async (): Promise => { - if (pointerFlush !== undefined) return pointerFlush; - pointerFlush = step1.visible && step1.hovered === false; - if (step1.visible && step1.hovered === null && node.frameId) { - // The target's frame cannot read a cross-origin parent; ask the tab. - const top = await deps.cdp - .send<{ result?: { value?: unknown } }>(target.tabId, "Runtime.evaluate", { - expression: "document.querySelector(':hover') === null", - returnByValue: true, - }) - .catch(() => undefined); - pointerFlush = top?.result?.value === true; - } - return pointerFlush; - }; - const focusDelivered = async (): Promise => { - const reply = await nodeCdp.send>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId, - functionDeclaration: "function() { return Boolean(this.ownerDocument?.hasFocus?.()); }", - returnByValue: true, - }, - ); - return !reply.exceptionDetails && reply.result?.value === true; - }; - const flushBackgroundFocus = async (): Promise => { - if (await canFlushWithPointer()) { - let timer: ReturnType | undefined; - const moved = await Promise.race([ - deps.cdp - .send(target.tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: -1, y: -1 }) - // A refused or late move falls back to the key press below. - .then( - () => true, - () => false, - ), - // The page became hidden after Step 1; its move waits for a frame. - new Promise((resolve) => { - timer = setTimeout(() => resolve(false), FILL_POINTER_FLUSH_TIMEOUT_MS); - }), - ]).finally(() => clearTimeout(timer)); - if (throwIfAborted(deps.signal)) return; - if (moved && (await focusDelivered())) return; - pointerFlush = false; - } - try { - await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { - type: "rawKeyDown", - commands: [], - }); - } finally { - await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" }); - } - }; + }; - const verifyRestoredFocus = async (expectedBefore: string): Promise => { - const focusErr = await focusViaCdp(); - if (focusErr) return focusErr; - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - if (!step1.hasFocus) { - await flushBackgroundFocus(); + const verifyRestoredFocus = async (expectedBefore: string): Promise => { + const focusErr = await focusViaCdp(); + if (focusErr) return focusErr; if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - } - const restored = await nodeCdp.send>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId, - functionDeclaration: `async function(before) { + if (!step1.hasFocus) { + await flushBackgroundFocus(); + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; + } + const restored = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: `async function(before) { await Promise.resolve(); await Promise.resolve(); if (!(${FILL_EDITABLE_FUNCTION}).call(this) || (${FILL_VALUE_FUNCTION}).call(this) !== before) return 'fill_target_changed'; if (this.getRootNode().activeElement !== this) return 'fill_focus_lost'; return this.ownerDocument.hasFocus() ? 'ready' : 'background'; }`, - arguments: [{ value: expectedBefore }], - awaitPromise: true, - returnByValue: true, - }, - ); - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - const state = restored.result?.value; - if (restored.exceptionDetails || (state !== "ready" && state !== "background")) { - return fillError( - state === "fill_target_changed" || state === "fill_focus_lost" ? state : "fill_failed", - "fill target changed or lost focus after restoring focus", + arguments: [{ value: expectedBefore }], + awaitPromise: true, + returnByValue: true, + }, ); - } - return null; - }; + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; + const state = restored.result?.value; + if (restored.exceptionDetails || (state !== "ready" && state !== "background")) { + return fillError( + state === "fill_target_changed" || state === "fill_focus_lost" ? state : "fill_failed", + "fill target changed or lost focus after restoring focus", + ); + } + return null; + }; - const runFallbackPrep = async (): Promise => { - const fallbackPrep = await nodeCdp.send>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId, - functionDeclaration: `async function(value, clearBefore) { + const runFallbackPrep = async (): Promise => { + const fallbackPrep = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: `async function(value, clearBefore) { await Promise.resolve(); await Promise.resolve(); const preparation = (${FILL_PREPARATION_FUNCTION}).call(this, value, clearBefore); @@ -1616,189 +1540,194 @@ export async function handleFill( } return { ...preparation, focused: this.getRootNode().activeElement === this }; }`, - arguments: [{ value: params.value }, { value: clearBefore }], - awaitPromise: true, - returnByValue: true, - }, - ); - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - if (fallbackPrep.exceptionDetails || !fallbackPrep.result?.value) { - return fillError("fill_failed", "fill preparation script failed"); - } - return fallbackPrep.result.value; - }; - - let cleared = step1.cleared; - let usedCdpFocus = false; - // Fallback path only when Step 1 could not finish preparation in-place - // (e.g. DOM.focus fallback, contenteditable range placement, number/email - // --no-clear append, or unfocused background tab requiring deferred-focus flush). - if (!step1.focused) { - usedCdpFocus = true; - if (cleared) { - // Clearing lost focus once; verify focus and empty value after restoring focus once. - const restoreErr = await verifyRestoredFocus(""); - if (restoreErr) return restoreErr; - } else { - const focusErr = await focusViaCdp(); - if (focusErr) return focusErr; - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - } - } - - if (!step1.readyForInsert || !step1.focused) { - if (!step1.hasFocus) { - await flushBackgroundFocus(); + arguments: [{ value: params.value }, { value: clearBefore }], + awaitPromise: true, + returnByValue: true, + }, + ); if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - } + if (fallbackPrep.exceptionDetails || !fallbackPrep.result?.value) { + return fillError("fill_failed", "fill preparation script failed"); + } + return fallbackPrep.result.value; + }; - if (!cleared && (clearBefore || !step1.focused || !step1.hasFocus)) { - let prepVal = await runFallbackPrep(); - if (isRpcError(prepVal)) return prepVal; - if ("error" in prepVal && prepVal.reason === "fill_focus_lost" && !usedCdpFocus) { - usedCdpFocus = true; + let cleared = step1.cleared; + let usedCdpFocus = false; + // Fallback path only when Step 1 could not finish preparation in-place + // (e.g. DOM.focus fallback, contenteditable range placement, number/email + // --no-clear append, or unfocused background tab requiring deferred-focus flush). + if (!step1.focused) { + usedCdpFocus = true; + if (cleared) { + // Clearing lost focus once; verify focus and empty value after restoring focus once. + const restoreErr = await verifyRestoredFocus(""); + if (restoreErr) return restoreErr; + } else { const focusErr = await focusViaCdp(); if (focusErr) return focusErr; if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - if (!step1.hasFocus) { - await flushBackgroundFocus(); - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - } - prepVal = await runFallbackPrep(); - if (isRpcError(prepVal)) return prepVal; } - if ("error" in prepVal) return fillError(prepVal.reason, prepVal.error); - step1.before = prepVal.before; - step1.expected = prepVal.expected; - cleared = clearBefore; + } - if (!prepVal.focused) { - const restoreErr = await verifyRestoredFocus(step1.before); - if (restoreErr) return restoreErr; + if (!step1.readyForInsert || !step1.focused) { + if (!step1.hasFocus) { + await flushBackgroundFocus(); + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; } - } - if (step1.write === "insert" && params.value !== "") { - if (step1.native && !clearBefore && !step1.supportsSelection) { - try { - await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { - type: "rawKeyDown", - commands: ["moveToEndOfDocument"], - }); - } finally { - await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" }); + if (!cleared && (clearBefore || !step1.focused || !step1.hasFocus)) { + let prepVal = await runFallbackPrep(); + if (isRpcError(prepVal)) return prepVal; + if ("error" in prepVal && prepVal.reason === "fill_focus_lost" && !usedCdpFocus) { + usedCdpFocus = true; + const focusErr = await focusViaCdp(); + if (focusErr) return focusErr; + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; + if (!step1.hasFocus) { + await flushBackgroundFocus(); + if (throwIfAborted(deps.signal)) + return { code: "cancelled", message: "fill aborted" }; + } + prepVal = await runFallbackPrep(); + if (isRpcError(prepVal)) return prepVal; + } + if ("error" in prepVal) return fillError(prepVal.reason, prepVal.error); + step1.before = prepVal.before; + step1.expected = prepVal.expected; + cleared = clearBefore; + + if (!prepVal.focused) { + const restoreErr = await verifyRestoredFocus(step1.before); + if (restoreErr) return restoreErr; } - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; } - if (!step1.native || (!clearBefore && !step1.supportsSelection)) { - let ready = await checkReady(!step1.native); - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - if (!step1.native && (ready === "ready" || ready === "background")) { - ready = await checkReady(false, true); + + if (step1.write === "insert" && params.value !== "") { + if (step1.native && !clearBefore && !step1.supportsSelection) { + try { + await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { + type: "rawKeyDown", + commands: ["moveToEndOfDocument"], + }); + } finally { + await deps.cdp.send(target.tabId, "Input.dispatchKeyEvent", { type: "keyUp" }); + } if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; } - if (typeof ready !== "string") return ready; - if (ready !== "ready" && ready !== "background") { - return fillError( - ready, - "fill target changed or lost focus while positioning the caret", - ); + if (!step1.native || (!clearBefore && !step1.supportsSelection)) { + let ready = await checkReady(!step1.native); + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; + if (!step1.native && (ready === "ready" || ready === "background")) { + ready = await checkReady(false, true); + if (throwIfAborted(deps.signal)) + return { code: "cancelled", message: "fill aborted" }; + } + if (typeof ready !== "string") return ready; + if (ready !== "ready" && ready !== "background") { + return fillError( + ready, + "fill target changed or lost focus while positioning the caret", + ); + } } } } - } - // Step 2: Core write. - let valueAssigned = false; - if (step1.write === "value") { - if (!(cleared && step1.expected === "")) { - const assigned = await nodeCdp.send>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId, - functionDeclaration: `function(expected) { + // Step 2: Core write. + let valueAssigned = false; + if (step1.write === "value") { + if (!(cleared && step1.expected === "")) { + if (usedCdpFocus || !step1.readyForInsert || !step1.focused) { + await deps.cdp.beginDomMutation?.(); + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; + } + const assigned = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: `function(expected) { const changed = this.value !== expected; Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set.call(this, expected); return changed; }`, - arguments: [{ value: step1.expected }], - returnByValue: true, - }, - ); - if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; - if (assigned.exceptionDetails) { - return fillError("fill_failed", "fill value assignment script failed"); + arguments: [{ value: step1.expected }], + returnByValue: true, + }, + ); + if (throwIfAborted(deps.signal)) return { code: "cancelled", message: "fill aborted" }; + if (assigned.exceptionDetails) { + return fillError("fill_failed", "fill value assignment script failed"); + } + valueAssigned = assigned.result?.value === true; } - valueAssigned = assigned.result?.value === true; + } else if (params.value !== "") { + await deps.cdp.send(target.tabId, "Input.insertText", { + text: params.value.replace(/\r\n?/g, "\n"), + }); } - } else if (params.value !== "") { - await deps.cdp.send(target.tabId, "Input.insertText", { - text: params.value.replace(/\r\n?/g, "\n"), - }); - } - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "fill aborted" }; - } + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "fill aborted" }; + } - // Step 3: Merged event dispatch + optimistic value/connection readback. - const finalValueAlreadyNotified = - (clearBefore && params.value === "" && !valueAssigned) || - (step1.write === "insert" && params.value !== ""); - const notifyInput = !finalValueAlreadyNotified; - const verified = await nodeCdp.send>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId, - functionDeclaration: FILL_STEP3_FUNCTION, - arguments: [ - { value: notifyInput }, - { value: step1.expected }, - { value: step1.isSensitive }, - { value: FILL_MAX_ACTUAL_VALUE_CHARS }, - ], - awaitPromise: true, - returnByValue: true, - }, - ); - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "fill aborted" }; - } - const verification = verified.result?.value; - if ( - verified.exceptionDetails || - typeof verification?.connected !== "boolean" || - typeof verification?.valueLength !== "number" || - typeof verification?.sensitive !== "boolean" - ) { - return fillError( - "fill_failed", - "fill verification script failed or returned an unexpected result", + // Step 3: Merged event dispatch + optimistic value/connection readback. + const finalValueAlreadyNotified = + (clearBefore && params.value === "" && !valueAssigned) || + (step1.write === "insert" && params.value !== ""); + const notifyInput = !finalValueAlreadyNotified; + const verified = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration: FILL_STEP3_FUNCTION, + arguments: [ + { value: notifyInput }, + { value: step1.expected }, + { value: step1.isSensitive }, + { value: FILL_MAX_ACTUAL_VALUE_CHARS }, + ], + awaitPromise: true, + returnByValue: true, + }, ); + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "fill aborted" }; + } + const verification = verified.result?.value; + if ( + verified.exceptionDetails || + typeof verification?.connected !== "boolean" || + typeof verification?.valueLength !== "number" || + typeof verification?.sensitive !== "boolean" + ) { + return fillError( + "fill_failed", + "fill verification script failed or returned an unexpected result", + ); + } + if (!verification.connected) { + return fillError("fill_target_changed", "fill target was removed or replaced during input"); + } + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + value_length: verification.valueLength, + ...(!verification.sensitive && typeof verification.actualValue === "string" + ? { actual_value: verification.actualValue } + : {}), + ...(verification.formatted ? { formatted: true } : {}), + }); + } catch (err) { + if (isStaleFrameHandleError(err)) { + return rpcError("not_found", "ref_not_found", err.message); + } + return fillError("fill_failed", err instanceof Error ? err.message : String(err)); } - if (!verification.connected) { - return fillError("fill_target_changed", "fill target was removed or replaced during input"); - } - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - tab_id: target.tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, - value_length: verification.valueLength, - ...(!verification.sensitive && typeof verification.actualValue === "string" - ? { actual_value: verification.actualValue } - : {}), - ...(verification.formatted ? { formatted: true } : {}), - }); - } catch (err) { - if (isStaleFrameHandleError(err)) { - return rpcError("not_found", "ref_not_found", err.message); - } - return fillError("fill_failed", err instanceof Error ? err.message : String(err)); - } finally { - await nodeCdp.send(target.tabId, "Runtime.releaseObject", { objectId }).catch(() => undefined); - } + }); } // --------------------------------------------------------------------------- @@ -2265,9 +2194,9 @@ export async function handleSelect( if (isRpcError(node)) return node; const nodeCdp = cdpRunnerForFrame(deps.cdp, node.handle); - let objectId: string; + let resolvedObjectId: string; try { - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + deps.cdp.trackSessionTab(ctx.sessionId, target.tabId); const described = await nodeCdp.send<{ node?: DescribedNode }>( target.tabId, "DOM.describeNode", @@ -2296,29 +2225,7 @@ export async function handleSelect( } const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); if (isRpcError(objectIdOrErr)) return objectIdOrErr; - objectId = objectIdOrErr; - // Validate the requested options before scrolling or focusing: moving focus - // can blur another field and run its change or autosave handlers, so only a - // rejection reported here is free of page effects. - const validated = await nodeCdp.send<{ - result?: { value?: SelectMutationResult | { ok: true; multiple: boolean } | null }; - }>(target.tabId, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: SELECT_OPTIONS_FUNCTION, - arguments: [{ value: params.values }, { value: false }], - returnByValue: true, - }); - const validation = validated.result?.value; - if (!validation?.ok) return selectFailure(validation, true); - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "select aborted" }; - } - const scrollErr = await scrollElementAndFramesIntoView(deps.cdp, target.tabId, node.handle); - if (scrollErr) return scrollErr; - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "select aborted" }; - } - await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + resolvedObjectId = objectIdOrErr; } catch (err) { if (isStaleFrameHandleError(err)) { return rpcError("not_found", "ref_not_found", err.message, PRE_WRITE_FAILURE); @@ -2329,38 +2236,73 @@ export async function handleSelect( }; } - if (throwIfAborted(deps.signal)) { - return { code: "cancelled", message: "select aborted" }; - } + return await withRemoteObject(deps.cdp, node.handle, resolvedObjectId, async (objectId) => { + try { + // Validate the requested options before scrolling or focusing: moving focus + // can blur another field and run its change or autosave handlers, so only a + // rejection reported here is free of page effects. + const validated = await nodeCdp.send<{ + result?: { value?: SelectMutationResult | { ok: true; multiple: boolean } | null }; + }>(target.tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: SELECT_OPTIONS_FUNCTION, + arguments: [{ value: params.values }, { value: false }], + returnByValue: true, + }); + const validation = validated.result?.value; + if (!validation?.ok) return selectFailure(validation, true); + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "select aborted" }; + } + const scrollErr = await scrollElementAndFramesIntoView(deps.cdp, target.tabId, node.handle); + if (scrollErr) return scrollErr; + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "select aborted" }; + } + await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + } catch (err) { + if (isStaleFrameHandleError(err)) { + return rpcError("not_found", "ref_not_found", err.message, PRE_WRITE_FAILURE); + } + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } - try { - await deps.cdp.beginDomMutation?.(); - const evaluated = await nodeCdp.send<{ - result?: { value?: SelectMutationResult | null }; - }>(target.tabId, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: SELECT_OPTIONS_FUNCTION, - arguments: [{ value: params.values }, { value: true }], - returnByValue: true, - }); - const mutation = evaluated.result?.value; - // The page changed after validation, and focus already moved; keep the - // effect unknown even though no option was changed. - if (!mutation?.ok) return selectFailure(mutation, false); - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - tab_id: target.tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, - multiple: mutation.multiple, - selected_values: mutation.selected_values, - selected_labels: mutation.selected_labels, - }); - } catch (err) { - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; - } + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "select aborted" }; + } + + try { + await deps.cdp.beginDomMutation?.(); + const evaluated = await nodeCdp.send<{ + result?: { value?: SelectMutationResult | null }; + }>(target.tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: SELECT_OPTIONS_FUNCTION, + arguments: [{ value: params.values }, { value: true }], + returnByValue: true, + }); + const mutation = evaluated.result?.value; + // The page changed after validation, and focus already moved; keep the + // effect unknown even though no option was changed. + if (!mutation?.ok) return selectFailure(mutation, false); + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + multiple: mutation.multiple, + selected_values: mutation.selected_values, + selected_labels: mutation.selected_labels, + }); + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + }); } /** Maps a rejected option match. `preWrite` only before scrolling or focusing. */ diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 0df92f544..c938e2f42 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -8,6 +8,7 @@ import { type ObservationOutput, prepareVisualObservation, publishObservationPage, + renderObservationOutput, } from "./vom/visual-observation"; export { parsePngDimensions } from "./png"; @@ -22,15 +23,13 @@ import { type ActiveScopeBlock, applyVomInteractionRecovery, type CondSurface, - estimateLineTokens, isVomReferenceNode, - renderVom, type VomNode, type VomOptions, type VomScene, } from "@browser-skill/vom"; import { readTimeoutDetails } from "@/browser-driver/command-deadline"; -import { type FrameNodeHandle, isStaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { type FrameNodeHandle, isStaleFrameHandleError } from "@/browser-driver/frame-handles"; import { type CaptureSuppressSendToTab, withExtensionOverlayHidden, @@ -756,6 +755,7 @@ export interface SnapshotDeps { /** A first policy negotiation publishes refs only after its sync succeeds. */ refStore?: RefStore; accessibilityLease?: AccessibilityLease; + includeVisualFacts?: boolean; cdp: CdpRunner; tabsApi: { get(tabId: number): Promise; @@ -944,6 +944,7 @@ export interface CaptureVomObservationOptions extends VomOptions { accessibilityLease?: AccessibilityLease; onScene?: (scene: VomScene) => void; includeVisualFacts?: boolean; + deferPagePublish?: boolean; skipRender?: boolean; conditionalSurfaceProbe?: boolean; hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; @@ -1060,23 +1061,45 @@ async function buildVomObservation( doc.identity ? [[doc.frame.frameId, doc.identity] as const] : [], ), ); - if (options.includeVisualFacts) { - const discovery = await deduplicateVisualCandidates( - await discoverVisualCandidates(facts, options.signal), - options.signal, - ); - const visualOutput = prepareVisualObservation( - decoratedScene, - structured, - discovery, + const rootKey = captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root"; + if (options.skipRender) { + throwIfAborted(options.signal, "observation"); + return projectRecordSafeObservation({ + rootFrameId: rootKey, + frameDocuments: normalizedDocuments, identities, - captured.rootFrameId ?? "root", - notices, - renderOptions, - ); + rendered: { text: "", refs: [], truncated: false }, + surfaceProbes: hoverProbes.surfaceProbes, + hoverProbe: { + performed: hoverProbes.performed, + revealedContent: hoverProbes.revealedContent, + }, + }); + } + const discovery = options.includeVisualFacts + ? await deduplicateVisualCandidates( + await discoverVisualCandidates(facts, options.signal), + options.signal, + ) + : { + candidates: [], + complete: true, + issues: [], + captureIssues: facts.issues, + }; + const visualOutput = prepareVisualObservation( + decoratedScene, + structured, + discovery, + identities, + rootKey, + notices, + renderOptions, + ); + if (options.deferPagePublish) { return { ...projectRecordSafeObservation({ - rootFrameId: captured.rootFrameId ?? "root", + rootFrameId: rootKey, frameDocuments: normalizedDocuments, identities, rendered: { text: "", refs: [], truncated: false }, @@ -1089,23 +1112,10 @@ async function buildVomObservation( visualOutput, }; } - const captureNotice = notices.join("\n"); - const rendered = options.skipRender - ? { text: "", refs: [], truncated: false } - : renderVom(decoratedScene, { - allocateRef: renderOptions.allocateRef, - maxDepth: options.maxDepth, - maxTokens: - !captureNotice || options.maxTokens === undefined - ? options.maxTokens - : Math.max(0, options.maxTokens - estimateLineTokens(captureNotice)), - redactValues: options.redactValues, - activeRegionPolicy: options.activeRegionPolicy, - }); - if (!options.skipRender && captureNotice) rendered.text += `\n${captureNotice}`; + const rendered = renderObservationOutput(visualOutput, { maxTokens: options.maxTokens }); throwIfAborted(options.signal, "observation"); return projectRecordSafeObservation({ - rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root", + rootFrameId: rootKey, frameDocuments: normalizedDocuments, identities, rendered, @@ -1146,6 +1156,20 @@ function observeIdentityKey(sessionId: string, tabId: number): string { return `${sessionId}:${tabId}`; } +export function clearObserveIdentityStreaksForSession(sessionId: string): void { + const prefix = `${sessionId}:`; + for (const key of observeIdentityStreaks.keys()) { + if (key.startsWith(prefix)) observeIdentityStreaks.delete(key); + } +} + +export function clearObserveIdentityStreaksForTab(tabId: number): void { + const suffix = `:${tabId}`; + for (const key of observeIdentityStreaks.keys()) { + if (key.endsWith(suffix)) observeIdentityStreaks.delete(key); + } +} + /** * Pure rule so the streak logic is testable without a browser: a failure * joins the previous streak only when it is on the **same page** (identical @@ -1279,7 +1303,8 @@ async function handleVomObservation( refStore, accessibilityLease: deps.accessibilityLease, onScene: deps.onScene, - includeVisualFacts: toolName === "observe", + includeVisualFacts: deps.includeVisualFacts ?? toolName === "observe", + deferPagePublish: true, maxDepth: params.max_depth, maxTokens: params.max_tokens, activeRegionPolicy: true, @@ -1301,71 +1326,20 @@ async function handleVomObservation( data: { reason: "ref_not_found" }, }; } - if (observation.visualOutput) { - const page = await publishObservationPage( - refStore, - deps.cdp, - target.tabId, - { output: observation.visualOutput }, - signal, - ); - if (isRpcError(page)) return page; - readSucceeded = true; - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - ...page, - ...(observation.hoverProbe?.performed - ? { - hover_probe: { - performed: true, - revealed_content: observation.hoverProbe.revealedContent, - }, - } - : {}), - ...((params as ObserveParams).debug_surfaces - ? { - debug: { - surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({ - trigger_backend_node_id: probe.triggerBackendNodeId, - trigger_action: probe.triggerAction, - sub_items: probe.subItems, - ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), - ...(probe.confidence ? { confidence: probe.confidence } : {}), - })), - }, - } - : {}), - }); - } - const handleByFrameId = new Map( - observation.frames.map((frame) => [frame.frameId, frame.handle]), - ); - refStore.replace( - observation.refs.map((ref) => { - const frameId = ref.frameId || observation.rootFrameId; - const frameHandle = frameId ? handleByFrameId.get(frameId) : undefined; - return [ - ref.ref, - { - ...(ref.name ? { name: ref.name } : {}), - ...(ref.rowAnchor ? { rowAnchor: ref.rowAnchor } : {}), - backendNodeId: ref.backendNodeId, - tabId: target.tabId, - ...(frameId ? { frameId } : {}), - ...(frameHandle?.loaderId ? { loaderId: frameHandle.loaderId } : {}), - ...(frameHandle?.documentElementBackendNodeId !== undefined - ? { documentElementBackendNodeId: frameHandle.documentElementBackendNodeId } - : {}), - }, - ] as const; - }), + const page = await publishObservationPage( + refStore, + deps.cdp, + target.tabId, + { + output: observation.visualOutput!, + allowContinuation: toolName === "observe", + }, + signal, ); + if (isRpcError(page)) return page; readSucceeded = true; return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - observation_id: refStore.observationId, - text: observation.text, - ref_count: observation.refs.length, - tab_id: target.tabId, - truncated: observation.truncated, + ...page, ...(toolName === "observe" && observation.hoverProbe?.performed ? { hover_probe: { @@ -1379,9 +1353,9 @@ async function handleVomObservation( debug: { surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({ trigger_backend_node_id: probe.triggerBackendNodeId, - ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), trigger_action: probe.triggerAction, sub_items: probe.subItems, + ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), ...(probe.confidence ? { confidence: probe.confidence } : {}), })), }, diff --git a/apps/extension/src/tools/pointer-transaction.ts b/apps/extension/src/tools/pointer-transaction.ts index b4795fedc..de7663877 100644 --- a/apps/extension/src/tools/pointer-transaction.ts +++ b/apps/extension/src/tools/pointer-transaction.ts @@ -1,7 +1,11 @@ import type { CdpDispatchGuard } from "@/browser-driver/chromium-cdp"; -import type { CdpTarget, FrameDocumentHandle, FrameNodeHandle } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentHandle, + type FrameNodeHandle, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import type { InteractionDeps } from "./interaction"; -import { type CdpRunner, delegateCdpRunner, sendToCdpTarget } from "./shared"; +import { type CdpRunner, delegateCdpRunner } from "./shared"; import { waitBounded } from "./transfer-transaction"; /** A retired pointer transaction must never resume input when a CDP reply arrives late. */ @@ -19,22 +23,27 @@ export function pointerTransactionDeps( if (retired || signal?.aborted) throw new DOMException("pointer transaction aborted", "AbortError"); }; - const send = async (target: CdpTarget, method: string, params?: object): Promise => { + const sendFrame = async ( + handle: FrameDocumentHandle | FrameNodeHandle, + method: string, + params?: object, + guard?: CdpDispatchGuard, + ): Promise => { check(); const mouse = params as { type?: string } | undefined; const onDispatch = () => { check(); if ( release?.attachmentId !== undefined && - deps.cdp.getAttachmentId?.(target.tabId) !== release.attachmentId + deps.cdp.getAttachmentId(handle.tabId) !== release.attachmentId ) throw new Error("pointer attachment changed"); if (method === "Input.dispatchMouseEvent") { if (mouse?.type === "mousePressed") { release = { - tabId: target.tabId, + tabId: handle.tabId, params: { ...params, type: "mouseReleased", buttons: 0 }, - attachmentId: deps.cdp.getAttachmentId?.(target.tabId), + attachmentId: deps.cdp.getAttachmentId(handle.tabId), }; } else if (mouse?.type === "mouseMoved" && release) { release.params = { @@ -47,17 +56,14 @@ export function pointerTransactionDeps( releasing = true; } } + guard?.onDispatch?.(); }; - const result = deps.cdp.sendGuarded - ? await deps.cdp.sendGuarded(target, method, params, { - signal, - attachmentId: release?.attachmentId, - onDispatch, - }) - : await (() => { - onDispatch(); - return sendToCdpTarget(deps.cdp, target, method, params); - })(); + const result = await deps.cdp.sendToFrame(handle, method, params, { + ...guard, + signal: guard?.signal ?? signal, + attachmentId: guard?.attachmentId ?? release?.attachmentId, + onDispatch, + }); if (method === "Input.dispatchMouseEvent" && mouse?.type === "mouseReleased") { release = undefined; releasing = false; @@ -65,84 +71,21 @@ export function pointerTransactionDeps( return result; }; const cdp: CdpRunner = delegateCdpRunner(deps.cdp, { - send: (tabId, method, params) => send({ tabId }, method, params), - sendToTarget: send, - ...(deps.cdp.sendGuarded - ? { - sendGuarded: ( - target: CdpTarget, - method: string, - params: object | undefined, - guard: CdpDispatchGuard, - ) => { - check(); - return deps.cdp.sendGuarded!(target, method, params, { - ...guard, - signal: guard.signal ?? signal, - attachmentId: guard.attachmentId ?? release?.attachmentId, - onDispatch: () => { - check(); - if ( - release?.attachmentId !== undefined && - deps.cdp.getAttachmentId?.(target.tabId) !== release.attachmentId - ) - throw new Error("pointer attachment changed"); - guard.onDispatch?.(); - }, - }); - }, - } - : {}), - ...(deps.cdp.sendToFrame - ? { - sendToFrame: ( - handle: FrameDocumentHandle | FrameNodeHandle, - method: string, - params?: object, - guard?: CdpDispatchGuard, - ) => { - check(); - return deps.cdp.sendToFrame!(handle, method, params, { - ...guard, - signal: guard?.signal ?? signal, - attachmentId: guard?.attachmentId ?? release?.attachmentId, - onDispatch: () => { - check(); - if ( - release?.attachmentId !== undefined && - deps.cdp.getAttachmentId?.(handle.tabId) !== release.attachmentId - ) - throw new Error("pointer attachment changed"); - guard?.onDispatch?.(); - }, - }); - }, - } - : {}), - ...(deps.cdp.ensureAttachedToUrl - ? { - ensureAttachedToUrl: (tabId, expectedUrl) => { - check(); - return deps.cdp.ensureAttachedToUrl!(tabId, expectedUrl); - }, - } - : {}), - ...(deps.cdp.getMainFrameHandle - ? { - getMainFrameHandle: (tabId, sig) => { - check(); - return deps.cdp.getMainFrameHandle!(tabId, sig ?? signal); - }, - } - : {}), - ...(deps.cdp.getFrameGraph - ? { - getFrameGraph: (tabId, sig) => { - check(); - return deps.cdp.getFrameGraph!(tabId, sig ?? signal); - }, - } - : {}), + send: (tabId: number, method: string, params?: object, sig?: AbortSignal) => + sendFrame(toRootFrameHandle(tabId), method, params, sig ? { signal: sig } : undefined), + sendToFrame: sendFrame, + ensureAttachedToUrl: (tabId, expectedUrl) => { + check(); + return deps.cdp.ensureAttachedToUrl(tabId, expectedUrl); + }, + getMainFrameHandle: (tabId, sig) => { + check(); + return deps.cdp.getMainFrameHandle(tabId, sig ?? signal); + }, + getFrameDocuments: (tabId, sig) => { + check(); + return deps.cdp.getFrameDocuments(tabId, sig ?? signal); + }, }); const cleanupInput = async (deadline: number): Promise => { if (!release) return; @@ -150,12 +93,12 @@ export function pointerTransactionDeps( release = undefined; const ownsAttachment = () => pending.attachmentId === undefined || - deps.cdp.getAttachmentId?.(pending.tabId) === pending.attachmentId; + deps.cdp.getAttachmentId(pending.tabId) === pending.attachmentId; const detach = () => { if (!ownsAttachment()) return Promise.resolve(); return pending.attachmentId === undefined - ? deps.cdp.detach?.(pending.tabId) - : deps.cdp.detach?.(pending.tabId, pending.attachmentId); + ? deps.cdp.detach(pending.tabId) + : deps.cdp.detach(pending.tabId, pending.attachmentId); }; if (!ownsAttachment()) return; try { @@ -167,14 +110,12 @@ export function pointerTransactionDeps( (pending.params as { button?: string }).button === "left" && !releasing ) { - const cancel = deps.cdp.sendGuarded - ? deps.cdp.sendGuarded( - { tabId: pending.tabId }, - "Input.cancelDragging", - {}, - { attachmentId: pending.attachmentId }, - ) - : deps.cdp.send(pending.tabId, "Input.cancelDragging", {}); + const cancel = deps.cdp.sendToFrame( + toRootFrameHandle(pending.tabId), + "Input.cancelDragging", + {}, + { attachmentId: pending.attachmentId, allowReattach: false }, + ); await waitBounded( cancel, Math.min(deadline, Date.now() + 300), @@ -186,16 +127,15 @@ export function pointerTransactionDeps( // Never replay an in-flight release or send cleanup on a new attachment. const cleanup = releasing ? detach() - : deps.cdp.sendGuarded - ? deps.cdp.sendGuarded( - { tabId: pending.tabId }, - "Input.dispatchMouseEvent", - pending.params, - { - attachmentId: pending.attachmentId, - }, - ) - : deps.cdp.send(pending.tabId, "Input.dispatchMouseEvent", pending.params); + : deps.cdp.sendToFrame( + toRootFrameHandle(pending.tabId), + "Input.dispatchMouseEvent", + pending.params, + { + attachmentId: pending.attachmentId, + allowReattach: false, + }, + ); if (!cleanup) throw new Error("pointer input cleanup unavailable"); await waitBounded(cleanup, deadline, undefined, "pointer input cleanup timed out"); } catch (error) { diff --git a/apps/extension/src/tools/read.ts b/apps/extension/src/tools/read.ts index 5083c6b0f..5e628de8c 100644 --- a/apps/extension/src/tools/read.ts +++ b/apps/extension/src/tools/read.ts @@ -1,4 +1,10 @@ -import type { CdpFrame, FrameDocumentHandle, FrameNodeHandle } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentHandle, + type FrameNodeHandle, + isStaleFrameHandleError, + toFrameDocumentHandle, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import type { SessionManager } from "@/session-manager/manager"; import type { RpcError } from "@/transport/types"; import { documentIdentity, type RuntimeReply, SERIALIZATION, unwrap } from "./dom-read-runtime"; @@ -6,14 +12,13 @@ import { collectDomRead } from "./read/collector"; import type { RawRead, ReadParams, ReadResult } from "./read/types"; import { ReadFailure, readError, validateRead } from "./read/validation"; import { + type CdpDispatchGuard, type CdpRunner, type ChromeTabsApi, delegateCdpRunner, isRpcError, lookupSession, resolveCdpAccessibleTargetTab, - sendToCdpTarget, - sendToFrameHandle, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; import { throwIfAborted } from "./vom/capture-abort"; @@ -44,10 +49,10 @@ export async function handleRead( "read", ); if (isRpcError(tab)) return tab; - deps.cdp.trackSessionTab?.(ctx.sessionId, tab.tabId); - await deps.cdp.ensureAttachedToUrl?.(tab.tabId, tab.url); + deps.cdp.trackSessionTab(ctx.sessionId, tab.tabId); + await deps.cdp.ensureAttachedToUrl(tab.tabId, tab.url); const documentRevision = ctx.refStore.documentRevision(tab.tabId); - const attachmentId = deps.cdp.getAttachmentId?.(tab.tabId); + const attachmentId = deps.cdp.getAttachmentId(tab.tabId); if (!attachmentId) return { code: "unsupported", @@ -58,7 +63,7 @@ export async function handleRead( throwIfAborted(signal); if ( manager.get(ctx.sessionId) !== ctx || - deps.cdp.getAttachmentId?.(tab.tabId) !== attachmentId || + deps.cdp.getAttachmentId(tab.tabId) !== attachmentId || ctx.refStore.documentRevision(tab.tabId) !== documentRevision ) stale(); @@ -68,51 +73,33 @@ export async function handleRead( check(); return { attachmentId, signal }; } - if (deps.cdp.getAttachmentId?.(tab.tabId) !== attachmentId) return null; - return { attachmentId }; + if (deps.cdp.getAttachmentId(tab.tabId) !== attachmentId) return null; + return { attachmentId, allowReattach: false }; }; - const knownFrameTargets = new Map(); const sendFrame = async ( handle: FrameDocumentHandle | FrameNodeHandle, method: string, args?: object, + extraGuard?: CdpDispatchGuard, ): Promise => { const guard = guardFor(method); - if (!guard) return undefined as T; - if (!deps.cdp.sendToFrame) { - const knownTarget = handle.frameId ? knownFrameTargets.get(handle.frameId) : undefined; - if (knownTarget) { - return deps.cdp.sendGuarded - ? deps.cdp.sendGuarded(knownTarget, method, args, guard) - : sendToCdpTarget(deps.cdp, knownTarget, method, args); - } - } - return sendToFrameHandle(deps.cdp, handle, method, args, guard); + if (!guard) return {} as T; + return deps.cdp.sendToFrame(handle, method, args, { + ...extraGuard, + ...guard, + signal: extraGuard?.signal ?? guard.signal, + }); }; const cdp = delegateCdpRunner(deps.cdp, { - send: async (tabId: number, method: string, args?: object): Promise => { - const guard = guardFor(method); - if (!guard) return undefined as T; - return deps.cdp.sendGuarded - ? deps.cdp.sendGuarded({ tabId }, method, args, guard) - : deps.cdp.send(tabId, method, args); - }, - sendToTarget: async ( - target: Parameters>[0], - method: string, - args?: object, - ): Promise => { - const guard = guardFor(method); - if (!guard) return undefined as T; - return deps.cdp.sendGuarded - ? deps.cdp.sendGuarded(target, method, args, guard) - : sendToCdpTarget(deps.cdp, target, method, args); - }, + resourceCleanup: undefined, + send: (tabId: number, method: string, args?: object, sig?: AbortSignal): Promise => + sendFrame(toRootFrameHandle(tabId), method, args, sig ? { signal: sig } : undefined), sendToFrame: ( handle: FrameDocumentHandle | FrameNodeHandle, method: string, args?: object, - ) => sendFrame(handle, method, args), + extraGuard?: CdpDispatchGuard, + ) => sendFrame(handle, method, args, extraGuard), }); // A main-document query must not collect unrelated OOPIF documents or wait // for their renderers. Only a ref in a child needs the complete frame graph. @@ -120,16 +107,14 @@ export async function handleRead( frameTree?: { frame: { id: string; loaderId?: string; url?: string } }; }>(tab.tabId, "Page.getFrameTree", {}); if (!tree.frameTree?.frame.id) return stale(); - const root: CdpFrame = { + const root = toFrameDocumentHandle(tab.tabId, { frameId: tree.frameTree.frame.id, - loaderId: tree.frameTree.frame.loaderId, - url: tree.frameTree.frame.url, - target: { tabId: tab.tabId }, - }; - knownFrameTargets.set(root.frameId, root.target); + loaderId: tree.frameTree.frame.loaderId ?? "", + }); + const rootUrl = tree.frameTree.frame.url; check(); const rootDoc = await documentIdentity(cdp, root, attachmentId, stale); - let frame = root; + let frame: FrameDocumentHandle = root; let backendNodeId: number | undefined; let expectedDocElementBackendNodeId: number | undefined; if (params.ref) { @@ -137,16 +122,15 @@ export async function handleRead( if (isRpcError(ref)) return ref; expectedDocElementBackendNodeId = ref.documentElementBackendNodeId; if (ref.frameId && ref.frameId !== root.frameId) { - const graph = await deps.cdp.getFrameGraph?.(tab.tabId); + const graph = await deps.cdp.getFrameDocuments(tab.tabId, signal); check(); - const selected = graph?.frames.find( + const selected = graph.frames.find( (f) => f.frameId === ref.frameId && (!ref.loaderId || !f.loaderId || f.loaderId === ref.loaderId), ); if (!selected) return stale(); - frame = selected; - knownFrameTargets.set(frame.frameId, frame.target); + frame = selected.handle; } else if (ref.loaderId && root.loaderId && ref.loaderId !== root.loaderId) { return stale(); } @@ -203,7 +187,7 @@ export async function handleRead( }); const result: ReadResult = { tab_id: tab.tabId, - page_url: frame === root ? raw.frame_url : (root.url ?? tab.url ?? ""), + page_url: frame === root ? raw.frame_url : (rootUrl ?? tab.url ?? ""), frame_id: frame.frameId, frame_url: raw.frame_url, title: raw.title, @@ -239,6 +223,13 @@ export async function handleRead( if (signal?.aborted || (error instanceof Error && error.name === "AbortError")) return { code: "cancelled", message: "DOM read cancelled", data: { effect_state: "none" } }; if (error instanceof ReadFailure) return readError(error); + if (isStaleFrameHandleError(error)) + return readError( + new ReadFailure( + "read_target_stale", + "The read document or target changed; inspect it again", + ), + ); return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error), diff --git a/apps/extension/src/tools/read/collector.ts b/apps/extension/src/tools/read/collector.ts index fd9b3c881..b749efb36 100644 --- a/apps/extension/src/tools/read/collector.ts +++ b/apps/extension/src/tools/read/collector.ts @@ -53,13 +53,10 @@ export function collectDomRead(this: Element, options: ReadOptions): RawRead { const sensitive = element.matches(options.sensitive) || (field.read === "text" && !!element.querySelector(options.sensitive)); - if ( - sensitive || - element.closest( - "script,style,template,noscript,[data-bsk-overlay],[data-bsk-overlay-host],browser-skill-overlay", - ) - ) - redacted.push(field.key); + const excludedAncestors = + options.excludedAncestors ?? + "script,style,template,noscript,[data-bsk-overlay],[data-bsk-overlay-host],browser-skill-overlay"; + if (sensitive || element.closest(excludedAncestors)) redacted.push(field.key); else if (field.read === "text") value = element.textContent; else if (field.read === "attribute") value = element.getAttribute(field.attribute!); else if (field.read === "checked") diff --git a/apps/extension/src/tools/read/read.test.ts b/apps/extension/src/tools/read/read.test.ts index f05718c18..e9a7c6933 100644 --- a/apps/extension/src/tools/read/read.test.ts +++ b/apps/extension/src/tools/read/read.test.ts @@ -1,7 +1,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { type FrameDocumentGraph, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; import { handleRead } from "../read"; -import type { CdpRunner } from "../shared"; +import { type CdpRunner, createStubCdpRunner } from "../shared"; import { collectDomRead } from "./collector"; import type { ReadParams } from "./types"; import { validateRead } from "./validation"; @@ -119,15 +120,31 @@ async function fixture() { } throw new Error(method); }); - const cdp: CdpRunner = { - send: send as CdpRunner["send"], - getAttachmentId: () => state.attachment, - ensureAttachedToUrl: vi.fn(async () => {}), - getFrameGraph: vi.fn(async () => ({ + const ensureAttachedToUrl = vi.fn(async () => {}); + const getFrameDocuments = vi.fn( + async (): Promise => ({ rootFrameId: "root", - frames: [{ frameId: "root", target: { tabId: 7 }, url: tab.url }], - })), - }; + frames: [ + { + handle: toFrameDocumentHandle(7, { frameId: "root", loaderId: "" }), + frameId: "root", + loaderId: "", + url: tab.url, + processKey: "root", + isOutOfProcess: false, + }, + ], + }), + ); + const cdp = Object.assign( + createStubCdpRunner({ + send: send as CdpRunner["send"], + getAttachmentId: () => state.attachment, + ensureAttachedToUrl, + getFrameDocuments, + }), + { ensureAttachedToUrl, getFrameDocuments }, + ); const run = (patch: object = {}, signal?: AbortSignal) => handleRead( manager, @@ -157,13 +174,15 @@ describe("DOM read identity and lifecycle", () => { const store = h.manager.get("s")!.refStore; if (frame === "child") { store.set("e1", 2, { tabId: 7, frameId: "child", loaderId: "child-loader" }); - vi.mocked(h.cdp.getFrameGraph!).mockResolvedValue({ + vi.mocked(h.cdp.getFrameDocuments).mockResolvedValue({ rootFrameId: "root", frames: [ { + handle: toFrameDocumentHandle(7, { frameId: "child", loaderId: "child-loader" }), frameId: "child", - target: { tabId: 7, sessionId: "child-session" }, loaderId: "child-loader", + processKey: "oopif:child", + isOutOfProcess: true, }, ], }); @@ -179,11 +198,11 @@ describe("DOM read identity and lifecycle", () => { }); it("does not collect or depend on unrelated child renderers for a main-document read", async () => { const h = await fixture(); - vi.mocked(h.cdp.getFrameGraph!).mockRejectedValue( + vi.mocked(h.cdp.getFrameDocuments).mockRejectedValue( new Error("unrelated OOPIF cannot be observed"), ); expect(await h.run()).toMatchObject({ scope: "current_dom" }); - expect(h.cdp.getFrameGraph).not.toHaveBeenCalled(); + expect(h.cdp.getFrameDocuments).not.toHaveBeenCalled(); }); it("rejects invalid read capabilities before touching CDP", async () => { const h = await fixture(); diff --git a/apps/extension/src/tools/read/types.ts b/apps/extension/src/tools/read/types.ts index db2cdda81..3e2dd65d3 100644 --- a/apps/extension/src/tools/read/types.ts +++ b/apps/extension/src/tools/read/types.ts @@ -8,6 +8,7 @@ export interface ReadOptions { max_bytes: number; timeout_ms: number; sensitive: string; + excludedAncestors?: string; } export interface RawRead { frame_url: string; diff --git a/apps/extension/src/tools/read/validation.ts b/apps/extension/src/tools/read/validation.ts index 828002264..565e3e7f1 100644 --- a/apps/extension/src/tools/read/validation.ts +++ b/apps/extension/src/tools/read/validation.ts @@ -1,7 +1,7 @@ import { - RECORD_PASSWORD_ATTRIBUTE, - SENSITIVE_AUTOCOMPLETE_SELECTOR, -} from "@/lib/sensitive-autocomplete"; + READ_REDACTED_ANCESTOR_SELECTOR, + SENSITIVE_READ_INPUT_SELECTOR, +} from "@/lib/dom-predicates"; import type { RpcError, RpcErrorReason } from "@/transport/types"; import type { ReadOptions, ReadParams } from "./types"; @@ -101,6 +101,7 @@ export function validateRead(params: ReadParams): ReadOptions { max_nodes: bound(params.max_nodes, 100, 1, 1000), max_bytes: bound(params.max_bytes, 262144, 1024, 1048576), timeout_ms: bound(params.timeout_ms, 3000, 100, 10000), - sensitive: `input[type="password"],input[type="file"],input[type="hidden"],[${RECORD_PASSWORD_ATTRIBUTE}],${SENSITIVE_AUTOCOMPLETE_SELECTOR}`, + sensitive: SENSITIVE_READ_INPUT_SELECTOR, + excludedAncestors: READ_REDACTED_ANCESTOR_SELECTOR, }; } diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 9671032fd..edec11d51 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -58,6 +58,9 @@ import { interface ActiveRecording { requestId: string; + deps: RecordDeps; + armedTabIds: Set; + bypassedTabIds: Set; tabs: RecordingTabCoordinator; agentWindowId: number; isTabAllowed: (tabId: number) => boolean; @@ -500,6 +503,9 @@ async function clearRearmTimersForRecording( deps: RecordDeps, ): Promise { clearRearmTimer(recording.tabs.currentTabId); + for (const tabId of recording.armedTabIds) { + clearRearmTimer(tabId); + } try { const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); for (const tab of tabs) { @@ -518,21 +524,22 @@ async function stopRecordingOnAllAgentTabs( throw new Error("one or more recording documents did not confirm their final steps"); } const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; - let tabIds = [recording.tabs.currentTabId]; + let tabIds = [...new Set([recording.tabs.currentTabId, ...recording.armedTabIds])]; try { const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); tabIds = [ ...new Set([ - recording.tabs.currentTabId, + ...tabIds, ...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])), ]), ]; } catch { - // Fall back to the current recording tab. + // Fall back to the known recording tabs. } for (const tabId of tabIds) { - if (!recording.isTabAllowed(tabId)) continue; + const wasArmed = recording.armedTabIds.delete(tabId); + if (!wasArmed && !recording.isTabAllowed(tabId)) continue; // RECORD_STOP only clears the tab's record overlay; captured steps are // flushed by the frame coordinator above. Tabs the overlay script cannot // enter (blob:, the PDF viewer, browser pages) have nothing to clear. @@ -541,7 +548,19 @@ async function stopRecordingOnAllAgentTabs( } catch { // Best-effort cleanup. } - if (deps.bypassOverlay) { + if (deps.bypassOverlay && recording.bypassedTabIds.delete(tabId)) { + try { + await deps.bypassOverlay(tabId, false); + } catch { + // Best-effort cleanup. + } + } + } + + if (deps.bypassOverlay && recording.bypassedTabIds.size > 0) { + const remainingBypassed = [...recording.bypassedTabIds]; + recording.bypassedTabIds.clear(); + for (const tabId of remainingBypassed) { try { await deps.bypassOverlay(tabId, false); } catch { @@ -574,6 +593,7 @@ async function rearmRecording( const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId); if (!frameStarted) throw new Error("recording document did not start"); } + recording.armedTabIds.add(targetTabId); await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing); if (isFinishing()) return false; if (activation) { @@ -899,8 +919,14 @@ export async function handleRecordStart( const sessionCdp = deps.cdp ? sessionCdpRunner(deps.cdp, params.session_id, () => manager.get(ctx.sessionId) === ctx) : undefined; + const recordingDeps: RecordDeps = { ...deps, signal: undefined }; + const armedTabIds = new Set(); + const bypassedTabIds = new Set(); recordings.set(params.session_id, { requestId, + deps: recordingDeps, + armedTabIds, + bypassedTabIds, tabs: new RecordingTabCoordinator(target.tabId, navigateUrl), agentWindowId: ctx.agentWindowId, isTabAllowed, @@ -957,6 +983,7 @@ export async function handleRecordStart( releaseBrowserObservationListenersIfIdle(); await notifyActiveRecordingsChanged(); if (notifyContent) { + armedTabIds.delete(target.tabId); try { await deps.sendToTab(target.tabId, { type: RECORD_CANCEL, @@ -966,7 +993,7 @@ export async function handleRecordStart( // Content may never have received RECORD_START. } } - if (deps.bypassOverlay) { + if (deps.bypassOverlay && bypassedTabIds.delete(target.tabId)) { try { await deps.bypassOverlay(target.tabId, false); } catch { @@ -1053,11 +1080,13 @@ export async function handleRecordStart( } if (deps.bypassOverlay) { + bypassedTabIds.add(target.tabId); try { // Single ref for the initial race before RecordOverlay mounts; rearm must // not stack additional refs (see rearmRecording). Cleared on stop. await deps.bypassOverlay(target.tabId, true); } catch { + bypassedTabIds.delete(target.tabId); // Best-effort; activeRecord also hides the control overlay. } } @@ -1074,6 +1103,7 @@ export async function handleRecordStart( const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId); if (!frameStarted) throw new Error("top recording document did not start"); } + armedTabIds.add(target.tabId); await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab); } catch { await abortPending(true); @@ -1185,16 +1215,25 @@ export async function handleRecordAwait( return "trace" in outcome ? { trace: outcome.trace } : outcome.error; } -export function clearRecordingForSession(sessionId: string): void { +export function clearRecordingForSession( + sessionId: string, + overrideDeps?: Partial, +): void { const recording = recordings.get(sessionId); if (!recording) { recordings.delete(sessionId); releaseBrowserObservationListenersIfIdle(); return; } - void clearRearmTimersForRecording(recording, getDefaultDeps()); + const effectiveDeps: RecordDeps = overrideDeps + ? { ...recording.deps, ...overrideDeps } + : recording.deps; + clearRearmTimer(recording.tabs.currentTabId); + for (const tabId of recording.armedTabIds) { + clearRearmTimer(tabId); + } if (!recording.settled) { - getDefaultDeps().frameCoordinator?.cancel(recording.requestId); + effectiveDeps.frameCoordinator?.cancel(recording.requestId); recording.observation?.cancel(); recording.settled = true; recording.rejectFinish(new Error("recording cleared")); @@ -1202,6 +1241,42 @@ export function clearRecordingForSession(sessionId: string): void { recordings.delete(sessionId); releaseBrowserObservationListenersIfIdle(); void notifyActiveRecordingsChanged(); + + const cancelMsg: RecordCancelMessage = { + type: RECORD_CANCEL, + requestId: recording.requestId, + }; + const immediateCancelTabs = new Set([ + recording.tabs.currentTabId, + ...recording.armedTabIds, + ]); + recording.armedTabIds.clear(); + for (const tabId of immediateCancelTabs) { + void effectiveDeps.sendToTab(tabId, cancelMsg).catch(() => {}); + } + + if (effectiveDeps.bypassOverlay && recording.bypassedTabIds.size > 0) { + const bypassed = [...recording.bypassedTabIds]; + recording.bypassedTabIds.clear(); + for (const tabId of bypassed) { + void effectiveDeps.bypassOverlay(tabId, false).catch(() => {}); + } + } + + void (async () => { + try { + const tabs = await effectiveDeps.tabsApi.query({ windowId: recording.agentWindowId }); + for (const tab of tabs) { + if (typeof tab.id !== "number") continue; + clearRearmTimer(tab.id); + if (!immediateCancelTabs.has(tab.id) && recording.isTabAllowed(tab.id)) { + void effectiveDeps.sendToTab(tab.id, cancelMsg).catch(() => {}); + } + } + } catch { + // Best-effort cleanup. + } + })(); } export type { RecordFinishMessage }; diff --git a/apps/extension/src/tools/scroll-visibility.ts b/apps/extension/src/tools/scroll-visibility.ts index 6c40dc527..3b20149fe 100644 --- a/apps/extension/src/tools/scroll-visibility.ts +++ b/apps/extension/src/tools/scroll-visibility.ts @@ -1,8 +1,8 @@ import { - cdpTargetKey, type FrameNodeHandle, + frameProcessBoundary, StaleFrameHandleError, -} from "@/browser-driver/frame-graph"; +} from "@/browser-driver/frame-handles"; import type { NodeAddress } from "./frame-geometry"; import { clipPolygon, @@ -101,14 +101,20 @@ async function projectRect( ); } const parent = frame.parentFrameId ? await context.frame(frame.parentFrameId) : undefined; - if (parent && cdpTargetKey(parent.target) === cdpTargetKey(frame.target)) { + if (parent && parent.processKey === frame.processKey) { // DOM rectangles are document-local. Same-target iframe quads are already // target-local, so cross this one boundary before the target projection. - const viewport = await context.viewport(frame.target); + const process = frameProcessBoundary(frame); + const viewport = await context.viewport(process); if (frame.ownerBackendNodeId === undefined || !viewport) { throw new Error("scroll-to could not resolve the frame viewport"); } - const local = await context.snapshotProjection(frame, frame.ownerBackendNodeId, [], viewport); + const local = await context.snapshotProjection( + { process, frameId: frame.frameId }, + frame.ownerBackendNodeId, + [], + viewport, + ); if (local.status !== "available") throw new Error("scroll-to could not project the frame"); region = projectRegionToViewport(region, local.projection.geometry); } diff --git a/apps/extension/src/tools/scroll.ts b/apps/extension/src/tools/scroll.ts index 5a9a59b97..46d5e5182 100644 --- a/apps/extension/src/tools/scroll.ts +++ b/apps/extension/src/tools/scroll.ts @@ -1,10 +1,10 @@ import type { CdpDispatchGuard } from "@/browser-driver/chromium-cdp"; import { - type CdpTarget, - cdpTargetKey, + type FrameDocumentGraph, type FrameDocumentHandle, type FrameNodeHandle, -} from "@/browser-driver/frame-graph"; + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import type { SessionManager } from "@/session-manager/manager"; import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; @@ -21,7 +21,6 @@ import { isRpcError, lookupSession, resolveTargetTab, - sendToCdpTarget, sendToFrameHandle, } from "./shared"; @@ -49,22 +48,7 @@ export async function handleScrollTo( if (Date.now() >= deadline) throw new DOMException("scroll-to timed out", "TimeoutError"); }; const objectGroup = `bsk-scroll-${crypto.randomUUID()}`; - const objectTargets = new Map(); const objectHandles = new Map(); - const send = async (target: CdpTarget, method: string, args?: object): Promise => { - checkActive(); - // Also own objects allocated by the shared scroll fallback. Record the - // target before awaiting so cancellation cannot skip their cleanup. - if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target); - const result = await sendToCdpTarget( - deps.cdp, - target, - method, - method === "DOM.resolveNode" ? { ...args, objectGroup } : args, - ); - checkActive(); - return result; - }; const sendFrame = async ( handle: FrameDocumentHandle | FrameNodeHandle, method: string, @@ -72,6 +56,8 @@ export async function handleScrollTo( guard?: CdpDispatchGuard, ): Promise => { checkActive(); + // Also own objects allocated by the shared scroll fallback. Record the + // handle before awaiting so cancellation cannot skip their cleanup. if (method === "DOM.resolveNode") { objectHandles.set(`${handle.tabId}:${handle.frameId}:${handle.loaderId}`, { tabId: handle.tabId, @@ -92,41 +78,29 @@ export async function handleScrollTo( checkActive(); return result; }; - let graph: ReturnType> | undefined; + let graph: Promise | undefined; const cdp: CdpRunner = delegateCdpRunner(deps.cdp, { - send: (tabId, method, args) => send({ tabId }, method, args), - sendToTarget: send, + send: (tabId: number, method: string, args?: object, sig?: AbortSignal) => + sendFrame(toRootFrameHandle(tabId), method, args, sig ? { signal: sig } : undefined), sendToFrame: sendFrame, - ...(deps.cdp.ensureAttachedToUrl - ? { - ensureAttachedToUrl: async (tabId, expectedUrl) => { - checkActive(); - await deps.cdp.ensureAttachedToUrl!(tabId, expectedUrl); - checkActive(); - }, - } - : {}), - ...(deps.cdp.getMainFrameHandle - ? { - getMainFrameHandle: async (tabId, sig) => { - checkActive(); - const result = await deps.cdp.getMainFrameHandle!(tabId, sig); - checkActive(); - return result; - }, - } - : {}), - ...(deps.cdp.getFrameGraph - ? { - getFrameGraph: async (tabId) => { - checkActive(); - graph ??= deps.cdp.getFrameGraph!(tabId); - const result = await graph; - checkActive(); - return result; - }, - } - : {}), + ensureAttachedToUrl: async (tabId, expectedUrl) => { + checkActive(); + await deps.cdp.ensureAttachedToUrl(tabId, expectedUrl); + checkActive(); + }, + getMainFrameHandle: async (tabId, sig) => { + checkActive(); + const result = await deps.cdp.getMainFrameHandle(tabId, sig); + checkActive(); + return result; + }, + getFrameDocuments: async (tabId, sig) => { + checkActive(); + graph ??= deps.cdp.getFrameDocuments(tabId, sig); + const result = await graph; + checkActive(); + return result; + }, }); try { checkActive(); @@ -166,15 +140,14 @@ export async function handleScrollTo( } finally { // Cleanup bypasses the cancellation guard and cannot replace the result. const cleanup = deps.cdp.resourceCleanup ?? deps.cdp; - for (const target of objectTargets.values()) { - await sendToCdpTarget(cleanup, target, "Runtime.releaseObjectGroup", { objectGroup }).catch( - () => {}, - ); - } for (const handle of objectHandles.values()) { - await sendToFrameHandle(cleanup, handle, "Runtime.releaseObjectGroup", { objectGroup }).catch( - () => {}, - ); + await sendToFrameHandle( + cleanup, + handle, + "Runtime.releaseObjectGroup", + { objectGroup }, + { allowReattach: false }, + ).catch(() => {}); } } } diff --git a/apps/extension/src/tools/selector-resolution.ts b/apps/extension/src/tools/selector-resolution.ts new file mode 100644 index 000000000..b7c35bef2 --- /dev/null +++ b/apps/extension/src/tools/selector-resolution.ts @@ -0,0 +1,151 @@ +import type { FrameNodeHandle } from "@/browser-driver/frame-handles"; +import type { RpcError } from "@/transport/types"; +import { rpcError } from "./errors"; +import { type CdpRunner, lookupPinnedSelectorTarget } from "./shared"; + +export interface ResolvedMainDocumentSelector { + handle: FrameNodeHandle; + backendNodeId: number; + frameId?: string; + usedSelector: string; +} + +export interface ResolveMainDocumentSelectorOptions { + toolName: string; + multiMatch?: "first" | "reject"; + signal?: AbortSignal; +} + +/** + * Resolve a CSS selector in the tab's main document to a concrete `FrameNodeHandle` + * (`backendNodeId`). Honors request-scoped pinned selector targets established by + * `CooperationController` without mutating `SessionContext.refStore`. + */ +export async function resolveMainDocumentSelector( + cdp: CdpRunner, + tabId: number, + selector: string, + options: ResolveMainDocumentSelectorOptions, +): Promise { + const pinned = lookupPinnedSelectorTarget(cdp, tabId, selector); + if (pinned) { + return { + handle: pinned.handle, + backendNodeId: pinned.backendNodeId, + ...(pinned.frameId ? { frameId: pinned.frameId } : {}), + usedSelector: pinned.usedSelector, + }; + } + + try { + const doc = await cdp.send<{ root?: { nodeId?: number } }>( + tabId, + "DOM.getDocument", + { depth: 0 }, + options.signal, + ); + const rootNodeId = doc.root?.nodeId; + if (typeof rootNodeId !== "number") { + return { + code: "cdp_failed", + message: "DOM.getDocument returned no root nodeId", + }; + } + + const query = { nodeId: rootNodeId, selector }; + let nodeId: number | undefined; + if (options.multiMatch === "reject") { + const found = await cdp.send<{ nodeIds: number[] }>( + tabId, + "DOM.querySelectorAll", + query, + options.signal, + ); + if (found.nodeIds.length > 1) { + return { + code: "invalid_params", + message: `${options.toolName}: selector matched ${found.nodeIds.length} elements; use a VOM ref or a narrower selector`, + }; + } + nodeId = found.nodeIds[0]; + } else { + nodeId = ( + await cdp.send<{ nodeId?: number }>(tabId, "DOM.querySelector", query, options.signal) + ).nodeId; + } + + if (typeof nodeId !== "number" || nodeId === 0) { + return rpcError( + "not_found", + "selector_not_found", + `selector ${selector} did not match any element`, + ); + } + + const described = await cdp.send<{ node?: { backendNodeId?: number } }>( + tabId, + "DOM.describeNode", + { nodeId }, + options.signal, + ); + const backendNodeId = described.node?.backendNodeId; + if (typeof backendNodeId !== "number") { + return { + code: "cdp_failed", + message: "DOM.describeNode returned no backendNodeId", + }; + } + + return { + handle: { + tabId, + frameId: "", + loaderId: "", + backendNodeId, + }, + backendNodeId, + usedSelector: selector, + }; + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } +} + +/** + * Probe whether a CSS selector currently matches at least one element in the tab's + * main document (reusing an already-fetched `rootNodeId` when available). + */ +export async function selectorExistsInMainDocument( + cdp: CdpRunner, + tabId: number, + selector: string, + rootNodeId?: number, + signal?: AbortSignal, +): Promise { + if (lookupPinnedSelectorTarget(cdp, tabId, selector)) return true; + try { + let resolvedRootId = rootNodeId; + if (typeof resolvedRootId !== "number") { + const doc = await cdp.send<{ root?: { nodeId?: number } }>( + tabId, + "DOM.getDocument", + { depth: 0 }, + signal, + ); + resolvedRootId = doc.root?.nodeId; + if (typeof resolvedRootId !== "number") return false; + } + const res = await cdp.send<{ nodeId?: number }>( + tabId, + "DOM.querySelector", + { nodeId: resolvedRootId, selector }, + signal, + ); + return typeof res.nodeId === "number" && res.nodeId !== 0; + } catch { + return false; + } +} diff --git a/apps/extension/src/tools/session.ts b/apps/extension/src/tools/session.ts index d6fb5e26e..67c7db14b 100644 --- a/apps/extension/src/tools/session.ts +++ b/apps/extension/src/tools/session.ts @@ -16,7 +16,8 @@ import type { } from "@/transport/types"; import { rpcError } from "./errors"; import { clearHelpForSession } from "./human-loop"; -import { clearRecordingForSession } from "./record"; +import { clearObserveIdentityStreaksForSession } from "./observation"; +import { clearRecordingForSession, type RecordDeps } from "./record"; import type { CdpRunner, ChromeTabsApi } from "./shared"; import { isRpcError } from "./shared"; import { chromeAgentOverlayResetApi, returnBorrowedTab, type TabManagementDeps } from "./tabs"; @@ -74,7 +75,7 @@ export interface SessionStartDeps { export interface SessionStopDeps { signal?: AbortSignal; - cdp?: Pick & { + cdp?: Partial> & { detachSession(sessionId: string): Promise; }; /** @@ -91,6 +92,8 @@ export interface SessionStopDeps { * pure. Tests can inject a fake `ChromeTabsApi` too. */ tabsQuery?: ChromeTabsApi; + /** Optional recording runtime deps forwarded to `clearRecordingForSession`. */ + recording?: Partial; } /** @@ -218,8 +221,10 @@ async function stopSession( // Remote access must end before returning a tab. Preserve the local recording // and help request when a failed return keeps its session alive for a retry. if (ctx.remote) { - clearRecordingForSession(params.session_id); + if (deps.recording) clearRecordingForSession(params.session_id, deps.recording); + else clearRecordingForSession(params.session_id); clearHelpForSession(params.session_id); + clearObserveIdentityStreaksForSession(params.session_id); } // Step 1: auto-return borrowed tabs. Iterate over a snapshot of the @@ -300,8 +305,10 @@ async function stopSession( ctx.refStore.clear(); if (!ctx.remote) { - clearRecordingForSession(params.session_id); + if (deps.recording) clearRecordingForSession(params.session_id, deps.recording); + else clearRecordingForSession(params.session_id); clearHelpForSession(params.session_id); + clearObserveIdentityStreaksForSession(params.session_id); } // Step 3: detach CDP sessions this session opened (no-op if none). diff --git a/apps/extension/src/tools/shared.ts b/apps/extension/src/tools/shared.ts index 28c3bdaa7..d3e06ac76 100644 --- a/apps/extension/src/tools/shared.ts +++ b/apps/extension/src/tools/shared.ts @@ -6,17 +6,19 @@ // exactly the same sandbox + visibility rules as the M6 observation // handlers (review parity). -import type { CdpDebuggee, CdpDispatchGuard, DialogCursor } from "@/browser-driver/chromium-cdp"; +import type { CdpDispatchGuard, DialogCursor } from "@/browser-driver/chromium-cdp"; export type { CdpDispatchGuard }; import { - type CdpFrameGraph, - type CdpTarget, + type FrameDocumentGraph, type FrameDocumentHandle, type FrameNodeHandle, StaleFrameHandleError, -} from "@/browser-driver/frame-graph"; + type TabEventSource, + toFrameDocumentHandle, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import type { NativeInputTimestamp } from "@/browser-driver/native-input-clock"; import { shareScreencasts } from "@/browser-driver/screencast-coordinator"; import { @@ -64,16 +66,17 @@ export interface ResolvedTargetTab { export type { DialogCursor }; /** - * Minimal CDP surface every tool handler needs. Backed by - * `ChromiumCdp` in production; tests inject a fake. + * Typed CDP driver surface every tool handler uses. Backed by `ChromiumCdp` + * in production; unit tests construct typed stubs via `createStubCdpRunner`. * - * `trackSessionTab` is optional so test doubles need not implement it - * — production `ChromiumCdp` always supplies it for ref-aware session - * teardown. + * Contextual `CooperationController` hooks (`runBrowserMutation`, `resourceCleanup`, + * `beginDomMutation`, `verifyPointerTarget`) and shared-tab-stripped background + * execution hooks (`acquireBackgroundExecution`, `ownsBackgroundExecution`, + * `restoreBackgroundExecution`) remain capability-gated. */ export interface CdpRunner { /** Measure the browser's native epoch conversion before arming page input. */ - prepareInputTimestamp?(): Promise; + prepareInputTimestamp(): Promise; /** Fence a browser API effect at invocation, after asynchronous ownership checks. */ runBrowserMutation?(operation: () => Promise): Promise; /** Only release resources acquired by this request; never restore page/user state. */ @@ -87,68 +90,65 @@ export interface CdpRunner { /** Intentional displacement from the observed action point, in viewport CSS pixels. */ offset?: { x: number; y: number }, ): Promise; - send(tabId: number, method: string, params?: object): Promise; - sendToTarget?(target: CdpTarget, method: string, params?: object): Promise; - /** Production dispatch boundary; optional for lightweight test runners. */ - sendGuarded?( - target: CdpTarget, + send( + tabId: number, method: string, - params: object | undefined, - guard: CdpDispatchGuard, + params?: object, + signal?: AbortSignal, ): Promise; /** Stable frame/node handle dispatch; validates `frameId` + `loaderId` inside `ChromiumCdp`. */ - sendToFrame?( + sendToFrame( handle: FrameDocumentHandle | FrameNodeHandle, method: string, params?: object, guard?: CdpDispatchGuard, ): Promise; - getMainFrameHandle?(tabId: number, signal?: AbortSignal): Promise; - isEventFromFrame?(source: CdpDebuggee, handle: FrameDocumentHandle): boolean; - hasOwnFrameTarget?(tabId: number, frameId: string, parentFrameId?: string): boolean; - recordFrameDocumentElement?( + getMainFrameHandle(tabId: number, signal?: AbortSignal): Promise; + getFrameDocuments(tabId: number, signal?: AbortSignal): Promise; + isEventFromFrame(source: TabEventSource, handle: FrameDocumentHandle): boolean; + hasOwnFrameTarget(tabId: number, frameId: string, parentFrameId?: string): boolean; + recordFrameDocumentElement( tabId: number, frameId: string, loaderId: string, documentElementBackendNodeId: number, ): void; - detach?(tabId: number, expectedAttachmentId?: string): Promise; - getFrameGraph?(tabId: number, signal?: AbortSignal): Promise; - getAttachmentId?(tabId: number): string | undefined; - retainTabAttachment?(tabId: number): () => void; - ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise; + detach(tabId: number, expectedAttachmentId?: string): Promise; + getAttachmentId(tabId: number): string | undefined; + retainTabAttachment(tabId: number): () => void; + ensureAttachedToUrl(tabId: number, expectedUrl: string | undefined): Promise; acquireBackgroundExecution?(sessionId: string, tabId: number): Promise; /** Whether this session requests the persistent override, regardless of applied CDP state. */ ownsBackgroundExecution?(sessionId: string, tabId: number): boolean; - releaseBackgroundExecution?(sessionId: string, tabId: number): Promise; + releaseBackgroundExecution(sessionId: string, tabId: number): Promise; /** Resend this session's owned override, even when the applied-state cache reports it active. */ restoreBackgroundExecution?(sessionId: string, tabId: number): Promise; - trackSessionTab?(sessionId: string, tabId: number): void; - releaseSessionTab?(sessionId: string, tabId: number): Promise; - onEvent?( + trackSessionTab(sessionId: string, tabId: number): void; + releaseSessionTab(sessionId: string, tabId: number): Promise; + onEvent( tabId: number, - handler: (source: CdpDebuggee, method: string, params: unknown) => void, + handler: (source: TabEventSource, method: string, params: unknown) => void, ): { dispose(): void; }; - dialogCursor?(tabId: number): DialogCursor; - dialogsSince?(tabId: number, cursor: DialogCursor): JavaScriptDialogInfo[]; - pendingDialogs?(tabId?: number): PendingJavaScriptDialog[]; - resolveDialog?( + dialogCursor(tabId: number): DialogCursor; + dialogsSince(tabId: number, cursor: DialogCursor): JavaScriptDialogInfo[]; + pendingDialogs(tabId?: number): PendingJavaScriptDialog[]; + resolveDialog( id: string, accept: boolean, text?: string, guard?: Pick, ): Promise; - onDialogChanged?( + onDialogChanged( handler: ( tabId: number, dialog: PendingJavaScriptDialog | null, reason?: "decision_timeout", ) => void, ): { dispose(): void }; - ensureConsoleCapture?(tabId: number, signal?: AbortSignal): Promise; - consoleEntriesSince?( + ensureConsoleCapture(tabId: number, signal?: AbortSignal): Promise; + consoleEntriesSince( tabId: number, since: number | undefined, limit: number, @@ -157,86 +157,292 @@ export interface CdpRunner { ): ConsoleResult; } +export interface PinnedSelectorTarget { + readonly handle: FrameNodeHandle; + readonly backendNodeId: number; + readonly frameId?: string; + readonly usedSelector: string; +} + +const pinnedSelectorTargets = new WeakMap>(); + +function sharePinnedSelectorTargets(source: object, target: object): void { + const existing = pinnedSelectorTargets.get(source); + if (existing) pinnedSelectorTargets.set(target, existing); +} + +export function withPinnedSelectorTargets( + cdp: T, + targets: ReadonlyMap, +): T { + const merged = new Map(pinnedSelectorTargets.get(cdp)); + for (const [selector, entry] of targets) merged.set(selector, entry); + const wrapped = delegateCdpRunner(cdp, {}); + pinnedSelectorTargets.set(wrapped, merged); + return wrapped; +} + +export function lookupPinnedSelectorTarget( + cdp: CdpRunner, + tabId: number, + selector: string, +): PinnedSelectorTarget | undefined { + const entry = pinnedSelectorTargets.get(cdp)?.get(selector); + return entry && entry.handle.tabId === tabId ? entry : undefined; +} + +export interface DelegateCdpRunnerOptions { + /** Keys hidden from both property lookup and the `in` operator. */ + omit?: ReadonlySet; + /** Methods whose `this` should be bound to the wrapper Proxy (so internal `this.send` calls stay guarded). */ + bindReceiver?: ReadonlySet; + /** Optional dynamic method interceptor (used by `sessionCdpRunner`). */ + intercept?: (target: T, key: PropertyKey, value: unknown, wrapper: T) => unknown; +} + /** * Wrap `base` via prototype-preserving delegation so overriding `send` / - * `sendToFrame` / hooks never silently strips optional `CdpRunner` capabilities. + * `sendToFrame` / hooks never silently strips `CdpRunner` capabilities or + * breaks `"method" in runner` invariants. */ -export function delegateCdpRunner(base: CdpRunner, overrides: Partial): CdpRunner { +export function delegateCdpRunner( + base: T, + overrides: Partial, + options?: DelegateCdpRunnerOptions, +): T { + const omit = options?.omit; + const bindReceiver = options?.bindReceiver; + const intercept = options?.intercept; const wrapper = new Proxy(base, { has(target, key) { - return ( - (Object.prototype.hasOwnProperty.call(overrides, key) && - overrides[key as keyof CdpRunner] !== undefined) || - Reflect.has(target, key) - ); + if (omit?.has(key)) return false; + if (Object.prototype.hasOwnProperty.call(overrides, key)) { + const desc = Object.getOwnPropertyDescriptor(overrides, key); + if (desc?.get || desc?.value !== undefined) return true; + return false; + } + return Reflect.has(target, key); }, get(target, key, receiver) { + if (omit?.has(key)) return undefined; if (Object.prototype.hasOwnProperty.call(overrides, key)) { - const override = overrides[key as keyof CdpRunner]; - if (override !== undefined) return override; + const desc = Object.getOwnPropertyDescriptor(overrides, key); + if (desc?.get) return desc.get.call(wrapper); + if (desc?.value !== undefined) return desc.value; } - const value = Reflect.get(target, key, receiver === wrapper ? target : receiver); - return typeof value === "function" ? value.bind(target) : value; + const raw = Reflect.get(target, key, receiver === wrapper ? target : receiver); + const intercepted = intercept ? intercept(target, key, raw, wrapper) : undefined; + if (intercepted !== undefined) return intercepted; + if (typeof raw !== "function") return raw; + return bindReceiver?.has(key) ? raw.bind(wrapper) : raw.bind(target); }, }); shareScreencasts(base, wrapper); + sharePinnedSelectorTargets(base, wrapper); return wrapper; } -export function sendToCdpTarget( - cdp: CdpRunner, - target: CdpTarget, - method: string, - params?: object, -): Promise { - if (target.sessionId && cdp.sendToTarget) { - return cdp.sendToTarget(target, method, params); - } - return cdp.send(target.tabId, method, params); +/** + * Construct a complete, strongly-typed `CdpRunner` test double from a minimal + * `{ send, ... }` partial so unit tests exercise the exact `sendToFrame` / + * `getFrameDocuments` driver contract without polluting production `CdpRunner` + * with optional mock-only methods. + */ +export type CdpRunnerStubOverrides = Partial> & { + send: (tabId: number, method: string, params?: object, signal?: AbortSignal) => Promise; +}; + +export function createStubCdpRunner( + overrides: T, +): CdpRunner & Omit, keyof CdpRunner> { + const { send, ...restOverrides } = overrides; + const typedSend = send as CdpRunner["send"]; + const stub: CdpRunner = { + prepareInputTimestamp: async () => ({ + timestamp: Date.now(), + nativeTimestamp: Date.now() / 1000, + assertValid: () => {}, + }), + send: typedSend, + sendToFrame: async ( + handle: FrameDocumentHandle | FrameNodeHandle, + method: string, + params?: object, + guard?: CdpDispatchGuard, + ): Promise => { + guard?.signal?.throwIfAborted(); + if (guard?.attachmentId !== undefined) { + const currentAttachment = stub.getAttachmentId(handle.tabId); + if (currentAttachment !== guard.attachmentId) { + throw new Error("Debugger attachment changed before dispatch"); + } + } + guard?.onDispatch?.(); + if (guard?.signal !== undefined) { + return stub.send(handle.tabId, method, params, guard.signal); + } + return params !== undefined + ? stub.send(handle.tabId, method, params) + : stub.send(handle.tabId, method); + }, + getMainFrameHandle: async (tabId: number, signal?: AbortSignal) => { + signal?.throwIfAborted(); + try { + const tree = await (signal !== undefined + ? stub.send<{ frameTree?: { frame?: { id?: string; loaderId?: string } } }>( + tabId, + "Page.getFrameTree", + {}, + signal, + ) + : stub.send<{ frameTree?: { frame?: { id?: string; loaderId?: string } } }>( + tabId, + "Page.getFrameTree", + {}, + )); + const rootFrame = tree?.frameTree?.frame; + if (rootFrame?.id) { + return toFrameDocumentHandle(tabId, { + frameId: rootFrame.id, + loaderId: rootFrame.loaderId ?? "", + }); + } + } catch (err) { + if ( + err instanceof Error && + (err.name === "AbortError" || err.name === "CdpReadTimeoutError") + ) { + throw err; + } + } + return toFrameDocumentHandle(tabId, { frameId: "root", loaderId: "" }); + }, + getFrameDocuments: async (tabId: number, signal?: AbortSignal): Promise => { + signal?.throwIfAborted(); + try { + type RawFrameNode = { + frame?: { + id?: string; + parentId?: string; + loaderId?: string; + url?: string; + urlFragment?: string; + }; + childFrames?: RawFrameNode[]; + }; + const tree = await (signal !== undefined + ? stub.send<{ frameTree?: RawFrameNode }>(tabId, "Page.getFrameTree", {}, signal) + : stub.send<{ frameTree?: RawFrameNode }>(tabId, "Page.getFrameTree", {})); + const rootFrame = tree?.frameTree?.frame; + if (rootFrame?.id) { + const rootLoaderId = rootFrame.loaderId ?? ""; + const frames: FrameDocumentGraph["frames"][number][] = [ + { + handle: toFrameDocumentHandle(tabId, { + frameId: rootFrame.id, + loaderId: rootLoaderId, + }), + frameId: rootFrame.id, + loaderId: rootLoaderId, + ...(rootFrame.url ? { url: rootFrame.url } : {}), + ...(rootFrame.urlFragment !== undefined + ? { urlFragment: rootFrame.urlFragment } + : {}), + processKey: "root", + isOutOfProcess: false, + }, + ]; + const walkChildren = (children: RawFrameNode[] | undefined, fallbackParentId: string) => { + for (const child of children ?? []) { + if (!child.frame?.id) continue; + const childLoader = child.frame.loaderId ?? ""; + frames.push({ + handle: toFrameDocumentHandle(tabId, { + frameId: child.frame.id, + loaderId: childLoader, + }), + frameId: child.frame.id, + loaderId: childLoader, + parentFrameId: child.frame.parentId ?? fallbackParentId, + ...(child.frame.url ? { url: child.frame.url } : {}), + ...(child.frame.urlFragment !== undefined + ? { urlFragment: child.frame.urlFragment } + : {}), + processKey: "root", + isOutOfProcess: false, + }); + walkChildren(child.childFrames, child.frame.id); + } + }; + walkChildren(tree.frameTree?.childFrames, rootFrame.id); + return { + ...(stub.getAttachmentId(tabId) !== undefined + ? { attachmentId: stub.getAttachmentId(tabId) } + : {}), + rootFrameId: rootFrame.id, + frames, + }; + } + } catch (err) { + if ( + err instanceof Error && + (err.name === "AbortError" || err.name === "CdpReadTimeoutError") + ) { + throw err; + } + } + return { + ...(stub.getAttachmentId(tabId) !== undefined + ? { attachmentId: stub.getAttachmentId(tabId) } + : {}), + rootFrameId: "", + frames: [], + }; + }, + isEventFromFrame: (source, handle) => + source.tabId === handle.tabId && (!handle.frameId ? source.isRootTarget : true), + hasOwnFrameTarget: () => false, + recordFrameDocumentElement: () => {}, + detach: async () => {}, + getAttachmentId: () => "stub-attachment", + retainTabAttachment: () => () => {}, + ensureAttachedToUrl: async () => {}, + releaseBackgroundExecution: async () => {}, + trackSessionTab: () => {}, + releaseSessionTab: async () => {}, + onEvent: () => ({ dispose: () => {} }), + dialogCursor: () => 0, + dialogsSince: () => [], + pendingDialogs: () => [], + resolveDialog: async () => { + throw new Error("No pending JavaScript dialog"); + }, + onDialogChanged: () => ({ dispose: () => {} }), + ensureConsoleCapture: async () => {}, + consoleEntriesSince: (tabId) => ({ + tab_id: tabId, + entries: [], + next_since: 0, + truncated: false, + }), + ...restOverrides, + }; + return stub as CdpRunner & NoInfer; } -export async function sendToFrameHandle( +export function sendToFrameHandle( cdp: CdpRunner, handle: FrameDocumentHandle | FrameNodeHandle, method: string, params?: object, guard?: CdpDispatchGuard, ): Promise { - if (cdp.sendToFrame) { + if (guard !== undefined) { return cdp.sendToFrame(handle, method, params, guard); } - guard?.signal?.throwIfAborted(); - if ( - guard?.attachmentId !== undefined && - cdp.getAttachmentId && - cdp.getAttachmentId(handle.tabId) !== guard.attachmentId - ) { - throw new Error("Debugger attachment changed before dispatch"); - } - if (handle.frameId && cdp.getFrameGraph) { - const graph = await cdp.getFrameGraph(handle.tabId); - const frame = graph.frames.find((entry) => entry.frameId === handle.frameId); - if (!frame) { - throw new StaleFrameHandleError(handle, `frame ${handle.frameId} is no longer attached`); - } - if (handle.loaderId && frame.loaderId && frame.loaderId !== handle.loaderId) { - throw new StaleFrameHandleError( - handle, - `frame ${handle.frameId} navigated (${handle.loaderId} -> ${frame.loaderId})`, - ); - } - guard?.onDispatch?.(); - return sendToCdpTarget(cdp, frame.target, method, params); - } - guard?.onDispatch?.(); - return cdp.send(handle.tabId, method, params); -} - -export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner { - return delegateCdpRunner(cdp, { - send: (_tabId: number, method: string, params?: object) => - sendToCdpTarget(cdp, target, method, params), - }); + return params !== undefined + ? cdp.sendToFrame(handle, method, params) + : cdp.sendToFrame(handle, method); } export function cdpRunnerForFrame( @@ -245,14 +451,82 @@ export function cdpRunnerForFrame( guard?: CdpDispatchGuard, ): CdpRunner { return delegateCdpRunner(cdp, { - send: (_tabId: number, method: string, params?: object) => - sendToFrameHandle(cdp, handle, method, params, guard), + send: (_tabId: number, method: string, params?: object, signal?: AbortSignal) => { + const effectiveGuard = signal && !guard?.signal ? { ...guard, signal } : guard; + if (effectiveGuard !== undefined) { + return cdp.sendToFrame(handle, method, params, effectiveGuard); + } + return params !== undefined + ? cdp.sendToFrame(handle, method, params) + : cdp.sendToFrame(handle, method); + }, }); } +/** + * Deterministic RAII scope for a CDP V8 `objectGroup`. Always calls + * `Runtime.releaseObjectGroup` via `cdp.resourceCleanup ?? cdp` on exit + * (including errors and cancellation) without resurrecting detached tabs. + */ +export async function withObjectGroup( + cdp: CdpRunner, + target: number | FrameDocumentHandle | FrameNodeHandle, + prefix: string, + fn: (objectGroup: string) => Promise, +): Promise { + const objectGroup = `${prefix}-${crypto.randomUUID()}`; + const cleanup = cdp.resourceCleanup ?? cdp; + const handle = typeof target === "number" ? toRootFrameHandle(target) : target; + try { + return await fn(objectGroup); + } finally { + try { + await cleanup.sendToFrame( + handle, + "Runtime.releaseObjectGroup", + { objectGroup }, + { allowReattach: false }, + ); + } catch { + // Never let cleanup failures (detached tab, unowned tab, or navigated frame) mask caller outcome. + } + } +} + +/** + * Deterministic RAII scope for an individual V8 `RemoteObject` (`objectId`). + * Always calls `Runtime.releaseObject` via `cdp.resourceCleanup ?? cdp` on exit + * without resurrecting detached tabs. + */ +export async function withRemoteObject( + cdp: CdpRunner, + target: number | FrameDocumentHandle | FrameNodeHandle, + objectId: string, + fn: (objectId: string) => Promise, +): Promise { + const cleanup = cdp.resourceCleanup ?? cdp; + const handle = typeof target === "number" ? toRootFrameHandle(target) : target; + try { + return await fn(objectId); + } finally { + try { + await cleanup.sendToFrame( + handle, + "Runtime.releaseObject", + { objectId }, + { allowReattach: false }, + ); + } catch { + // Never let cleanup failures (detached tab, unowned tab, or navigated frame) mask caller outcome. + } + } +} + +export { toFrameDocumentHandle, toRootFrameHandle }; + const sessionRunners = new WeakMap>(); -const TAB_SCOPED_CDP_METHODS = new Set([ - "getFrameGraph", +const TAB_SCOPED_CDP_METHODS = new Set([ + "getFrameDocuments", "getMainFrameHandle", "ensureAttached", "ensureAttachedToUrl", @@ -279,69 +553,57 @@ export function sessionCdpRunner( const track = (tabId: number) => { if (typeof tabId === "number" && (!isLive || isLive())) { - cdp.trackSessionTab?.(sessionId, tabId); + cdp.trackSessionTab(sessionId, tabId); } }; - const wrapper = new Proxy(cdp, { - get(target, key, receiver) { - if (key === "send") { - return (tabId: number, ...rest: [string, object?]) => { + const wrapper = delegateCdpRunner( + cdp, + { + send: ( + tabId: number, + method: string, + params?: object, + signal?: AbortSignal, + ): Promise => { + track(tabId); + if (signal !== undefined) { + return cdp.send(tabId, method, params, signal); + } + return params !== undefined + ? cdp.send(tabId, method, params) + : cdp.send(tabId, method); + }, + sendToFrame: ( + handle: FrameDocumentHandle | FrameNodeHandle, + method: string, + params?: object, + guard?: CdpDispatchGuard, + ): Promise => { + if (guard?.attachmentId === undefined && guard?.allowReattach !== false) { + track(handle.tabId); + } + if (guard !== undefined) { + return cdp.sendToFrame(handle, method, params, guard); + } + return params !== undefined + ? cdp.sendToFrame(handle, method, params) + : cdp.sendToFrame(handle, method); + }, + }, + { + intercept: (target, key, raw, self) => { + if (!TAB_SCOPED_CDP_METHODS.has(key) || typeof raw !== "function") return undefined; + return function (this: unknown, tabId: number, ...rest: unknown[]) { track(tabId); - return (target.send as (tabId: number, ...args: unknown[]) => Promise).call( - target, + return (raw as (tabId: number, ...args: unknown[]) => unknown).call( + this === self ? target : this, tabId, ...rest, ); }; - } - if (key === "sendToTarget" && typeof target.sendToTarget === "function") { - return (cdpTarget: CdpTarget, ...rest: [string, object?]) => { - track(cdpTarget.tabId); - return ( - target.sendToTarget as (target: CdpTarget, ...args: unknown[]) => Promise - ).call(target, cdpTarget, ...rest); - }; - } - if (key === "sendGuarded" && typeof target.sendGuarded === "function") { - return ( - cdpTarget: CdpTarget, - method: string, - params: object | undefined, - guard: CdpDispatchGuard, - ) => { - if (guard?.attachmentId === undefined) track(cdpTarget.tabId); - return target.sendGuarded!(cdpTarget, method, params, guard); - }; - } - if (key === "sendToFrame" && typeof target.sendToFrame === "function") { - return ( - handle: FrameDocumentHandle | FrameNodeHandle, - method: string, - params?: object, - guard?: CdpDispatchGuard, - ) => { - if (guard?.attachmentId === undefined) track(handle.tabId); - return target.sendToFrame!(handle, method, params, guard); - }; - } - if (typeof key === "string" && TAB_SCOPED_CDP_METHODS.has(key)) { - const fn = Reflect.get(target, key, receiver); - if (typeof fn === "function") { - return function (this: unknown, tabId: number, ...rest: unknown[]) { - track(tabId); - return (fn as (tabId: number, ...args: unknown[]) => unknown).call( - this === wrapper ? target : this, - tabId, - ...rest, - ); - }; - } - } - const value = Reflect.get(target, key, receiver); - return typeof value === "function" ? value.bind(target) : value; + }, }, - }); - shareScreencasts(cdp, wrapper); + ); if (!bySession) { bySession = new Map(); sessionRunners.set(cdp, bySession); diff --git a/apps/extension/src/tools/snapshot-ref.ts b/apps/extension/src/tools/snapshot-ref.ts index a8e48a978..64ed525c0 100644 --- a/apps/extension/src/tools/snapshot-ref.ts +++ b/apps/extension/src/tools/snapshot-ref.ts @@ -2,7 +2,7 @@ // compound CDP node identity via the session RefStore, and build stable // `ref_not_found` errors for hard-failure tool paths. -import type { FrameNodeHandle } from "@/browser-driver/frame-graph"; +import type { FrameNodeHandle } from "@/browser-driver/frame-handles"; import type { SessionContext } from "@/session-manager/manager"; import { normaliseRef, type RefEntry } from "@/session-manager/ref-store"; import type { RpcError } from "@/transport/types"; diff --git a/apps/extension/src/tools/tabs.ts b/apps/extension/src/tools/tabs.ts index 6a080cc82..50fe15177 100644 --- a/apps/extension/src/tools/tabs.ts +++ b/apps/extension/src/tools/tabs.ts @@ -327,21 +327,25 @@ export interface TabManagementDeps { /** Clears Agent-scoped overlays after a borrowed tab is returned. */ agentOverlayReset?: AgentOverlayResetApi; /** Releases this session's CDP claim after a borrowed tab is returned. */ - cdp?: Pick; + cdp?: Partial< + Pick + >; /** Runs after tab_return validation, before moving the borrowed tab. */ beforeReturn?: (sessionId: string, tabId: number) => Promise; /** * Reports whether `windowId` is any live session's Agent Window. - * `tab_return`'s fallback window picker uses this to avoid moving a - * returned borrowed tab into *another* session's Agent Window (Agent - * Windows are created with `type: "normal"`, so they otherwise match - * the `getLastFocused({ windowTypes: ["normal"] })` query). Production - * callers derive this from the `SessionManager`; the `() => false` - * default only preserves legacy behaviour for direct/test callers. + * `tab_return`'s fallback window picker requires this predicate so returning + * a borrowed tab can never move it into another session's Agent Window + * (Agent Windows are created with `type: "normal"`, so they otherwise match + * the `getLastFocused({ windowTypes: ["normal"] })` query). */ isAgentWindowId?: (windowId: number) => boolean; } +export type ReturnBorrowedTabDeps = TabManagementDeps & { + isAgentWindowId: (windowId: number) => boolean; +}; + function mutateBrowser(deps: TabManagementDeps, operation: () => Promise): Promise { return deps.cdp?.runBrowserMutation ? deps.cdp.runBrowserMutation(operation) : operation(); } @@ -369,10 +373,6 @@ function getTabGroupsApi(deps: TabManagementDeps): ChromeTabGroupsApi | undefine return undefined; } -function getIsAgentWindowId(deps: TabManagementDeps): (windowId: number) => boolean { - return deps.isAgentWindowId ?? (() => false); -} - function getAgentOverlayResetApi(deps: TabManagementDeps): AgentOverlayResetApi { return deps.agentOverlayReset ?? chromeAgentOverlayResetApi; } @@ -1302,7 +1302,9 @@ export async function handleTabBorrow( const rollback = await returnBorrowedTab(ctx, params.tab_id, { ...deps, // Returning an already moved tab is this transaction's rollback, not new input. - cdp: { releaseSessionTab: deps.cdp?.releaseSessionTab?.bind(deps.cdp) }, + cdp: deps.cdp?.releaseSessionTab + ? { releaseSessionTab: deps.cdp.releaseSessionTab.bind(deps.cdp) } + : undefined, signal: undefined, isSessionLive: () => manager.get(ctx.sessionId) === ctx, isAgentWindowId: (id) => manager.findByWindowId(id) !== null, @@ -1781,7 +1783,7 @@ async function restoreReturnedTabAttributes( export async function returnBorrowedTab( ctx: SessionContext, tabId: number, - deps: TabManagementDeps, + deps: ReturnBorrowedTabDeps, ): Promise { return withUiTeardown(ctx, tabId, () => returnBorrowedTabCore(ctx, tabId, deps)); } @@ -1789,7 +1791,7 @@ export async function returnBorrowedTab( async function returnBorrowedTabCore( ctx: SessionContext, tabId: number, - deps: TabManagementDeps, + deps: ReturnBorrowedTabDeps, ): Promise { const entry = ctx.borrowedTabs.get(tabId); if (!entry) { @@ -1801,7 +1803,7 @@ async function returnBorrowedTabCore( const tabsApi = getTabsApi(deps); const windowsApi = getWindowsApi(deps); const tabGroupsApi = getTabGroupsApi(deps); - const isAgentWindowId = getIsAgentWindowId(deps); + const { isAgentWindowId } = deps; const alreadyCancelled = aborted(deps.signal, "tab_return", deps.isSessionLive); if (alreadyCancelled) return alreadyCancelled; diff --git a/apps/extension/src/tools/target-hit.ts b/apps/extension/src/tools/target-hit.ts index 8452ed40f..ab9df8e0d 100644 --- a/apps/extension/src/tools/target-hit.ts +++ b/apps/extension/src/tools/target-hit.ts @@ -1,9 +1,4 @@ -import { - type CdpFrameTreeNode, - cdpTargetKey, - type FrameDocumentHandle, - type FrameNodeHandle, -} from "@/browser-driver/frame-graph"; +import type { FrameDocumentHandle, FrameNodeHandle } from "@/browser-driver/frame-handles"; import { type Point, unprojectPoint } from "./geometry"; import { cssViewport, GeometryContext, type LayoutMetrics } from "./geometry/frame-context"; import { type CdpRunner, sendToFrameHandle } from "./shared"; @@ -53,13 +48,13 @@ export async function pointBelongsToTarget( const frameId = handle.frameId || ( - await sendToFrameHandle<{ frameTree?: CdpFrameTreeNode }>( + await sendToFrameHandle<{ frameTree?: { frame?: { id?: string } } }>( cdp, handle, "Page.getFrameTree", {}, ) - ).frameTree?.frame.id; + ).frameTree?.frame?.id; if (!frameId || hit.frameId !== frameId) return false; const [resolvedTarget, resolvedHit] = await Promise.all([ @@ -120,14 +115,14 @@ export async function pointReachesTarget( }; try { let local = point; - if (target.handle.frameId && cdp.getFrameGraph) { + if (target.handle.frameId) { const context = new GeometryContext(cdp, target.handle.tabId); const frame = await context.frame(target.handle.frameId); if (!frame) return false; if (target.handle.loaderId && frame.loaderId && frame.loaderId !== target.handle.loaderId) { return false; } - if (frame.target.sessionId) { + if (frame.isOutOfProcess) { const ancestry = await context.ancestry(target.handle.frameId); const projection = await context.targetProjection(target.handle.frameId); if (!ancestry || !projection) return false; @@ -135,7 +130,7 @@ export async function pointReachesTarget( for (const child of ancestry) { const parent = child.parentFrameId && (await context.frame(child.parentFrameId)); if (!parent) return false; - if (cdpTargetKey(child.target) === cdpTargetKey(parent.target)) continue; + if (child.processKey === parent.processKey) continue; if (child.ownerBackendNodeId === undefined) return false; owners.push({ handle: { @@ -162,9 +157,13 @@ export async function pointReachesTarget( const cleanup = cdp.resourceCleanup ?? cdp; await Promise.all( [...touched.values()].map((address) => - sendToFrameHandle(cleanup, address, "Runtime.releaseObjectGroup", { objectGroup }).catch( - () => {}, - ), + sendToFrameHandle( + cleanup, + address, + "Runtime.releaseObjectGroup", + { objectGroup }, + { allowReattach: false }, + ).catch(() => {}), ), ); } diff --git a/apps/extension/src/tools/viewport-screenshot.ts b/apps/extension/src/tools/viewport-screenshot.ts index 6c10412dc..b49322e46 100644 --- a/apps/extension/src/tools/viewport-screenshot.ts +++ b/apps/extension/src/tools/viewport-screenshot.ts @@ -83,29 +83,25 @@ function changed(): RpcError { function boundToAttachment(cdp: CdpRunner, tabId: number, attachmentId: string): CdpRunner { return delegateCdpRunner(cdp, { send: (id: number, method: string, params?: object) => { - if (id !== tabId || cdp.getAttachmentId?.(id) !== attachmentId) { + if (id !== tabId || cdp.getAttachmentId(id) !== attachmentId) { return Promise.reject(new Error("Screenshot attachment changed")); } return cdp.send(id, method, params); }, - ...(cdp.sendToFrame - ? { - sendToFrame: ( - handle: Parameters>[0], - method: string, - params?: object, - guard?: Parameters>[3], - ) => { - if (handle.tabId !== tabId || cdp.getAttachmentId?.(handle.tabId) !== attachmentId) { - return Promise.reject(new Error("Screenshot attachment changed")); - } - return cdp.sendToFrame!(handle, method, params, { - ...guard, - attachmentId, - }); - }, - } - : {}), + sendToFrame: ( + handle: Parameters[0], + method: string, + params?: object, + guard?: Parameters[3], + ) => { + if (handle.tabId !== tabId || cdp.getAttachmentId(handle.tabId) !== attachmentId) { + return Promise.reject(new Error("Screenshot attachment changed")); + } + return cdp.sendToFrame(handle, method, params, { + ...guard, + attachmentId, + }); + }, }); } @@ -190,20 +186,16 @@ export async function captureControlledViewport( if (!cleanup) check(); return wait(cdp.send(id, method, params), cleanup); }, - ...(cdp.sendToFrame - ? { - sendToFrame: ( - handle: Parameters>[0], - method: string, - params?: object, - guard?: Parameters>[3], - ) => { - const cleanup = method === "Runtime.releaseObjectGroup"; - if (!cleanup) check(); - return wait(cdp.sendToFrame!(handle, method, params, guard), cleanup); - }, - } - : {}), + sendToFrame: ( + handle: Parameters[0], + method: string, + params?: object, + guard?: Parameters[3], + ) => { + const cleanup = method === "Runtime.releaseObjectGroup"; + if (!cleanup) check(); + return wait(cdp.sendToFrame(handle, method, params, guard), cleanup); + }, }); const controlled = () => { check(); diff --git a/apps/extension/src/tools/visual-target.ts b/apps/extension/src/tools/visual-target.ts index 62dfc6cfb..9804b45ad 100644 --- a/apps/extension/src/tools/visual-target.ts +++ b/apps/extension/src/tools/visual-target.ts @@ -1,8 +1,9 @@ import { - type CdpFrame, type FrameDocumentHandle, + type FrameDocumentInfo, isStaleFrameHandleError, -} from "@/browser-driver/frame-graph"; + rootProcessBoundary, +} from "@/browser-driver/frame-handles"; import type { RpcError } from "@/transport/types"; import { rpcError } from "./errors"; import { @@ -272,10 +273,12 @@ export async function resolveVisualRegionNow( const frameHandles = new Map( path.map(({ frame: item }) => [item.document.frameId, item.document]), ); - const frames: CdpFrame[] = path.map(({ frame: item }) => ({ + const frames: FrameDocumentInfo[] = path.map(({ frame: item }) => ({ + handle: item.document, frameId: item.document.frameId, loaderId: item.document.loaderId, - target: { tabId }, + processKey: "root", + isOutOfProcess: false, ...(item.parent ? { parentFrameId: item.parent.frame.document.frameId, @@ -347,7 +350,7 @@ export async function resolveVisualRegionNow( const parentViewport = await geometry.viewport(parent); if (!parentViewport) return stale("parent viewport unavailable"); const local = await geometry.snapshotProjection( - { target: { tabId }, frameId: item.document.frameId }, + { process: rootProcessBoundary(tabId), frameId: item.document.frameId }, item.parent.ownerBackendNodeId, [], parentViewport, diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts index c7d6cf84d..0f158b5fc 100644 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -6,12 +6,19 @@ import { CdpReadTimeoutError, READ_TIMEOUT_MS, } from "@/browser-driver/command-deadline"; -import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; -import { cdpTargetKey } from "@/browser-driver/frame-graph"; +import { + createFrameDocumentInfo, + type FrameDocumentHandle, + type FrameDocumentInfo, + type FrameNodeHandle, + frameProcessBoundary, + rootProcessBoundary, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; import { RefStore } from "@/session-manager/ref-store"; import { captureVomObservation } from "../../observation"; -import type { CdpRunner } from "../../shared"; +import { type CdpDispatchGuard, type CdpRunner, createStubCdpRunner } from "../../shared"; import { ACCESSIBILITY_LEASE_TTL_MS, AccessibilityLease, @@ -24,9 +31,30 @@ import { deduplicateVisualCandidates } from "../visual-dedup"; import { discoverVisualCandidates } from "../visual-discovery"; import { publishObservationPage } from "../visual-observation"; +interface FixtureFrameSpec { + frameId: string; + parentFrameId?: string; + ownerBackendNodeId?: number; + processKey?: string; + url?: string; +} + +function makeFrame(spec: FixtureFrameSpec): FrameDocumentInfo { + const processKey = spec.processKey ?? "root"; + return createFrameDocumentInfo({ + tabId: 4, + frameId: spec.frameId, + parentFrameId: spec.parentFrameId, + ownerBackendNodeId: spec.ownerBackendNodeId, + url: spec.url, + processKey, + isOutOfProcess: processKey !== "root", + }); +} + function fixture( options: { - frames?: CdpFrame[]; + frames?: FixtureFrameSpec[]; canvas?: boolean; form?: boolean; after?: Record; @@ -37,40 +65,48 @@ function fixture( overlay?: string; } = {}, ) { - const frames = options.frames ?? [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "same", - parentFrameId: "main", - ownerBackendNodeId: 3, - target: { tabId: 4 }, - }, - { - frameId: "nested", - parentFrameId: "same", - ownerBackendNodeId: 13, - target: { tabId: 4 }, - }, - { - frameId: "remote", - parentFrameId: "main", - ownerBackendNodeId: 4, - target: { tabId: 4, sessionId: "remote" }, - }, - ]; + const frames: FrameDocumentInfo[] = ( + options.frames ?? [ + { frameId: "main" }, + { frameId: "same", parentFrameId: "main", ownerBackendNodeId: 3 }, + { frameId: "nested", parentFrameId: "same", ownerBackendNodeId: 13 }, + { + frameId: "remote", + parentFrameId: "main", + ownerBackendNodeId: 4, + processKey: "oopif:remote", + }, + ] + ).map(makeFrame); + const processForHandle = (handle: FrameDocumentHandle | FrameNodeHandle): string => { + if (!handle.frameId) return "root"; + return frames.find((f) => f.frameId === handle.frameId)?.processKey ?? "root"; + }; const elements = new Map( - frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]), + frames.map((frame, i) => [frame.frameId, frame.isOutOfProcess ? 1 : i * 10 + 1]), ); const contexts = new Map(frames.map((frame, i) => [i + 1, frame.frameId])); - const logs: Array<{ target: CdpTarget; method: string; params: Record }> = []; + const logs: Array<{ + processKey: string; + handle: FrameDocumentHandle | FrameNodeHandle; + method: string; + params: Record; + guard?: CdpDispatchGuard; + }> = []; let snapshots = 0; - const targetCount = new Set(frames.map((frame) => cdpTargetKey(frame.target))).size; - const send = vi.fn( - async (target: CdpTarget, method: string, params: object = {}): Promise => { - logs.push({ target, method, params: params as Record }); + const targetCount = new Set(frames.map((frame) => frame.processKey)).size; + const sendToFrame = vi.fn( + async ( + handle: FrameDocumentHandle | FrameNodeHandle, + method: string, + params: object = {}, + guard?: CdpDispatchGuard, + ): Promise => { + const processKey = processForHandle(handle); + logs.push({ processKey, handle, method, params: params as Record, guard }); const args = params as Record; - if (options.fail === `${target.sessionId ?? "main"}:${method}`) - throw new Error("fixture failure"); + const failKey = processKey === "root" ? "main" : processKey.replace(/^oopif:/, ""); + if (options.fail === `${failKey}:${method}`) throw new Error("fixture failure"); let result: unknown = {}; if (method === "Page.createIsolatedWorld") result = { @@ -99,7 +135,7 @@ function fixture( deepSerializedValue: { type: "array", value: frames - .filter((frame) => cdpTargetKey(frame.target) === cdpTargetKey(target)) + .filter((frame) => frame.processKey === processKey) .map((frame) => ({ type: "array", value: [ @@ -152,16 +188,12 @@ function fixture( ], documents: frames .filter( - (frame) => - cdpTargetKey(frame.target) === cdpTargetKey(target) && - frame.frameId !== options.omitDocument, + (frame) => frame.processKey === processKey && frame.frameId !== options.omitDocument, ) .map((frame) => { const element = elements.get(frame.frameId)!; const localFrames = frames.filter( - (f) => - cdpTargetKey(f.target) === cdpTargetKey(target) && - f.frameId !== options.omitDocument, + (f) => f.processKey === processKey && f.frameId !== options.omitDocument, ); const children = localFrames.filter((f) => f.parentFrameId === frame.frameId); @@ -233,19 +265,27 @@ function fixture( return result as T; }, ); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ getAttachmentId: () => options.missingIdentity ? undefined : options.attachmentChanged && snapshots === targetCount ? "new-attachment" : "attachment", - getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }), - send: (tabId, method, params) => - (send as NonNullable)({ tabId }, method, params), - sendToTarget: send as NonNullable, - }; - return { cdp, logs, elements }; + getFrameDocuments: async () => ({ rootFrameId: frames[0].frameId, frames }), + getMainFrameHandle: async () => frames[0].handle, + hasOwnFrameTarget: (_tabId, frameId) => + frames.some((f) => f.frameId === frameId && f.isOutOfProcess), + send: ((tabId: number, method: string, params?: object, signal?: AbortSignal) => + sendToFrame( + toRootFrameHandle(tabId), + method, + params, + signal ? { signal } : undefined, + ) as Promise) as CdpRunner["send"], + sendToFrame: sendToFrame as CdpRunner["sendToFrame"], + }); + return { cdp, logs, elements, frames, processForHandle }; } function pendingLayoutFixture(options: Parameters[0] = {}) { @@ -254,7 +294,7 @@ function pendingLayoutFixture(options: Parameters[0] = {}) { vi.spyOn(console, "debug").mockImplementation(() => {}); const f = fixture(options); const gate = new CdpReadGate(); - const original = f.cdp.sendToTarget!; + const original = f.cdp.sendToFrame; const attempted: string[] = []; let entered!: () => void; const layoutStarted = new Promise((resolve) => { @@ -264,17 +304,19 @@ function pendingLayoutFixture(options: Parameters[0] = {}) { const pending = new Promise((resolve) => { finish = resolve; }); - f.cdp.sendToTarget = (target, method, params) => { + f.cdp.sendToFrame = (handle, method, params, guard) => { + const processKey = f.processForHandle(handle); attempted.push(method); - return gate.run(target, method, () => { - if (method === "Page.getLayoutMetrics" && !target.sessionId) { + return gate.run({ tabId: handle.tabId, processKey }, method, () => { + if (method === "Page.getLayoutMetrics" && processKey === "root") { entered(); return pending as never; } - return original(target, method, params); + return original(handle, method, params, guard); }); }; - f.cdp.send = (tabId, method, params) => f.cdp.sendToTarget!({ tabId }, method, params); + f.cdp.send = (tabId, method, params, signal) => + f.cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); return { ...f, gate, attempted, layoutStarted, finish }; } @@ -398,33 +440,35 @@ describe("captureObservationFacts", () => { it("isolates an ancestor session timeout encountered while projecting a nested OOPIF", async () => { const f = fixture({ frames: [ - { frameId: "main", target: { tabId: 4 } }, + { frameId: "main" }, { frameId: "outer", parentFrameId: "main", ownerBackendNodeId: 4, - target: { tabId: 4, sessionId: "outer" }, + processKey: "oopif:outer", }, { frameId: "inner", parentFrameId: "outer", ownerBackendNodeId: 2, - target: { tabId: 4, sessionId: "inner" }, + processKey: "oopif:inner", }, ], }); - const original = f.cdp.sendToTarget!; - f.cdp.sendToTarget = async (target, method, params) => { - if (target.sessionId === "outer" && method === "DOM.getBoxModel") - throw new CdpReadTimeoutError(method, 4, CAPTURE_READ_TIMEOUT_MS, "outer"); - return original(target, method, params); + const original = f.cdp.sendToFrame; + f.cdp.sendToFrame = async (handle, method, params, guard) => { + if (f.processForHandle(handle) === "oopif:outer" && method === "DOM.getBoxModel") + throw new CdpReadTimeoutError(method, 4, CAPTURE_READ_TIMEOUT_MS, "oopif:outer"); + return original(handle, method, params, guard); }; - f.cdp.send = (tabId, method, params) => f.cdp.sendToTarget!({ tabId }, method, params); + f.cdp.send = (tabId, method, params, signal) => + f.cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); const facts = await captureObservationFacts(f.cdp, 4); expect(facts.documents.map((document) => document.frame.frameId)).toEqual(["main"]); expect(facts.issues).toContainEqual( expect.objectContaining({ - target: { tabId: 4, sessionId: "outer" }, + tabId: 4, + processKey: "oopif:outer", reason: "capture-unavailable", }), ); @@ -434,17 +478,18 @@ describe("captureObservationFacts", () => { "Accessibility.getFullAXTree", ])("keeps healthy documents after a child %s timeout without fallback reads", async (method) => { const f = fixture(); - const original = f.cdp.sendToTarget!; + const original = f.cdp.sendToFrame; const attempted: string[] = []; - f.cdp.sendToTarget = async (target, name, params) => { - if (target.sessionId === "remote") { + f.cdp.sendToFrame = async (handle, name, params, guard) => { + if (f.processForHandle(handle) === "oopif:remote") { attempted.push(name); if (name === method) - throw new CdpReadTimeoutError(name, 4, CAPTURE_READ_TIMEOUT_MS, "remote"); + throw new CdpReadTimeoutError(name, 4, CAPTURE_READ_TIMEOUT_MS, "oopif:remote"); } - return original(target, name, params); + return original(handle, name, params, guard); }; - f.cdp.send = (tabId, name, params) => f.cdp.sendToTarget!({ tabId }, name, params); + f.cdp.send = (tabId, name, params, signal) => + f.cdp.sendToFrame(toRootFrameHandle(tabId), name, params, signal ? { signal } : undefined); const facts = await captureObservationFacts(f.cdp, 4); expect(facts.documents.map((document) => document.frame.frameId)).toEqual([ "main", @@ -453,7 +498,8 @@ describe("captureObservationFacts", () => { ]); expect(facts.issues).toContainEqual( expect.objectContaining({ - target: { tabId: 4, sessionId: "remote" }, + tabId: 4, + processKey: "oopif:remote", reason: "capture-unavailable", }), ); @@ -463,34 +509,43 @@ describe("captureObservationFacts", () => { it("reports omitted frame-tree subtrees without recapturing them", async () => { const f = fixture(); - const graph = await f.cdp.getFrameGraph!(4); + const graph = await f.cdp.getFrameDocuments(4); const remote = graph.frames.find((frame) => frame.frameId === "remote")!; - f.cdp.getFrameGraph = async () => ({ + f.cdp.getFrameDocuments = async () => ({ ...graph, frames: graph.frames.filter((frame) => frame !== remote), - unavailableFrames: [remote], + unavailableFrames: [ + { + frameId: remote.frameId, + parentFrameId: remote.parentFrameId, + url: remote.url, + processKey: remote.processKey, + isOutOfProcess: remote.isOutOfProcess, + }, + ], }); const facts = await captureObservationFacts(f.cdp, 4); expect(facts.documents.some((document) => document.frame.frameId === "remote")).toBe(false); expect(facts.issues).toContainEqual({ - target: remote.target, + tabId: remote.handle.tabId, + processKey: remote.processKey, frameId: "remote", stage: "dom", reason: "capture-unavailable", }); - expect(f.logs.some((call) => call.target.sessionId === "remote")).toBe(false); + expect(f.logs.some((call) => call.processKey === "oopif:remote")).toBe(false); }); it("normalizes snapshot-owned root scroll instead of mixing in stale CSS scroll", async () => { - const { cdp } = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] }); - const original = cdp.sendToTarget!; - cdp.sendToTarget = async (target, method, params) => { + const { cdp } = fixture({ frames: [{ frameId: "main" }] }); + const original = cdp.sendToFrame; + cdp.sendToFrame = async (handle, method, params, guard) => { if (method === "Page.getLayoutMetrics") return { visualViewport: { clientWidth: 2000 }, cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 }, } as never; - const reply = await original(target, method, params); + const reply = await original(handle, method, params, guard); if (method === "DOMSnapshot.captureSnapshot") { const doc = (reply as SnapshotReply).documents![0]; doc.scrollOffsetX = 40; @@ -499,7 +554,8 @@ describe("captureObservationFacts", () => { } return reply as never; }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); const facts = await captureObservationFacts(cdp, 4); expect(facts.documents[0].domNodes.find((node) => node.backendNodeId === 2)).toMatchObject({ localRect: { x: 80, y: 300, w: 120, h: 40 }, @@ -567,8 +623,8 @@ describe("captureObservationFacts", () => { it("validates newly discovered snapshot documents without a pre-snapshot identity read", async () => { const { cdp, logs } = fixture(); - const graph = await cdp.getFrameGraph!(4); - cdp.getFrameGraph = async () => ({ ...graph, frames: graph.frames.slice(0, 1) }); + const graph = await cdp.getFrameDocuments(4); + cdp.getFrameDocuments = async () => ({ ...graph, frames: graph.frames.slice(0, 1) }); const facts = await captureObservationFacts(cdp, 4); expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "same", "nested"]); expect(facts.documents.every((doc) => doc.identity)).toBe(true); @@ -579,11 +635,12 @@ describe("captureObservationFacts", () => { }); it("rejects a snapshot root that no longer matches the current frame root", async () => { - const { cdp } = fixture(); - const original = cdp.sendToTarget!; - cdp.sendToTarget = async (target, method, params) => { - const result = await original(target, method, params); - if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId) { + const f = fixture(); + const { cdp } = f; + const original = cdp.sendToFrame; + cdp.sendToFrame = async (handle, method, params, guard) => { + const result = await original(handle, method, params, guard); + if (method === "DOMSnapshot.captureSnapshot" && f.processForHandle(handle) === "root") { const snapshot = result as { documents: { frameId: string; nodes: { backendNodeId: number[] } }[]; }; @@ -591,34 +648,37 @@ describe("captureObservationFacts", () => { } return result as never; }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity"); }); it("starts after-identity reads only after every target finishes and propagates cancellation during identity cleanup", async () => { - const { cdp, logs } = fixture(); + const f = fixture(); + const { cdp, logs } = f; const controller = new AbortController(); - const original = cdp.sendToTarget!; + const original = cdp.sendToFrame; let identityCleanup = 0; - cdp.sendToTarget = async (target, method, params) => { + cdp.sendToFrame = async (handle, method, params, guard) => { if (method === "Page.createIsolatedWorld") { expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength( 4, ); } - const result = await original(target, method, params); + const result = await original(handle, method, params, guard); if ( method === "Runtime.releaseObjectGroup" && String((params as { objectGroup?: string })?.objectGroup).startsWith( "bsk-document-identity-", ) && - target.sessionId === "remote" + f.processForHandle(handle) === "oopif:remote" ) { if (++identityCleanup === 1) controller.abort(); } return result as never; }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({ name: "AbortError", }); @@ -628,17 +688,16 @@ describe("captureObservationFacts", () => { it.each([ 1, 10, 100, ])("bounds identity reads for %i same-target documents after snapshot and AX collection", async (count) => { - const frames: CdpFrame[] = Array.from({ length: count }, (_, i) => ({ + const frames: FixtureFrameSpec[] = Array.from({ length: count }, (_, i) => ({ frameId: `frame-${i}`, - target: { tabId: 4 }, ...(i ? { parentFrameId: "frame-0", ownerBackendNodeId: 90000 + i } : {}), })); const { cdp, logs } = fixture({ frames }); - const original = cdp.sendToTarget!; + const original = cdp.sendToFrame; let active = 0; let peak = 0; let released = 0; - cdp.sendToTarget = async (target, method, params) => { + cdp.sendToFrame = async (handle, method, params, guard) => { if (method === "Page.createIsolatedWorld") { peak = Math.max(peak, ++active); await new Promise((resolve) => setTimeout(resolve, 1)); @@ -652,7 +711,7 @@ describe("captureObservationFacts", () => { count, ); } - const result = await original(target, method, params); + const result = await original(handle, method, params, guard); if ( method === "Runtime.releaseObjectGroup" && String((params as { objectGroup?: string })?.objectGroup).startsWith( @@ -664,7 +723,8 @@ describe("captureObservationFacts", () => { } return result as never; }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); const facts = await captureObservationFacts(cdp, 4); expect(facts.documents.every((doc) => doc.identity)).toBe(true); expect(peak).toBe(Math.min(count, 4)); @@ -681,13 +741,12 @@ describe("captureObservationFacts", () => { }); it("joins identity cleanup on cancellation without starting queued frames", async () => { - const frames: CdpFrame[] = Array.from({ length: 5 }, (_, i) => ({ + const frames: FixtureFrameSpec[] = Array.from({ length: 5 }, (_, i) => ({ frameId: `frame-${i}`, - target: { tabId: 4 }, ...(i ? { parentFrameId: "frame-0", ownerBackendNodeId: 90000 + i } : {}), })); const { cdp, logs } = fixture({ frames }); - const original = cdp.sendToTarget!; + const original = cdp.sendToFrame; const controller = new AbortController(); let release!: () => void; const gate = new Promise((resolve) => { @@ -698,8 +757,8 @@ describe("captureObservationFacts", () => { ready = resolve; }); let held = 0; - cdp.sendToTarget = async (target, method, params) => { - const result = await original(target, method, params); + cdp.sendToFrame = async (handle, method, params, guard) => { + const result = await original(handle, method, params, guard); if ( method === "Runtime.releaseObjectGroup" && String((params as { objectGroup?: string })?.objectGroup).startsWith( @@ -711,7 +770,8 @@ describe("captureObservationFacts", () => { } return result as never; }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); let settled = false; const pending = captureObservationFacts(cdp, 4, controller.signal).finally(() => { settled = true; @@ -755,7 +815,11 @@ describe("captureObservationFacts", () => { expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }), ); expect(facts.issues).toContainEqual( - expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }), + expect.objectContaining({ + tabId: 4, + processKey: "oopif:remote", + stage: "dom", + }), ); }); @@ -788,26 +852,28 @@ describe("captureObservationFacts", () => { "cancel", "worker-abort", ] as const)("joins target cleanup and stops claiming targets after %s", async (mode) => { - const frames = Array.from({ length: 5 }, (_, i) => ({ + const frames: FixtureFrameSpec[] = Array.from({ length: 5 }, (_, i) => ({ frameId: `f${i}`, - target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) }, + ...(i ? { processKey: `oopif:s${i}` } : {}), })); - const { cdp } = fixture({ frames }); - const original = cdp.sendToTarget!; + const f = fixture({ frames }); + const { cdp } = f; + const original = cdp.sendToFrame; const pending = new Map void; reject: (error: Error) => void }>(); const calls: Array<{ target: string; method: string }> = []; let release: (() => void) | undefined; let released = false; - const send: NonNullable = async ( - target: CdpTarget, + const sendToFrame: CdpRunner["sendToFrame"] = async ( + handle: FrameDocumentHandle | FrameNodeHandle, method: string, params?: object, + guard?: CdpDispatchGuard, ) => { - calls.push({ target: target.sessionId ?? "main", method }); - if (method === "DOMSnapshot.captureSnapshot" && target.sessionId) { - await new Promise((resolve, reject) => - pending.set(target.sessionId!, { resolve, reject }), - ); + const processKey = f.processForHandle(handle); + const shortKey = processKey === "root" ? "main" : processKey.replace(/^oopif:/, ""); + calls.push({ target: shortKey, method }); + if (method === "DOMSnapshot.captureSnapshot" && processKey !== "root") { + await new Promise((resolve, reject) => pending.set(shortKey, { resolve, reject })); } if (method === "Runtime.releaseObjectGroup") { await new Promise((resolve) => { @@ -815,13 +881,14 @@ describe("captureObservationFacts", () => { }); released = true; } - const result = await original(target, method, params); - if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId) + const result = await original(handle, method, params, guard); + if (method === "DOMSnapshot.captureSnapshot" && processKey === "root") (result as { strings: string[] }).strings[2] = "input"; return result; }; - cdp.sendToTarget = send; - cdp.send = (tabId, method, params) => send({ tabId }, method, params); + cdp.sendToFrame = sendToFrame; + cdp.send = (tabId, method, params, signal) => + sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); const controller = new AbortController(); let settled = false; const capture = captureObservationFacts(cdp, 4, controller.signal); @@ -839,8 +906,6 @@ describe("captureObservationFacts", () => { expect(release).toBeDefined(); }); if (mode === "cancel") controller.abort(); - // Cancellation may race with an ordinary transport rejection. Without an - // external signal, an escaping AbortError still terminates the worker pool. pending .get("s1")! .reject( @@ -862,8 +927,6 @@ describe("captureObservationFacts", () => { ), ).toBe(true); }); - // Allow worker rejection and the outer promise chain to run while cleanup - // is deliberately held. The old fail-fast pool settles here. await new Promise((resolve) => setTimeout(resolve, 0)); expect(settled).toBe(false); expect(released).toBe(false); @@ -881,26 +944,27 @@ describe("captureObservationFacts", () => { }); it("bounds collection across many targets and stops scheduling after cancellation", async () => { - const frames = Array.from({ length: 12 }, (_, i) => ({ + const frames: FixtureFrameSpec[] = Array.from({ length: 12 }, (_, i) => ({ frameId: `f${i}`, - target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) }, + ...(i ? { processKey: `oopif:s${i}` } : {}), })); const { cdp } = fixture({ frames }); let active = 0, peak = 0; - const original = cdp.sendToTarget!; - const send: NonNullable = async (target, method, params) => { + const original = cdp.sendToFrame; + const sendToFrame: CdpRunner["sendToFrame"] = async (handle, method, params, guard) => { active++; peak = Math.max(peak, active); try { await new Promise((resolve) => setTimeout(resolve, 1)); - return await original(target, method, params); + return await original(handle, method, params, guard); } finally { active--; } }; - cdp.sendToTarget = send; - cdp.send = (tabId, method, params) => send({ tabId }, method, params); + cdp.sendToFrame = sendToFrame; + cdp.send = (tabId, method, params, signal) => + sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); await captureObservationFacts(cdp, 4); // This fixture has no frame edges, so no concurrent owner requests. expect(peak).toBeLessThanOrEqual(4); @@ -948,63 +1012,82 @@ function childSnapshot(frameId: string, backendNodeId: number) { describe("OOPIF capture", () => { it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => { - const sendToTarget = vi.fn(async (target, method) => { - if (method === "Runtime.evaluate") return { result: { value: { width: 300, height: 200 } } }; + const rootSend = vi.fn(async (_tabId: number, method: string, params?: object) => { + if ( + method === "Accessibility.enable" || + method === "DOMSnapshot.enable" || + method === "Accessibility.disable" + ) + return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.getBoxModel") { + const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId; + const x = backendNodeId === 10 ? 50 : 500; + return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } }; + } if (method === "Page.getLayoutMetrics") { return { visualViewport: { clientWidth: 1000 }, cssVisualViewport: { clientWidth: 1000, scale: 1 }, - cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, }; } - if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") { - return target.sessionId === "left-session" - ? childSnapshot("left", 101) - : childSnapshot("right", 201); - } - if (method === "Accessibility.getFullAXTree") return { nodes: [] }; - throw new Error(`unexpected ${method}`); + if (method === "Runtime.evaluate") return { result: { value: { width: 300, height: 200 } } }; + if (method === "Runtime.releaseObjectGroup" || method === "Runtime.releaseObject") return {}; + if (method === "DOM.resolveNode") return { object: { objectId: "iframe-owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 300, height: 200 } } }; + throw new Error(`unexpected root ${method}`); }); - const cdp: CdpRunner = { - send: vi.fn(async (_tabId, method, params) => { - if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); - if (method === "Accessibility.getFullAXTree") return { nodes: [] }; - if (method === "DOM.getBoxModel") { - const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId; - const x = backendNodeId === 10 ? 50 : 500; - return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } }; - } + const sendToFrame = vi.fn( + async (handle: FrameDocumentHandle | FrameNodeHandle, method: string, params?: object) => { + if (!handle.frameId || handle.frameId === "main") + return rootSend(handle.tabId, method, params); + if (method === "Runtime.evaluate") + return { result: { value: { width: 300, height: 200 } } }; + if (method === "Runtime.releaseObjectGroup" || method === "Accessibility.disable") + return {}; if (method === "Page.getLayoutMetrics") { return { visualViewport: { clientWidth: 1000 }, cssVisualViewport: { clientWidth: 1000, scale: 1 }, - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, + cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 }, }; } - throw new Error(`unexpected root ${method}`); - }) as CdpRunner["send"], - sendToTarget: sendToTarget as unknown as NonNullable, - getFrameGraph: vi.fn(async () => ({ + if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") { + return handle.frameId === "left" + ? childSnapshot("left", 101) + : childSnapshot("right", 201); + } + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + throw new Error(`unexpected ${method}`); + }, + ); + const cdp: CdpRunner = createStubCdpRunner({ + send: rootSend as CdpRunner["send"], + sendToFrame: sendToFrame as unknown as CdpRunner["sendToFrame"], + hasOwnFrameTarget: (_tabId, frameId) => frameId === "left" || frameId === "right", + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + makeFrame({ frameId: "main" }), + makeFrame({ frameId: "left", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "left-session" }, - }, - { + processKey: "oopif:left", + }), + makeFrame({ frameId: "right", parentFrameId: "main", ownerBackendNodeId: 20, - target: { tabId: 4, sessionId: "right-session" }, - }, + processKey: "oopif:right", + }), ], })), - }; + }); const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4)); @@ -1057,7 +1140,11 @@ describe("OOPIF capture", () => { nested.documents[0], ], }; - const reply = async (_target: unknown, method: string) => { + const reply = async (target: unknown, method: string) => { + const isRoot = + typeof target === "number" || + !(target as FrameDocumentHandle).frameId || + (target as FrameDocumentHandle).frameId === "main"; if (method === "Page.getLayoutMetrics") return { visualViewport: { clientWidth: 1000 }, @@ -1065,34 +1152,38 @@ describe("OOPIF capture", () => { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 }, }; if (method === "DOMSnapshot.captureSnapshot") - return typeof _target === "number" ? childSnapshot("main", 11) : snapshot; + return isRoot ? childSnapshot("main", 11) : snapshot; if (method === "DOM.getBoxModel") return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } }; - if (method === "DOM.resolveNode") throw new Error("nested owner replaced"); + if (method === "DOM.resolveNode" && !isRoot) throw new Error("nested owner replaced"); + if (method === "DOM.resolveNode") return { object: { objectId: "root-owner" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 300, height: 200 } } }; return {}; }; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(reply) as CdpRunner["send"], - sendToTarget: vi.fn(reply) as NonNullable, - getFrameGraph: vi.fn(async () => ({ + sendToFrame: vi.fn(reply) as unknown as CdpRunner["sendToFrame"], + hasOwnFrameTarget: (_tabId, frameId) => frameId === "child" || frameId === "nested", + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + makeFrame({ frameId: "main" }), + makeFrame({ frameId: "nested", parentFrameId: "child", ownerBackendNodeId: 101, - target: { tabId: 4, sessionId: "child-session" }, - }, - { + processKey: "oopif:child", + }), + makeFrame({ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child-session" }, - }, + processKey: "oopif:child", + }), ], })), - }; + }); const captured = await captureObservationFacts(cdp, 4); expect( captured.issues @@ -1101,7 +1192,17 @@ describe("OOPIF capture", () => { ).toEqual([ { status: "unavailable", - source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" }, + source: { + process: frameProcessBoundary( + makeFrame({ + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + processKey: "oopif:child", + }), + ), + frameId: "nested", + }, ownerBackendNodeId: 101, }, ]); @@ -1114,24 +1215,40 @@ describe("OOPIF capture", () => { }); it("keeps OOPIF semantics when viewport projection is unavailable", async () => { - const cdp: CdpRunner = { - send: vi.fn(async (_tabId, method) => { - if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); - if (method === "Accessibility.getFullAXTree") return { nodes: [] }; - if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable"); - if (method === "Page.getLayoutMetrics") { - return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000, scale: 1 }, - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, - }; - } - throw new Error(`unexpected root ${method}`); - }) as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method) => { + const rootSend = vi.fn(async (_tabId: number, method: string) => { + if ( + method === "Accessibility.enable" || + method === "DOMSnapshot.enable" || + method === "Accessibility.disable" || + method === "Runtime.releaseObjectGroup" + ) + return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable"); + if (method === "Runtime.evaluate") return {}; + if (method === "Page.getLayoutMetrics") { + return { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, + }; + } + throw new Error(`unexpected root ${method}`); + }); + const cdp: CdpRunner = createStubCdpRunner({ + send: rootSend as CdpRunner["send"], + sendToFrame: vi.fn(async (handle: FrameDocumentHandle | FrameNodeHandle, method: string) => { + if (!handle.frameId || handle.frameId === "main") return rootSend(handle.tabId, method); if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable"); - if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; + if ( + method === "DOMSnapshot.enable" || + method === "Accessibility.enable" || + method === "Accessibility.disable" || + method === "Runtime.releaseObjectGroup" || + method === "Runtime.evaluate" + ) + return {}; if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101); if (method === "Accessibility.getFullAXTree") { return { @@ -1146,20 +1263,21 @@ describe("OOPIF capture", () => { }; } throw new Error(`unexpected ${method}`); - }) as unknown as NonNullable, - getFrameGraph: vi.fn(async () => ({ + }) as unknown as CdpRunner["sendToFrame"], + hasOwnFrameTarget: (_tabId, frameId) => frameId === "child", + getFrameDocuments: vi.fn(async () => ({ rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + makeFrame({ frameId: "main" }), + makeFrame({ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child-session" }, - }, + processKey: "oopif:child", + }), ], })), - }; + }); const { documents } = semanticCapture(await captureObservationFacts(cdp, 4)); const childDocument = documents.find((document) => document.frameId === "child"); @@ -1226,9 +1344,10 @@ function siblingCaptureFixture( active--; } }); + const cdp = createStubCdpRunner({ send: send as CdpRunner["send"] }); return { snapshot, - cdp: { send: send as CdpRunner["send"] }, + cdp, send, peak: () => peak, active: () => active, @@ -1248,18 +1367,17 @@ describe("snapshot document provenance", () => { const f = siblingCaptureFixture(); const cdp: CdpRunner = { ...f.cdp, - getFrameGraph: async () => ({ + getFrameDocuments: async () => ({ rootFrameId: mode === "new-root" ? "old-root" : "frame-0", frames: [ - { frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } }, + makeFrame({ frameId: mode === "new-root" ? "old-root" : "frame-0" }), ...(mode === "stale-owner" ? [ - { + makeFrame({ frameId: "frame-1", parentFrameId: "frame-0", ownerBackendNodeId: 999, - target: { tabId: 4 }, - }, + }), ] : []), ], @@ -1298,9 +1416,9 @@ describe("snapshot document provenance", () => { const facts = await captureObservationFacts( { ...f.cdp, - getFrameGraph: async () => ({ + getFrameDocuments: async () => ({ rootFrameId: "frame-0", - frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }], + frames: [makeFrame({ frameId: "frame-0", url: "https://old.test/" })], }), }, 4, @@ -1322,26 +1440,37 @@ describe("snapshot document provenance", () => { const f = siblingCaptureFixture(); const cdp: CdpRunner = { ...f.cdp, - getFrameGraph: async () => ({ + hasOwnFrameTarget: (_tabId, frameId) => frameId === "frame-1", + getFrameDocuments: async () => ({ rootFrameId: "frame-0", frames: [ - { frameId: "frame-0", target: { tabId: 4 } }, - { + makeFrame({ frameId: "frame-0" }), + makeFrame({ frameId: "frame-1", parentFrameId: "frame-0", ownerBackendNodeId: 100, - target: { tabId: 4, sessionId: "old" }, - }, + processKey: "oopif:old", + }), ], }), - sendToTarget: async (_target: CdpTarget, method: string): Promise => - (method === "Accessibility.getFullAXTree" - ? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] } - : {}) as T, + sendToFrame: async ( + handle: FrameDocumentHandle | FrameNodeHandle, + method: string, + params?: object, + guard?: CdpDispatchGuard, + ): Promise => { + if (!handle.frameId || handle.frameId !== "frame-1") + return f.cdp.send(handle.tabId, method, params, guard?.signal); + return ( + method === "Accessibility.getFullAXTree" + ? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] } + : {} + ) as T; + }, }; const facts = await captureObservationFacts(cdp, 4); const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1"); - expect(current?.frame.target).toEqual({ tabId: 4 }); + expect(current?.frame.processKey).toBe("root"); expect(current?.domNodes.length).toBeGreaterThan(0); expect(current?.axNodes).toEqual([]); }); @@ -1366,8 +1495,8 @@ describe("snapshot document provenance", () => { }); it("retains the caller URL in the root AX-only fallback", async () => { - const cdp: CdpRunner = { - getFrameGraph: async () => { + const cdp: CdpRunner = createStubCdpRunner({ + getFrameDocuments: async () => { throw new Error("no graph"); }, send: async (_tabId: number, method: string): Promise => { @@ -1378,7 +1507,7 @@ describe("snapshot document provenance", () => { : {} ) as T; }, - }; + }); const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/"); expect(facts.documents[0].frame.url).toBe("https://page.test/"); expect(facts.documents[0].axNodes).toHaveLength(1); @@ -1402,29 +1531,40 @@ describe("snapshot document provenance", () => { true, ])("isolates competing target claims independent of batch order %s", async (reverse) => { const f = siblingCaptureFixture(); - const remote = { + const remote = makeFrame({ frameId: "frame-1", parentFrameId: "frame-0", ownerBackendNodeId: 100, - target: { tabId: 4, sessionId: "remote" }, - }; - const main = { frameId: "frame-0", target: { tabId: 4 } }; + processKey: "oopif:remote", + }); + const main = makeFrame({ frameId: "frame-0" }); const cdp: CdpRunner = { ...f.cdp, - getFrameGraph: async () => ({ + hasOwnFrameTarget: (_tabId, frameId) => frameId === "frame-1", + getFrameDocuments: async () => ({ rootFrameId: "frame-0", frames: reverse ? [remote, main] : [main, remote], }), - sendToTarget: async (_target: CdpTarget, method: string): Promise => - (method === "DOMSnapshot.captureSnapshot" - ? { ...f.snapshot, documents: [f.snapshot.documents[1]] } - : method === "Page.getLayoutMetrics" - ? { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000, scale: 1 }, - cssLayoutViewport: { clientWidth: 200, clientHeight: 100 }, - } - : {}) as T, + sendToFrame: async ( + handle: FrameDocumentHandle | FrameNodeHandle, + method: string, + params?: object, + guard?: CdpDispatchGuard, + ): Promise => { + if (!handle.frameId || handle.frameId !== "frame-1") + return f.cdp.send(handle.tabId, method, params, guard?.signal); + return ( + method === "DOMSnapshot.captureSnapshot" + ? { ...f.snapshot, documents: [f.snapshot.documents[1]] } + : method === "Page.getLayoutMetrics" + ? { + visualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, + cssLayoutViewport: { clientWidth: 200, clientHeight: 100 }, + } + : {} + ) as T; + }, }; const facts = await captureObservationFacts(cdp, 4); expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1"); @@ -1462,6 +1602,8 @@ describe("sibling frame measurement scheduling", () => { } as never; return original(tabId, method, params); }; + fixture.cdp.sendToFrame = (handle, method, params, guard) => + fixture.cdp.send(handle.tabId, method, params, guard?.signal); const captured = await captureObservationFacts(fixture.cdp, 4); expect(captured.issues.filter((issue) => issue.stage === "geometry")).toEqual([]); expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual( @@ -1529,12 +1671,12 @@ describe("sibling frame measurement scheduling", () => { ).toEqual([ { status: "unavailable", - source: { target: { tabId: 4 }, frameId: "frame-1" }, + source: { process: rootProcessBoundary(4), frameId: "frame-1" }, ownerBackendNodeId: 100, }, { status: "blocked", - source: { target: { tabId: 4 }, frameId: "frame-7" }, + source: { process: rootProcessBoundary(4), frameId: "frame-7" }, cause: captured.issues.find( (issue) => issue.frameId === "frame-1" && issue.stage === "geometry", )?.projectionIssue, @@ -1617,6 +1759,8 @@ describe("AX frame scheduling", () => { active--; } }; + f.cdp.sendToFrame = (handle, method, params, guard) => + f.cdp.send(handle.tabId, method, params, guard?.signal); const controller = new AbortController(); let settled = false; const capture = captureObservationFacts(f.cdp, 4, controller.signal); @@ -1666,13 +1810,16 @@ describe("Accessibility domain lifetime", () => { vi.restoreAllMocks(); }); - function pendingCleanup(sessionId?: string) { + function pendingCleanup(processKey = "root") { vi.useFakeTimers(); vi.spyOn(console, "warn").mockImplementation(() => {}); const f = fixture(); const gate = new CdpReadGate(); - const original = f.cdp.sendToTarget!; - const target = { tabId: 4, sessionId }; + const original = f.cdp.sendToFrame; + const handle = + processKey === "root" + ? f.frames[0].handle + : f.frames.find((frame) => frame.processKey === processKey)!.handle; const attempted: string[] = []; let enter!: () => void; const started = new Promise((resolve) => { @@ -1682,29 +1829,34 @@ describe("Accessibility domain lifetime", () => { const pending = new Promise((resolve) => { finish = resolve; }); - f.cdp.sendToTarget = (target, method, params) => { + f.cdp.sendToFrame = (targetHandle, method, params, guard) => { + if (guard?.attachmentId !== undefined && guard.attachmentId !== f.cdp.getAttachmentId(4)) + return Promise.reject(new Error("Debugger attachment changed before dispatch")); + const targetProcessKey = f.processForHandle(targetHandle); attempted.push(method); - return gate.run(target, method, () => { - if (method === "Accessibility.disable" && target.sessionId === sessionId) { - f.logs.push({ target, method, params: (params ?? {}) as Record }); + return gate.run({ tabId: targetHandle.tabId, processKey: targetProcessKey }, method, () => { + if (method === "Accessibility.disable" && targetProcessKey === processKey) { + f.logs.push({ + processKey: targetProcessKey, + handle: targetHandle, + method, + params: (params ?? {}) as Record, + guard, + }); enter(); return pending as never; } - return original(target, method, params); + return original(targetHandle, method, params, guard); }); }; - f.cdp.send = (tabId, method, params) => f.cdp.sendToTarget!({ tabId }, method, params); - f.cdp.sendGuarded = (target, method, params, guard) => { - if (guard.attachmentId !== f.cdp.getAttachmentId?.(4)) - return Promise.reject(new Error("Debugger attachment changed before dispatch")); - return f.cdp.sendToTarget!(target, method, params); - }; - return { ...f, gate, target, attempted, started, finish }; + f.cdp.send = (tabId, method, params, signal) => + f.cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); + return { ...f, gate, handle, processKey, attempted, started, finish }; } - describe.each([undefined, "remote"])("pending cleanup (session=%s)", (sessionId) => { + describe.each(["root", "oopif:remote"])("pending cleanup (process=%s)", (processKey) => { it("retains every captured document and releases only after Chrome replies", async () => { - const f = pendingCleanup(sessionId); + const f = pendingCleanup(processKey); const work = captureObservationFacts(f.cdp, 4); await f.started; await vi.advanceTimersByTimeAsync(ACCESSIBILITY_CLEANUP_TIMEOUT_MS); @@ -1719,12 +1871,12 @@ describe("Accessibility domain lifetime", () => { expect( f.attempted.slice(firstCleanup).every((method) => method === "Accessibility.disable"), ).toBe(true); - await expect(f.cdp.sendToTarget!(f.target, "Runtime.evaluate")).rejects.toMatchObject({ + await expect(f.cdp.sendToFrame(f.handle, "Runtime.evaluate")).rejects.toMatchObject({ phase: "blocked", }); f.finish({}); await vi.advanceTimersByTimeAsync(0); - await expect(f.cdp.sendToTarget!(f.target, "Runtime.evaluate")).resolves.toEqual({}); + await expect(f.cdp.sendToFrame(f.handle, "Runtime.evaluate")).resolves.toEqual({}); expect(vi.getTimerCount()).toBe(0); }); @@ -1732,7 +1884,7 @@ describe("Accessibility domain lifetime", () => { false, true, ])("finishes hover reads before cleanup times out (visual=%s)", async (includeVisualFacts) => { - const f = pendingCleanup(sessionId); + const f = pendingCleanup(processKey); const outcome = captureVomObservation(f.cdp, 4, "https://example.test", { conditionalSurfaceProbe: true, includeVisualFacts, @@ -1761,7 +1913,7 @@ describe("Accessibility domain lifetime", () => { }); it("rejects cancellation during cleanup within its deadline", async () => { - const f = pendingCleanup(sessionId); + const f = pendingCleanup(processKey); const controller = new AbortController(); const work = captureVomObservation(f.cdp, 4, "https://example.test", { conditionalSurfaceProbe: true, @@ -1776,14 +1928,14 @@ describe("Accessibility domain lifetime", () => { }); it("keeps a replacement gate closed when the old cleanup replies late", async () => { - const f = pendingCleanup(sessionId); + const f = pendingCleanup(processKey); const work = captureObservationFacts(f.cdp, 4); await f.started; await vi.advanceTimersByTimeAsync(ACCESSIBILITY_CLEANUP_TIMEOUT_MS); await work; f.gate.reset(4); const replacement = f.gate.run( - f.target, + { tabId: 4, processKey: f.processKey }, "Accessibility.disable", () => new Promise(() => {}), ); @@ -1792,17 +1944,15 @@ describe("Accessibility domain lifetime", () => { await rejected; f.finish({}); await vi.advanceTimersByTimeAsync(0); - await expect(f.cdp.sendToTarget!(f.target, "Runtime.evaluate")).rejects.toMatchObject({ + await expect(f.cdp.sendToFrame(f.handle, "Runtime.evaluate")).rejects.toMatchObject({ phase: "blocked", }); }); }); - const axCalls = (logs: ReturnType["logs"], sessionId?: string) => + const axCalls = (logs: ReturnType["logs"], processKey = "root") => logs - .filter( - (call) => call.target.sessionId === sessionId && call.method.startsWith("Accessibility."), - ) + .filter((call) => call.processKey === processKey && call.method.startsWith("Accessibility.")) .map((call) => call.method); it("disables AX on each target once its trees are read", async () => { @@ -1815,7 +1965,7 @@ describe("Accessibility domain lifetime", () => { "Accessibility.getFullAXTree", "Accessibility.disable", ]); - expect(axCalls(logs, "remote")).toEqual([ + expect(axCalls(logs, "oopif:remote")).toEqual([ "Accessibility.enable", "Accessibility.getFullAXTree", "Accessibility.disable", @@ -1828,31 +1978,31 @@ describe("Accessibility domain lifetime", () => { it("sends the cleanup only on the attachment that enabled AX", async () => { const { cdp, logs } = fixture(); - const sendGuarded = vi.fn(async () => ({}) as never); - cdp.sendGuarded = sendGuarded; await captureObservationFacts(cdp, 4); - expect(sendGuarded).toHaveBeenCalledTimes(2); - for (const target of [{ tabId: 4 }, { tabId: 4, sessionId: "remote" }]) - expect(sendGuarded).toHaveBeenCalledWith( - target, - "Accessibility.disable", - {}, - { attachmentId: "attachment" }, - ); - expect(logs.some((call) => call.method === "Accessibility.disable")).toBe(false); + const disables = logs.filter((call) => call.method === "Accessibility.disable"); + expect(disables).toHaveLength(2); + expect(disables.map((call) => call.processKey).sort()).toEqual(["oopif:remote", "root"]); + for (const disable of disables) { + expect(disable.guard).toMatchObject({ + attachmentId: "attachment", + allowReattach: false, + ignoreFrameNavigation: true, + }); + } }); it("leaves a replaced attachment alone", async () => { const { cdp, logs } = fixture(); - const original = cdp.sendToTarget!; + const original = cdp.sendToFrame; let attachment = "attachment"; cdp.getAttachmentId = () => attachment; - cdp.sendToTarget = async (target, method, params) => { - const reply = await original(target, method, params); + cdp.sendToFrame = async (handle, method, params, guard) => { + const reply = await original(handle, method, params, guard); if (method === "Accessibility.getFullAXTree") attachment = "replacement"; return reply as never; }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity"); expect(logs.some((call) => call.method === "Accessibility.disable")).toBe(false); }); @@ -1863,21 +2013,35 @@ describe("Accessibility domain lifetime", () => { try { const f = fixture(); const gate = new CdpReadGate(); - const original = f.cdp.sendToTarget!; - f.cdp.sendToTarget = (target, method, params) => - gate.run(target, method, () => { - if (target.sessionId !== "remote" || method !== "Accessibility.getFullAXTree") - return original(target, method, params); - f.logs.push({ target, method, params: (params ?? {}) as Record }); + const original = f.cdp.sendToFrame; + f.cdp.sendToFrame = (handle, method, params, guard) => { + const processKey = f.processForHandle(handle); + return gate.run({ tabId: handle.tabId, processKey }, method, () => { + if (processKey !== "oopif:remote" || method !== "Accessibility.getFullAXTree") + return original(handle, method, params, guard); + f.logs.push({ + processKey, + handle, + method, + params: (params ?? {}) as Record, + guard, + }); return new Promise(() => {}); }); - f.cdp.send = (tabId, method, params) => f.cdp.sendToTarget!({ tabId }, method, params); + }; + f.cdp.send = (tabId, method, params, signal) => + f.cdp.sendToFrame( + toRootFrameHandle(tabId), + method, + params, + signal ? { signal } : undefined, + ); const work = captureObservationFacts(f.cdp, 4); await vi.advanceTimersByTimeAsync(CAPTURE_READ_TIMEOUT_MS); const facts = await work; expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "same", "nested"]); expect(axCalls(f.logs).at(-1)).toBe("Accessibility.disable"); - expect(axCalls(f.logs, "remote")).toEqual([ + expect(axCalls(f.logs, "oopif:remote")).toEqual([ "Accessibility.enable", "Accessibility.getFullAXTree", ]); @@ -1890,29 +2054,31 @@ describe("Accessibility domain lifetime", () => { it("releases AX when the capture is cancelled during AX reads", async () => { const { cdp, logs } = fixture(); const controller = new AbortController(); - const original = cdp.sendToTarget!; - cdp.sendToTarget = async (target, method, params) => { + const original = cdp.sendToFrame; + cdp.sendToFrame = async (handle, method, params, guard) => { if (method === "Accessibility.getFullAXTree") controller.abort(); - return original(target, method, params); + return original(handle, method, params, guard); }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({ name: "AbortError", }); expect(axCalls(logs).at(-1)).toBe("Accessibility.disable"); - expect(axCalls(logs, "remote").at(-1)).toBe("Accessibility.disable"); + expect(axCalls(logs, "oopif:remote").at(-1)).toBe("Accessibility.disable"); }); it("releases AX when observation is cancelled during hover reads", async () => { const { cdp, logs } = fixture(); const controller = new AbortController(); - const original = cdp.sendToTarget!; - cdp.sendToTarget = async (target, method, params) => { + const original = cdp.sendToFrame; + cdp.sendToFrame = async (handle, method, params, guard) => { if (method === "Runtime.evaluate" && (params as { returnByValue?: boolean }).returnByValue) controller.abort(); - return original(target, method, params); + return original(handle, method, params, guard); }; - cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + cdp.send = (tabId, method, params, signal) => + cdp.sendToFrame(toRootFrameHandle(tabId), method, params, signal ? { signal } : undefined); await expect( captureVomObservation(cdp, 4, "https://example.test", { conditionalSurfaceProbe: true, @@ -1920,7 +2086,7 @@ describe("Accessibility domain lifetime", () => { }), ).rejects.toMatchObject({ name: "AbortError" }); expect(axCalls(logs).at(-1)).toBe("Accessibility.disable"); - expect(axCalls(logs, "remote").at(-1)).toBe("Accessibility.disable"); + expect(axCalls(logs, "oopif:remote").at(-1)).toBe("Accessibility.disable"); }); it("releases AX without replacing an observation consumer's failure", async () => { @@ -1934,42 +2100,50 @@ describe("Accessibility domain lifetime", () => { }), ).rejects.toBe(error); expect(axCalls(logs).at(-1)).toBe("Accessibility.disable"); - expect(axCalls(logs, "remote").at(-1)).toBe("Accessibility.disable"); + expect(axCalls(logs, "oopif:remote").at(-1)).toBe("Accessibility.disable"); }); it("reuses AX across back-to-back leased captures and disables after sliding idle TTL", async () => { vi.useFakeTimers(); try { const { cdp, logs } = fixture(); - const sendGuarded = vi.fn(async () => ({}) as never); - cdp.sendGuarded = sendGuarded; const lease = new AccessibilityLease(4); + const disableCalls = () => logs.filter((call) => call.method === "Accessibility.disable"); await captureObservationFacts(cdp, 4, undefined, undefined, { accessibilityLease: lease }); - expect(sendGuarded).not.toHaveBeenCalled(); + expect(disableCalls()).toHaveLength(0); expect(axCalls(logs).filter((m) => m === "Accessibility.enable")).toHaveLength(1); - expect(axCalls(logs, "remote").filter((m) => m === "Accessibility.enable")).toHaveLength(1); + expect( + axCalls(logs, "oopif:remote").filter((m) => m === "Accessibility.enable"), + ).toHaveLength(1); await vi.advanceTimersByTimeAsync(ACCESSIBILITY_LEASE_TTL_MS - 1_000); - expect(sendGuarded).not.toHaveBeenCalled(); + expect(disableCalls()).toHaveLength(0); // A second capture resets the sliding TTL without re-enabling or disabling AX. await captureObservationFacts(cdp, 4, undefined, undefined, { accessibilityLease: lease }); expect(axCalls(logs).filter((m) => m === "Accessibility.enable")).toHaveLength(1); - expect(axCalls(logs, "remote").filter((m) => m === "Accessibility.enable")).toHaveLength(1); + expect( + axCalls(logs, "oopif:remote").filter((m) => m === "Accessibility.enable"), + ).toHaveLength(1); await vi.advanceTimersByTimeAsync(2_000); - expect(sendGuarded).not.toHaveBeenCalled(); + expect(disableCalls()).toHaveLength(0); await vi.advanceTimersByTimeAsync(ACCESSIBILITY_LEASE_TTL_MS - 2_000); - expect(sendGuarded).toHaveBeenCalledTimes(2); - for (const target of [{ tabId: 4 }, { tabId: 4, sessionId: "remote" }]) - expect(sendGuarded).toHaveBeenCalledWith( - target, - "Accessibility.disable", - {}, - { attachmentId: "attachment" }, - ); + expect(disableCalls()).toHaveLength(2); + expect( + disableCalls() + .map((c) => c.processKey) + .sort(), + ).toEqual(["oopif:remote", "root"]); + for (const disable of disableCalls()) { + expect(disable.guard).toMatchObject({ + attachmentId: "attachment", + allowReattach: false, + ignoreFrameNavigation: true, + }); + } expect(vi.getTimerCount()).toBe(0); } finally { vi.useRealTimers(); @@ -1998,7 +2172,7 @@ describe("Accessibility domain lifetime", () => { await lease.release(); expect(axCalls(logs).at(-1)).toBe("Accessibility.disable"); - expect(axCalls(logs, "remote").at(-1)).toBe("Accessibility.disable"); + expect(axCalls(logs, "oopif:remote").at(-1)).toBe("Accessibility.disable"); expect(vi.getTimerCount()).toBe(0); } finally { vi.useRealTimers(); @@ -2011,21 +2185,35 @@ describe("Accessibility domain lifetime", () => { try { const f = fixture(); const gate = new CdpReadGate(); - const original = f.cdp.sendToTarget!; + const original = f.cdp.sendToFrame; let hangRemote = false; - f.cdp.sendToTarget = (target, method, params) => - gate.run(target, method, () => { + f.cdp.sendToFrame = (handle, method, params, guard) => { + const processKey = f.processForHandle(handle); + return gate.run({ tabId: handle.tabId, processKey }, method, () => { if ( hangRemote && - target.sessionId === "remote" && + processKey === "oopif:remote" && method === "Accessibility.getFullAXTree" ) { - f.logs.push({ target, method, params: (params ?? {}) as Record }); + f.logs.push({ + processKey, + handle, + method, + params: (params ?? {}) as Record, + guard, + }); return new Promise(() => {}); } - return original(target, method, params); + return original(handle, method, params, guard); }); - f.cdp.send = (tabId, method, params) => f.cdp.sendToTarget!({ tabId }, method, params); + }; + f.cdp.send = (tabId, method, params, signal) => + f.cdp.sendToFrame( + toRootFrameHandle(tabId), + method, + params, + signal ? { signal } : undefined, + ); const lease = new AccessibilityLease(4); await captureObservationFacts(f.cdp, 4, undefined, undefined, { accessibilityLease: lease }); @@ -2039,7 +2227,7 @@ describe("Accessibility domain lifetime", () => { await vi.advanceTimersByTimeAsync(ACCESSIBILITY_LEASE_TTL_MS); expect(axCalls(f.logs).at(-1)).toBe("Accessibility.disable"); - expect(axCalls(f.logs, "remote")).toEqual([ + expect(axCalls(f.logs, "oopif:remote")).toEqual([ "Accessibility.enable", "Accessibility.getFullAXTree", "Accessibility.getFullAXTree", @@ -2095,8 +2283,8 @@ describe("visual facts integration", () => { expect(calls).toHaveLength(2); for (const call of calls) expect(call.params.computedStyles).toEqual(expected); } - expect(base.logs.map((c) => [c.target, c.method])).toEqual( - visual.logs.map((c) => [c.target, c.method]), + expect(base.logs.map((c) => [c.processKey, c.method])).toEqual( + visual.logs.map((c) => [c.processKey, c.method]), ); const scene = (input: typeof baseFacts) => buildSemanticVomScene({ @@ -2118,7 +2306,7 @@ describe("visual facts integration", () => { it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => { const { cdp, logs } = fixture({ canvas: true, - frames: [{ frameId: "main", target: { tabId: 4 } }], + frames: [{ frameId: "main" }], }); const facts = await captureObservationFacts(cdp, 4, undefined, undefined, { includeVisualFacts: true, @@ -2144,7 +2332,7 @@ describe("visual facts integration", () => { expect(logs).toHaveLength(before); expect(facts.documents[0].index.nodes.get(2)?.layout?.clientRect).toEqual([0, 0, 100, 40]); expect(facts.documents[0].geometry?.pageScale).toBe(1); - const baseline = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] }); + const baseline = fixture({ frames: [{ frameId: "main" }] }); await captureObservationFacts(baseline.cdp, 4); expect(logs.map((call) => call.method)).toEqual(baseline.logs.map((call) => call.method)); }); diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 503dd210c..c63ab1f61 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; +import { createStubCdpRunner } from "../../shared"; import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; @@ -210,7 +211,7 @@ function nestedHoverTriggerSnapshotReply() { } function makeCdp(snapshot: unknown) { - return { + return createStubCdpRunner({ send: vi.fn(async (_tab: number, method: string) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; @@ -249,7 +250,7 @@ function makeCdp(snapshot: unknown) { if (method === "Runtime.releaseObjectGroup") return {}; throw new Error(`unexpected ${method}`); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); } describe("snapshot capture and hover", () => { @@ -275,7 +276,7 @@ describe("snapshot capture and hover", () => { it("runs hover surface probes when explicitly enabled", async () => { let hoverStateCalls = 0; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply(); @@ -312,7 +313,7 @@ describe("snapshot capture and hover", () => { if (method === "Input.dispatchMouseEvent") return {}; throw new Error(`unexpected ${method}`); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); @@ -340,7 +341,7 @@ describe("snapshot capture and hover", () => { it("uses a fresh baseline for each hover candidate", async () => { let hoverStateCalls = 0; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply(); @@ -374,7 +375,7 @@ describe("snapshot capture and hover", () => { if (method === "Input.dispatchMouseEvent") return {}; throw new Error(`unexpected ${method}`); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); @@ -390,7 +391,7 @@ describe("snapshot capture and hover", () => { it("deduplicates nested hover candidates for one visual trigger", async () => { let hoverMoves = 0; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply(); @@ -421,7 +422,7 @@ describe("snapshot capture and hover", () => { } throw new Error(`unexpected ${method}`); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; await probeHoverSurfaces(cdp, 4, captured.nodes); @@ -737,7 +738,7 @@ describe("snapshot capture and hover", () => { }); it("subtracts scroll offset to make rects viewport-relative", async () => { - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_t: number, method: string) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); @@ -750,7 +751,7 @@ describe("snapshot capture and hover", () => { } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const div = nodes.find((n) => n.backendNodeId === 12); expect(div?.localRect?.y).toBe(-200); @@ -786,7 +787,7 @@ describe("snapshot capture and hover", () => { }, ], }; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_t: number, method: string) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; @@ -800,7 +801,7 @@ describe("snapshot capture and hover", () => { } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ x: 0, @@ -864,7 +865,7 @@ describe("snapshot capture and hover", () => { }, ], }; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_t: number, method: string) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; @@ -877,7 +878,7 @@ describe("snapshot capture and hover", () => { } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const btn = nodes.find((n) => n.backendNodeId === 12); expect(btn?.textContent).toBe("登录"); @@ -956,7 +957,7 @@ describe("snapshot capture and hover", () => { }, ], }; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_t: number, method: string) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; @@ -975,7 +976,7 @@ describe("snapshot capture and hover", () => { if (method === "Runtime.releaseObject") return {}; throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const facts = await captureObservationFacts(cdp, 4); const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!; const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!; @@ -1047,7 +1048,7 @@ describe("snapshot capture and hover", () => { }, ], }; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_t: number, method: string) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; @@ -1066,7 +1067,7 @@ describe("snapshot capture and hover", () => { if (method === "Runtime.releaseObject") return {}; throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const { documents } = await captureObservationFacts(cdp, 4); const input = documents @@ -1159,7 +1160,7 @@ describe("snapshot capture and hover", () => { }, ], }; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn( async ( _t: number, @@ -1199,7 +1200,7 @@ describe("snapshot capture and hover", () => { throw new Error(method); }, ) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const { documents } = await captureObservationFacts(cdp, 4); const nestedIframe = documents @@ -1245,7 +1246,7 @@ describe("snapshot capture and hover", () => { }, ], }; - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_t: number, method: string) => { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; @@ -1259,13 +1260,13 @@ describe("snapshot capture and hover", () => { } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100); }); it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => { - const cdp = { + const cdp = createStubCdpRunner({ send: vi.fn(async (_t: number, method: string) => { if (method === "DOM.getDocument") return { root: { nodeId: 1 } }; if (method === "DOM.querySelector") return { nodeId: 99 }; @@ -1281,7 +1282,7 @@ describe("snapshot capture and hover", () => { } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, - }; + }); const excluded = await collectOverlayExcludedBackendIds(cdp, 4); expect(excluded).toEqual(new Set([200, 201, 202])); }); diff --git a/apps/extension/src/tools/vom/__tests__/document-identity.test.ts b/apps/extension/src/tools/vom/__tests__/document-identity.test.ts index 3f2b5bfa2..ea0c0d35c 100644 --- a/apps/extension/src/tools/vom/__tests__/document-identity.test.ts +++ b/apps/extension/src/tools/vom/__tests__/document-identity.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { StaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { StaleFrameHandleError } from "@/browser-driver/frame-handles"; import type { CdpRunner } from "../../shared"; import { verifyDocumentIdentity, verifyVisualTargetIdentity } from "../document-identity"; import type { VisualCandidate } from "../visual-discovery"; diff --git a/apps/extension/src/tools/vom/__tests__/form-state.test.ts b/apps/extension/src/tools/vom/__tests__/form-state.test.ts index 7340355b8..f6fcbe966 100644 --- a/apps/extension/src/tools/vom/__tests__/form-state.test.ts +++ b/apps/extension/src/tools/vom/__tests__/form-state.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; -import { type CdpRunner, cdpRunnerForTarget } from "../../shared"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; +import { type CdpRunner, cdpRunnerForFrame, createStubCdpRunner } from "../../shared"; import { enrichFormControlStates } from "../form-capture"; import { decodeDocument } from "../snapshot"; @@ -92,7 +93,7 @@ async function fakeCdp( return {}; }); const nodes = await decodeControls(snap); - return { send, cdp: { send: send as CdpRunner["send"] }, nodes }; + return { send, cdp: createStubCdpRunner({ send: send as CdpRunner["send"] }), nodes }; } async function decodeControls(snap: ReturnType) { @@ -188,20 +189,24 @@ describe("form state identity", () => { }); it("keeps identical backend ids in different OOPIF targets separate", async () => { - const sendToTarget = vi.fn(async (target, method) => { - if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]); + const sendToFrame = vi.fn(async (handle, method) => { + if (method === "Runtime.evaluate") return reply([[11, state(handle.frameId)]]); return {}; }); const send = vi.fn(async () => { throw new Error("unexpected top-level target"); }); - const cdp = { send, sendToTarget } as unknown as CdpRunner; - for (const sessionId of ["left-frame", "right-frame"]) { + const cdp = createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], + sendToFrame: sendToFrame as unknown as CdpRunner["sendToFrame"], + }); + for (const frameId of ["left-frame", "right-frame"]) { + const handle = toFrameDocumentHandle(7, { frameId }); const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]])); - await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes); - expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId); - expect(sendToTarget).toHaveBeenCalledWith( - { tabId: 7, sessionId }, + await enrichFormControlStates(cdpRunnerForFrame(cdp, handle), 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(frameId); + expect(sendToFrame).toHaveBeenCalledWith( + handle, "Runtime.releaseObjectGroup", expect.anything(), ); @@ -248,13 +253,13 @@ describe("form state identity", () => { it("does not report success when cancelled during object cleanup", async () => { const controller = new AbortController(); const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]])); - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tabId, method) => { if (method === "Runtime.evaluate") return reply([[11, state("new")]]); if (method === "Runtime.releaseObjectGroup") controller.abort(); return {}; }) as CdpRunner["send"], - }; + }); await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({ name: "AbortError", }); @@ -386,7 +391,7 @@ it.each([ if (method === "Runtime.callFunctionOn") return { result: { value: inForm } }; return {}; }); - await enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes); + await enrichFormControlStates(createStubCdpRunner({ send: send as CdpRunner["send"] }), 7, nodes); expect(send).toHaveBeenCalledWith( 7, "DOM.resolveNode", @@ -406,7 +411,7 @@ it("bounds missing range-owner reads and keeps the uninspected remainder unknown if (method === "Runtime.callFunctionOn") return { result: { value: false } }; return {}; }); - await enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes); + await enrichFormControlStates(createStubCdpRunner({ send: send as CdpRunner["send"] }), 7, nodes); expect(send.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(64); expect(nodes[0].find((n) => n.backendNodeId === 64)?.inForm).toBe(false); expect(nodes[0].find((n) => n.backendNodeId === 65)?.inForm).toBeUndefined(); @@ -433,12 +438,14 @@ it("drains in-flight range reads before releasing objects after a terminal error return {}; }); let settled = false; - const result = enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes).catch( - (error) => { - settled = true; - return error; - }, - ); + const result = enrichFormControlStates( + createStubCdpRunner({ send: send as CdpRunner["send"] }), + 7, + nodes, + ).catch((error) => { + settled = true; + return error; + }); await vi.waitFor(() => expect(resolveSecond).toBeTypeOf("function")); const error = Object.assign(new Error("renderer timed out"), { name: "CdpReadTimeoutError" }); rejectFirst(error); @@ -452,23 +459,27 @@ it("drains in-flight range reads before releasing objects after a terminal error }); it("keeps fallback owner reads with identical backend ids scoped to their OOPIF target", async () => { - const sendToTarget = vi.fn(async (target, method) => { + const sendToFrame = vi.fn(async (handle, method) => { if (method === "Runtime.evaluate") return reply([]); if (method === "DOM.resolveNode") return { object: { objectId: "range" } }; if (method === "Runtime.callFunctionOn") - return { result: { value: target.sessionId === "owned-frame" } }; + return { result: { value: handle.frameId === "owned-frame" } }; return {}; }); const send = vi.fn(async () => { throw new Error("unexpected root target"); }); - const cdp = { send, sendToTarget } as unknown as CdpRunner; - for (const sessionId of ["owned-frame", "free-frame"]) { + const cdp = createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], + sendToFrame: sendToFrame as unknown as CdpRunner["sendToFrame"], + }); + for (const frameId of ["owned-frame", "free-frame"]) { + const handle = toFrameDocumentHandle(7, { frameId }); const nodes = await decodeControls(snapshot([[{ id: 11, type: "range", value: "0" }]])); - await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes); - expect(nodes[0].find((n) => n.backendNodeId === 11)?.inForm).toBe(sessionId === "owned-frame"); - expect(sendToTarget).toHaveBeenCalledWith( - { tabId: 7, sessionId }, + await enrichFormControlStates(cdpRunnerForFrame(cdp, handle), 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.inForm).toBe(frameId === "owned-frame"); + expect(sendToFrame).toHaveBeenCalledWith( + handle, "Runtime.callFunctionOn", expect.objectContaining({ objectId: "range" }), ); diff --git a/apps/extension/src/tools/vom/__tests__/frame-document.test.ts b/apps/extension/src/tools/vom/__tests__/frame-document.test.ts index b62ef5644..0125b002f 100644 --- a/apps/extension/src/tools/vom/__tests__/frame-document.test.ts +++ b/apps/extension/src/tools/vom/__tests__/frame-document.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { createFrameDocumentInfo, type FrameDocumentInfo } from "@/browser-driver/frame-handles"; import type { CapturedNode } from "../capture"; import { buildFrameDocuments, @@ -32,11 +32,11 @@ function captured(frameNodes: Map): FrameDomInput { describe("buildFrameDocuments", () => { it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => { - const graph: CdpFrameGraph = { + const graph: { rootFrameId: string; frames: FrameDocumentInfo[] } = { rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { frameId: "child", parentFrameId: "main", target: { tabId: 4 } }, + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ tabId: 4, frameId: "child", parentFrameId: "main" }), ], }; const batches: FrameAxBatch[] = [ @@ -75,11 +75,11 @@ describe("buildFrameDocuments", () => { }); it("deduplicates overlapping same-target AX results using the strongest ownership", async () => { - const graph: CdpFrameGraph = { + const graph: { rootFrameId: string; frames: FrameDocumentInfo[] } = { rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { frameId: "child", parentFrameId: "main", target: { tabId: 4 } }, + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ tabId: 4, frameId: "child", parentFrameId: "main" }), ], }; const childNode = { nodeId: "shared", backendDOMNodeId: 100 }; @@ -110,20 +110,24 @@ describe("buildFrameDocuments", () => { }); it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => { - const graph: CdpFrameGraph = { + const graph: { rootFrameId: string; frames: FrameDocumentInfo[] } = { rootFrameId: "main", frames: [ - { frameId: "main", target: { tabId: 4 } }, - { + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, frameId: "left", parentFrameId: "main", - target: { tabId: 4, sessionId: "left-session" }, - }, - { + processKey: "oopif:left", + isOutOfProcess: true, + }), + createFrameDocumentInfo({ + tabId: 4, frameId: "right", parentFrameId: "main", - target: { tabId: 4, sessionId: "right-session" }, - }, + processKey: "oopif:right", + isOutOfProcess: true, + }), ], }; const batches: FrameAxBatch[] = graph.frames.slice(1).map((frame) => ({ @@ -160,11 +164,11 @@ describe("buildFrameDocuments", () => { ]); const capture = captured(frameNodes); const frames = ["main", "left", "right", "child"].map((frameId) => ({ - frame: { + frame: createFrameDocumentInfo({ + tabId: 4, frameId, - target: { tabId: 4 }, ...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}), - }, + }), nodes: [], })); @@ -177,8 +181,13 @@ describe("buildFrameDocuments", () => { }); it("rejects explicit AX frame ownership in a different target", async () => { const frames = [ - { frameId: "main", target: { tabId: 4 } }, - { frameId: "remote", target: { tabId: 4, sessionId: "remote" } }, + createFrameDocumentInfo({ tabId: 4, frameId: "main" }), + createFrameDocumentInfo({ + tabId: 4, + frameId: "remote", + processKey: "oopif:remote", + isOutOfProcess: true, + }), ]; const unresolved = vi.fn(); const documents = await buildFrameDocuments( @@ -193,7 +202,7 @@ describe("buildFrameDocuments", () => { }); it("resolves a reverse-ordered deep AX parent chain without recursion", async () => { - const frame = { frameId: "main", target: { tabId: 4 } }; + const frame = createFrameDocumentInfo({ tabId: 4, frameId: "main" }); const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({ nodeId: String(i), ...(i ? { parentId: String(i - 1) } : { frameId: "main" }), diff --git a/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts index b4b2e4db7..da973f0c7 100644 --- a/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts +++ b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; -import type { CdpTarget } from "@/browser-driver/frame-graph"; -import type { CdpRunner } from "../../shared"; +import { type FrameDocumentHandle, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; +import { type CdpRunner, createStubCdpRunner } from "../../shared"; import type { CapturedNode } from "../capture"; import type { FrameDocument, FrameOwnedAxNode } from "../frame-document"; import { probeTooltipNames } from "../name-enrichment"; @@ -9,10 +9,10 @@ import { stableIdentifierName } from "../semantic-graph/name-evidence"; import { frameBackendKey } from "../semantic-graph/types"; function document( - target: CdpTarget, + handle: FrameDocumentHandle, backendNodeId: number, accessibleName?: string, - frameId = "child", + frameId = handle.frameId || "child", x = 10, ): FrameDocument { const node: CapturedNode = { @@ -34,9 +34,12 @@ function document( }; return { frameId, + loaderId: handle.loaderId, + processKey: "root", + isOutOfProcess: false, parentFrameId: "main", contextScopeId: "child", - target, + handle, axNodes: [], domNodes: [node], url: "https://example.com/frame", @@ -56,66 +59,66 @@ function semantics(documents: FrameDocument[]) { describe("tooltip name enrichment", () => { it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => { - const target = { tabId: 7, sessionId: "oopif-session" }; + const handle = toFrameDocumentHandle(7, { frameId: "child", loaderId: "oopif-loader" }); let hovered = false; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tabId, method, params) => { if (method === "Input.dispatchMouseEvent") { hovered = (params as { x?: number }).x === 30; } return {}; }) as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method) => { + sendToFrame: vi.fn(async (_handle, method) => { if (method === "Page.createIsolatedWorld") return { executionContextId: 11 }; if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } }; if (method === "Runtime.callFunctionOn") { return { result: { value: hovered ? ["插入行"] : [] } }; } return {}; - }) as CdpRunner["sendToTarget"], - }; + }) as CdpRunner["sendToFrame"], + }); - const documents = [document(target, 501)]; + const documents = [document(handle, 501)]; const graph = semantics(documents); const names = await probeTooltipNames(cdp, 7, documents, graph); expect(names.get(frameBackendKey("child", 501))).toBe("插入行"); - expect(cdp.sendToTarget).toHaveBeenCalledWith( - target, + expect(cdp.sendToFrame).toHaveBeenCalledWith( + handle, "Page.createIsolatedWorld", expect.objectContaining({ frameId: "child" }), ); const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length; - const targetSendsBeforeCache = vi.mocked( - cdp.sendToTarget as NonNullable, - ).mock.calls.length; + const frameSendsBeforeCache = vi.mocked(cdp.sendToFrame).mock.calls.length; const cached = await probeTooltipNames(cdp, 7, documents, graph); expect(cached.get(frameBackendKey("child", 501))).toBe("插入行"); expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache); - expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache); + expect(cdp.sendToFrame).toHaveBeenCalledTimes(frameSendsBeforeCache); }); it("does not probe controls that already have a higher-confidence name", async () => { - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], - sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"], - }; + sendToFrame: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToFrame"], + }); - const documents = [document({ tabId: 8 }, 601, "Add a new record")]; + const documents = [ + document(toFrameDocumentHandle(8, { frameId: "child" }), 601, "Add a new record"), + ]; const names = await probeTooltipNames(cdp, 8, documents, semantics(documents)); expect(names.size).toBe(0); expect(cdp.send).not.toHaveBeenCalled(); - expect(cdp.sendToTarget).not.toHaveBeenCalled(); + expect(cdp.sendToFrame).not.toHaveBeenCalled(); }); it("does not collect supplemental names from form controls", async () => { - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], - sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"], - }; - const inputDocument = document({ tabId: 8 }, 602); + sendToFrame: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToFrame"], + }); + const inputDocument = document(toFrameDocumentHandle(8, { frameId: "child" }), 602); inputDocument.domNodes[0] = { ...inputDocument.domNodes[0], tag: "input", @@ -129,22 +132,27 @@ describe("tooltip name enrichment", () => { expect(names.size).toBe(0); expect(cdp.send).not.toHaveBeenCalled(); - expect(cdp.sendToTarget).not.toHaveBeenCalled(); + expect(cdp.sendToFrame).not.toHaveBeenCalled(); }); it("remembers controls that revealed no tooltip so the miss is paid once", async () => { - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async () => ({})) as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method) => { + sendToFrame: vi.fn(async (_handle, method) => { if (method === "Page.createIsolatedWorld") return { executionContextId: 31 }; if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } }; if (method === "Runtime.callFunctionOn") return { result: { value: [] } }; return {}; - }) as CdpRunner["sendToTarget"], - }; + }) as CdpRunner["sendToFrame"], + }); const documents = [ - document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"), + document( + toFrameDocumentHandle(11, { frameId: "miss-frame", loaderId: "miss-loader" }), + 801, + undefined, + "miss-frame", + ), ]; const graph = semantics(documents); @@ -165,19 +173,19 @@ describe("tooltip name enrichment", () => { it("keeps tooltip evidence isolated across sibling frame sessions", async () => { let hoveredX = -10; - const cdp: CdpRunner = { + const cdp: CdpRunner = createStubCdpRunner({ send: vi.fn(async (_tabId, method, params) => { if (method === "Input.dispatchMouseEvent") { hoveredX = (params as { x: number }).x; } return {}; }) as CdpRunner["send"], - sendToTarget: vi.fn(async (target, method, params) => { + sendToFrame: vi.fn(async (handle, method, params) => { if (method === "Page.createIsolatedWorld") { - return { executionContextId: target.sessionId === "left-session" ? 21 : 22 }; + return { executionContextId: handle.frameId === "left" ? 21 : 22 }; } if (method === "DOM.resolveNode") { - return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } }; + return { object: { objectId: handle.frameId === "left" ? "left" : "right" } }; } if (method === "Runtime.callFunctionOn") { const objectId = (params as { objectId: string }).objectId; @@ -190,11 +198,23 @@ describe("tooltip name enrichment", () => { return { result: { value } }; } return {}; - }) as CdpRunner["sendToTarget"], - }; + }) as CdpRunner["sendToFrame"], + }); const documents = [ - document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10), - document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110), + document( + toFrameDocumentHandle(9, { frameId: "left", loaderId: "left-loader" }), + 701, + undefined, + "left", + 10, + ), + document( + toFrameDocumentHandle(9, { frameId: "right", loaderId: "right-loader" }), + 701, + undefined, + "right", + 110, + ), ]; const names = await probeTooltipNames(cdp, 9, documents, semantics(documents)); diff --git a/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts b/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts index 77a08d490..fab9a78eb 100644 --- a/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts +++ b/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { rootProcessBoundary, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { normalizeDocument } from "../normalize"; import { buildSemanticVomScene } from "../semantic-graph"; import { BASIC_SNAPSHOT, type SnapshotProfile, VISUAL_SNAPSHOT } from "../snapshot"; @@ -37,7 +38,7 @@ async function normalize(input: Spec[], profile: SnapshotProfile = BASIC_SNAPSHO }, strings, { - target: { tabId: 1 }, + process: rootProcessBoundary(1), frameId: "main", ownerFrameBackendNodeId: null, projection: null, @@ -79,8 +80,11 @@ describe("custom progress form membership", () => { documents: [ { frameId: "main", + loaderId: "", + processKey: "root", + isOutOfProcess: false, contextScopeId: "main", - target: { tabId: 1 }, + handle: toFrameDocumentHandle(1, { frameId: "main" }), axNodes: [], domNodes: normalized.nodes, }, diff --git a/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts index 84e556618..ce125e857 100644 --- a/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts +++ b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import { RECORD_DOCUMENT_ATTRIBUTE, recordingDocumentMarkerValue, @@ -13,7 +14,10 @@ describe("projectRecordSafeObservation", () => { frameDocuments: [ { frameId: "child", - target: { tabId: 4, sessionId: "oopif-session" }, + loaderId: "", + processKey: "root", + isOutOfProcess: false, + handle: toFrameDocumentHandle(4, { frameId: "child" }), contextScopeId: "scope-1", axNodes: [], domNodes: [ @@ -56,7 +60,10 @@ describe("projectRecordSafeObservation", () => { frameDocuments: [ { frameId: "root", - target: { tabId: 4 }, + loaderId: "", + processKey: "root", + isOutOfProcess: false, + handle: toFrameDocumentHandle(4, { frameId: "root" }), contextScopeId: "scope-1", axNodes: [], domNodes: [ diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index 2de7b1d45..6c8890689 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -1,5 +1,6 @@ import { isVomReferenceNode, renderVom } from "@browser-skill/vom"; import { describe, expect, it } from "vitest"; +import { rootProcessBoundary, toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import type { CapturedNode } from "../capture"; import type { FrameDocument } from "../frame-document"; import { normalizeDocument } from "../normalize"; @@ -39,8 +40,11 @@ function document( ): FrameDocument { return { frameId, + loaderId: "", + processKey: "root", + isOutOfProcess: false, contextScopeId: frameId, - target: { tabId: 1 }, + handle: toFrameDocumentHandle(1, { frameId }), axNodes, domNodes, ...frame, @@ -1168,12 +1172,12 @@ it("strips direct hidden #text from textstyle stealth containers while preservin strings, { frameId: "main", - target: { tabId: 1 }, + process: rootProcessBoundary(1), ownerFrameBackendNodeId: null, projection: { status: "available", projection: { - source: { target: { tabId: 1 }, frameId: "main" }, + source: { process: rootProcessBoundary(1), frameId: "main" }, geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } }, }, }, @@ -1220,8 +1224,11 @@ it("strips direct hidden #text from textstyle stealth containers while preservin const frameDoc: FrameDocument = { frameId: "main", + loaderId: "", + processKey: "root", + isOutOfProcess: false, contextScopeId: "main", - target: { tabId: 1 }, + handle: toFrameDocumentHandle(1, { frameId: "main" }), axNodes, domNodes: normalized.nodes, excludedBackendNodeIds: normalized.index.excludedBackendNodeIds, diff --git a/apps/extension/src/tools/vom/__tests__/semantic-resolve.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-resolve.test.ts index 7f72b49c8..3de6f3822 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-resolve.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-resolve.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { toFrameDocumentHandle } from "@/browser-driver/frame-handles"; import type { CapturedNode } from "../capture"; import { buildSemanticGraph, resolveSemanticGraph, type SemanticAxNode } from "../semantic-graph"; @@ -21,7 +22,16 @@ function graph(domNodes: CapturedNode[], axNodes: SemanticAxNode[] = []) { viewport: { width: 800, height: 600 }, rootFrameId: "main", documents: [ - { frameId: "main", contextScopeId: "main", target: { tabId: 1 }, domNodes, axNodes }, + { + frameId: "main", + loaderId: "", + processKey: "root", + isOutOfProcess: false, + contextScopeId: "main", + handle: toFrameDocumentHandle(1, { frameId: "main" }), + domNodes, + axNodes, + }, ], }); } @@ -209,4 +219,32 @@ describe("semantic sibling context", () => { expect(byBackend(resolved, 18)?.rowAnchor).toBe("settlement-2026Q4.csv"); expect(byBackend(resolved, 25)?.rowAnchor).toBeUndefined(); }); + + it("keeps popup state in structured hasPopup/expanded fields without mutating vom.name", () => { + const resolved = resolveSemanticGraph( + graph([ + dom(1, "main"), + { + ...dom(10, "div", "Filter"), + attrs: { role: "combobox", "aria-haspopup": "listbox", "aria-expanded": "true" }, + }, + { + ...dom(11, "button", "More actions"), + attrs: { "aria-haspopup": "menu", "aria-expanded": "false" }, + }, + ]), + ); + expect(byBackend(resolved, 10)).toMatchObject({ + role: "combobox", + name: "Filter", + hasPopup: true, + expanded: true, + }); + expect(byBackend(resolved, 11)).toMatchObject({ + role: "button", + name: "More actions", + hasPopup: true, + expanded: false, + }); + }); }); diff --git a/apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts b/apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts index fd391ddb0..e935ea704 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts @@ -117,8 +117,18 @@ describe("visual candidate deduplication", () => { const input: VisualDiscoveryResult = { ...discovery([candidate(2)]), complete: false, - issues: [{ reason: "geometry-unavailable", target: { tabId: 1 }, frameId: "missing" }], - captureIssues: [{ stage: "ax", reason: "capture-unavailable", target: { tabId: 1 } }], + issues: [ + { reason: "geometry-unavailable", tabId: 1, processKey: "root", frameId: "missing" }, + ], + captureIssues: [ + { + stage: "ax", + reason: "capture-unavailable", + tabId: 1, + processKey: "root", + frameId: "main", + }, + ], }; const result = await deduplicateVisualCandidates(input); expect(result.complete).toBe(false); diff --git a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts index 5b24cf0c0..c61410cce 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import type { CdpFrame } from "@/browser-driver/frame-graph"; +import { + createFrameDocumentInfo, + type FrameDocumentInfo, + frameProcessBoundary, +} from "@/browser-driver/frame-handles"; import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; import type { GeometryProjection } from "../../geometry"; import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts"; @@ -49,14 +53,14 @@ interface Spec { async function document( nodes: Spec[], options: { - frame?: CdpFrame; + frame?: FrameDocumentInfo; projection?: GeometryProjection; pageScale?: number; scrollY?: number; layoutScale?: number; } = {}, ): Promise> { - const frame = options.frame ?? { frameId: "main", target: { tabId: 1 } }; + const frame = options.frame ?? createFrameDocumentInfo({ tabId: 1, frameId: "main" }); const strings: string[] = []; const str = (value: string) => { const index = strings.indexOf(value); @@ -98,11 +102,11 @@ async function document( { frameId: frame.frameId, ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null, - target: frame.target, + process: frameProcessBoundary(frame), projection: { status: "available", projection: { - source: { target: frame.target, frameId: frame.frameId }, + source: { process: frameProcessBoundary(frame), frameId: frame.frameId }, geometry: options.projection ?? projection, }, }, @@ -119,9 +123,9 @@ async function document( frame, identity: { attachmentId: "attached", - tabId: frame.target.tabId, + tabId: frame.handle.tabId, frameId: frame.frameId, - loaderId: frame.loaderId ?? "loader-1", + loaderId: frame.loaderId || "loader-1", documentElementBackendNodeId: 1, }, index: normalized.index, @@ -154,12 +158,12 @@ describe("Canvas discovery", () => { { id: 4, parent: 1 }, ], { - frame: { + frame: createFrameDocumentInfo({ + tabId: 1, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 2, - target: { tabId: 1 }, - }, + }), }, ); const result = await discoverVisualCandidates(facts([child, parent])); @@ -198,12 +202,12 @@ describe("Canvas discovery", () => { }); const parent = await document([{ id: 2, parent: 1, tag: "iframe", styles: {} }]); const child = await document([{ id: 3, parent: 1 }], { - frame: { + frame: createFrameDocumentInfo({ + tabId: 1, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 2, - target: { tabId: 1 }, - }, + }), }); expect(await discoverVisualCandidates(facts([parent, child]))).toMatchObject({ candidates: [{ backendNodeId: 3 }], @@ -346,12 +350,14 @@ describe("Canvas discovery", () => { ]); parent.domNodes.splice(0); // Visual discovery never reads the semantic subset. const child = await document([{ id: 2, parent: 1, bounds: [0, 0, 200, 200] }], { - frame: { + frame: createFrameDocumentInfo({ + tabId: 1, frameId: "child", parentFrameId: "main", ownerBackendNodeId: 3, - target: { tabId: 1, sessionId: "remote" }, - }, + processKey: "oopif:remote", + isOutOfProcess: true, + }), projection: { ...projection, edges: [ @@ -438,7 +444,15 @@ describe("Canvas discovery", () => { const doc = await document([{ id: 2, parent: 1 }]); const partial = facts( [doc], - [{ target: doc.frame.target, frameId: "main", stage: "ax", reason: "capture-unavailable" }], + [ + { + tabId: doc.frame.handle.tabId, + processKey: doc.frame.processKey, + frameId: "main", + stage: "ax", + reason: "capture-unavailable", + }, + ], ); const result = await discoverVisualCandidates(partial); expect(result.candidates).toHaveLength(1); diff --git a/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts b/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts index 708a84f31..8637370b3 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-observation.test.ts @@ -10,7 +10,7 @@ import { RefStore } from "@/session-manager/ref-store"; import type { ObserveResult } from "@/transport/types"; import { auditContext } from "../../audit-context"; import { handleObserve, handleSnapshot } from "../../observation"; -import type { CdpRunner } from "../../shared"; +import { type CdpRunner, createStubCdpRunner } from "../../shared"; import type { DocumentIdentity } from "../facts"; import { buildSemanticGraph, @@ -66,6 +66,7 @@ function fixture(count = 20, maxTokens?: number) { visuals: candidates.map((_, key) => ({ key, parentId: 1, frameId: "top" })), }; const output: ObservationOutput = { + rootFrameId: "top", candidates, identities: new Map([["top", identity]]), rootIdentity: identity, @@ -86,7 +87,10 @@ function fixture(count = 20, maxTokens?: number) { if (method === "Runtime.releaseObjectGroup") return {}; throw new Error(`unexpected ${method}`); }); - const cdp = { send, getAttachmentId: () => "a" } as unknown as CdpRunner; + const cdp = createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], + getAttachmentId: () => "a", + }); return { output, scene, @@ -122,9 +126,11 @@ describe("visual observation output", () => { viewport: { width: 1200, height: 800 }, documents: [ { - target: { tabId: 4 }, + handle: { tabId: 4, frameId: "top", loaderId: identity.loaderId }, frameId: "top", loaderId: identity.loaderId, + processKey: "root", + isOutOfProcess: false, contextScopeId: "top", axNodes: [], domNodes: [1, 100, 101].map((id) => ({ @@ -290,7 +296,10 @@ describe("visual observation output", () => { const docs: Parameters[0]["documents"] = [ { frameId: "top", - target: { tabId: 4 }, + loaderId: "", + processKey: "root", + isOutOfProcess: false, + handle: { tabId: 4, frameId: "top", loaderId: "" }, contextScopeId: "top", domNodes: [ { @@ -395,7 +404,10 @@ describe("visual observation output", () => { documents: [ { frameId: "top", - target: { tabId: 4 }, + loaderId: "", + processKey: "root", + isOutOfProcess: false, + handle: { tabId: 4, frameId: "top", loaderId: "" }, contextScopeId: "top", axNodes: [], domNodes: [ @@ -423,9 +435,12 @@ describe("visual observation output", () => { }, { frameId: "child", + loaderId: "", + processKey: "root", + isOutOfProcess: false, parentFrameId: "top", ownerBackendNodeId: 2, - target: { tabId: 4, sessionId: "oopif" }, + handle: { tabId: 4, frameId: "child", loaderId: "" }, contextScopeId: "child", axNodes: [], domNodes: [ @@ -581,14 +596,24 @@ it.each([ if (method === "Runtime.releaseObjectGroup" || method.endsWith(".enable")) return {}; throw new Error(`unexpected ${method}`); }); - const cdp = { - send, + const cdp: CdpRunner = createStubCdpRunner({ + send: send as unknown as CdpRunner["send"], + sendToFrame: ((_handle: unknown, method: string, params?: Record) => + send(4, method, params)) as CdpRunner["sendToFrame"], getAttachmentId: () => "a", - getFrameGraph: async () => ({ + getFrameDocuments: async () => ({ rootFrameId: "top", - frames: [{ frameId: "top", target: { tabId: 4 } }], + frames: [ + { + frameId: "top", + loaderId: "", + handle: { tabId: 4, frameId: "top", loaderId: "" }, + processKey: "root", + isOutOfProcess: false, + }, + ], }), - } as unknown as CdpRunner; + }); const tab = { id: 4, windowId: 100, active: true, url: "http://fixture.test" } as chrome.tabs.Tab; const deps = { cdp, tabsApi: { get: async () => tab, query: async () => [tab] } }; const observed = await handleObserve(manager, { session_id: "test" }, deps); diff --git a/apps/extension/src/tools/vom/capture-coordinator.ts b/apps/extension/src/tools/vom/capture-coordinator.ts index d2665960e..a49e96780 100644 --- a/apps/extension/src/tools/vom/capture-coordinator.ts +++ b/apps/extension/src/tools/vom/capture-coordinator.ts @@ -1,19 +1,17 @@ import { CdpReadTimeoutError } from "@/browser-driver/command-deadline"; import { - type CdpFrame, - type CdpFrameGraph, - type CdpTarget, - cdpTargetKey, - omitFrameSubtrees, -} from "@/browser-driver/frame-graph"; + type FrameDocumentGraph, + type FrameDocumentInfo, + type FrameProcessBoundary, + frameProcessBoundary, + frameProcessIdentityKey, + omitFrameDocumentSubtrees, + rootProcessBoundary, + toFrameDocumentHandle, + type UnavailableFrameDocument, +} from "@/browser-driver/frame-handles"; import { cssViewport, GeometryContext } from "../geometry/frame-context"; -import { - type CdpDispatchGuard, - type CdpRunner, - cdpRunnerForTarget, - delegateCdpRunner, - sendToCdpTarget, -} from "../shared"; +import { type CdpRunner, cdpRunnerForFrame } from "../shared"; import { collectOverlayExcludedBackendIds } from "./capture"; import { isCaptureTerminalError as isCaptureAbort, @@ -38,8 +36,8 @@ import { } from "./snapshot"; interface TargetBatch { - target: CdpTarget; - frames: CdpFrame[]; + process: FrameProcessBoundary; + frames: FrameDocumentInfo[]; snapshotAttachmentId?: string; documents: PreparedFrameDocument[]; ax: FrameAxBatch[]; @@ -89,7 +87,7 @@ async function collectTasks( export const ACCESSIBILITY_LEASE_TTL_MS = 10_000; interface LeasedAccessibilityTarget { - target: CdpTarget; + process: FrameProcessBoundary; attachmentId: string; needsEnable: boolean; } @@ -112,11 +110,15 @@ function unregisterTabLease(tabId: number, lease: AccessibilityLease): void { if (set.size === 0) leasesByTab.delete(tabId); } -function isTargetLeased(tabId: number, target: CdpTarget, attachmentId: string): boolean { +function isProcessLeased( + tabId: number, + process: FrameProcessBoundary, + attachmentId: string, +): boolean { const set = leasesByTab.get(tabId); if (!set) return false; for (const lease of set) { - if (lease.holdsAttachment(target, attachmentId)) return true; + if (lease.holdsAttachment(process, attachmentId)) return true; } return false; } @@ -139,15 +141,15 @@ export class AccessibilityLease { private readonly ttlMs = ACCESSIBILITY_LEASE_TTL_MS, ) {} - hasActive(target: CdpTarget, attachmentId: string | undefined): boolean { + hasActive(process: FrameProcessBoundary, attachmentId: string | undefined): boolean { if (this.disposed || attachmentId === undefined) return false; - const entry = this.entries.get(cdpTargetKey(target)); + const entry = this.entries.get(frameProcessIdentityKey(process)); return entry !== undefined && entry.attachmentId === attachmentId && !entry.needsEnable; } - holdsAttachment(target: CdpTarget, attachmentId: string): boolean { + holdsAttachment(process: FrameProcessBoundary, attachmentId: string): boolean { if (this.disposed) return false; - const entry = this.entries.get(cdpTargetKey(target)); + const entry = this.entries.get(frameProcessIdentityKey(process)); return entry !== undefined && entry.attachmentId === attachmentId; } @@ -170,7 +172,11 @@ export class AccessibilityLease { forgetTimedOut(error: CdpReadTimeoutError): void { if (error.tabId !== this.tabId) return; for (const [key, entry] of this.entries) { - if (!error.sessionId || entry.target.sessionId === error.sessionId) { + if ( + !error.processKey || + error.processKey === "root" || + entry.process.processKey === error.processKey + ) { this.entries.delete(key); } } @@ -179,29 +185,29 @@ export class AccessibilityLease { retain( cdp: CdpRunner, - enabled: ReadonlyMap, - ): Map { - const unretained = new Map(); + enabled: ReadonlyMap, + ): Map { + const unretained = new Map(); if (this.disposed) { - for (const [target, attachmentId] of enabled) unretained.set(target, attachmentId); + for (const [process, attachmentId] of enabled) unretained.set(process, attachmentId); return unretained; } - const currentAttachmentId = cdp.getAttachmentId?.(this.tabId); + const currentAttachmentId = cdp.getAttachmentId(this.tabId); for (const [key, entry] of this.entries) { if (entry.attachmentId !== currentAttachmentId) { this.entries.delete(key); - unretained.set(entry.target, entry.attachmentId); + unretained.set(entry.process, entry.attachmentId); } } - for (const [target, attachmentId] of enabled) { + for (const [process, attachmentId] of enabled) { if (attachmentId !== undefined && attachmentId === currentAttachmentId) { - this.entries.set(cdpTargetKey(target), { - target, + this.entries.set(frameProcessIdentityKey(process), { + process, attachmentId, needsEnable: false, }); } else { - unretained.set(target, attachmentId); + unretained.set(process, attachmentId); } } this.cdp = cdp; @@ -209,12 +215,12 @@ export class AccessibilityLease { return unretained; } - takeEntries(): Map { + takeEntries(): Map { this.clearTimer(); unregisterTabLease(this.tabId, this); - const drained = new Map(); + const drained = new Map(); for (const entry of this.entries.values()) { - drained.set(entry.target, entry.attachmentId); + drained.set(entry.process, entry.attachmentId); } this.entries.clear(); return drained; @@ -270,17 +276,24 @@ export class AccessibilityLease { async function releaseAccessibility( cdp: CdpRunner, tabId: number, - enabled: ReadonlyMap, + enabled: ReadonlyMap, ): Promise { - await collectTasks([...enabled], async ([target, attachmentId]) => { + await collectTasks([...enabled], async ([process, attachmentId]) => { // A detach already disabled every domain of that attachment. - if (cdp.getAttachmentId?.(tabId) !== attachmentId) return; - if (attachmentId !== undefined && isTargetLeased(tabId, target, attachmentId)) return; - const disabled = - cdp.sendGuarded && attachmentId !== undefined - ? cdp.sendGuarded(target, "Accessibility.disable", {}, { attachmentId }) - : sendToCdpTarget(cdp, target, "Accessibility.disable", {}); - await disabled.catch(() => {}); + if (cdp.getAttachmentId(tabId) !== attachmentId) return; + if (attachmentId !== undefined && isProcessLeased(tabId, process, attachmentId)) return; + await cdp + .sendToFrame( + process.rootHandle, + "Accessibility.disable", + {}, + { + ...(attachmentId !== undefined ? { attachmentId } : {}), + allowReattach: false, + ignoreFrameNavigation: true, + }, + ) + .catch(() => {}); }); } @@ -288,12 +301,19 @@ async function releaseAccessibility( * every frame on the tab; a child command gates only its original target. */ function forgetPendingAccessibility( tabId: number, - enabled: Map, + enabled: Map, error: unknown, ): void { if (!(error instanceof CdpReadTimeoutError)) return; - for (const target of enabled.keys()) - if (!error.sessionId || target.sessionId === error.sessionId) enabled.delete(target); + for (const process of enabled.keys()) { + if ( + !error.processKey || + error.processKey === "root" || + process.processKey === error.processKey + ) { + enabled.delete(process); + } + } const leases = leasesByTab.get(tabId); if (leases) { for (const lease of [...leases]) lease.forgetTimedOut(error); @@ -328,7 +348,7 @@ export async function withObservationFacts( pageUrl?: string, options: { includeVisualFacts?: boolean; accessibilityLease?: AccessibilityLease } = {}, ): Promise { - const axEnabled = new Map(); + const axEnabled = new Map(); const lease = options.accessibilityLease; lease?.beginUse(); let toRelease = axEnabled; @@ -344,10 +364,10 @@ export async function withObservationFacts( } catch (error) { forgetPendingAccessibility(tabId, axEnabled, error); if (lease) { - const seenKeys = new Set([...axEnabled.keys()].map(cdpTargetKey)); - for (const [target, attachmentId] of lease.takeEntries()) { - if (!seenKeys.has(cdpTargetKey(target))) { - axEnabled.set(target, attachmentId); + const seenKeys = new Set([...axEnabled.keys()].map(frameProcessIdentityKey)); + for (const [process, attachmentId] of lease.takeEntries()) { + if (!seenKeys.has(frameProcessIdentityKey(process))) { + axEnabled.set(process, attachmentId); } } } @@ -368,7 +388,7 @@ export async function withObservationFacts( async function collectObservationFacts( cdp: CdpRunner, tabId: number, - axEnabled: Map, + axEnabled: Map, signal: AbortSignal | undefined, pageUrl: string | undefined, options: { includeVisualFacts?: boolean; accessibilityLease?: AccessibilityLease }, @@ -376,9 +396,9 @@ async function collectObservationFacts( const profile = options.includeVisualFacts ? VISUAL_SNAPSHOT : BASIC_SNAPSHOT; const startedAt = Date.now(); throwCaptureAborted(signal); - let graph: CdpFrameGraph | undefined; + let graph: FrameDocumentGraph | undefined; try { - graph = await cdp.getFrameGraph?.(tabId, signal); + graph = await cdp.getFrameDocuments(tabId, signal); } catch (error) { forgetPendingAccessibility(tabId, axEnabled, error); if (isCaptureAbort(error)) throw error; @@ -386,7 +406,8 @@ async function collectObservationFacts( throwCaptureAborted(signal); const geometry = new GeometryContext(cdp, tabId, graph, signal); const issues: CaptureIssue[] = (graph?.unavailableFrames ?? []).map((frame) => ({ - target: frame.target, + tabId, + processKey: frame.processKey, frameId: frame.frameId, stage: "dom", reason: "capture-unavailable", @@ -395,56 +416,81 @@ async function collectObservationFacts( const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []); const groups = new Map>(); for (const frame of graph?.frames ?? []) { - const key = cdpTargetKey(frame.target); + const process = frameProcessBoundary(frame); + const key = frameProcessIdentityKey(process); + const isTargetRoot = + !frame.parentFrameId || graphFrames.get(frame.parentFrameId)?.processKey !== frame.processKey; let group = groups.get(key); if (!group) { group = { - target: frame.target, + process, frames: [], documents: [], ax: [], fallbackExcluded: new Set(), }; groups.set(key, group); + } else if (process.isOutOfProcess && isTargetRoot) { + group.process = process; } group.frames.push( - !frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl + !frame.isOutOfProcess && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl ? { ...frame, url: pageUrl } : frame, ); } - if (!groups.has(cdpTargetKey({ tabId }))) - groups.set(cdpTargetKey({ tabId }), { - target: { tabId }, + const rootProcess = rootProcessBoundary(tabId); + if (!groups.has(frameProcessIdentityKey(rootProcess))) { + groups.set(frameProcessIdentityKey(rootProcess), { + process: rootProcess, frames: [], documents: [], ax: [], fallbackExcluded: new Set(), }); + } const batches = [...groups.values()]; const unavailable = (batch: TargetBatch, stage: CaptureIssue["stage"], frameId?: string) => - issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" }); + issues.push({ + tabId: batch.process.tabId, + processKey: batch.process.processKey, + frameId, + stage, + reason: "capture-unavailable", + }); let firstFailure: unknown; const stopped = new Set>(); const isolateChildTimeout = (error: unknown, batch: TargetBatch): boolean => { if ( !(error instanceof CdpReadTimeoutError) || - !batch.target.sessionId || - !error.sessionId || + !batch.process.isOutOfProcess || + !error.processKey || + error.processKey === "root" || error.tabId !== tabId ) return false; - const failed = batches.find((item) => item.target.sessionId === error.sessionId); + const failed = batches.find((item) => item.process.processKey === error.processKey); if (!failed) return false; // Projection of a nested OOPIF can fail in an ancestor's session. Omit // that dependency subtree as well, without turning it into a root failure. - const omitted = omitFrameSubtrees( + const omitted = omitFrameDocumentSubtrees( { rootFrameId: graph?.rootFrameId ?? "root", frames: batches.flatMap((item) => item.frames) }, - new Map([...failed.frames, ...batch.frames].map((frame) => [frame.frameId, frame.target])), + new Map( + [...failed.frames, ...batch.frames].map((frame) => [ + frame.frameId, + { + frameId: frame.frameId, + ...(frame.parentFrameId ? { parentFrameId: frame.parentFrameId } : {}), + ...(frame.url ? { url: frame.url } : {}), + processKey: frame.processKey, + isOutOfProcess: frame.isOutOfProcess, + }, + ]), + ), ); for (const frame of omitted.unavailableFrames ?? []) unavailableFrameIds.add(frame.frameId); for (const item of batches) { - if (!item.target.sessionId) continue; + if (!item.process.isOutOfProcess) continue; if (item !== batch && !item.frames.some((frame) => unavailableFrameIds.has(frame.frameId))) continue; if (!stopped.has(item)) unavailable(item, "dom"); @@ -456,19 +502,22 @@ async function collectObservationFacts( batches, async (batch) => { if (stopped.has(batch)) return; - const scoped = cdpRunnerForTarget(cdp, batch.target); + const scoped = cdpRunnerForFrame(cdp, batch.process.rootHandle, { + ignoreFrameNavigation: true, + signal, + }); try { throwCaptureAborted(signal); await scoped.send(tabId, "DOMSnapshot.enable", {}); throwCaptureAborted(signal); - batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId); + batch.snapshotAttachmentId = cdp.getAttachmentId(tabId); const snapshot = await scoped.send(tabId, "DOMSnapshot.captureSnapshot", { computedStyles: profile.computedStyles, includePaintOrder: true, includeDOMRects: true, }); throwCaptureAborted(signal); - const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl); + const observed = describeSnapshotFrames(snapshot, batch.process, graphFrames, pageUrl); batch.snapshotFrameIds = new Set( [...observed.ids].filter((id) => !unavailableFrameIds.has(id)), ); @@ -480,7 +529,7 @@ async function collectObservationFacts( ]; batch.documents = await prepareSnapshot( snapshot, - batch.target, + batch.process, batch.frames.filter((frame) => observed.ids.has(frame.frameId)), issues, signal, @@ -507,18 +556,22 @@ async function collectObservationFacts( batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal); } if (stopped.has(batch)) return; - if (!batch.frames.length && !graph) + if (!batch.frames.length && !graph?.frames.length) { batch.frames = [ { + handle: toFrameDocumentHandle(tabId, { frameId: "root", loaderId: "" }), frameId: "root", - target: batch.target, - ...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}), + loaderId: "", + processKey: batch.process.processKey, + isOutOfProcess: batch.process.isOutOfProcess, + ...(!batch.process.isOutOfProcess && pageUrl ? { url: pageUrl } : {}), }, ]; + } try { throwCaptureAborted(signal); - const currentAttachmentId = cdp.getAttachmentId?.(tabId); - if (!options.accessibilityLease?.hasActive(batch.target, currentAttachmentId)) { + const currentAttachmentId = cdp.getAttachmentId(tabId); + if (!options.accessibilityLease?.hasActive(batch.process, currentAttachmentId)) { await scoped.send(tabId, "Accessibility.enable", {}); } } catch (error) { @@ -530,16 +583,16 @@ async function collectObservationFacts( unavailable(batch, "ax"); return; } - axEnabled.set(batch.target, cdp.getAttachmentId?.(tabId)); + axEnabled.set(batch.process, cdp.getAttachmentId(tabId)); }, signal, ); // Schedule AX by frame across all ready targets, rather than nesting pools. // Keep result slots in source order even when CDP replies arrive out of order. - const axTasks: { batch: TargetBatch; frame: CdpFrame; result?: FrameAxBatch }[] = + const axTasks: { batch: TargetBatch; frame: FrameDocumentInfo; result?: FrameAxBatch }[] = batches.flatMap((batch) => - axEnabled.has(batch.target) ? batch.frames.map((frame) => ({ batch, frame })) : [], + axEnabled.has(batch.process) ? batch.frames.map((frame) => ({ batch, frame })) : [], ); await collectTasks( axTasks, @@ -548,10 +601,11 @@ async function collectObservationFacts( if (stopped.has(batch)) return; try { throwCaptureAborted(signal); - const result = await cdpRunnerForTarget(cdp, batch.target).send<{ nodes?: T[] }>( - tabId, + const result = await cdp.sendToFrame<{ nodes?: T[] }>( + batch.process.rootHandle, "Accessibility.getFullAXTree", frame.frameId === "root" ? {} : { frameId: frame.frameId }, + { ignoreFrameNavigation: true, signal }, ); throwCaptureAborted(signal); task.result = { frame, nodes: result.nodes ?? [] }; @@ -574,11 +628,11 @@ async function collectObservationFacts( const claims = new Map(); const invalid = new Set(unavailableFrameIds); for (const batch of batches) { - const targetKey = cdpTargetKey(batch.target); + const processIdentity = frameProcessIdentityKey(batch.process); for (const id of batch.snapshotFrameIds ?? []) { const previous = claims.get(id); - if (previous !== undefined && previous !== targetKey) invalid.add(id); - claims.set(id, targetKey); + if (previous !== undefined && previous !== processIdentity) invalid.add(id); + claims.set(id, processIdentity); } const accepted = new Set(batch.frames.map((frame) => frame.frameId)); for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id); @@ -586,23 +640,28 @@ async function collectObservationFacts( const currentFrames = batches.flatMap((batch) => batch.frames.filter( (frame) => - !claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target), + !claims.has(frame.frameId) || + claims.get(frame.frameId) === frameProcessIdentityKey(batch.process), ), ); - const rootBatch = batches.find((batch) => !batch.target.sessionId); + const rootBatch = batches.find((batch) => !batch.process.isOutOfProcess); const rootFrameId = - rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root"; + rootBatch?.rootFrameId || graph?.rootFrameId || rootBatch?.frames[0]?.frameId || "root"; const ownershipInvalid = new Set(invalid); const identities = new Map(); const selected = new Map(currentFrames.map((frame) => [frame.frameId, frame])); const checks = batches.flatMap((batch) => { const docs = new Map(batch.documents.map((doc) => [doc.frame.frameId, doc])); return batch.frames - .filter( - (frame) => + .filter((frame) => { + const chosen = selected.get(frame.frameId); + return ( !invalid.has(frame.frameId) && - cdpTargetKey(selected.get(frame.frameId)!.target) === cdpTargetKey(batch.target), - ) + chosen !== undefined && + frameProcessIdentityKey({ tabId: chosen.handle.tabId, processKey: chosen.processKey }) === + frameProcessIdentityKey(batch.process) + ); + }) .map((frame) => ({ batch, frame, doc: docs.get(frame.frameId) })); }); const checkIdentities = (recheck = false) => @@ -618,7 +677,8 @@ async function collectObservationFacts( return; if (!batch.snapshotAttachmentId || doc?.documentElementBackendNodeId === undefined) { issues.push({ - target: frame.target, + tabId: frame.handle.tabId, + processKey: frame.processKey, frameId: frame.frameId, stage: "identity", reason: "identity-unverified", @@ -627,35 +687,14 @@ async function collectObservationFacts( } const identity: DocumentIdentity = { attachmentId: batch.snapshotAttachmentId, - tabId: frame.target.tabId, + tabId: frame.handle.tabId, frameId: frame.frameId, loaderId: frame.loaderId ?? "", documentElementBackendNodeId: doc.documentElementBackendNodeId, }; - const identityCdp = cdp.sendToFrame - ? cdp - : delegateCdpRunner(cdp, { - sendToFrame: ( - _handle: unknown, - method: string, - params?: object, - guard?: CdpDispatchGuard, - ) => { - guard?.signal?.throwIfAborted(); - if ( - guard?.attachmentId !== undefined && - cdp.getAttachmentId && - cdp.getAttachmentId(tabId) !== guard.attachmentId - ) { - throw new Error("Debugger attachment changed before dispatch"); - } - guard?.onDispatch?.(); - return sendToCdpTarget(cdp, batch.target, method, params); - }, - }); let verified: Awaited>; try { - verified = await verifyDocumentIdentity(identityCdp, identity, signal); + verified = await verifyDocumentIdentity(cdp, identity, signal); } catch (error) { forgetPendingAccessibility(tabId, axEnabled, error); throwCaptureAborted(signal); @@ -663,10 +702,9 @@ async function collectObservationFacts( for (const id of unavailableFrameIds) invalid.add(id); return; } - const status = - cdp.getAttachmentId?.(tabId) === identity.attachmentId ? verified : "changed"; + const status = cdp.getAttachmentId(tabId) === identity.attachmentId ? verified : "changed"; if (status === "current") { - cdp.recordFrameDocumentElement?.( + cdp.recordFrameDocumentElement( tabId, frame.frameId, identity.loaderId, @@ -676,7 +714,8 @@ async function collectObservationFacts( } else { invalid.add(frame.frameId); issues.push({ - target: frame.target, + tabId: frame.handle.tabId, + processKey: frame.processKey, frameId: frame.frameId, stage: "identity", reason: status === "changed" ? "document-changed" : "identity-unavailable", @@ -690,7 +729,7 @@ async function collectObservationFacts( let layoutTimedOut = false; if (!invalid.has(rootFrameId)) { try { - const measured = cssViewport(await geometry.layoutMetrics({ tabId })); + const measured = cssViewport(await geometry.layoutMetrics(rootProcessBoundary(tabId))); viewport = { width: measured.width, height: measured.height }; } catch (error) { forgetPendingAccessibility(tabId, axEnabled, error); @@ -704,7 +743,8 @@ async function collectObservationFacts( for (const frame of currentFrames) if (!invalid.has(frame.frameId)) issues.push({ - target: frame.target, + tabId: frame.handle.tabId, + processKey: frame.processKey, frameId: frame.frameId, stage: "geometry", reason: "geometry-unavailable", @@ -717,7 +757,7 @@ async function collectObservationFacts( try { await projectSnapshot( batch.documents, - batch.target, + batch.process, batch.frames.filter((frame) => !invalid.has(frame.frameId)), geometry, issues, @@ -742,10 +782,12 @@ async function collectObservationFacts( // Reattachment during optional layout work invalidates even an earlier // successful identity proof. This check requires no renderer command. for (const [frameId, identity] of identities) { - if (cdp.getAttachmentId?.(tabId) !== identity.attachmentId) { + if (cdp.getAttachmentId(tabId) !== identity.attachmentId) { invalid.add(frameId); + const selectedFrame = selected.get(frameId); issues.push({ - target: selected.get(frameId)?.target ?? { tabId }, + tabId, + processKey: selectedFrame?.processKey ?? "root", frameId, stage: "identity", reason: "document-changed", @@ -771,7 +813,8 @@ async function collectObservationFacts( for (const frame of currentFrames) { if (ownershipInvalid.has(frame.frameId)) issues.push({ - target: frame.target, + tabId: frame.handle.tabId, + processKey: frame.processKey, frameId: frame.frameId, stage: "ownership", reason: "frame-ownership-unresolved", @@ -785,9 +828,13 @@ async function collectObservationFacts( ); const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId)); const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); - const belongs = (frame: CdpFrame) => { + const belongs = (frame: FrameDocumentInfo) => { const current = frameById.get(frame.frameId); - return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target); + return ( + current && + frameProcessIdentityKey({ tabId: current.handle.tabId, processKey: current.processKey }) === + frameProcessIdentityKey({ tabId: frame.handle.tabId, processKey: frame.processKey }) + ); }; const decoded = new Map( batches @@ -810,7 +857,8 @@ async function collectObservationFacts( signal, (frame) => issues.push({ - target: frame.target, + tabId: frame.handle.tabId, + processKey: frame.processKey, frameId: frame.frameId, stage: "ownership", reason: "frame-ownership-unresolved", @@ -820,7 +868,9 @@ async function collectObservationFacts( for (const document of documents) { const doc = decoded.get(document.frameId); const index = doc?.index ?? (await buildDocumentIndex([], signal)); - const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded; + const fallback = groups.get( + frameProcessIdentityKey({ tabId: document.handle.tabId, processKey: document.processKey }), + )?.fallbackExcluded; const { domNodes, axNodes, diff --git a/apps/extension/src/tools/vom/document-identity.ts b/apps/extension/src/tools/vom/document-identity.ts index 21b1a1f8b..6b13a2974 100644 --- a/apps/extension/src/tools/vom/document-identity.ts +++ b/apps/extension/src/tools/vom/document-identity.ts @@ -1,4 +1,4 @@ -import { isStaleFrameHandleError } from "@/browser-driver/frame-graph"; +import { isStaleFrameHandleError } from "@/browser-driver/frame-handles"; import { type CdpRunner, sendToFrameHandle } from "../shared"; import { isCaptureTerminalError, throwIfAborted } from "./capture-abort"; import type { DocumentIdentity } from "./facts"; diff --git a/apps/extension/src/tools/vom/facts.ts b/apps/extension/src/tools/vom/facts.ts index 9a7454434..00369fe12 100644 --- a/apps/extension/src/tools/vom/facts.ts +++ b/apps/extension/src/tools/vom/facts.ts @@ -1,5 +1,6 @@ import type { Viewport } from "@browser-skill/vom"; -import type { CdpFrame, CdpTarget, FrameDocumentHandle } from "@/browser-driver/frame-graph"; +import type { FrameDocumentHandle, FrameDocumentInfo } from "@/browser-driver/frame-handles"; +import { INTERACTIVE_ARIA_ROLES } from "@/lib/dom-predicates"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge"; import type { GeometryProjection } from "../geometry"; import type { FrameProjectionIssue, SnapshotCoordinates } from "../geometry/coordinate-types"; @@ -60,26 +61,6 @@ const INTERACTIVE_OR_VISUAL_TAGS = new Set([ "dialog", ]); -const INTERACTIVE_ARIA_ROLES = new Set([ - "button", - "checkbox", - "combobox", - "link", - "listbox", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "option", - "radio", - "searchbox", - "slider", - "spinbutton", - "switch", - "tab", - "textbox", - "treeitem", -]); - const INLINE_OPACITY_RE = /(?:^|;)\s*opacity\s*:\s*([+-]?\d*\.?\d+)/i; const INLINE_FONT_SIZE_RE = /(?:^|;)\s*font-size\s*:\s*([+-]?\d*\.?\d+)\s*px\b/i; const INLINE_CLIP_OVERFLOW_RE = /(?:^|;)\s*overflow(?:-[xy])?\s*:\s*(?:hidden|clip)\b/i; @@ -1760,7 +1741,8 @@ export interface CapturedSceneInput { export interface CaptureIssue { /** Owner failure or inherited blockage; absent for other geometry failures. */ projectionIssue?: FrameProjectionIssue; - target: CdpTarget; + tabId: number; + processKey: string; frameId?: string; stage: "dom" | "ax" | "identity" | "ownership" | "geometry" | "forms"; reason: @@ -1782,7 +1764,7 @@ export interface DocumentGeometry { export interface DocumentFacts { readonly geometry?: DocumentGeometry; - readonly frame: CdpFrame; + readonly frame: FrameDocumentInfo; readonly identity?: DocumentIdentity; readonly index: DocumentIndex; readonly domNodes: CapturedNode[]; diff --git a/apps/extension/src/tools/vom/frame-document.ts b/apps/extension/src/tools/vom/frame-document.ts index d19a2af53..8d5fbf7db 100644 --- a/apps/extension/src/tools/vom/frame-document.ts +++ b/apps/extension/src/tools/vom/frame-document.ts @@ -1,9 +1,8 @@ import { - type CdpFrame, - type CdpFrameGraph, - type CdpTarget, - cdpTargetKey, -} from "@/browser-driver/frame-graph"; + type FrameDocumentGraph, + type FrameDocumentInfo, + frameProcessIdentityKey, +} from "@/browser-driver/frame-handles"; import { createCaptureCheckpoint } from "./capture-abort"; import type { CapturedNode } from "./capture-types"; @@ -16,11 +15,11 @@ export interface FrameOwnedAxNode { } export interface FrameAxBatch { - frame: CdpFrame; + frame: FrameDocumentInfo; nodes: T[]; } -export interface FrameDocument extends CdpFrame { +export interface FrameDocument extends FrameDocumentInfo { contextScopeId: string; axNodes: T[]; domNodes: CapturedNode[]; @@ -42,23 +41,27 @@ interface Ownership { interface OwnedCandidate { node: T; - target: CdpTarget; + processIdentity: string; ownership: Ownership; } -function targetNodeKey(target: CdpTarget, nodeId: string): string { - return `${cdpTargetKey(target)}:${nodeId}`; +function frameProcessKeyOf(frame: FrameDocumentInfo): string { + return frameProcessIdentityKey({ tabId: frame.handle.tabId, processKey: frame.processKey }); } -function targetBackendKey(target: CdpTarget, backendNodeId: number): string { - return `${cdpTargetKey(target)}:${backendNodeId}`; +function targetNodeKey(processIdentity: string, nodeId: string): string { + return `${processIdentity}:${nodeId}`; +} + +function targetBackendKey(processIdentity: string, backendNodeId: number): string { + return `${processIdentity}:${backendNodeId}`; } function frameList( - graph: CdpFrameGraph | null, + graph: Pick | null, batches: FrameAxBatch[], -): CdpFrame[] { - const frames = new Map(); +): FrameDocumentInfo[] { + const frames = new Map(); for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame); for (const batch of batches) { if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame); @@ -67,11 +70,11 @@ function frameList( } export async function buildFrameDocuments( - graph: CdpFrameGraph | null, + graph: Pick | null, batches: FrameAxBatch[], captured: FrameDomInput, signal?: AbortSignal, - unresolved?: (frame: CdpFrame) => void, + unresolved?: (frame: FrameDocumentInfo) => void, ): Promise[]> { const checkpoint = createCaptureCheckpoint(signal); let work = 0; @@ -82,17 +85,19 @@ export async function buildFrameDocuments( captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []); const backendOwner = new Map(); for (const frame of frames) { + const processIdentity = frameProcessKeyOf(frame); for (const node of domNodesForFrame(frame.frameId)) { if (work++ % 256 === 0) { const pending = checkpoint(); if (pending) await pending; } - backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId); + backendOwner.set(targetBackendKey(processIdentity, node.backendNodeId), frame.frameId); } } const candidates = new Map>(); for (const batch of batches) { + const batchProcessIdentity = frameProcessKeyOf(batch.frame); const nodeById = new Map(); for (const node of batch.nodes) { if (work++ % 256 === 0) { @@ -123,7 +128,7 @@ export async function buildFrameDocuments( if (current.frameId) { const frame = frameById.get(current.frameId); // An explicit contradiction must not fall through to a guessed owner. - if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) { + if (!frame || frameProcessKeyOf(frame) !== batchProcessIdentity) { if (!reported) { unresolved?.(batch.frame); reported = true; @@ -137,7 +142,7 @@ export async function buildFrameDocuments( } const frameId = typeof current.backendDOMNodeId === "number" - ? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId)) + ? backendOwner.get(targetBackendKey(batchProcessIdentity, current.backendDOMNodeId)) : undefined; if (frameId) { ownership = { frameId, strength: 3 }; @@ -157,10 +162,10 @@ export async function buildFrameDocuments( } if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership); if (!ownership.frameId) continue; - const key = targetNodeKey(batch.frame.target, node.nodeId); + const key = targetNodeKey(batchProcessIdentity, node.nodeId); const existing = candidates.get(key); if (!existing || ownership.strength > existing.ownership.strength) { - candidates.set(key, { node, target: batch.frame.target, ownership }); + candidates.set(key, { node, processIdentity: batchProcessIdentity, ownership }); } } } @@ -174,12 +179,13 @@ export async function buildFrameDocuments( const pending = checkpoint(); if (pending) await pending; } - const { node, target, ownership } = candidate; + const { node, processIdentity, ownership } = candidate; const parentFrameId = node.parentId - ? ownershipByTargetNode.get(targetNodeKey(target, node.parentId)) + ? ownershipByTargetNode.get(targetNodeKey(processIdentity, node.parentId)) : undefined; const childIds = node.childIds?.filter( - (childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId, + (childId) => + ownershipByTargetNode.get(targetNodeKey(processIdentity, childId)) === ownership.frameId, ); const ownedNode = { ...node, diff --git a/apps/extension/src/tools/vom/name-enrichment.ts b/apps/extension/src/tools/vom/name-enrichment.ts index acff86a1e..0a4c8817e 100644 --- a/apps/extension/src/tools/vom/name-enrichment.ts +++ b/apps/extension/src/tools/vom/name-enrichment.ts @@ -1,5 +1,5 @@ import { isVomReferenceNode } from "@browser-skill/vom"; -import { type CdpRunner, sendToCdpTarget } from "../shared"; +import { type CdpRunner, withRemoteObject } from "../shared"; import type { FrameDocument, FrameOwnedAxNode } from "./frame-document"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; import { stableIdentifierName } from "./semantic-graph/name-evidence"; @@ -42,7 +42,7 @@ function cacheKey( document: FrameDocument, backendNodeId: number, ): string { - return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`; + return `${document.handle.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`; } /** Returns the cache entry, or `undefined` when this control was never probed. */ @@ -181,9 +181,8 @@ async function executionContextId( cdp: CdpRunner, document: FrameDocument, ): Promise { - const result = await sendToCdpTarget<{ executionContextId?: number }>( - cdp, - document.target, + const result = await cdp.sendToFrame<{ executionContextId?: number }>( + document.handle, "Page.createIsolatedWorld", { frameId: document.frameId, @@ -200,9 +199,9 @@ async function resolvedObjectId( contextId: number, backendNodeId: number, ): Promise { - const result = await sendToCdpTarget<{ + const result = await cdp.sendToFrame<{ object?: { objectId?: string }; - }>(cdp, document.target, "DOM.resolveNode", { + }>(document.handle, "DOM.resolveNode", { backendNodeId, executionContextId: contextId, }); @@ -214,9 +213,9 @@ async function tooltipTexts( document: FrameDocument, objectId: string, ): Promise { - const result = await sendToCdpTarget<{ + const result = await cdp.sendToFrame<{ result?: { value?: string[] }; - }>(cdp, document.target, "Runtime.callFunctionOn", { + }>(document.handle, "Runtime.callFunctionOn", { functionDeclaration: tooltipCollectorFunction(), objectId, returnByValue: true, @@ -224,16 +223,6 @@ async function tooltipTexts( return result.result?.value ?? []; } -async function releaseObject( - cdp: CdpRunner, - document: FrameDocument, - objectId: string, -): Promise { - await sendToCdpTarget(cdp, document.target, "Runtime.releaseObject", { objectId }).catch( - () => undefined, - ); -} - export async function probeTooltipNames( cdp: CdpRunner, tabId: number, @@ -265,7 +254,6 @@ export async function probeTooltipNames( for (const candidate of pending) { if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError"); if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break; - let objectId: string | undefined; try { let contextId = contexts.get(candidate.document.frameId); if (contextId === undefined) { @@ -273,30 +261,32 @@ export async function probeTooltipNames( if (contextId === undefined) continue; contexts.set(candidate.document.frameId, contextId); } - objectId = await resolvedObjectId( + const objectId = await resolvedObjectId( cdp, candidate.document, contextId, candidate.backendNodeId, ); if (!objectId) continue; - await clearHover(cdp, tabId); - const before = await tooltipTexts(cdp, candidate.document, objectId); - await cdp.send(tabId, "Input.dispatchMouseEvent", { - type: "mouseMoved", - x: candidate.x, - y: candidate.y, + await withRemoteObject(cdp, candidate.document.handle, objectId, async (id) => { + await clearHover(cdp, tabId); + const before = await tooltipTexts(cdp, candidate.document, id); + await cdp.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: candidate.x, + y: candidate.y, + }); + await waitForHover(TOOLTIP_SETTLE_MS, options.signal); + const after = await tooltipTexts(cdp, candidate.document, id); + const name = cleanTooltipDiff(before, after); + storeName(candidate.document, candidate.backendNodeId, name ?? null); + if (name) { + names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name); + } }); - await waitForHover(TOOLTIP_SETTLE_MS, options.signal); - const after = await tooltipTexts(cdp, candidate.document, objectId); - const name = cleanTooltipDiff(before, after); - storeName(candidate.document, candidate.backendNodeId, name ?? null); - if (!name) continue; - names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name); } catch (error) { if (error instanceof Error && error.name === "AbortError") throw error; } finally { - if (objectId) await releaseObject(cdp, candidate.document, objectId); await clearHover(cdp, tabId); } } diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index a05fa43b0..31e0ac3cb 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -1,5 +1,9 @@ import type { Viewport } from "@browser-skill/vom"; -import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentInfo, + type FrameProcessBoundary, + frameProcessIdentityKey, +} from "@/browser-driver/frame-handles"; import { type GeometryProjection, projectRectToViewport } from "../geometry"; import { type FrameProjectionState, @@ -39,7 +43,7 @@ export interface FrameContext { ownerFrameBackendNodeId: number | null; projection: FrameProjectionState | null; targetProjection?: GeometryProjection | null; - target: CdpTarget; + process: FrameProcessBoundary; coordinates: SnapshotCoordinates | null; pageScale?: number; } @@ -193,7 +197,7 @@ async function projectDocument( const bounds = node.layout?.bounds ?? []; const input = snapshotViewportRect( bounds, - { target: context.target, frameId: context.frameId }, + { process: context.process, frameId: context.frameId }, context.coordinates, ); const local = input?.rect; @@ -229,7 +233,7 @@ async function projectDocument( } export interface NormalizedFrameDocument extends NormalizedDocument { - frame: CdpFrame; + frame: FrameDocumentInfo; } export interface PreparedFrameDocument extends NormalizedFrameDocument { @@ -240,8 +244,8 @@ export interface PreparedFrameDocument extends NormalizedFrameDocument { * before optional geometry is allowed to time out and close the read gate. */ export async function prepareSnapshot( snapshot: SnapshotReply, - target: CdpTarget, - frames: readonly CdpFrame[], + process: FrameProcessBoundary, + frames: readonly FrameDocumentInfo[], issues: CaptureIssue[], signal?: AbortSignal, profile: SnapshotProfile = BASIC_SNAPSHOT, @@ -255,7 +259,8 @@ export async function prepareSnapshot( const frame = id ? frameById.get(id) : undefined; if (!id || !frame || seen.has(id)) { issues.push({ - target, + tabId: process.tabId, + processKey: process.processKey, frameId: id, stage: "ownership", reason: "frame-ownership-unresolved", @@ -268,7 +273,7 @@ export async function prepareSnapshot( strings, { frameId: id, - target, + process, ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null, projection: null, coordinates: null, @@ -284,8 +289,8 @@ export async function prepareSnapshot( /** Apply optional geometry after collection and document verification. */ export async function projectSnapshot( documents: readonly PreparedFrameDocument[], - target: CdpTarget, - frames: readonly CdpFrame[], + process: FrameProcessBoundary, + frames: readonly FrameDocumentInfo[], geometry: GeometryContext, issues: CaptureIssue[], signal?: AbortSignal, @@ -299,13 +304,13 @@ export async function projectSnapshot( if (id !== undefined) ownerIds.add(id); } } - geometry.registerSnapshotOwners(target, ownerIds); + geometry.registerSnapshotOwners(process, ownerIds); const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); const sources = new Map(documents.map((doc) => [doc.frame.frameId, doc.snapshot])); const prepared = new Map(documents.map((doc) => [doc.frame.frameId, doc])); let metrics: LayoutMetrics = {}; try { - metrics = await geometry.layoutMetrics(target); + metrics = await geometry.layoutMetrics(process); } catch (error) { if (isCaptureAbort(error)) throw error; } @@ -318,8 +323,8 @@ export async function projectSnapshot( (size) => Number.isFinite(size) && size > 0, ); - const children = new Map(); - const pending: CdpFrame[] = []; + const children = new Map(); + const pending: FrameDocumentInfo[] = []; for (const frame of frames) { if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame); else { @@ -331,10 +336,14 @@ export async function projectSnapshot( const projections = new Map(); let targetProjection: GeometryProjection | null = null; const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined; - if (target.sessionId && targetRoot) { + if (process.isOutOfProcess && targetRoot) { try { const known = await geometry.frame(targetRoot.frameId); - if (known && cdpTargetKey(known.target) === cdpTargetKey(target)) + if ( + known && + frameProcessIdentityKey({ tabId: known.handle.tabId, processKey: known.processKey }) === + frameProcessIdentityKey(process) + ) targetProjection = await geometry.targetProjection(targetRoot.frameId); } catch (error) { if (isCaptureAbort(error)) throw error; @@ -360,7 +369,7 @@ export async function projectSnapshot( for (const child of children.get(frame.frameId) ?? []) pending.push(child); const doc = sources.get(frame.frameId); if (!doc) continue; - const source = { target, frameId: frame.frameId }; + const source = { process, frameId: frame.frameId }; const coordinates = snapshotCoordinates( doc, layoutUnitsPerCssPixel, @@ -382,7 +391,8 @@ export async function projectSnapshot( : (projections.get(frame.frameId) ?? null); if (!state && frame.frameId !== rootFrameId) issues.push({ - target, + tabId: process.tabId, + processKey: process.processKey, frameId: frame.frameId, stage: "ownership", reason: "frame-ownership-unresolved", @@ -391,9 +401,10 @@ export async function projectSnapshot( state = { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }; projections.set(frame.frameId, state); const projection = state?.status === "available" ? state.projection : null; - if (!projection || (target.sessionId && !targetProjection)) + if (!projection || (process.isOutOfProcess && !targetProjection)) issues.push({ - target, + tabId: process.tabId, + processKey: process.processKey, frameId: frame.frameId, stage: "geometry", reason: "geometry-unavailable", @@ -415,7 +426,7 @@ export async function projectSnapshot( Array.from({ length: Math.min(4, siblings.length) }, async () => { while (next < siblings.length && !signal?.aborted && !failure) { const child = siblings[next++]; - const childSource = { target, frameId: child.frameId }; + const childSource = { process, frameId: child.frameId }; if (!projection) { projections.set( child.frameId, @@ -469,8 +480,8 @@ export async function projectSnapshot( ...(profile.includeVisualFacts ? { pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale } : {}), - target, - ...(target.sessionId ? { targetProjection } : {}), + process, + ...(process.isOutOfProcess ? { targetProjection } : {}), coordinates, }, signal, diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts index 3b261f917..f176a2f9e 100644 --- a/apps/extension/src/tools/vom/record-safe-observation.ts +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -1,5 +1,5 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom"; -import type { FrameDocumentHandle } from "@/browser-driver/frame-graph"; +import type { FrameDocumentHandle } from "@/browser-driver/frame-handles"; import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity"; import type { CapturedSurfaceProbe } from "./capture"; import type { @@ -69,7 +69,7 @@ function projectFrames( return { frameId: document.frameId, handle: { - tabId: document.target.tabId, + tabId: document.handle.tabId, frameId: document.frameId, loaderId: document.loaderId ?? "", ...(documentElementBackendNodeId !== undefined ? { documentElementBackendNodeId } : {}), diff --git a/apps/extension/src/tools/vom/semantic-graph/build.ts b/apps/extension/src/tools/vom/semantic-graph/build.ts index 6ebe66418..59edfc552 100644 --- a/apps/extension/src/tools/vom/semantic-graph/build.ts +++ b/apps/extension/src/tools/vom/semantic-graph/build.ts @@ -46,7 +46,8 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap ? { ownerBackendNodeId: document.ownerBackendNodeId } : {}), contextScopeId: document.contextScopeId, - target: document.target, + handle: document.handle, + processKey: document.processKey, ...(document.url ? { url: document.url } : {}), order: frameOrder, }); diff --git a/apps/extension/src/tools/vom/semantic-graph/resolve.ts b/apps/extension/src/tools/vom/semantic-graph/resolve.ts index 0140f27c5..532b4071e 100644 --- a/apps/extension/src/tools/vom/semantic-graph/resolve.ts +++ b/apps/extension/src/tools/vom/semantic-graph/resolve.ts @@ -1,5 +1,9 @@ -import type { VomNode, VomSelectOption } from "@browser-skill/vom"; -import { RECORD_PASSWORD_ATTRIBUTE, sensitiveAutocomplete } from "@/lib/sensitive-autocomplete"; +import { VOM_DERIVED_NAME_CAP, type VomNode, type VomSelectOption } from "@browser-skill/vom"; +import { + INTERACTIVE_ARIA_ROLES, + RECORD_PASSWORD_ATTRIBUTE, + sensitiveAutocomplete, +} from "@/lib/dom-predicates"; import type { NodeFacts } from "../facts"; import { MAX_ROW_ANCHOR_CHARS, @@ -27,25 +31,7 @@ import { frameBackendKey } from "./types"; const FORM_TAGS = new Set(["input", "textarea", "select"]); const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]); const TEXT_AX_ROLES = new Set(["statictext", "inlinetextbox", "labeltext", "text"]); -const INTERACTIVE_ROLES = new Set([ - "button", - "checkbox", - "combobox", - "link", - "listbox", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "option", - "radio", - "searchbox", - "slider", - "spinbutton", - "switch", - "tab", - "textbox", - "treeitem", -]); +const INTERACTIVE_ROLES = INTERACTIVE_ARIA_ROLES; const SENSITIVE_INPUT_TYPES = new Set([ "password", "credit-card", @@ -354,10 +340,11 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes { joinedLength += (parts.length > 0 ? 1 : 0) + childText.length; parts.push(childText); } - if (joinedLength >= 240) break; + if (joinedLength >= VOM_DERIVED_NAME_CAP) break; } resolvingText.delete(nodeId); - const result = parts.length === 0 ? undefined : clean(parts.join(" ").slice(0, 240)); + const result = + parts.length === 0 ? undefined : clean(parts.join(" ").slice(0, VOM_DERIVED_NAME_CAP)); textMemo.set(nodeId, result); return result; }; @@ -395,9 +382,11 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes { joinedLength += (parts.length > 0 ? 1 : 0) + childText.length; parts.push(childText); } - if (joinedLength >= 240) break; + if (joinedLength >= VOM_DERIVED_NAME_CAP) break; } - return complete(parts.length === 0 ? undefined : clean(parts.join(" ").slice(0, 240))); + return complete( + parts.length === 0 ? undefined : clean(parts.join(" ").slice(0, VOM_DERIVED_NAME_CAP)), + ); }; const excludedAxSubtreeMemo = new Map(); @@ -607,7 +596,6 @@ function resolvedName( node: SemanticGraphNode, role: string | undefined, tag: string, - axProps: ExtractedAxProps | undefined, graph: SemanticGraph, indexes: ResolveIndexes, supplementalNames: ReadonlyMap, @@ -650,10 +638,6 @@ function resolvedName( } } } - const hasPopup = axProps?.hasPopup; - if (name && hasPopup && hasPopup !== "false") { - name = axProps?.expanded === "true" ? `${name} [expanded]` : `${name} [has-submenu]`; - } if (!name && role?.toLowerCase() === "iframe") return clean(attrs.id); return name; } @@ -1009,7 +993,6 @@ export function resolveSemanticGraph( node, role, tag, - axProps, graph, indexes, supplementalNames, @@ -1084,7 +1067,31 @@ export function resolveSemanticGraph( }; const selected = axIgnored ? undefined : axProps?.selected; const expanded = axIgnored ? undefined : axProps?.expanded; + const hasPopup = axIgnored ? undefined : axProps?.hasPopup; const controls = axIgnored ? undefined : axProps?.controls; + const ariaHasPopup = clean(attrs["aria-haspopup"])?.toLowerCase(); + const ariaExpanded = clean(attrs["aria-expanded"])?.toLowerCase(); + if ( + (hasPopup !== undefined && hasPopup !== "false") || + (hasPopup === undefined && ariaHasPopup !== undefined && ariaHasPopup !== "false") + ) { + vom.hasPopup = true; + } + const expandedBool = + expanded === "true" + ? true + : expanded === "false" + ? false + : expanded === undefined + ? ariaExpanded === "true" + ? true + : ariaExpanded === "false" + ? false + : undefined + : undefined; + if (expandedBool !== undefined) { + vom.expanded = expandedBool; + } if ((selected === "true" || selected === "false") && !vom.attrs?.["aria-selected"]) { vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": selected }; } diff --git a/apps/extension/src/tools/vom/semantic-graph/types.ts b/apps/extension/src/tools/vom/semantic-graph/types.ts index a656933ad..5d5ee3c6c 100644 --- a/apps/extension/src/tools/vom/semantic-graph/types.ts +++ b/apps/extension/src/tools/vom/semantic-graph/types.ts @@ -1,5 +1,5 @@ import type { Viewport, VomNode } from "@browser-skill/vom"; -import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { FrameDocumentHandle } from "@/browser-driver/frame-handles"; import type { CapturedNode } from "../capture"; export interface SemanticAxNode { @@ -25,7 +25,8 @@ export interface SemanticFrame { ownerBackendNodeId?: number; ownerNodeId?: SemanticNodeId; contextScopeId: string; - target: CdpTarget; + handle: FrameDocumentHandle; + processKey: string; url?: string; order: number; } diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index 62c1a57cb..ffcac2f36 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -1,4 +1,9 @@ -import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentInfo, + type FrameProcessBoundary, + frameProcessIdentityKey, + toFrameDocumentHandle, +} from "@/browser-driver/frame-handles"; import { RECORD_PASSWORD_ATTRIBUTE, sensitiveAutocomplete } from "@/lib/sensitive-autocomplete"; import { createCaptureCheckpoint } from "./capture-abort"; import type { CapturedNode } from "./capture-types"; @@ -350,14 +355,15 @@ export async function decodeDocument( * The earlier graph only supplies compatible metadata and cross-target edges. */ export function describeSnapshotFrames( snapshot: SnapshotReply, - target: CdpTarget, - hintById: ReadonlyMap, + process: FrameProcessBoundary, + hintById: ReadonlyMap, pageUrl?: string, -): { frames: CdpFrame[]; ids: Set; rootFrameId?: string } { +): { frames: FrameDocumentInfo[]; ids: Set; rootFrameId?: string } { const raw = snapshot.documents ?? []; const strings = snapshot.strings ?? []; const ids = new Set(); const invalid = new Set(); + const processIdentity = frameProcessIdentityKey(process); const frames = raw.map((doc) => { const frameId = snapshotFrameId(doc, strings); if (!frameId) return undefined; @@ -365,32 +371,52 @@ export function describeSnapshotFrames( ids.add(frameId); const hint = hintById.get(frameId); const compatible = - hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined; + hint && + frameProcessIdentityKey({ tabId: hint.handle.tabId, processKey: hint.processKey }) === + processIdentity + ? hint + : undefined; + const loaderId = compatible?.loaderId ?? ""; const url = (doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url; - return { + const next: { + handle: FrameDocumentInfo["handle"]; + frameId: string; + loaderId: string; + parentFrameId?: string; + ownerBackendNodeId?: number; + url?: string; + urlFragment?: string; + processKey: string; + isOutOfProcess: boolean; + } = { + handle: compatible?.handle ?? toFrameDocumentHandle(process.tabId, { frameId, loaderId }), frameId, - target, - ...(compatible?.loaderId !== undefined ? { loaderId: compatible.loaderId } : {}), + loaderId, ...(url ? { url } : {}), ...(compatible?.urlFragment !== undefined ? { urlFragment: compatible.urlFragment } : {}), - } as CdpFrame; + processKey: process.processKey, + isOutOfProcess: process.isOutOfProcess, + }; + return next; }); const root = frames[0]; if (root) { const hint = hintById.get(root.frameId); const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined; if ( - target.sessionId && + process.isOutOfProcess && hint && - cdpTargetKey(hint.target) === cdpTargetKey(target) && + frameProcessIdentityKey({ tabId: hint.handle.tabId, processKey: hint.processKey }) === + processIdentity && parent && - cdpTargetKey(parent.target) !== cdpTargetKey(target) + frameProcessIdentityKey({ tabId: parent.handle.tabId, processKey: parent.processKey }) !== + processIdentity ) { root.parentFrameId = parent.frameId; root.ownerBackendNodeId = hint.ownerBackendNodeId; } - if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl; + if (!process.isOutOfProcess && !root.url && pageUrl) root.url = pageUrl; } const owned = new Set(); for (let i = 0; i < raw.length; i++) { @@ -411,7 +437,9 @@ export function describeSnapshotFrames( } } return { - frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)), + frames: frames.filter( + (frame): frame is FrameDocumentInfo => !!frame && !invalid.has(frame.frameId), + ), ids, rootFrameId: root?.frameId, }; diff --git a/apps/extension/src/tools/vom/visual-discovery.ts b/apps/extension/src/tools/vom/visual-discovery.ts index fe6e92458..4f4c59297 100644 --- a/apps/extension/src/tools/vom/visual-discovery.ts +++ b/apps/extension/src/tools/vom/visual-discovery.ts @@ -1,4 +1,3 @@ -import { type CdpTarget } from "@/browser-driver/frame-graph"; import type { ViewportRect } from "../geometry"; import { createCaptureCheckpoint } from "./capture-abort"; import type { @@ -40,12 +39,14 @@ export interface VisualCandidate { export type VisualDiscoveryIssue = | { readonly reason: "visual-facts-not-collected"; - readonly target?: never; + readonly tabId?: never; + readonly processKey?: never; readonly frameId?: never; readonly backendNodeId?: never; } | { - readonly target: CdpTarget; + readonly tabId: number; + readonly processKey: string; readonly frameId: string; readonly backendNodeId?: number; readonly reason: VisualIssueReason; @@ -322,7 +323,8 @@ export async function discoverVisualCandidates( }); if (region.status === "unavailable") { issues.push({ - target: document.frame.target, + tabId: document.frame.handle.tabId, + processKey: document.frame.processKey, frameId: document.frame.frameId, backendNodeId: node.backendNodeId, reason: region.reason, @@ -349,7 +351,12 @@ export async function discoverVisualCandidates( !reportedFrames.has(document.frame.frameId) && (!document.identity || !document.index.nodes.size) ) - issues.push({ target: document.frame.target, frameId: document.frame.frameId, reason }); + issues.push({ + tabId: document.frame.handle.tabId, + processKey: document.frame.processKey, + frameId: document.frame.frameId, + reason, + }); } const pending = checkpoint(); if (pending) await pending; diff --git a/apps/extension/src/tools/vom/visual-observation.ts b/apps/extension/src/tools/vom/visual-observation.ts index c80507ffa..7289666bc 100644 --- a/apps/extension/src/tools/vom/visual-observation.ts +++ b/apps/extension/src/tools/vom/visual-observation.ts @@ -114,8 +114,16 @@ export interface ObservationOutput { render: ReturnType; candidates: readonly VisualCandidate[]; identities: Map; + rootFrameId: string; rootIdentity?: DocumentIdentity; - frames: Map; + frames: Map< + string, + { + loaderId?: string; + parentFrameId?: string; + documentElementBackendNodeId?: number; + } + >; notices: string[]; maxTokens?: number; } @@ -149,6 +157,7 @@ export function prepareVisualObservation( render: prepareObservationRender(attachVisualEntries(scene, graph, candidates), options), candidates, identities, + rootFrameId, rootIdentity: identities.get(rootFrameId), frames: new Map( [...graph.frames].map(([id, frame]) => [ @@ -156,6 +165,9 @@ export function prepareVisualObservation( { ...(frame.loaderId !== undefined ? { loaderId: frame.loaderId } : {}), ...(frame.parentFrameId ? { parentFrameId: frame.parentFrameId } : {}), + ...(frame.handle.documentElementBackendNodeId !== undefined + ? { documentElementBackendNodeId: frame.handle.documentElementBackendNodeId } + : {}), }, ]), ), @@ -199,6 +211,7 @@ function page( budget: number | undefined, continued: boolean, nextToken: string, + allowContinuation = true, ): Page | RpcError { const max = budget ?? Infinity; if (!(max >= 0) || Number.isNaN(max)) return failure("max_tokens must be nonnegative"); @@ -216,7 +229,8 @@ function page( if (continued && state.buffer[0]?.context) headers.push(state.buffer[0].context); const fixed = [...headers, ...state.output.notices]; let used = fixed.reduce((n, line) => n + cost(line), 0); - if (used > max) return failure("max_tokens is too small for observation headers and notices"); + if (allowContinuation && used > max) + return failure("max_tokens is too small for observation headers and notices"); const lines = [...headers]; const refs: RenderedRef[] = []; const frameIds = new Set(); @@ -230,11 +244,15 @@ function page( } const row = state.buffer[emitted]; if (!row) break; - const reserve = state.buffer.length > emitted + 1 || !state.exhausted ? cost(footer) : 0; + const reserve = + allowContinuation && (state.buffer.length > emitted + 1 || !state.exhausted) + ? cost(footer) + : 0; let text = row.text; if (used + cost(text) + reserve > max && row.minimal) text = row.minimal; if (used + cost(text) + reserve > max) { - if (!emitted) return failure("max_tokens is too small to advance; increase the budget"); + if (allowContinuation && !emitted) + return failure("max_tokens is too small to advance; increase the budget"); break; } if (row.ref) refs.push({ ...row.ref, line: lines.length }); @@ -245,17 +263,58 @@ function page( } lines.push(...state.output.notices); const more = state.buffer.length > emitted || !state.exhausted; - if (more) lines.push(footer); + if (allowContinuation && more) lines.push(footer); return { text: lines.join("\n"), refs, more, frameIds, consumed: emitted }; } +export function renderObservationOutput( + output: ObservationOutput, + options: { maxTokens?: number } = {}, +): { text: string; refs: RenderedRef[]; truncated: boolean } { + const budget = options.maxTokens ?? output.maxTokens; + const clampedBudget = + budget !== undefined && (!Number.isFinite(budget) || budget < 0) + ? Number.isNaN(budget) + ? 0 + : Math.max(0, budget) + : budget; + const state: Continuation = { + stableRefs: false, + output, + tabId: 0, + revision: 0, + token: "", + exhausted: false, + buffer: [], + }; + const rendered = page(state, clampedBudget, false, "", false); + if ("code" in rendered) { + return { + text: [...output.render.headers, ...output.notices].join("\n"), + refs: [], + truncated: true, + }; + } + return { + text: rendered.text, + refs: rendered.refs, + truncated: rendered.more || output.render.truncated(), + }; +} + export async function publishObservationPage( store: RefStore, cdp: CdpRunner, tabId: number, - input: { output?: ObservationOutput; cursor?: string; maxTokens?: number }, + input: { + output?: ObservationOutput; + cursor?: string; + maxTokens?: number; + allowContinuation?: boolean; + }, signal?: AbortSignal, ): Promise { + const allowContinuation = input.allowContinuation ?? true; let state: Continuation; if (input.output) state = { @@ -284,7 +343,9 @@ export async function publishObservationPage( // Retry retains exactly the latest output; it cannot advance the iterator twice. const retry = input.cursor && state.last?.input === input.cursor ? state.last : undefined; const nextToken = crypto.randomUUID(); - const rendered = retry ? undefined : page(state, budget, !!input.cursor, nextToken); + const rendered = retry + ? undefined + : page(state, budget, !!input.cursor, nextToken, allowContinuation); if (rendered && "code" in rendered) return rendered; const identities = new Map(retry?.identities); if (state.output.rootIdentity) @@ -324,11 +385,15 @@ export async function publishObservationPage( ref.ref, { kind: "visual-region", candidate: state.output.candidates[ref.visualKey] }, ]; - const frameMeta = ref.frameId ? state.output.frames.get(ref.frameId) : undefined; - const frameIdentity = ref.frameId - ? state.output.identities.get(ref.frameId) + const resolvedFrameId = + ref.frameId || state.output.rootIdentity?.frameId || state.output.rootFrameId; + const frameMeta = resolvedFrameId ? state.output.frames.get(resolvedFrameId) : undefined; + const frameIdentity = resolvedFrameId + ? state.output.identities.get(resolvedFrameId) : state.output.rootIdentity; const loaderId = frameIdentity?.loaderId ?? frameMeta?.loaderId; + const documentElementBackendNodeId = + frameIdentity?.documentElementBackendNodeId ?? frameMeta?.documentElementBackendNodeId; return [ ref.ref, { @@ -336,11 +401,9 @@ export async function publishObservationPage( ...(ref.rowAnchor ? { rowAnchor: ref.rowAnchor } : {}), backendNodeId: ref.backendNodeId, tabId, - frameId: ref.frameId ?? frameIdentity?.frameId, + ...(resolvedFrameId ? { frameId: resolvedFrameId } : {}), ...(loaderId !== undefined ? { loaderId } : {}), - ...(frameIdentity?.documentElementBackendNodeId !== undefined - ? { documentElementBackendNodeId: frameIdentity.documentElementBackendNodeId } - : {}), + ...(documentElementBackendNodeId !== undefined ? { documentElementBackendNodeId } : {}), }, ]; }); @@ -355,10 +418,11 @@ export async function publishObservationPage( ref_count: refs.length, tab_id: tabId, truncated: rendered!.more || state.output.render.truncated(), - ...(rendered!.more ? { next_cursor: state.token } : {}), + ...(allowContinuation && rendered!.more ? { next_cursor: state.token } : {}), }; - if (input.cursor) state.last = { input: input.cursor, budget, result, identities }; - if (!rendered!.more) { + if (allowContinuation && input.cursor) + state.last = { input: input.cursor, budget, result, identities }; + if (!allowContinuation || !rendered!.more) { // Keep only the latest response/identity proof for transport retry, not the full observation. state.output = { ...state.output, @@ -369,7 +433,7 @@ export async function publishObservationPage( render: { headers: [], rows: [][Symbol.iterator](), truncated: () => false }, }; } - if (rendered!.more || input.cursor) continuations.set(store, state); + if (allowContinuation && (rendered!.more || input.cursor)) continuations.set(store, state); else continuations.delete(store); return result; } diff --git a/apps/extension/src/tools/waits.ts b/apps/extension/src/tools/waits.ts index 3cc649cd7..9dd86ae39 100644 --- a/apps/extension/src/tools/waits.ts +++ b/apps/extension/src/tools/waits.ts @@ -27,7 +27,11 @@ import { isMissingNode } from "./errors"; // selector / frame handling identical to every other tool. import type { CdpDispatchGuard } from "@/browser-driver/chromium-cdp"; -import type { CdpTarget, FrameDocumentHandle, FrameNodeHandle } from "@/browser-driver/frame-graph"; +import { + type FrameDocumentHandle, + type FrameNodeHandle, + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import { normaliseRef } from "@/session-manager/ref-store"; import type { @@ -51,7 +55,6 @@ import { isRpcError, lookupSession, resolveTargetTab, - sendToCdpTarget, sendToFrameHandle, } from "./shared"; @@ -312,38 +315,11 @@ export async function handleWaitForElement( const objectGroup = `bsk-wait-for-element-${crypto.randomUUID()}`; const objectHandles = new Map(); let finished = false; - const releaseTarget = (objectTarget: CdpTarget) => { - void sendToCdpTarget( - deps.cdp.resourceCleanup ?? deps.cdp, - objectTarget, - "Runtime.releaseObjectGroup", - { - objectGroup, - }, - ).catch(() => {}); - }; const releaseHandle = (handle: FrameDocumentHandle) => { // Cleanup must not extend the wait's deadline or delay cancellation. - void sendToFrameHandle( - deps.cdp.resourceCleanup ?? deps.cdp, - handle, - "Runtime.releaseObjectGroup", - { objectGroup }, - ).catch(() => {}); - }; - const send = async (cdpTarget: CdpTarget, method: string, args?: object): Promise => { - checkBudget(deadline, deps.signal); - try { - return await (deps.cdp.sendGuarded - ? deps.cdp.sendGuarded(cdpTarget, method, args, { - signal: deps.signal, - onDispatch: () => checkBudget(deadline, deps.signal), - }) - : sendToCdpTarget(deps.cdp, cdpTarget, method, args)); - } finally { - // A resolveNode reply may allocate a handle after timeout/cancellation. - if (finished && method === "DOM.resolveNode") releaseTarget(cdpTarget); - } + void (deps.cdp.resourceCleanup ?? deps.cdp) + .sendToFrame(handle, "Runtime.releaseObjectGroup", { objectGroup }, undefined) + .catch(() => {}); }; const sendFrame = async ( handle: FrameDocumentHandle | FrameNodeHandle, @@ -352,55 +328,57 @@ export async function handleWaitForElement( guard?: CdpDispatchGuard, ): Promise => { checkBudget(deadline, deps.signal); - if (!deps.cdp.sendToFrame && deps.cdp.sendGuarded && !handle.frameId) { - return send({ tabId: handle.tabId }, method, args); - } + const docHandle: FrameDocumentHandle = { + tabId: handle.tabId, + frameId: handle.frameId, + loaderId: handle.loaderId, + ...(handle.documentElementBackendNodeId !== undefined + ? { documentElementBackendNodeId: handle.documentElementBackendNodeId } + : {}), + }; + let dispatchedResolve = false; try { return await sendToFrameHandle(deps.cdp, handle, method, args, { ...guard, signal: guard?.signal ?? deps.signal, onDispatch: () => { checkBudget(deadline, deps.signal); + if (method === "DOM.resolveNode") { + dispatchedResolve = true; + objectHandles.set( + `${docHandle.tabId}:${docHandle.frameId}:${docHandle.loaderId}`, + docHandle, + ); + } guard?.onDispatch?.(); }, }); } finally { - if (finished && method === "DOM.resolveNode") releaseHandle(handle); + // A resolveNode reply may allocate a handle after timeout/cancellation. + if (finished && dispatchedResolve) releaseHandle(docHandle); } }; const probeCdp: CdpRunner = delegateCdpRunner(deps.cdp, { - send: (tabId, method, args) => send({ tabId }, method, args), - sendToTarget: send, + send: (tabId: number, method: string, args?: object, sig?: AbortSignal) => + sendFrame(toRootFrameHandle(tabId), method, args, sig ? { signal: sig } : undefined), sendToFrame: sendFrame, - ...(deps.cdp.ensureAttachedToUrl - ? { - ensureAttachedToUrl: async (tabId, expectedUrl) => { - checkBudget(deadline, deps.signal); - await deps.cdp.ensureAttachedToUrl!(tabId, expectedUrl); - checkBudget(deadline, deps.signal); - }, - } - : {}), - ...(deps.cdp.getMainFrameHandle - ? { - getMainFrameHandle: async (tabId, sig) => { - checkBudget(deadline, deps.signal); - const result = await deps.cdp.getMainFrameHandle!(tabId, sig ?? deps.signal); - checkBudget(deadline, deps.signal); - return result; - }, - } - : {}), - ...(deps.cdp.getFrameGraph - ? { - getFrameGraph: async (tabId, sig) => { - checkBudget(deadline, deps.signal); - const result = await deps.cdp.getFrameGraph!(tabId, sig ?? deps.signal); - checkBudget(deadline, deps.signal); - return result; - }, - } - : {}), + ensureAttachedToUrl: async (tabId, expectedUrl) => { + checkBudget(deadline, deps.signal); + await deps.cdp.ensureAttachedToUrl(tabId, expectedUrl); + checkBudget(deadline, deps.signal); + }, + getMainFrameHandle: async (tabId, sig) => { + checkBudget(deadline, deps.signal); + const result = await deps.cdp.getMainFrameHandle(tabId, sig ?? deps.signal); + checkBudget(deadline, deps.signal); + return result; + }, + getFrameDocuments: async (tabId, sig) => { + checkBudget(deadline, deps.signal); + const result = await deps.cdp.getFrameDocuments(tabId, sig ?? deps.signal); + checkBudget(deadline, deps.signal); + return result; + }, }); let probe: Probe | undefined; @@ -490,7 +468,6 @@ async function probeState( ? { documentElementBackendNodeId: node.handle.documentElementBackendNodeId } : {}), }; - objectHandles.set(`${docHandle.tabId}:${docHandle.frameId}:${docHandle.loaderId}`, docHandle); try { const resolved = await sendToFrameHandle<{ object?: { objectId?: string } }>( cdp, @@ -499,6 +476,9 @@ async function probeState( { backendNodeId: node.backendNodeId, objectGroup }, ); objectId = resolved.object?.objectId; + if (objectId) { + objectHandles.set(`${docHandle.tabId}:${docHandle.frameId}:${docHandle.loaderId}`, docHandle); + } } catch (error) { // A missing ref identifies a removed node. A selector may already match // its replacement, so only a fresh lookup can establish its absence. diff --git a/apps/extension/src/tools/wheel.ts b/apps/extension/src/tools/wheel.ts index a58d5b814..78cc324da 100644 --- a/apps/extension/src/tools/wheel.ts +++ b/apps/extension/src/tools/wheel.ts @@ -1,17 +1,17 @@ import type { CdpDispatchGuard } from "@/browser-driver/chromium-cdp"; import { - type CdpTarget, - cdpTargetKey, + type FrameDocumentGraph, type FrameDocumentHandle, type FrameNodeHandle, -} from "@/browser-driver/frame-graph"; + toRootFrameHandle, +} from "@/browser-driver/frame-handles"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { RpcError, WheelParams, WheelResult } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; import { cdpError, rpcError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; import { clipPolygon, polygonArea, polygonCentroid, rectPolygon } from "./geometry"; -import { waitForInputPaint, withInputReady } from "./input-readiness"; +import { type InputWindowsApi, waitForInputPaint, withInputReady } from "./input-readiness"; import { modifiersBitfield, resolveBackendNode } from "./interaction"; import { scrollVisibleBounds } from "./scroll-visibility"; import { @@ -22,13 +22,13 @@ import { isRpcError, lookupSession, resolveTargetTab, - sendToCdpTarget, sendToFrameHandle, } from "./shared"; export interface WheelDeps { cdp: CdpRunner; tabsApi: ChromeTabsApi; + windowsApi?: InputWindowsApi; signal?: AbortSignal; /** Temporarily disable the Agent Window overlay's input blocker. */ bypassOverlay?: (tabId: number, enabled: boolean) => Promise; @@ -85,20 +85,7 @@ export async function handleWheel( // Guard the shared target/geometry helpers without changing other tools. // Own fallback scrolling and visibility objects, even if allocation is cancelled. const objectGroup = `bsk-wheel-${crypto.randomUUID()}`; - const objectTargets = new Map(); const objectHandles = new Map(); - const send = async (target: CdpTarget, method: string, args?: object): Promise => { - checkActive(); - if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target); - const result = await sendToCdpTarget( - deps.cdp, - target, - method, - method === "DOM.resolveNode" ? { ...args, objectGroup } : args, - ); - checkActive(); - return result; - }; const sendFrame = async ( handle: FrameDocumentHandle | FrameNodeHandle, method: string, @@ -126,41 +113,29 @@ export async function handleWheel( checkActive(); return result; }; - let graph: ReturnType> | undefined; + let graph: Promise | undefined; const cdp: CdpRunner = delegateCdpRunner(deps.cdp, { - send: (tabId, method, args) => send({ tabId }, method, args), - sendToTarget: send, + send: (tabId: number, method: string, args?: object, sig?: AbortSignal) => + sendFrame(toRootFrameHandle(tabId), method, args, sig ? { signal: sig } : undefined), sendToFrame: sendFrame, - ...(deps.cdp.ensureAttachedToUrl - ? { - ensureAttachedToUrl: async (tabId, expectedUrl) => { - checkActive(); - await deps.cdp.ensureAttachedToUrl!(tabId, expectedUrl); - checkActive(); - }, - } - : {}), - ...(deps.cdp.getMainFrameHandle - ? { - getMainFrameHandle: async (tabId, sig) => { - checkActive(); - const result = await deps.cdp.getMainFrameHandle!(tabId, sig); - checkActive(); - return result; - }, - } - : {}), - ...(deps.cdp.getFrameGraph - ? { - getFrameGraph: async (tabId) => { - checkActive(); - graph ??= deps.cdp.getFrameGraph!(tabId); - const result = await graph; - checkActive(); - return result; - }, - } - : {}), + ensureAttachedToUrl: async (tabId, expectedUrl) => { + checkActive(); + await deps.cdp.ensureAttachedToUrl(tabId, expectedUrl); + checkActive(); + }, + getMainFrameHandle: async (tabId, sig) => { + checkActive(); + const result = await deps.cdp.getMainFrameHandle(tabId, sig); + checkActive(); + return result; + }, + getFrameDocuments: async (tabId, sig) => { + checkActive(); + graph ??= deps.cdp.getFrameDocuments(tabId, sig); + const result = await graph; + checkActive(); + return result; + }, }); let bypassTab: number | undefined; @@ -232,15 +207,14 @@ export async function handleWheel( } } const cleanup = deps.cdp.resourceCleanup ?? deps.cdp; - for (const target of objectTargets.values()) { - await sendToCdpTarget(cleanup, target, "Runtime.releaseObjectGroup", { objectGroup }).catch( - () => {}, - ); - } for (const handle of objectHandles.values()) { - await sendToFrameHandle(cleanup, handle, "Runtime.releaseObjectGroup", { objectGroup }).catch( - () => {}, - ); + await sendToFrameHandle( + cleanup, + handle, + "Runtime.releaseObjectGroup", + { objectGroup }, + { allowReattach: false }, + ).catch(() => {}); } } } diff --git a/apps/extension/src/video/manager.test.ts b/apps/extension/src/video/manager.test.ts index f14c26299..bfe13e8b8 100644 --- a/apps/extension/src/video/manager.test.ts +++ b/apps/extension/src/video/manager.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { rootEventSource } from "@/browser-driver/frame-handles"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import type { VideoHost, VideoHostCommand } from "./host-protocol"; import { VIDEO_NAVIGATION_TIMEOUT_MS, VideoManager } from "./manager"; import { VideoArtifactStore } from "./store"; @@ -46,16 +47,17 @@ async function setup() { const store = new MemoryStore(); let owner = "local"; let event: Parameters>[1]; + const send = vi.fn( + async (_tab: number, method: string): Promise => + method === "Page.captureScreenshot" ? { data: "image" } : {}, + ); const cdp = { getAttachmentId: () => "one", onEvent: (_tabId: number, handler: typeof event) => { event = handler; return { dispose: vi.fn() }; }, - send: vi.fn( - async (_tab: number, method: string): Promise => - (method === "Page.captureScreenshot" ? { data: "image" } : {}) as T, - ), + send, }; const request = vi.fn(async (command: VideoHostCommand): Promise => { if (command.action === "start") { @@ -85,7 +87,7 @@ async function setup() { const changed = vi.fn(); const video = new VideoManager({ sessions, - cdp: cdp as CdpRunner, + cdp: createStubCdpRunner({ ...cdp, send: send as CdpRunner["send"] }), store, owner: () => owner, tabs: { query: async () => [tab], get: async () => tab }, @@ -108,7 +110,11 @@ async function setup() { owner = value; }, frame: (data: string, timestamp?: number) => - event({ tabId: 7 }, "Page.screencastFrame", { data, metadata: { timestamp }, sessionId: 1 }), + event(rootEventSource(7), "Page.screencastFrame", { + data, + metadata: { timestamp }, + sessionId: 1, + }), }; } diff --git a/crates/bsk-cli/src/cli/recipe/vom.rs b/crates/bsk-cli/src/cli/recipe/vom.rs index e78ad813b..b8d102697 100644 --- a/crates/bsk-cli/src/cli/recipe/vom.rs +++ b/crates/bsk-cli/src/cli/recipe/vom.rs @@ -331,9 +331,20 @@ mod tests { #[test] fn a_menu_is_the_same_control_whether_or_not_it_is_open() { - let closed = parse("@vom 1\n @e1 combobox \"Region [has-submenu]\""); - let open = parse("@vom 1\n @e1 combobox \"Region [expanded]\""); - for page in [closed, open] { + let closed = parse("@vom 1\n @e1 combobox \"Region\" [has-submenu]"); + let open = parse("@vom 1\n @e1 combobox \"Region\" [expanded]"); + let custom_closed = parse("@vom 1\n @e1 combobox \"Region\" [custom-dropdown: click to open]"); + let custom_open = parse("@vom 1\n @e1 combobox \"Region\" [custom-dropdown: expanded]"); + let legacy_closed = parse("@vom 1\n @e1 combobox \"Region [has-submenu]\""); + let legacy_open = parse("@vom 1\n @e1 combobox \"Region [expanded]\""); + for page in [ + closed, + open, + custom_closed, + custom_open, + legacy_closed, + legacy_open, + ] { assert_eq!( page.candidates(&target("combobox", "Region", None)).len(), 1 diff --git a/crates/bsk-cli/tests/fixtures/vom_contract.json b/crates/bsk-cli/tests/fixtures/vom_contract.json index bcc021fae..ab762b3b3 100644 --- a/crates/bsk-cli/tests/fixtures/vom_contract.json +++ b/crates/bsk-cli/tests/fixtures/vom_contract.json @@ -24,9 +24,10 @@ { "id": 3, "parentId": 1, - "tag": "select", + "tag": "input", "role": "combobox", - "name": "Region [has-submenu]" + "name": "Region", + "hasPopup": true }, { "id": 4, @@ -166,7 +167,7 @@ } ] }, - "text": "@vom 1\n@view 1280x720\n@layers 1 focus=L1\nL1 page\n main\n heading \"Sales report\"\n @e1 combobox \"Region [has-submenu]\"\n @e2 textbox \"Note\" [filled] placeholder=\"a ] =\\\"b\\\"\" =\"hello \\\"x\\\"\"\n @e3 textbox \"Password\" [filled] =\"•••\"\n @e4 textbox \"Empty\" [empty]\n @e5 checkbox \"Agree\" [checked]\n @e6 link \"Docs\" [→ https://x.test/a]b?c=[1]]\n @e7 button \"Search\" [disabled]\n @e8 button \"Say \\\"hi\\\" \\\\ now\"\n @e9 button\n @e10 listitem [draggable] \"Card\"\n row \"ORD-1\"\n @e11 button \"Approve\" [ctx: ORD-1]\n row \"ORD-2\"\n @e12 button \"Approve\" [ctx: ORD-2]\n status \"Total $42\"\n @e13 button \"Menu\"\n [§ active: Menu]\n @e9 button \"Pay\"\n button \"Pay\"\n StaticText \"Plain text\"", + "text": "@vom 1\n@view 1280x720\n@layers 1 focus=L1\nL1 page\n main\n heading \"Sales report\"\n @e1 combobox \"Region\" [has-submenu]\n @e2 textbox \"Note\" [filled] placeholder=\"a ] =\\\"b\\\"\" =\"hello \\\"x\\\"\"\n @e3 textbox \"Password\" [filled] =\"•••\"\n @e4 textbox \"Empty\" [empty]\n @e5 checkbox \"Agree\" [checked]\n @e6 link \"Docs\" [→ https://x.test/a]b?c=[1]]\n @e7 button \"Search\" [disabled]\n @e8 button \"Say \\\"hi\\\" \\\\ now\"\n @e9 button\n @e10 listitem [draggable] \"Card\"\n row \"ORD-1\"\n @e11 button \"Approve\" [ctx: ORD-1]\n row \"ORD-2\"\n @e12 button \"Approve\" [ctx: ORD-2]\n status \"Total $42\"\n @e13 button \"Menu\"\n [§ active: Menu]\n @e9 button \"Pay\"\n button \"Pay\"\n StaticText \"Plain text\"", "lines": [ { "ref": null, diff --git a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py index 109bcadda..500adf078 100644 --- a/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py +++ b/evals/browser/skills/browserskill-release-test/scripts/test_release_audit.py @@ -968,12 +968,19 @@ def test_nested_menu_vom_assertions_and_gate_contract(self): elif cmd in ("sync", "observe", "snapshot"): phase = "post" if "--post" in args else "pre" if phase == "pre": - vom = ( - "@vom 1\\nmenubar \\"Console navigation\\"\\n" - " @e1 menuitem \\"Overview\\"\\n" - " @e2 menuitem \\"Workspace [has-submenu]\\"" - + ("\\n @e9 menuitem \\"Audit export\\"" if mode == "leaked_pre" else "") - ) + if mode == "v050": + vom = ( + "@vom 1\\nmenubar \\"Console navigation\\"\\n" + " @e1 menuitem \\"Overview\\"\\n" + " @e2 menuitem \\"Workspace [has-submenu]\\"" + ) + else: + vom = ( + "@vom 1\\nmenubar \\"Console navigation\\"\\n" + " @e1 menuitem \\"Overview\\"\\n" + " @e2 menuitem \\"Workspace\\" [has-submenu]" + + ("\\n @e9 menuitem \\"Audit export\\"" if mode == "leaked_pre" else "") + ) elif mode == "v050": vom = ( "@vom 1\\nmenubar \\"Console navigation\\"\\n" @@ -986,7 +993,7 @@ def test_nested_menu_vom_assertions_and_gate_contract(self): elif mode == "dup_ref": vom = ( "@vom 1\\nmenu\\n" - " @e4 menuitem \\"Advanced Audit export Report history [expanded]\\"\\n" + " @e4 menuitem \\"Advanced Audit export Report history\\" [expanded]\\n" " @e5 menuitem \\"Audit export\\"\\n" " @e5 menuitem \\"Report history\\"" ) @@ -995,10 +1002,10 @@ def test_nested_menu_vom_assertions_and_gate_contract(self): vom = ( "@vom 1\\nmenubar \\"Console navigation\\"\\n" " @e1 menuitem \\"Overview\\"\\n" - " @e2 menuitem \\"Workspace [expanded]\\"\\n" + " @e2 menuitem \\"Workspace\\" [expanded]\\n" "menu\\n" " @e3 menuitem \\"Members\\"\\n" - " @e4 menuitem \\"Advanced Audit export Report history [expanded]\\"\\n" + " @e4 menuitem \\"Advanced Audit export Report history\\" [expanded]\\n" f" {audit_ref} menuitem \\"Audit export\\"\\n" " @e8 menuitem \\"Report history\\"" ) diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index 35a6ea446..966da720e 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -2528,5 +2528,121 @@ describe("selection evidence", () => { { ref: "e2", backendNodeId: 399 }, ]); }); + + it("budgets the L2 occlusion footer line in renderDoubleLayer, preserves generic Chinese headings, and renders popup tags without duplication", () => { + // 1. Two-layer modal scene: maxTokens budget includes the `L2 page — occluded by L1` footer. + const modalNodes: VomNode[] = [ + node({ id: 1, tag: "main", role: "main", rect: { x: 0, y: 0, w: 1280, h: 720 } }), + ...Array.from({ length: 12 }, (_, idx) => + node({ + id: 10 + idx, + parentId: 1, + backendNodeId: 100 + idx, + tag: "button", + role: "button", + name: `Background action item ${idx + 1}`, + rect: { x: 10, y: idx * 40, w: 200, h: 32 }, + }), + ), + node({ + id: 50, + tag: "div", + role: "dialog", + name: "Confirm action", + modal: true, + position: "fixed", + paintOrder: 10, + rect: { x: 200, y: 100, w: 600, h: 400 }, + }), + ...Array.from({ length: 8 }, (_, idx) => + node({ + id: 60 + idx, + parentId: 50, + backendNodeId: 200 + idx, + tag: "button", + role: "button", + name: `Modal option button ${idx + 1} with detailed description`, + paintOrder: 10, + rect: { x: 220, y: 120 + idx * 36, w: 300, h: 30 }, + }), + ), + ]; + for (const budget of [45, 65, 90, 120]) { + const out = renderVom(scene(modalNodes), { maxTokens: budget }); + // Estimate matches render.ts estimateVomTokens (~chars / 3.5 per line); ensure L2 footer never pushes output unboundedly past budget. + const bodyLines = out.text.split("\n").filter((line) => !line.startsWith("[truncated:")); + const approxTokens = bodyLines.reduce( + (sum, line) => sum + Math.max(1, Math.ceil((line.length + 1) / 3.5)), + 0, + ); + expect(approxTokens).toBeLessThanOrEqual(budget); + } + + // 2. Generic Chinese headings ("欢迎", "联系方式") are preserved as section context rather than suppressed by benchmark filters. + const headingScene = scene([ + node({ id: 1, tag: "section", role: "region" }), + node({ id: 2, parentId: 1, tag: "h2", role: "heading", name: "欢迎" }), + node({ + id: 3, + parentId: 1, + backendNodeId: 11, + tag: "button", + role: "button", + name: "详情", + }), + node({ id: 4, tag: "section", role: "region" }), + node({ id: 5, parentId: 4, tag: "h2", role: "heading", name: "联系方式" }), + node({ + id: 6, + parentId: 4, + backendNodeId: 12, + tag: "button", + role: "button", + name: "详情", + }), + ]); + const headingOut = renderVom(headingScene); + expect(headingOut.text).toContain("[ctx: 欢迎]"); + expect(headingOut.text).toContain("[ctx: 联系方式]"); + + // 3. Popup state tags: custom combobox gets `[custom-dropdown: expanded]` without `[expanded]` inside quotes; + // popup buttons/menuitems emit post-quote `[expanded]` / `[has-submenu]`. + const popupOut = renderVom( + scene([ + node({ + id: 1, + backendNodeId: 21, + tag: "div", + role: "combobox", + name: "Filter", + expanded: true, + hasPopup: "listbox", + attrs: { "aria-expanded": "true", "aria-haspopup": "listbox" }, + }), + node({ + id: 2, + backendNodeId: 22, + tag: "button", + role: "button", + name: "Products", + expanded: true, + hasPopup: "menu", + }), + node({ + id: 3, + backendNodeId: 23, + tag: "div", + role: "menuitem", + name: "Export", + expanded: false, + hasPopup: "menu", + }), + ]), + ); + expect(popupOut.text).toContain('@e1 combobox "Filter" [custom-dropdown: expanded]'); + expect(popupOut.text).not.toContain('"Filter [expanded]"'); + expect(popupOut.text).toContain('@e2 button "Products" [expanded]'); + expect(popupOut.text).toContain('@e3 menuitem "Export" [has-submenu]'); + }); }); }); diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index 6ee31e624..f4d8c609c 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -1,11 +1,14 @@ export { applyVomInteractionRecovery, + encodeVomNameContent, + inspectVomLine, isVomReferenceNode, isVomStructuralRole, prepareObservationRender, type RenderRow, renderVom, selectionMarker, + VOM_DERIVED_NAME_CAP, } from "./render"; export { estimateLineTokens, estimateTokens, TOKEN_ESTIMATOR } from "./tokens"; export type { @@ -17,6 +20,7 @@ export type { RenderedRef, Viewport, VisualEntry, + VomLineMetadata, VomNode, VomOptions, VomRef, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 5710e7117..6b574db1c 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -6,6 +6,7 @@ import type { CondSurface, Rect, RenderedRef, + VomLineMetadata, VomNode, VomOptions, VomResult, @@ -77,10 +78,126 @@ const STRUCTURAL_ROLES = new Set([ "webarea", ]); +/** Maximum length for names derived by joining descendant text without an ellipsis mark. */ +export const VOM_DERIVED_NAME_CAP = 240; + export function isVomStructuralRole(role: string | undefined): boolean { return STRUCTURAL_ROLES.has(role?.toLowerCase() ?? ""); } +/** Normalize and JSON-escape a text string exactly as VOM renders node names. */ +export function encodeVomNameContent(value: string): string { + return JSON.stringify(value.replace(/\s+/g, " ").trim()).slice(1, -1); +} + +const PRE_NAME_DRAG_HINTS = [" [draggable]", " [drop-target]", " [drag-handle]"] as const; + +function readLeadingJsonEscapedContent( + text: string, +): { escaped: string; rest: string } | undefined { + if (!text.startsWith('"')) return undefined; + let escaped = false; + for (let i = 1; i < text.length; i += 1) { + const ch = text[i]; + if (ch === "\\" && !escaped) { + escaped = true; + continue; + } + if (ch === '"' && !escaped) { + return { escaped: text.slice(1, i), rest: text.slice(i + 1) }; + } + escaped = false; + } + return undefined; +} + +/** Structured inspection of one rendered VOM line shared with incremental text-delta patching. */ +export function inspectVomLine(line: string): VomLineMetadata { + const trimmed = line.replace(/^ +/, ""); + const indent = line.length - trimmed.length; + if (trimmed.startsWith("@context ")) { + const body = trimmed.slice("@context ".length); + const spaceAt = body.indexOf(" "); + const role = spaceAt >= 0 ? body.slice(0, spaceAt) : body; + const quoted = + spaceAt >= 0 ? readLeadingJsonEscapedContent(body.slice(spaceAt + 1)) : undefined; + return { + indent, + kind: "context", + ...(role ? { role } : {}), + ...(quoted && quoted.rest.length === 0 ? { escapedName: quoted.escaped } : {}), + plainNamedNode: false, + silentlyTruncatesLabel: true, + }; + } + if ( + trimmed.startsWith("@warning ") || + trimmed.startsWith("@more ") || + trimmed.startsWith("@continued ") + ) { + return { indent, kind: "notice", plainNamedNode: false, silentlyTruncatesLabel: false }; + } + if ( + trimmed.startsWith("@vom ") || + trimmed.startsWith("@view ") || + trimmed.startsWith("@layers ") + ) { + return { indent, kind: "header", plainNamedNode: false, silentlyTruncatesLabel: false }; + } + if (/^L\d+\s/.test(trimmed)) { + return { indent, kind: "layer", plainNamedNode: false, silentlyTruncatesLabel: false }; + } + if (trimmed.startsWith("[§ active:")) { + return { indent, kind: "active-scope", plainNamedNode: false, silentlyTruncatesLabel: false }; + } + + let rest = trimmed; + let ref: string | undefined; + if (rest.startsWith("@e")) { + const spaceAt = rest.indexOf(" "); + if (spaceAt > 2) { + const digits = rest.slice(2, spaceAt); + if (/^\d+$/.test(digits)) { + ref = `e${digits}`; + rest = rest.slice(spaceAt + 1); + } + } + } + const roleMatch = /^[A-Za-z][\w-]*/.exec(rest); + if (!roleMatch) { + return { indent, kind: "node", plainNamedNode: false, silentlyTruncatesLabel: false }; + } + const role = roleMatch[0]; + rest = rest.slice(role.length); + let hadPreNameAnnotation = false; + for (const hint of PRE_NAME_DRAG_HINTS) { + if (rest.startsWith(hint)) { + hadPreNameAnnotation = true; + rest = rest.slice(hint.length); + break; + } + } + let escapedName: string | undefined; + if (rest.startsWith(' "')) { + const parsed = readLeadingJsonEscapedContent(rest.slice(1)); + if (parsed) { + escapedName = parsed.escaped; + rest = parsed.rest; + } + } + const plainNamedNode = + ref === undefined && !hadPreNameAnnotation && escapedName !== undefined && rest.length === 0; + return { + indent, + kind: "node", + ...(ref ? { ref } : {}), + role, + ...(escapedName !== undefined ? { escapedName } : {}), + plainNamedNode, + silentlyTruncatesLabel: role === "canvas", + }; +} + const SENSITIVE_MASK = "•••"; const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY; const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY; @@ -169,14 +286,14 @@ function cleanSemanticText(value: string | undefined): string | undefined { } function isLowValueContext(value: string): boolean { - const clean = value.trim(); - const key = clean.toLowerCase(); - if (clean === "欢迎") return true; - if (key.includes("icp") || key.includes("备案")) return true; - if (clean.includes("联系方式") || clean.includes("政府网站标识")) return true; - if (/^\d{5,}$/.test(clean)) return true; - if (/北京市小客车指标调控管理信息系统|beijing municipal commission/i.test(clean)) return true; - return false; + return /^\d{5,}$/.test(value.trim()); +} + +function isDocumentRootContextStop(node: VomNode): boolean { + return ( + ["rootwebarea", "webarea", "document"].includes(normalizedRole(node)) || + ["body", "html"].includes(normalizedTag(node)) + ); } function normalizedRole(node: VomNode): string { @@ -688,7 +805,7 @@ function contextLabel(node: VomNode, targetName: string): string | undefined { } function weakLabelContext(node: VomNode, targetName: string): string | undefined { - if (isVomReferenceNode(node)) return undefined; + if (isVomReferenceNode(node) || isDocumentRootContextStop(node)) return undefined; return weakContextText( cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), targetName, @@ -798,7 +915,7 @@ function collectOwnedContext( let guard = 0; while (current !== null && guard <= state.parentMap.size) { const parent = state.nodesById.get(current); - if (!parent) break; + if (!parent || isDocumentRootContextStop(parent)) break; if (!sharesContextScope(node, parent)) break; const label = contextLabel(parent, nodeName); if (label) contexts.push({ text: label, source: "owned", order: contexts.length }); @@ -847,7 +964,13 @@ function collectSameContainerContext( cache.set(key, prefix); const parent = state.nodesById.get(parentId); - if (!parent || !sharesContextScope(node, parent) || isContextBoundary(parent)) break; + if ( + !parent || + !sharesContextScope(node, parent) || + isContextBoundary(parent) || + isDocumentRootContextStop(parent) + ) + break; childId = parentId; parentId = state.parentMap.get(parentId) ?? null; guard += 1; @@ -1018,8 +1141,13 @@ function renderNodeLine( const inlineOpts = formatInlineSelectOptions(node, state); if (inlineOpts) line += ` [options: ${inlineOpts}]`; } else if (role === "combobox" && !isNativeSelect && !isEditableCombobox(node)) { - const expanded = node.attrs?.["aria-expanded"]?.toLowerCase() === "true"; + const expanded = + node.expanded === true || node.attrs?.["aria-expanded"]?.toLowerCase() === "true"; line += expanded ? " [custom-dropdown: expanded]" : " [custom-dropdown: click to open]"; + } else if (node.hasPopup && name) { + const expanded = + node.expanded === true || node.attrs?.["aria-expanded"]?.toLowerCase() === "true"; + line += expanded ? " [expanded]" : " [has-submenu]"; } if (surface && surface.subItems.length > 0) { @@ -1814,8 +1942,25 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp "@layers 2 focus=L1", `L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`, ]; - const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks); - state.lines.push(renderPageOcclusionLine(hiddenCount)); + const occlusionLine = renderPageOcclusionLine(hiddenCount); + const occlusionCost = estimateLineTokens(occlusionLine); + const budgetFinite = options.maxTokens !== undefined && Number.isFinite(options.maxTokens); + const nodeOptions = budgetFinite + ? { ...options, maxTokens: Math.max(0, options.maxTokens! - occlusionCost) } + : options; + const state = renderNodes( + visibleNodes, + nodeOptions, + header, + scene.surfaces, + scene.activeScopeBlocks, + ); + if (!budgetFinite || state.tokens + occlusionCost <= options.maxTokens!) { + state.lines.push(occlusionLine); + state.tokens += occlusionCost; + } else { + state.truncated = true; + } return { text: state.lines.join("\n"), diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 74b7457bd..7bea9ad2a 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -69,6 +69,10 @@ export interface VomNode { domAncestorIds?: number[]; modal?: boolean; + /** Accessibility/ARIA popup trigger state (`aria-haspopup` / AX `hasPopup`). */ + hasPopup?: boolean; + /** Accessibility/ARIA expansion state (`aria-expanded` / AX `expanded`). */ + expanded?: boolean; sensitive?: boolean; disabled?: boolean; inert?: boolean; @@ -145,6 +149,19 @@ export interface RenderedRef extends VomRef { line: number; } +export interface VomLineMetadata { + indent: number; + kind: "header" | "layer" | "context" | "active-scope" | "notice" | "node"; + ref?: string; + role?: string; + /** JSON-escaped inner text of the line's primary quoted name, if present. */ + escapedName?: string; + /** True when the line is strictly `role "name"` without `@eN` or trailing annotations. */ + plainNamedNode: boolean; + /** True when the renderer may cut the line's primary label without appending `…`. */ + silentlyTruncatesLabel: boolean; +} + export interface VomResult { text: string; refs: RenderedRef[]; From 82d453ee181d92a56b9b38ff1dec2b0a5e1f2ebd Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Sun, 15 Nov 2026 12:44:58 -0600 Subject: [PATCH 2/3] fix(extension,cli): close Wave 4C geometry, stub dispatch-guard, RAII cleanup, and rustfmt gaps Why: - `GeometryContext.endpointForFrame` returned frame-scoped `frame.handle` instead of `frameProcessBoundary(frame)`, splitting cache keys from process-scoped `layoutMetrics` queries and doubling `Page.getLayoutMetrics` IPC calls on multi-frame / OOPIF pages. - `createStubCdpRunner` dropped `CdpDispatchGuard` (`signal`, `attachmentId`, `onDispatch`) when 3-argument `sendToFrame` overrides were supplied and closed over `stub.send` instead of `this.send` on spread clones, breaking `CooperationController` dispatch accounting, `pointerTransactionDeps` mouse-release tracking, and mid-read `resolveVisualRegionNow` overrides. - `ChromiumCdp.sendToFrame` / `resolveValidatedFrameRoute` checked `ignoreFrameNavigation` instead of `isCleanup` before dispatching `Runtime.releaseObjectGroup` / `Runtime.releaseObject`, risking redundant `Page.getFrameTree` refreshes or `StaleFrameHandleError` after same-process frame navigation; `cdpRunnerForFrame` forwarded aborted `guard.signal` into `Runtime.releaseObjectGroup` on cancelled VOM captures; `waits`, `element-geometry` (`scrollNodeIntoView` / `descendantBoundingRect`), `visual-target`, `vom/document-identity`, and `viewport-screenshot` had remaining V8 handle release / `allowReattach: false` gaps; and `vom.rs` had an unformatted line failing `cargo fmt --check`. What: - Route `GeometryContext.endpointForFrame` via `frameProcessBoundary(frame)` so `Page.getLayoutMetrics` and owner caches share one key per renderer process. - Update `createStubCdpRunner` to resolve `this` on spread clones and enforce `CdpDispatchGuard` (`signal`, `allowReattach`, `attachmentId`, `onDispatch`) on `<4`-argument `sendToFrame` overrides (including Vitest `vi.fn` spies). - Harden `ChromiumCdp.sendToFrame` / `resolveValidatedFrameRoute`, `cdpRunnerForFrame`, `waits`, `element-geometry`, `scroll`, `wheel`, `pointer-transaction`, `file-input-transaction`, `file-drop-transaction`, `visual-target`, `vom/document-identity`, and `viewport-screenshot` so V8 handle cleanup bypasses aborted signals, skips stale-loader rejections, and never reattaches detached tabs or masks caller outcomes. - Format `crates/bsk-cli/src/cli/recipe/vom.rs` (`cargo fmt --all`), add regression tests across `chromium-cdp.test.ts`, `geometry-context.test.ts`, `form-state.test.ts`, `element-geometry.test.ts`, and `waits.test.ts`, and sync bilingual `CHANGELOG.md` / `CHANGELOG.zh-CN.md`. --- CHANGELOG.md | 2 +- CHANGELOG.zh-CN.md | 2 +- .../__tests__/chromium-cdp.test.ts | 74 +++++++ .../src/browser-driver/chromium-cdp.ts | 24 +- .../tools/__tests__/element-geometry.test.ts | 1 + .../tools/__tests__/geometry-context.test.ts | 12 + .../src/tools/__tests__/waits.test.ts | 2 +- apps/extension/src/tools/element-geometry.ts | 64 +++--- .../src/tools/file-drop-transaction.ts | 5 +- .../src/tools/file-input-transaction.ts | 5 +- .../src/tools/geometry/frame-context.ts | 3 +- .../src/tools/pointer-transaction.ts | 1 + apps/extension/src/tools/scroll.ts | 1 + apps/extension/src/tools/shared.ts | 205 +++++++++++++++--- .../src/tools/viewport-screenshot.ts | 27 ++- apps/extension/src/tools/visual-target.ts | 12 +- .../tools/vom/__tests__/form-state.test.ts | 32 +++ .../src/tools/vom/document-identity.ts | 4 +- apps/extension/src/tools/waits.ts | 4 +- apps/extension/src/tools/wheel.ts | 1 + crates/bsk-cli/src/cli/recipe/vom.rs | 3 +- 21 files changed, 387 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80c2107d8..fe9a04007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ determines which versions can connect. ### Changed -- **Complete `browser-driver/` boundary encapsulation, strict `CdpRunner` RAII contract, unified `--selector` DOM predicates, and structured VOM metadata**: Sealed `CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, and `getFrameGraph` inside `apps/extension/src/browser-driver/` behind `FrameDocumentHandle`, `FrameNodeHandle`, `FrameProcessBoundary`, and `TabEventSource` across all remaining upper-layer modules (`read`, `extract`, `waits`, `wheel`, `drag`, `action-guard`, `vom/facts`, `AccessibilityLease`, `debug/`, `recording-runtime`, and `geometry/frame-context`); promoted all `CdpRunner` methods to a non-optional interface backed by `delegateCdpRunner`, `createStubCdpRunner`, and deterministic RAII `withObjectGroup` / `withRemoteObject` V8 handle release (`handleSelect`, `handleFill`, `handleFocus`/`handleBlur`, `action-checks`, `file-input-transaction`, and `evaluate` with `return_by_value: false`, without resurrecting detached tabs or masking pre-dispatch errors); unified cross-frame, open Shadow DOM, `[data-bsk-overlay]` exclusion, sensitive `autocomplete`, and interactive ARIA role predicates across `interaction`, `CooperationController`, `human-loop`, `read`, and `extract`; and converged `cooperate sync` and `observe` onto structured `@browser-skill/vom` `VomLineMetadata` (`inspectVomLine` and `VOM_DERIVED_NAME_CAP`). +- **Complete `browser-driver/` boundary encapsulation, strict `CdpRunner` RAII contract, unified `--selector` DOM predicates, and structured VOM metadata**: Sealed `CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, and `getFrameGraph` inside `apps/extension/src/browser-driver/` behind `FrameDocumentHandle`, `FrameNodeHandle`, `FrameProcessBoundary`, and `TabEventSource` across all remaining upper-layer modules (`read`, `extract`, `waits`, `wheel`, `drag`, `action-guard`, `vom/facts`, `AccessibilityLease`, `debug/`, `recording-runtime`, and `geometry/frame-context`); promoted all `CdpRunner` methods to a non-optional interface backed by `delegateCdpRunner`, `createStubCdpRunner`, and deterministic RAII `withObjectGroup` / `withRemoteObject` V8 handle release (`handleSelect`, `handleFill`, `handleFocus`/`handleBlur`, `action-checks`, `file-input-transaction`, `file-drop-transaction`, `waits`, `element-geometry`, `cdpRunnerForFrame`, and `evaluate` with `return_by_value: false`, without resurrecting detached tabs or masking pre-dispatch errors); unified cross-frame, open Shadow DOM, `[data-bsk-overlay]` exclusion, sensitive `autocomplete`, and interactive ARIA role predicates across `interaction`, `CooperationController`, `human-loop`, `read`, and `extract`; and converged `cooperate sync` and `observe` onto structured `@browser-skill/vom` `VomLineMetadata` (`inspectVomLine` and `VOM_DERIVED_NAME_CAP`). - **Lightweight `@wxg-prc-cpg/browser-skill-dsh-plugin` on the unified `bsk` CLI runtime**: Replaced the ~22k-LOC bespoke `bsk_*` wrapper and Web UI bundle in `packages/dsh-plugin-browserskill` with a lightweight Cordis plugin (free of `@deepseek-ai/dsh-*` host `rc` peer locks) that packages the canonical `crates/bsk-cli/skill` bundle, automatically heals host `PATH` for `~/.local/bin` (`$BSK_INSTALL_DIR`) and configured `bskPath`, and bundles an on-demand cross-platform `bsk` CLI + browser extension setup workflow (`bin/install-bsk.mjs` + `AGENT_INSTALL.md`) sharing the standard `bsk` CLI and `DSH_SESSION_ID` daemon scoping alongside restored `dsh-plugin-v*` release automation. - **Zero-tax Chrome MV3 `registration: "runtime"` dual-tier content script injection, SW-startup orphan Agent Window reaping, and shared `content/dom-primitives.ts`**: Migrated all four MV3 content-script entrypoints (`content`, `cooperation.content`, `record-frame.content`, and `long-screenshot-page.content`) out of static `manifest.json` `content_scripts` into idempotent runtime injection (`claimIdempotentContentScript`) with `noScriptStartedPostMessage: true` so ordinary browsing incurs zero content-script, `window.postMessage`, or React 19 overhead; injected Tier A (`content.js` and `long-screenshot-page.js`) per `tabId`/`documentId` via idempotent `ensureOverlayInjected` across controlled task tabs, `webNavigation.onCommitted` (including Speculation Rules prerender outermost frames, sequenced strictly after Tier B `cooperation.js` arming/repair), pre-confirmation `tab_borrow` candidate user tabs, `request_help`, `record`, and active `video` overlays while skipping uninjected tabs on `hidden` / `CAPTURE_SUPPRESS` / `INPUT_PASSTHROUGH`; dynamically registered/unregistered pure-TypeScript Tier B (`cooperation.js` and `record-frame.js`) with `persistAcrossSessions: false` when active sessions or recordings transition between `0` and `>= 1`, loading silently at `document_start` with 0 DOM listeners and 0 startup IPC until armed by Background (`COOPERATION_ARM_FRAME` / `RECORD_FRAME_START` plus 50ms per-tab subframe coalescing, `cooperation.js` co-injection in `injectRecordFrameAllFrames`, and non-destructive per-frame `injectCooperationAllFrames` repair across `tab_borrow`, `ack === undefined` frames, and dynamically inserted light-DOM or open/closed `ShadowRoot` `about:blank` / `srcdoc` subframes); tracked dedicated Agent Window `ownedTabIds` in `chrome.storage.session` (`bsk.orphan_agent_windows`) to reap orphaned pure Agent Windows on Service Worker restart without ever closing windows containing user, borrowed, or human-preserved tabs; and extracted `apps/extension/src/content/dom-primitives.ts`. - **Single daemon `tool_pipeline`, 4-module `daemon/ipc` decomposition, non-blocking persistence, and shared `cli::orchestration` kernel**: Decomposed 3,489-line `daemon/ipc.rs` into `daemon/tool_pipeline.rs` (single unified `tool.*` pipeline with RAII `AuditTicketGuard`, `SessionActivityGuard`, `ToolInflightRegistry`, `IntentWal`, and post-settlement scope/run-log reconciliation), `daemon/ipc_sessions.rs`, `daemon/ipc_transport.rs`, and a slim `daemon/ipc.rs` router; promoted exhaustive `OutcomeKind` (`bsk-protocol`) / `Outcome` (`daemon/tool_policy.rs`) routing in `daemon/queue.rs`; moved `TransferRegistry` staging/import/session-teardown I/O, `AuditStore` JSONL `sync_data()` writes (including `session.user_interrupt` markers), and monotonic-sequence `IntentWal` flushes/atomic `sync_data()` compactions out of global state `Mutex`es and Tokio worker threads via `spawn_blocking`; and extracted `cli/orchestration.rs` to share Canvas screenshot point-click validation, `open`/`sync`/action/positive timeout bounds, explicit `--session` + `--browser` conflict checks, cached VOM hydration, and 2-second `CANCEL_RESPONSE_GRACE` settlement across `bsk` CLI and `bsk mcp`. diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index e9aeecf59..13d8095a3 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -17,7 +17,7 @@ CLI 和扩展可以分别发布,能否连接由协议兼容性决定。更新 ### 变更 -- **`browser-driver/` 彻底封盒、必填 `CdpRunner` 与确定性 RAII V8 句柄释放、统一 `--selector` DOM 谓词及结构化 VOM 行元数据**:将 `CdpTarget`、`CdpFrame`、`CdpFrameGraph`、`sendToTarget`、`sendGuarded`、`sendAttached` 与 `getFrameGraph` 彻底封入 `apps/extension/src/browser-driver/` 内部,通过 `FrameDocumentHandle`、`FrameNodeHandle`、`FrameProcessBoundary` 与 `TabEventSource` 统一收口剩余全部上层模块(`read`、`extract`、`waits`、`wheel`、`drag`、`action-guard`、`vom/facts`、`AccessibilityLease`、`debug/`、`recording-runtime` 与 `geometry/frame-context`);将 `CdpRunner` 接口方法全面提升为必填非可选契约,配套 `delegateCdpRunner`、`createStubCdpRunner` 以及确定性 RAII `withObjectGroup` / `withRemoteObject` V8 句柄释放(覆盖 `handleSelect`、`handleFill`、`handleFocus`/`handleBlur`、`action-checks`、`file-input-transaction` 与 `evaluate(return_by_value: false)`,不唤醒已断开标签页且不掩盖派发前错误);在 `interaction`、`CooperationController`、`human-loop`、`read` 与 `extract` 间统一跨帧、open Shadow DOM、`[data-bsk-overlay]` 排除、敏感 `autocomplete` 与交互式 ARIA 角色谓词;并将 `cooperate sync` 与 `observe` 统一收敛至 `@browser-skill/vom` 的结构化 `VomLineMetadata`(`inspectVomLine` 与 `VOM_DERIVED_NAME_CAP`)。 +- **`browser-driver/` 彻底封盒、必填 `CdpRunner` 与确定性 RAII V8 句柄释放、统一 `--selector` DOM 谓词及结构化 VOM 行元数据**:将 `CdpTarget`、`CdpFrame`、`CdpFrameGraph`、`sendToTarget`、`sendGuarded`、`sendAttached` 与 `getFrameGraph` 彻底封入 `apps/extension/src/browser-driver/` 内部,通过 `FrameDocumentHandle`、`FrameNodeHandle`、`FrameProcessBoundary` 与 `TabEventSource` 统一收口剩余全部上层模块(`read`、`extract`、`waits`、`wheel`、`drag`、`action-guard`、`vom/facts`、`AccessibilityLease`、`debug/`、`recording-runtime` 与 `geometry/frame-context`);将 `CdpRunner` 接口方法全面提升为必填非可选契约,配套 `delegateCdpRunner`、`createStubCdpRunner` 以及确定性 RAII `withObjectGroup` / `withRemoteObject` V8 句柄释放(覆盖 `handleSelect`、`handleFill`、`handleFocus`/`handleBlur`、`action-checks`、`file-input-transaction`、`file-drop-transaction`、`waits`、`element-geometry`、`cdpRunnerForFrame` 与 `evaluate(return_by_value: false)`,不唤醒已断开标签页且不掩盖派发前错误);在 `interaction`、`CooperationController`、`human-loop`、`read` 与 `extract` 间统一跨帧、open Shadow DOM、`[data-bsk-overlay]` 排除、敏感 `autocomplete` 与交互式 ARIA 角色谓词;并将 `cooperate sync` 与 `observe` 统一收敛至 `@browser-skill/vom` 的结构化 `VomLineMetadata`(`inspectVomLine` 与 `VOM_DERIVED_NAME_CAP`)。 - **基于统一 `bsk` CLI 运行时的轻量 `@wxg-prc-cpg/browser-skill-dsh-plugin` 插件**:将 `packages/dsh-plugin-browserskill` 从原先约 2.2 万行的自定义 `bsk_*` 工具包装器与 Web UI 前端组件重构为极简 Cordis 插件(彻底移除 `@deepseek-ai/dsh-*` 宿主 `rc` 版本对等依赖锁),直接内嵌主仓唯一的 `crates/bsk-cli/skill` 资源包,自动为宿主补齐 `~/.local/bin`(`$BSK_INSTALL_DIR`)与自定义 `bskPath` 的 `PATH` 解析,并打包跨平台按需 `bsk` CLI 安装器与浏览器扩展连接引导流程(`bin/install-bsk.mjs` + `AGENT_INSTALL.md`),底层与其他所有 Agent 100% 共用标准 `bsk` CLI 与 `DSH_SESSION_ID` 自动作用域机制,同时恢复 `dsh-plugin-v*` 独立发版流水线。 - **Chrome MV3 `registration: "runtime"` 零税双层按需内容脚本注入、SW 启动期孤儿 Agent 窗口兜底回收与共享 `content/dom-primitives.ts`**:将全部 4 个 MV3 内容脚本入口(`content`、`cooperation.content`、`record-frame.content` 与 `long-screenshot-page.content`)以 `noScriptStartedPostMessage: true` 与 `claimIdempotentContentScript` 幂等守卫迁出静态 `manifest.json` `content_scripts`,彻底消除日常浏览普通网页时的全局 `` 内容脚本、`window.postMessage` 广播与 React 19 加载税;通过幂等 `ensureOverlayInjected` 对受控任务标签页、`webNavigation.onCommitted`(含 Speculation Rules 预渲染顶层帧,且严格排在 Tier B `cooperation.js` 激活/补注入之后)、确认前 `tab_borrow` 候选用户标签页、`request_help`、`record` 与活跃 `video` 浮层按 `tabId`/`documentId` 单点注入 Tier A(`content.js` 与 `long-screenshot-page.js`),并在 `hidden` / `CAPTURE_SUPPRESS` / `INPUT_PASSTHROUGH` 时跳过未注入标签页;在活跃会话或录制于 `0` 与 `>= 1` 间切换时以 `persistAcrossSessions: false` 动态启停纯 TS 轻量 Tier B(`cooperation.js` 与 `record-frame.js`),使其在 `document_start` 零 DOM 监听、零启动唤醒 IPC 静默加载并由后台按需点名激活(`COOPERATION_ARM_FRAME` / `RECORD_FRAME_START` 配合单标签页 50ms 子帧合并去抖、`injectRecordFrameAllFrames` 协同注入 `cooperation.js`,以及在 `tab_borrow`、`ack === undefined` 帧与动态插入的普通 DOM 或 open/closed `ShadowRoot` 内同源 `about:blank` / `srcdoc` 子帧上进行无损按帧探测补注入);通过 `chrome.storage.session`(`bsk.orphan_agent_windows`)追踪独占 Agent 窗口创建的标签页集合,在 Service Worker 重启后精准回收无活跃会话的纯孤儿 Agent 窗口(严格保留含用户、借用或人工接管标签页的窗口);并抽离共享 DOM 原语模块 `apps/extension/src/content/dom-primitives.ts`。 - **守护进程单一 `tool_pipeline` 工具管线、`daemon/ipc` 四模块拆解、无锁阻塞持久化与 `cli::orchestration` 客户端共享编排内核**:将 3,489 行的 `daemon/ipc.rs` 拆解为 `daemon/tool_pipeline.rs`(统一收口全部 `tool.*` 执行路径的 RAII `AuditTicketGuard`、`SessionActivityGuard`、`ToolInflightRegistry`、`IntentWal` 与作用域/运行日志结算)、`daemon/ipc_sessions.rs`、`daemon/ipc_transport.rs` 及精简路由层 `daemon/ipc.rs`;在 `bsk-protocol` 与 `daemon/tool_policy.rs` / `daemon/queue.rs` 中建立穷尽式 `OutcomeKind` / `Outcome` 结果与超时归因路由;通过局部状态分离与 `spawn_blocking` 将 `TransferRegistry` 大文件暂存/导入复制/会话销毁目录清理、`AuditStore` JSONL `sync_data()` 刷盘(含 `session.user_interrupt` 标记)与单调序号 `IntentWal` 刷盘及 `sync_data()` 原子压缩彻底移出全局状态 `Mutex` 与 Tokio 工作线程阻塞路径;并提取 `cli/orchestration.rs` 统一 `bsk` CLI 与 `bsk mcp` 的 Canvas 截图坐标点击校验、`open`/`sync`/动作/正数超时边界、显式 `--session` + `--browser` 冲突校验、缓存 VOM 回填以及 2 秒 `CANCEL_RESPONSE_GRACE` 优雅取消结算。 diff --git a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts index 86bec1a9d..c30c557cb 100644 --- a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts +++ b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts @@ -3431,4 +3431,78 @@ describe("idle soft-detach and immediate close/return detach", () => { cdp.dispose(); } }); + + it("releases object groups on navigated frame handles without refreshing frame routes or reattaching", async () => { + const { api, onEvent } = fakeApi(); + let frameTreeCalls = 0; + vi.mocked(api.sendCommand).mockImplementation(async (_target, method) => { + if (method === "Page.getFrameTree") { + frameTreeCalls++; + return { + frameTree: { + frame: { id: "root", loaderId: "loader-1", url: "https://example.test/" }, + childFrames: [ + { + frame: { + id: "child", + parentId: "root", + loaderId: "child-loader-1", + url: "https://example.test/child", + }, + }, + ], + }, + }; + } + return {}; + }); + const cdp = new ChromiumCdp(api); + try { + await cdp.ensureAttached(58); + await cdp.getFrameDocuments(58); + const initialTreeCalls = frameTreeCalls; + + // Simulate same-process child frame navigation updating `loaderId`. + onEvent.fire({ tabId: 58 }, "Page.frameNavigated", { + frame: { + id: "child", + parentId: "root", + loaderId: "child-loader-2", + url: "https://example.test/child-next", + }, + }); + + const staleChildHandle = { + tabId: 58, + frameId: "child", + loaderId: "child-loader-1", + backendNodeId: 77, + }; + // Normal reads/writes on stale loaderId deterministically reject. + await expect( + cdp.sendToFrame(staleChildHandle, "DOM.resolveNode", { backendNodeId: 77 }), + ).rejects.toMatchObject({ name: "StaleFrameHandleError" }); + + // Cleanup (`Runtime.releaseObjectGroup` / `Runtime.releaseObject`) still dispatches + // to the session without calling `Page.getFrameTree` or throwing `StaleFrameHandleError`. + await expect( + cdp.sendToFrame( + staleChildHandle, + "Runtime.releaseObjectGroup", + { objectGroup: "bsk-test-group" }, + { allowReattach: false }, + ), + ).resolves.toEqual({}); + expect(frameTreeCalls).toBe(initialTreeCalls); + expect(api.sendCommand).toHaveBeenLastCalledWith( + { tabId: 58 }, + "Runtime.releaseObjectGroup", + { + objectGroup: "bsk-test-group", + }, + ); + } finally { + cdp.dispose(); + } + }); }); diff --git a/apps/extension/src/browser-driver/chromium-cdp.ts b/apps/extension/src/browser-driver/chromium-cdp.ts index 236c0b7ed..233b42d9b 100644 --- a/apps/extension/src/browser-driver/chromium-cdp.ts +++ b/apps/extension/src/browser-driver/chromium-cdp.ts @@ -1045,20 +1045,15 @@ export class ChromiumCdp { return this.sendToTarget({ tabId }, method, params); } - const route = await this.resolveValidatedFrameRoute( - handle, - method, - guard?.signal, - guard?.ignoreFrameNavigation === true, - ); - const target: CdpTarget = { - tabId, - ...(route.sessionId ? { sessionId: route.sessionId } : {}), - }; const ignoreNav = guard?.ignoreFrameNavigation === true || method === "Runtime.releaseObjectGroup" || method === "Runtime.releaseObject"; + const route = await this.resolveValidatedFrameRoute(handle, method, guard?.signal, ignoreNav); + const target: CdpTarget = { + tabId, + ...(route.sessionId ? { sessionId: route.sessionId } : {}), + }; try { return (await this.command( @@ -1122,7 +1117,7 @@ export class ChromiumCdp { let route = entry.frames.get(frameId); const needsRefresh = - !ignoreFrameNavigation && + !isCleanup && (!route || !route.authoritative || (entry.frameDiscovery.pending.size > 0 && !route.sessionId)); @@ -1132,7 +1127,7 @@ export class ChromiumCdp { if (entry?.kind !== "attached") throw new Error("debugger detached"); route = entry.frames.get(frameId); } - if (!route && ignoreFrameNavigation) { + if (!route && isCleanup) { for (const [sessionId, mappedFrameId] of entry.frameDiscovery.frameIds) { if (mappedFrameId === frameId) { return { sessionId, loaderId: "", authoritative: false }; @@ -1151,10 +1146,7 @@ export class ChromiumCdp { ); } const requiresNonEmptyLoader = "backendNodeId" in handle && route.sessionId !== undefined; - if ( - !ignoreFrameNavigation && - (route.loaderId !== loaderId || (requiresNonEmptyLoader && !loaderId)) - ) { + if (!isCleanup && (route.loaderId !== loaderId || (requiresNonEmptyLoader && !loaderId))) { throw new StaleFrameHandleError( tabId, frameId, diff --git a/apps/extension/src/tools/__tests__/element-geometry.test.ts b/apps/extension/src/tools/__tests__/element-geometry.test.ts index 8ad1ebc64..3847a7927 100644 --- a/apps/extension/src/tools/__tests__/element-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/element-geometry.test.ts @@ -83,6 +83,7 @@ describe("nodeContentRegion", () => { "DOM.getContentQuads", "DOM.resolveNode", "Runtime.callFunctionOn", + "Runtime.releaseObject", ]); }); }); diff --git a/apps/extension/src/tools/__tests__/geometry-context.test.ts b/apps/extension/src/tools/__tests__/geometry-context.test.ts index f6d713a96..40d17e682 100644 --- a/apps/extension/src/tools/__tests__/geometry-context.test.ts +++ b/apps/extension/src/tools/__tests__/geometry-context.test.ts @@ -476,4 +476,16 @@ describe("measurement geometry context", () => { height: 1000, }); }); + + it("shares Page.getLayoutMetrics cache between process boundary and frame projections", async () => { + const cdp = driver(); + const context = new GeometryContext(cdp, 4, graph); + await context.layoutMetrics(process); + await context.topLayoutMetrics(process); + await context.targetProjection("child"); + await context.targetViewport("child"); + expect( + cdp.calls.mock.calls.filter(([, method]) => method === "Page.getLayoutMetrics"), + ).toHaveLength(1); + }); }); diff --git a/apps/extension/src/tools/__tests__/waits.test.ts b/apps/extension/src/tools/__tests__/waits.test.ts index 7c6d940be..069962cc1 100644 --- a/apps/extension/src/tools/__tests__/waits.test.ts +++ b/apps/extension/src/tools/__tests__/waits.test.ts @@ -989,7 +989,7 @@ describe("wait-for-element regression coverage", () => { { tabId: 4, frameId: "child-frame", loaderId: "child-loader" }, "Runtime.releaseObjectGroup", { objectGroup: expect.any(String) }, - undefined, + { allowReattach: false }, ); expect(fake.sent).toHaveLength(0); }); diff --git a/apps/extension/src/tools/element-geometry.ts b/apps/extension/src/tools/element-geometry.ts index 9cde37c28..4e250c277 100644 --- a/apps/extension/src/tools/element-geometry.ts +++ b/apps/extension/src/tools/element-geometry.ts @@ -11,7 +11,7 @@ import { rectPolygon, type ViewportRect, } from "./geometry"; -import { type CdpRunner, isRpcError } from "./shared"; +import { type CdpRunner, isRpcError, withRemoteObject } from "./shared"; const ELEMENT_NOT_VISIBLE_MESSAGE = "element not visible (no content quads, box model, or visible descendant bounds)"; @@ -65,22 +65,24 @@ export async function scrollNodeIntoView( const objectIdOrErr = await backendNodeToObject(cdp, tabId, backendNodeId); if (isRpcError(objectIdOrErr)) return objectIdOrErr; - try { - await cdp.send(tabId, "Runtime.callFunctionOn", { - objectId: objectIdOrErr, - functionDeclaration: `function() { - this.scrollIntoView({ block: 'center', inline: 'center' }); - }`, - returnByValue: true, - }); - return null; - } catch (err) { - if (isStaleFrameHandleError(err)) throw err; - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; - } + return withRemoteObject(cdp, tabId, objectIdOrErr, async (objectId) => { + try { + await cdp.send(tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function() { + this.scrollIntoView({ block: 'center', inline: 'center' }); + }`, + returnByValue: true, + }); + return null; + } catch (err) { + if (isStaleFrameHandleError(err)) throw err; + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + }); } function validCdpRegion(quads: number[][] | undefined): Region { @@ -152,12 +154,13 @@ async function descendantBoundingRect( console.debug("[bsk element-geometry] resolveNode for descendant bounds failed", objectIdOrErr); return null; } - try { - const evaluated = await cdp.send<{ - result?: { value?: ViewportRect | null }; - }>(tabId, "Runtime.callFunctionOn", { - objectId: objectIdOrErr, - functionDeclaration: `function() { + return withRemoteObject(cdp, tabId, objectIdOrErr, async (objectId) => { + try { + const evaluated = await cdp.send<{ + result?: { value?: ViewportRect | null }; + }>(tabId, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: `function() { const viewportWidth = window.innerWidth || document.documentElement.clientWidth || 0; const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 0; const rects = []; @@ -188,13 +191,14 @@ async function descendantBoundingRect( const bottom = Math.max(...rects.map((r) => r.bottom)); return { x: left, y: top, width: right - left, height: bottom - top }; }`, - returnByValue: true, - }); - return parseViewportRect(evaluated.result?.value); - } catch (err) { - console.debug("[bsk element-geometry] descendant bounds failed", err); - return null; - } + returnByValue: true, + }); + return parseViewportRect(evaluated.result?.value); + } catch (err) { + console.debug("[bsk element-geometry] descendant bounds failed", err); + return null; + } + }); } function parseViewportRect(value: unknown): ViewportRect | null { diff --git a/apps/extension/src/tools/file-drop-transaction.ts b/apps/extension/src/tools/file-drop-transaction.ts index da97a45c1..72577e67d 100644 --- a/apps/extension/src/tools/file-drop-transaction.ts +++ b/apps/extension/src/tools/file-drop-transaction.ts @@ -285,7 +285,8 @@ export async function uploadThroughFileDrop( !( err instanceof Error && (err.message === "No resources owned on this tab" || - err.message === "debugger detached") + err.message === "debugger detached" || + err.message === "Debugger attachment changed before dispatch") ) && !isStaleFrameHandleError(err) ) { @@ -298,7 +299,7 @@ export async function uploadThroughFileDrop( ? (outcome.data?.effect_state as TransferEffectState | undefined) : "committed"; if (effect === "unknown" || cleanupFailed) { - await options.cdp.detach?.(handle.tabId); + await options.cdp.detach?.(handle.tabId).catch(() => {}); } if (cleanupFailed && isRpcError(outcome)) { outcome.data = { ...outcome.data, cleanup_state: "failed" }; diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts index 3d5688a3b..e5bf71a39 100644 --- a/apps/extension/src/tools/file-input-transaction.ts +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -488,7 +488,8 @@ export async function uploadThroughActivatedFileInput( !( err instanceof Error && (err.message === "No resources owned on this tab" || - err.message === "debugger detached") + err.message === "debugger detached" || + err.message === "Debugger attachment changed before dispatch") ) && !isStaleFrameHandleError(err) ) { @@ -501,7 +502,7 @@ export async function uploadThroughActivatedFileInput( ? (outcome.data?.effect_state as TransferEffectState | undefined) : "committed"; if (effect === "unknown" || cleanupFailed) { - await options.cdp.detach?.(handle.tabId); + await options.cdp.detach?.(handle.tabId).catch(() => {}); } if (cleanupFailed && isRpcError(outcome)) { outcome.data = { ...outcome.data, cleanup_state: "failed" }; diff --git a/apps/extension/src/tools/geometry/frame-context.ts b/apps/extension/src/tools/geometry/frame-context.ts index 0292f5863..6e01c8984 100644 --- a/apps/extension/src/tools/geometry/frame-context.ts +++ b/apps/extension/src/tools/geometry/frame-context.ts @@ -3,6 +3,7 @@ import { type FrameDocumentHandle, type FrameDocumentInfo, type FrameProcessBoundary, + frameProcessBoundary, frameProcessIdentityKey, } from "@/browser-driver/frame-handles"; import { @@ -108,7 +109,7 @@ export class GeometryContext { } private endpointForFrame(frame: FrameDocumentInfo): GeometryEndpoint { - return this.frameHandles?.get(frame.frameId) ?? frame.handle; + return this.frameHandles?.get(frame.frameId) ?? frameProcessBoundary(frame); } private async request( diff --git a/apps/extension/src/tools/pointer-transaction.ts b/apps/extension/src/tools/pointer-transaction.ts index de7663877..7b3f0171e 100644 --- a/apps/extension/src/tools/pointer-transaction.ts +++ b/apps/extension/src/tools/pointer-transaction.ts @@ -71,6 +71,7 @@ export function pointerTransactionDeps( return result; }; const cdp: CdpRunner = delegateCdpRunner(deps.cdp, { + resourceCleanup: deps.cdp.resourceCleanup ?? deps.cdp, send: (tabId: number, method: string, params?: object, sig?: AbortSignal) => sendFrame(toRootFrameHandle(tabId), method, params, sig ? { signal: sig } : undefined), sendToFrame: sendFrame, diff --git a/apps/extension/src/tools/scroll.ts b/apps/extension/src/tools/scroll.ts index 46d5e5182..628033daf 100644 --- a/apps/extension/src/tools/scroll.ts +++ b/apps/extension/src/tools/scroll.ts @@ -80,6 +80,7 @@ export async function handleScrollTo( }; let graph: Promise | undefined; const cdp: CdpRunner = delegateCdpRunner(deps.cdp, { + resourceCleanup: deps.cdp.resourceCleanup ?? deps.cdp, send: (tabId: number, method: string, args?: object, sig?: AbortSignal) => sendFrame(toRootFrameHandle(tabId), method, args, sig ? { signal: sig } : undefined), sendToFrame: sendFrame, diff --git a/apps/extension/src/tools/shared.ts b/apps/extension/src/tools/shared.ts index d3e06ac76..f11825edd 100644 --- a/apps/extension/src/tools/shared.ts +++ b/apps/extension/src/tools/shared.ts @@ -255,8 +255,79 @@ export type CdpRunnerStubOverrides = Partial> & { export function createStubCdpRunner( overrides: T, ): CdpRunner & Omit, keyof CdpRunner> { - const { send, ...restOverrides } = overrides; + const { send, sendToFrame: rawSendToFrame, ...restOverrides } = overrides; const typedSend = send as CdpRunner["send"]; + const resolveSelf = (self: unknown): CdpRunner => + self !== null && + typeof self === "object" && + typeof (self as Partial).send === "function" && + typeof (self as Partial).getAttachmentId === "function" + ? (self as CdpRunner) + : stub; + const applyDispatchGuard = ( + self: CdpRunner, + handle: FrameDocumentHandle | FrameNodeHandle, + guard?: CdpDispatchGuard, + ) => { + guard?.signal?.throwIfAborted(); + if (guard?.allowReattach === false && self.getAttachmentId(handle.tabId) === undefined) { + throw new Error("debugger detached"); + } + if (guard?.attachmentId !== undefined) { + const currentAttachment = self.getAttachmentId(handle.tabId); + if (currentAttachment !== guard.attachmentId) { + throw new Error("Debugger attachment changed before dispatch"); + } + } + guard?.onDispatch?.(); + }; + + let wrappedSendToFrame: CdpRunner["sendToFrame"] | undefined; + if (rawSendToFrame) { + const mockFn = rawSendToFrame as CdpRunner["sendToFrame"] & { + getMockImplementation?: () => CdpRunner["sendToFrame"] | undefined; + mockImplementation?: (fn: CdpRunner["sendToFrame"]) => unknown; + }; + if ( + typeof mockFn.getMockImplementation === "function" && + typeof mockFn.mockImplementation === "function" + ) { + const impl = mockFn.getMockImplementation(); + if (impl && impl.length < 4) { + mockFn.mockImplementation(async function ( + this: unknown, + handle: FrameDocumentHandle | FrameNodeHandle, + _method: string, + _params?: object, + guard?: CdpDispatchGuard, + ): Promise { + applyDispatchGuard(resolveSelf(this), handle, guard); + return (await impl.apply( + this, + arguments as unknown as Parameters, + )) as R; + }); + } + wrappedSendToFrame = mockFn; + } else if (rawSendToFrame.length < 4) { + wrappedSendToFrame = async function ( + this: unknown, + handle: FrameDocumentHandle | FrameNodeHandle, + _method: string, + _params?: object, + guard?: CdpDispatchGuard, + ): Promise { + applyDispatchGuard(resolveSelf(this), handle, guard); + return (await rawSendToFrame.apply( + this, + arguments as unknown as Parameters, + )) as R; + }; + } else { + wrappedSendToFrame = rawSendToFrame; + } + } + const stub: CdpRunner = { prepareInputTimestamp: async () => ({ timestamp: Date.now(), @@ -264,38 +335,34 @@ export function createStubCdpRunner( assertValid: () => {}, }), send: typedSend, - sendToFrame: async ( + sendToFrame: async function ( + this: unknown, handle: FrameDocumentHandle | FrameNodeHandle, method: string, params?: object, guard?: CdpDispatchGuard, - ): Promise => { - guard?.signal?.throwIfAborted(); - if (guard?.attachmentId !== undefined) { - const currentAttachment = stub.getAttachmentId(handle.tabId); - if (currentAttachment !== guard.attachmentId) { - throw new Error("Debugger attachment changed before dispatch"); - } - } - guard?.onDispatch?.(); + ): Promise { + const self = resolveSelf(this); + applyDispatchGuard(self, handle, guard); if (guard?.signal !== undefined) { - return stub.send(handle.tabId, method, params, guard.signal); + return self.send(handle.tabId, method, params, guard.signal); } return params !== undefined - ? stub.send(handle.tabId, method, params) - : stub.send(handle.tabId, method); + ? self.send(handle.tabId, method, params) + : self.send(handle.tabId, method); }, - getMainFrameHandle: async (tabId: number, signal?: AbortSignal) => { + getMainFrameHandle: async function (this: unknown, tabId: number, signal?: AbortSignal) { + const self = resolveSelf(this); signal?.throwIfAborted(); try { const tree = await (signal !== undefined - ? stub.send<{ frameTree?: { frame?: { id?: string; loaderId?: string } } }>( + ? self.send<{ frameTree?: { frame?: { id?: string; loaderId?: string } } }>( tabId, "Page.getFrameTree", {}, signal, ) - : stub.send<{ frameTree?: { frame?: { id?: string; loaderId?: string } } }>( + : self.send<{ frameTree?: { frame?: { id?: string; loaderId?: string } } }>( tabId, "Page.getFrameTree", {}, @@ -317,7 +384,12 @@ export function createStubCdpRunner( } return toFrameDocumentHandle(tabId, { frameId: "root", loaderId: "" }); }, - getFrameDocuments: async (tabId: number, signal?: AbortSignal): Promise => { + getFrameDocuments: async function ( + this: unknown, + tabId: number, + signal?: AbortSignal, + ): Promise { + const self = resolveSelf(this); signal?.throwIfAborted(); try { type RawFrameNode = { @@ -331,8 +403,8 @@ export function createStubCdpRunner( childFrames?: RawFrameNode[]; }; const tree = await (signal !== undefined - ? stub.send<{ frameTree?: RawFrameNode }>(tabId, "Page.getFrameTree", {}, signal) - : stub.send<{ frameTree?: RawFrameNode }>(tabId, "Page.getFrameTree", {})); + ? self.send<{ frameTree?: RawFrameNode }>(tabId, "Page.getFrameTree", {}, signal) + : self.send<{ frameTree?: RawFrameNode }>(tabId, "Page.getFrameTree", {})); const rootFrame = tree?.frameTree?.frame; if (rootFrame?.id) { const rootLoaderId = rootFrame.loaderId ?? ""; @@ -375,10 +447,9 @@ export function createStubCdpRunner( } }; walkChildren(tree.frameTree?.childFrames, rootFrame.id); + const attachmentId = self.getAttachmentId(tabId); return { - ...(stub.getAttachmentId(tabId) !== undefined - ? { attachmentId: stub.getAttachmentId(tabId) } - : {}), + ...(attachmentId !== undefined ? { attachmentId } : {}), rootFrameId: rootFrame.id, frames, }; @@ -391,10 +462,9 @@ export function createStubCdpRunner( throw err; } } + const attachmentId = self.getAttachmentId(tabId); return { - ...(stub.getAttachmentId(tabId) !== undefined - ? { attachmentId: stub.getAttachmentId(tabId) } - : {}), + ...(attachmentId !== undefined ? { attachmentId } : {}), rootFrameId: "", frames: [], }; @@ -426,6 +496,7 @@ export function createStubCdpRunner( truncated: false, }), ...restOverrides, + ...(wrappedSendToFrame ? { sendToFrame: wrappedSendToFrame } : {}), }; return stub as CdpRunner & NoInfer; } @@ -445,13 +516,65 @@ export function sendToFrameHandle( : cdp.sendToFrame(handle, method); } +const RELEASE_OBJECT_METHODS = new Set(["Runtime.releaseObjectGroup", "Runtime.releaseObject"]); + +function buildFrameCleanupGuard( + baseGuard: CdpDispatchGuard | undefined, + callGuard: CdpDispatchGuard | undefined, +): CdpDispatchGuard { + const attachmentId = callGuard?.attachmentId ?? baseGuard?.attachmentId; + const ignoreFrameNavigation = + callGuard?.ignoreFrameNavigation ?? baseGuard?.ignoreFrameNavigation; + return { + ...(attachmentId !== undefined ? { attachmentId } : {}), + ...(ignoreFrameNavigation !== undefined ? { ignoreFrameNavigation } : {}), + allowReattach: false, + }; +} + export function cdpRunnerForFrame( cdp: CdpRunner, handle: FrameDocumentHandle | FrameNodeHandle, guard?: CdpDispatchGuard, ): CdpRunner { + const remapTarget = ( + target: FrameDocumentHandle | FrameNodeHandle, + ): FrameDocumentHandle | FrameNodeHandle => + target.tabId === handle.tabId && !target.frameId && !target.loaderId ? handle : target; + const resourceCleanup = cdp.resourceCleanup + ? delegateCdpRunner(cdp.resourceCleanup, { + send: (_tabId: number, method: string, params?: object) => + cdp.resourceCleanup!.sendToFrame( + handle, + method, + params, + buildFrameCleanupGuard(guard, undefined), + ), + sendToFrame: ( + target: FrameDocumentHandle | FrameNodeHandle, + method: string, + params?: object, + callGuard?: CdpDispatchGuard, + ) => + cdp.resourceCleanup!.sendToFrame( + remapTarget(target), + method, + params, + buildFrameCleanupGuard(guard, callGuard), + ), + }) + : undefined; return delegateCdpRunner(cdp, { + ...(resourceCleanup ? { resourceCleanup } : {}), send: (_tabId: number, method: string, params?: object, signal?: AbortSignal) => { + if (RELEASE_OBJECT_METHODS.has(method)) { + return (cdp.resourceCleanup ?? cdp).sendToFrame( + handle, + method, + params, + buildFrameCleanupGuard(guard, undefined), + ); + } const effectiveGuard = signal && !guard?.signal ? { ...guard, signal } : guard; if (effectiveGuard !== undefined) { return cdp.sendToFrame(handle, method, params, effectiveGuard); @@ -460,6 +583,36 @@ export function cdpRunnerForFrame( ? cdp.sendToFrame(handle, method, params) : cdp.sendToFrame(handle, method); }, + sendToFrame: ( + target: FrameDocumentHandle | FrameNodeHandle, + method: string, + params?: object, + callGuard?: CdpDispatchGuard, + ) => { + const remapped = remapTarget(target); + if (RELEASE_OBJECT_METHODS.has(method)) { + return (cdp.resourceCleanup ?? cdp).sendToFrame( + remapped, + method, + params, + buildFrameCleanupGuard(guard, callGuard), + ); + } + const mergedGuard = + guard || callGuard + ? { + ...guard, + ...callGuard, + signal: callGuard?.signal ?? guard?.signal, + } + : undefined; + if (mergedGuard !== undefined) { + return cdp.sendToFrame(remapped, method, params, mergedGuard); + } + return params !== undefined + ? cdp.sendToFrame(remapped, method, params) + : cdp.sendToFrame(remapped, method); + }, }); } diff --git a/apps/extension/src/tools/viewport-screenshot.ts b/apps/extension/src/tools/viewport-screenshot.ts index b49322e46..681c1266a 100644 --- a/apps/extension/src/tools/viewport-screenshot.ts +++ b/apps/extension/src/tools/viewport-screenshot.ts @@ -6,7 +6,7 @@ import { import type { RpcError } from "@/transport/types"; import { rpcError } from "./errors"; import { parsePngDimensions } from "./png"; -import { type CdpRunner, delegateCdpRunner } from "./shared"; +import { type CdpRunner, delegateCdpRunner, toRootFrameHandle } from "./shared"; import { BoundedWaitError, waitBounded } from "./transfer-transaction"; import { isAbortError, throwIfAborted } from "./vom/capture-abort"; import { verifyDocumentIdentity } from "./vom/document-identity"; @@ -97,10 +97,18 @@ function boundToAttachment(cdp: CdpRunner, tabId: number, attachmentId: string): if (handle.tabId !== tabId || cdp.getAttachmentId(handle.tabId) !== attachmentId) { return Promise.reject(new Error("Screenshot attachment changed")); } - return cdp.sendToFrame(handle, method, params, { - ...guard, - attachmentId, - }); + const isCleanup = + method === "Runtime.releaseObjectGroup" || method === "Runtime.releaseObject"; + return (isCleanup ? (cdp.resourceCleanup ?? cdp) : cdp).sendToFrame( + handle, + method, + params, + { + ...guard, + attachmentId, + ...(isCleanup ? { allowReattach: false } : {}), + }, + ); }, }); } @@ -152,7 +160,14 @@ async function readDocumentIdentity(cdp: CdpRunner, tabId: number, signal?: Abor documentElementBackendNodeId: backendNodeId, } satisfies DocumentIdentity; } finally { - await bound.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => {}); + await bound + .sendToFrame( + toRootFrameHandle(tabId), + "Runtime.releaseObjectGroup", + { objectGroup }, + { allowReattach: false }, + ) + .catch(() => {}); } } diff --git a/apps/extension/src/tools/visual-target.ts b/apps/extension/src/tools/visual-target.ts index 9804b45ad..632d91eea 100644 --- a/apps/extension/src/tools/visual-target.ts +++ b/apps/extension/src/tools/visual-target.ts @@ -230,11 +230,11 @@ async function readFrame( } finally { if (cdp.getAttachmentId?.(document.tabId) === document.attachmentId) { await sendToFrameHandle( - cdp, + cdp.resourceCleanup ?? cdp, document, "Runtime.releaseObjectGroup", { objectGroup: verified.objectGroup }, - { attachmentId: document.attachmentId }, + { attachmentId: document.attachmentId, allowReattach: false }, ).catch(() => {}); } throwIfAborted(signal); @@ -470,11 +470,11 @@ export async function verifyCapturedTarget( const verified = await resolveVerifiedNode(cdp, frame.document, anchor, signal); if (verified.status !== "current") return stale(`visual DOM identity ${verified.status}`); await sendToFrameHandle( - cdp, + cdp.resourceCleanup ?? cdp, frame.document, "Runtime.releaseObjectGroup", { objectGroup: verified.objectGroup }, - { attachmentId: frame.document.attachmentId }, + { attachmentId: frame.document.attachmentId, allowReattach: false }, ).catch(() => {}); throwIfAborted(signal); if (frame.parent) { @@ -599,11 +599,11 @@ export async function verifyVisualHit( } finally { if (cdp.getAttachmentId?.(document.tabId) === document.attachmentId) { await sendToFrameHandle( - cdp, + cdp.resourceCleanup ?? cdp, document, "Runtime.releaseObjectGroup", { objectGroup: verified.objectGroup }, - { attachmentId: document.attachmentId }, + { attachmentId: document.attachmentId, allowReattach: false }, ).catch(() => {}); } throwIfAborted(signal); diff --git a/apps/extension/src/tools/vom/__tests__/form-state.test.ts b/apps/extension/src/tools/vom/__tests__/form-state.test.ts index f6fcbe966..692644369 100644 --- a/apps/extension/src/tools/vom/__tests__/form-state.test.ts +++ b/apps/extension/src/tools/vom/__tests__/form-state.test.ts @@ -209,6 +209,7 @@ describe("form state identity", () => { handle, "Runtime.releaseObjectGroup", expect.anything(), + { allowReattach: false }, ); } expect(send).not.toHaveBeenCalled(); @@ -250,6 +251,37 @@ describe("form state identity", () => { expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); }); + it("releases OOPIF remote objects through cdpRunnerForFrame even after guard.signal aborts", async () => { + const controller = new AbortController(); + const handle = toFrameDocumentHandle(7, { frameId: "oopif-frame", loaderId: "loader-1" }); + const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]])); + const sendToFrame = vi.fn(async (_h, method, _params, guard) => { + guard?.signal?.throwIfAborted(); + if (method === "Runtime.evaluate") { + controller.abort(); + return reply([[11, state("new")]]); + } + return {}; + }); + const cdp = createStubCdpRunner({ + send: vi.fn(async () => ({})) as CdpRunner["send"], + sendToFrame: sendToFrame as CdpRunner["sendToFrame"], + }); + const scoped = cdpRunnerForFrame(cdp, handle, { + ignoreFrameNavigation: true, + signal: controller.signal, + }); + await expect( + enrichFormControlStates(scoped, 7, nodes, controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }); + expect(sendToFrame).toHaveBeenLastCalledWith( + handle, + "Runtime.releaseObjectGroup", + expect.objectContaining({ objectGroup: expect.any(String) }), + { ignoreFrameNavigation: true, allowReattach: false }, + ); + }); + it("does not report success when cancelled during object cleanup", async () => { const controller = new AbortController(); const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]])); diff --git a/apps/extension/src/tools/vom/document-identity.ts b/apps/extension/src/tools/vom/document-identity.ts index 6b13a2974..096929d8b 100644 --- a/apps/extension/src/tools/vom/document-identity.ts +++ b/apps/extension/src/tools/vom/document-identity.ts @@ -134,11 +134,11 @@ async function verifyIdentity( } finally { if (!retained && dispatched && attached()) { await sendToFrameHandle( - cdp, + cdp.resourceCleanup ?? cdp, identity, "Runtime.releaseObjectGroup", { objectGroup }, - { attachmentId: identity.attachmentId }, + { attachmentId: identity.attachmentId, allowReattach: false }, ).catch(() => {}); } throwIfAborted(signal); diff --git a/apps/extension/src/tools/waits.ts b/apps/extension/src/tools/waits.ts index 9dd86ae39..0980359be 100644 --- a/apps/extension/src/tools/waits.ts +++ b/apps/extension/src/tools/waits.ts @@ -316,9 +316,9 @@ export async function handleWaitForElement( const objectHandles = new Map(); let finished = false; const releaseHandle = (handle: FrameDocumentHandle) => { - // Cleanup must not extend the wait's deadline or delay cancellation. + // Cleanup must not extend the wait's deadline, delay cancellation, or resurrect detached tabs. void (deps.cdp.resourceCleanup ?? deps.cdp) - .sendToFrame(handle, "Runtime.releaseObjectGroup", { objectGroup }, undefined) + .sendToFrame(handle, "Runtime.releaseObjectGroup", { objectGroup }, { allowReattach: false }) .catch(() => {}); }; const sendFrame = async ( diff --git a/apps/extension/src/tools/wheel.ts b/apps/extension/src/tools/wheel.ts index 78cc324da..146e8e46c 100644 --- a/apps/extension/src/tools/wheel.ts +++ b/apps/extension/src/tools/wheel.ts @@ -115,6 +115,7 @@ export async function handleWheel( }; let graph: Promise | undefined; const cdp: CdpRunner = delegateCdpRunner(deps.cdp, { + resourceCleanup: deps.cdp.resourceCleanup ?? deps.cdp, send: (tabId: number, method: string, args?: object, sig?: AbortSignal) => sendFrame(toRootFrameHandle(tabId), method, args, sig ? { signal: sig } : undefined), sendToFrame: sendFrame, diff --git a/crates/bsk-cli/src/cli/recipe/vom.rs b/crates/bsk-cli/src/cli/recipe/vom.rs index b8d102697..df03bdd02 100644 --- a/crates/bsk-cli/src/cli/recipe/vom.rs +++ b/crates/bsk-cli/src/cli/recipe/vom.rs @@ -333,7 +333,8 @@ mod tests { fn a_menu_is_the_same_control_whether_or_not_it_is_open() { let closed = parse("@vom 1\n @e1 combobox \"Region\" [has-submenu]"); let open = parse("@vom 1\n @e1 combobox \"Region\" [expanded]"); - let custom_closed = parse("@vom 1\n @e1 combobox \"Region\" [custom-dropdown: click to open]"); + let custom_closed = + parse("@vom 1\n @e1 combobox \"Region\" [custom-dropdown: click to open]"); let custom_open = parse("@vom 1\n @e1 combobox \"Region\" [custom-dropdown: expanded]"); let legacy_closed = parse("@vom 1\n @e1 combobox \"Region [has-submenu]\""); let legacy_open = parse("@vom 1\n @e1 combobox \"Region [expanded]\""); From 1e98f45e67f624bed583b00a2e61284e022d3692 Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Tue, 17 Nov 2026 18:55:35 +0800 Subject: [PATCH 3/3] fix(extension): restore legacy overlay-host exclusion, preserve selector receipts cleanly, and pin happy-dom MO callbacks Why: 1. `BSK_OVERLAY_SELECTOR` interpolated `[data-bsk-overlay],[${OVERLAY_HOST_MARKER_ATTR}]` where `OVERLAY_HOST_MARKER_ATTR` is already `"data-bsk-overlay"`, silently dropping legacy `[data-bsk-overlay-host]` exclusion from `READ_REDACTED_ANCESTOR_SELECTOR` (`tool.read`) and `extract` compared to pre-unification predicates, while `content/dom-primitives.ts` still used `OVERLAY_HOST_SELECTOR`. 2. Switching `target`-policy `--selector` actions to request-scoped `withPinnedSelectorTargets` avoided `used_ref` pollution, but `CooperationController.runAction` needed to preserve `used_selector` on `--selector` action receipts (`click`, `fill`, `hover`, `select`, `focus`, `blur`, `scroll_to`, `wheel`, `upload`, `download`) without injecting an undocumented `used_selector` field onto `tool.drag`'s `DragResult` (`params.source` / `params.target`) or `tool.press`'s `PressResult` (where `PressParams` accepts `params.selector` as an optional focus target while `PressResult` omits `used_selector`). 3. `evaluate.test.ts` still imported internal `type CdpDebuggee` from `@/browser-driver/chromium-cdp`. 4. `happy-dom` 15.x constructs `MutationObserverListener` with `new WeakRef((record) => this.report(record))` without retaining the closure, causing V8 young-gen scavenges across async turns to drop active `MutationObserver` callbacks in `unit-dom` tests (`record-frame-agent.test.ts`), while `WindowContextClassExtender` subclasses `MutationObserverImplementation` per `Window`. What: - Add `OVERLAY_HOST_LEGACY_ATTR = "data-bsk-overlay-host"` to `BSK_OVERLAY_SELECTOR` in `lib/dom-predicates.ts` and reuse it in `content/dom-primitives.ts` (`isExtensionOverlayElement` / `isExtensionOverlayNode`), with regression coverage in `tools/read/read.test.ts`. - Populate `used_selector` on object action results in `CooperationController.runAction` when `params.selector` was supplied and `req.method !== "tool.press"` (including `tool.download` before its early return) without polluting `DragResult` or `PressResult`, covered in `cooperation/target-policy.test.ts` and real-browser `cooperation/live-text.browser.test.ts`. - Derive the `sendCommand` target parameter type via `Parameters[0]` in `tools/__tests__/evaluate.test.ts`. - Patch `happy-dom`'s shared `MutationObserverImplementation` prototype once per V8 isolate in `src/vitest-setup.ts` so both top-level and subframe `Window` `MutationObserver` instances pin active callbacks in a `WeakMap` keyed by each `mutationListener` record while still releasing upon `disconnect()`. - Sync `CHANGELOG.md` and `CHANGELOG.zh-CN.md`. --- CHANGELOG.md | 4 +- CHANGELOG.zh-CN.md | 4 +- apps/extension/src/content/dom-primitives.ts | 17 +- apps/extension/src/cooperation/controller.ts | 24 ++- .../src/cooperation/live-text.browser.test.ts | 16 ++ .../src/cooperation/target-policy.test.ts | 162 ++++++++++++++++++ apps/extension/src/lib/dom-predicates.ts | 4 +- .../src/tools/__tests__/evaluate.test.ts | 34 ++-- apps/extension/src/tools/read/read.test.ts | 2 + apps/extension/src/vitest-setup.ts | 38 ++++ 10 files changed, 264 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe9a04007..03f7b2b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ determines which versions can connect. ### Changed -- **Complete `browser-driver/` boundary encapsulation, strict `CdpRunner` RAII contract, unified `--selector` DOM predicates, and structured VOM metadata**: Sealed `CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, and `getFrameGraph` inside `apps/extension/src/browser-driver/` behind `FrameDocumentHandle`, `FrameNodeHandle`, `FrameProcessBoundary`, and `TabEventSource` across all remaining upper-layer modules (`read`, `extract`, `waits`, `wheel`, `drag`, `action-guard`, `vom/facts`, `AccessibilityLease`, `debug/`, `recording-runtime`, and `geometry/frame-context`); promoted all `CdpRunner` methods to a non-optional interface backed by `delegateCdpRunner`, `createStubCdpRunner`, and deterministic RAII `withObjectGroup` / `withRemoteObject` V8 handle release (`handleSelect`, `handleFill`, `handleFocus`/`handleBlur`, `action-checks`, `file-input-transaction`, `file-drop-transaction`, `waits`, `element-geometry`, `cdpRunnerForFrame`, and `evaluate` with `return_by_value: false`, without resurrecting detached tabs or masking pre-dispatch errors); unified cross-frame, open Shadow DOM, `[data-bsk-overlay]` exclusion, sensitive `autocomplete`, and interactive ARIA role predicates across `interaction`, `CooperationController`, `human-loop`, `read`, and `extract`; and converged `cooperate sync` and `observe` onto structured `@browser-skill/vom` `VomLineMetadata` (`inspectVomLine` and `VOM_DERIVED_NAME_CAP`). +- **Complete `browser-driver/` boundary encapsulation, strict `CdpRunner` RAII contract, unified `--selector` DOM predicates, and structured VOM metadata**: Sealed `CdpTarget`, `CdpFrame`, `CdpFrameGraph`, `sendToTarget`, `sendGuarded`, `sendAttached`, and `getFrameGraph` inside `apps/extension/src/browser-driver/` behind `FrameDocumentHandle`, `FrameNodeHandle`, `FrameProcessBoundary`, and `TabEventSource` across all remaining upper-layer modules (`read`, `extract`, `waits`, `wheel`, `drag`, `action-guard`, `vom/facts`, `AccessibilityLease`, `debug/`, `recording-runtime`, and `geometry/frame-context`); promoted all `CdpRunner` methods to a non-optional interface backed by `delegateCdpRunner`, `createStubCdpRunner`, and deterministic RAII `withObjectGroup` / `withRemoteObject` V8 handle release (`handleSelect`, `handleFill`, `handleFocus`/`handleBlur`, `action-checks`, `file-input-transaction`, `file-drop-transaction`, `waits`, `element-geometry`, `cdpRunnerForFrame`, and `evaluate` with `return_by_value: false`, without resurrecting detached tabs or masking pre-dispatch errors); unified cross-frame, open Shadow DOM, `[data-bsk-overlay]` / `[data-bsk-overlay-host]` exclusion, sensitive `autocomplete`, and interactive ARIA role predicates across `interaction`, `CooperationController`, `human-loop`, `read`, `extract`, and `content/dom-primitives`; and converged `cooperate sync` and `observe` onto structured `@browser-skill/vom` `VomLineMetadata` (`inspectVomLine` and `VOM_DERIVED_NAME_CAP`). - **Lightweight `@wxg-prc-cpg/browser-skill-dsh-plugin` on the unified `bsk` CLI runtime**: Replaced the ~22k-LOC bespoke `bsk_*` wrapper and Web UI bundle in `packages/dsh-plugin-browserskill` with a lightweight Cordis plugin (free of `@deepseek-ai/dsh-*` host `rc` peer locks) that packages the canonical `crates/bsk-cli/skill` bundle, automatically heals host `PATH` for `~/.local/bin` (`$BSK_INSTALL_DIR`) and configured `bskPath`, and bundles an on-demand cross-platform `bsk` CLI + browser extension setup workflow (`bin/install-bsk.mjs` + `AGENT_INSTALL.md`) sharing the standard `bsk` CLI and `DSH_SESSION_ID` daemon scoping alongside restored `dsh-plugin-v*` release automation. - **Zero-tax Chrome MV3 `registration: "runtime"` dual-tier content script injection, SW-startup orphan Agent Window reaping, and shared `content/dom-primitives.ts`**: Migrated all four MV3 content-script entrypoints (`content`, `cooperation.content`, `record-frame.content`, and `long-screenshot-page.content`) out of static `manifest.json` `content_scripts` into idempotent runtime injection (`claimIdempotentContentScript`) with `noScriptStartedPostMessage: true` so ordinary browsing incurs zero content-script, `window.postMessage`, or React 19 overhead; injected Tier A (`content.js` and `long-screenshot-page.js`) per `tabId`/`documentId` via idempotent `ensureOverlayInjected` across controlled task tabs, `webNavigation.onCommitted` (including Speculation Rules prerender outermost frames, sequenced strictly after Tier B `cooperation.js` arming/repair), pre-confirmation `tab_borrow` candidate user tabs, `request_help`, `record`, and active `video` overlays while skipping uninjected tabs on `hidden` / `CAPTURE_SUPPRESS` / `INPUT_PASSTHROUGH`; dynamically registered/unregistered pure-TypeScript Tier B (`cooperation.js` and `record-frame.js`) with `persistAcrossSessions: false` when active sessions or recordings transition between `0` and `>= 1`, loading silently at `document_start` with 0 DOM listeners and 0 startup IPC until armed by Background (`COOPERATION_ARM_FRAME` / `RECORD_FRAME_START` plus 50ms per-tab subframe coalescing, `cooperation.js` co-injection in `injectRecordFrameAllFrames`, and non-destructive per-frame `injectCooperationAllFrames` repair across `tab_borrow`, `ack === undefined` frames, and dynamically inserted light-DOM or open/closed `ShadowRoot` `about:blank` / `srcdoc` subframes); tracked dedicated Agent Window `ownedTabIds` in `chrome.storage.session` (`bsk.orphan_agent_windows`) to reap orphaned pure Agent Windows on Service Worker restart without ever closing windows containing user, borrowed, or human-preserved tabs; and extracted `apps/extension/src/content/dom-primitives.ts`. - **Single daemon `tool_pipeline`, 4-module `daemon/ipc` decomposition, non-blocking persistence, and shared `cli::orchestration` kernel**: Decomposed 3,489-line `daemon/ipc.rs` into `daemon/tool_pipeline.rs` (single unified `tool.*` pipeline with RAII `AuditTicketGuard`, `SessionActivityGuard`, `ToolInflightRegistry`, `IntentWal`, and post-settlement scope/run-log reconciliation), `daemon/ipc_sessions.rs`, `daemon/ipc_transport.rs`, and a slim `daemon/ipc.rs` router; promoted exhaustive `OutcomeKind` (`bsk-protocol`) / `Outcome` (`daemon/tool_policy.rs`) routing in `daemon/queue.rs`; moved `TransferRegistry` staging/import/session-teardown I/O, `AuditStore` JSONL `sync_data()` writes (including `session.user_interrupt` markers), and monotonic-sequence `IntentWal` flushes/atomic `sync_data()` compactions out of global state `Mutex`es and Tokio worker threads via `spawn_blocking`; and extracted `cli/orchestration.rs` to share Canvas screenshot point-click validation, `open`/`sync`/action/positive timeout bounds, explicit `--session` + `--browser` conflict checks, cached VOM hydration, and 2-second `CANCEL_RESPONSE_GRACE` settlement across `bsk` CLI and `bsk mcp`. @@ -44,7 +44,7 @@ determines which versions can connect. ### Fixed -- Fixed `tool.wheel` skipping compositor paint readiness (`waitForInputPaint`) under `ToolDispatcher` persistent background leases on unfocused or minimized windows by decoupling `ReadyInput.hidden` (`lacksForegroundPresentation`) from temporary `restoreFocus` and wiring `windowsApi` in `ToolDispatcher`; stopped `target`-policy `--selector` actions in `CooperationController` from polluting `session.refStore`, emitting `used_ref`, or leaking hallucinated `@eN` references in `handleSelect` custom-dropdown error messages (`withPinnedSelectorTargets`); refreshed live frame activity inside `CooperationController.beginDomMutation()` under `target` policy before scripted DOM writes while preserving `state.baseline` (`observation_id`) when pre-write validation fails with `effect_state: "none"`; fixed `clearRecordingForSession` to use the caller/dispatcher's `deps` (`RECORD_CANCEL` + `bypassOverlay(tabId, false)`), required explicit `isAgentWindowId` in `returnBorrowedTab` so returning a borrowed user tab never routes into another session's Agent Window, replaced module-global download locking with instance-scoped `DownloadConcurrencyGate` (`session_busy`, `effect_state: "none"`, `dispatched: false`), and cleaned up `observeIdentityStreaks`, `hoverLatches`, and `hoverBypassTabs` on session teardown and `tabs.onRemoved`; enabled Canvas `[visual:screenshot]` `@eN` parity on `cooperate sync`, budgeted the `renderDoubleLayer` `L2 page … occluded by L1` footer line within `maxTokens`, and moved popup `[expanded]` / `[has-submenu]` tags out of `vom.name` to post-quote VOM tags so custom dropdowns no longer emit duplicate `[expanded]` suffixes. +- Fixed `tool.wheel` skipping compositor paint readiness (`waitForInputPaint`) under `ToolDispatcher` persistent background leases on unfocused or minimized windows by decoupling `ReadyInput.hidden` (`lacksForegroundPresentation`) from temporary `restoreFocus` and wiring `windowsApi` in `ToolDispatcher`; stopped `target`-policy `--selector` actions in `CooperationController` from polluting `session.refStore`, emitting `used_ref` (while preserving `used_selector` on `--selector` action receipts without polluting `tool.drag`'s `DragResult` or `tool.press`'s `PressResult`), or leaking hallucinated `@eN` references in `handleSelect` custom-dropdown error messages (`withPinnedSelectorTargets`); refreshed live frame activity inside `CooperationController.beginDomMutation()` under `target` policy before scripted DOM writes while preserving `state.baseline` (`observation_id`) when pre-write validation fails with `effect_state: "none"`; fixed `clearRecordingForSession` to use the caller/dispatcher's `deps` (`RECORD_CANCEL` + `bypassOverlay(tabId, false)`), required explicit `isAgentWindowId` in `returnBorrowedTab` so returning a borrowed user tab never routes into another session's Agent Window, replaced module-global download locking with instance-scoped `DownloadConcurrencyGate` (`session_busy`, `effect_state: "none"`, `dispatched: false`), and cleaned up `observeIdentityStreaks`, `hoverLatches`, and `hoverBypassTabs` on session teardown and `tabs.onRemoved`; enabled Canvas `[visual:screenshot]` `@eN` parity on `cooperate sync`, budgeted the `renderDoubleLayer` `L2 page … occluded by L1` footer line within `maxTokens`, and moved popup `[expanded]` / `[has-submenu]` tags out of `vom.name` to post-quote VOM tags so custom dropdowns no longer emit duplicate `[expanded]` suffixes. - Eliminated redundant back-to-back frame collection sweeps in `guardedRunner.runBrowserMutation` (`action-guard.ts`), skipped mutation fencing on passive `inspect` probes in `handleFullPageScreenshot`, and raised the per-frame cooperation IPC reply deadline (`FRAME_REPLY_TIMEOUT_MS` / `OWNER_REPLY_MS`) from `1200ms` to `3000ms` so cold high-DPR / software-rasterized full-page captures and Tier-A overlay mounts do not spuriously fail with `FrameUnresponsiveError` (#761). - Added `--max-tokens` (`max_tokens` in MCP) to `bsk step`, `bsk plan`, `bsk_interact` (with `sync_after` or `until_visible`), and `bsk_plan`, forwarding it to the post-action `tool.cooperate` sync instead of hardcoding `None` so `--max-tokens` baselines stay reusable across `delta` and `unchanged` syncs without forcing an unbounded page re-capture (#755). - Ignored internal `data-bsk-*` attributes and `` (`[data-bsk-overlay]`) ShadowHost mutations in `cooperation-capture.ts` so active recording and sensitive-field markers no longer advance `FrameState.structure`, excluded extension ShadowRoot UI controls (`HelpRequestOverlay`, `RecordOverlay`, `BorrowConfirmationOverlay`) across `record-capture.ts`, `record-hover-surface.ts`, and `cooperation-capture.ts` so overlay interactions and hover surfaces are never mis-recorded as webpage steps, and re-evaluated tab control ownership after awaiting `ensureOverlayInjected` so mid-injection tab detachments never flash stale `control` state. diff --git a/CHANGELOG.zh-CN.md b/CHANGELOG.zh-CN.md index 13d8095a3..baabed307 100644 --- a/CHANGELOG.zh-CN.md +++ b/CHANGELOG.zh-CN.md @@ -17,7 +17,7 @@ CLI 和扩展可以分别发布,能否连接由协议兼容性决定。更新 ### 变更 -- **`browser-driver/` 彻底封盒、必填 `CdpRunner` 与确定性 RAII V8 句柄释放、统一 `--selector` DOM 谓词及结构化 VOM 行元数据**:将 `CdpTarget`、`CdpFrame`、`CdpFrameGraph`、`sendToTarget`、`sendGuarded`、`sendAttached` 与 `getFrameGraph` 彻底封入 `apps/extension/src/browser-driver/` 内部,通过 `FrameDocumentHandle`、`FrameNodeHandle`、`FrameProcessBoundary` 与 `TabEventSource` 统一收口剩余全部上层模块(`read`、`extract`、`waits`、`wheel`、`drag`、`action-guard`、`vom/facts`、`AccessibilityLease`、`debug/`、`recording-runtime` 与 `geometry/frame-context`);将 `CdpRunner` 接口方法全面提升为必填非可选契约,配套 `delegateCdpRunner`、`createStubCdpRunner` 以及确定性 RAII `withObjectGroup` / `withRemoteObject` V8 句柄释放(覆盖 `handleSelect`、`handleFill`、`handleFocus`/`handleBlur`、`action-checks`、`file-input-transaction`、`file-drop-transaction`、`waits`、`element-geometry`、`cdpRunnerForFrame` 与 `evaluate(return_by_value: false)`,不唤醒已断开标签页且不掩盖派发前错误);在 `interaction`、`CooperationController`、`human-loop`、`read` 与 `extract` 间统一跨帧、open Shadow DOM、`[data-bsk-overlay]` 排除、敏感 `autocomplete` 与交互式 ARIA 角色谓词;并将 `cooperate sync` 与 `observe` 统一收敛至 `@browser-skill/vom` 的结构化 `VomLineMetadata`(`inspectVomLine` 与 `VOM_DERIVED_NAME_CAP`)。 +- **`browser-driver/` 彻底封盒、必填 `CdpRunner` 与确定性 RAII V8 句柄释放、统一 `--selector` DOM 谓词及结构化 VOM 行元数据**:将 `CdpTarget`、`CdpFrame`、`CdpFrameGraph`、`sendToTarget`、`sendGuarded`、`sendAttached` 与 `getFrameGraph` 彻底封入 `apps/extension/src/browser-driver/` 内部,通过 `FrameDocumentHandle`、`FrameNodeHandle`、`FrameProcessBoundary` 与 `TabEventSource` 统一收口剩余全部上层模块(`read`、`extract`、`waits`、`wheel`、`drag`、`action-guard`、`vom/facts`、`AccessibilityLease`、`debug/`、`recording-runtime` 与 `geometry/frame-context`);将 `CdpRunner` 接口方法全面提升为必填非可选契约,配套 `delegateCdpRunner`、`createStubCdpRunner` 以及确定性 RAII `withObjectGroup` / `withRemoteObject` V8 句柄释放(覆盖 `handleSelect`、`handleFill`、`handleFocus`/`handleBlur`、`action-checks`、`file-input-transaction`、`file-drop-transaction`、`waits`、`element-geometry`、`cdpRunnerForFrame` 与 `evaluate(return_by_value: false)`,不唤醒已断开标签页且不掩盖派发前错误);在 `interaction`、`CooperationController`、`human-loop`、`read`、`extract` 与 `content/dom-primitives` 间统一跨帧、open Shadow DOM、`[data-bsk-overlay]` / `[data-bsk-overlay-host]` 排除、敏感 `autocomplete` 与交互式 ARIA 角色谓词;并将 `cooperate sync` 与 `observe` 统一收敛至 `@browser-skill/vom` 的结构化 `VomLineMetadata`(`inspectVomLine` 与 `VOM_DERIVED_NAME_CAP`)。 - **基于统一 `bsk` CLI 运行时的轻量 `@wxg-prc-cpg/browser-skill-dsh-plugin` 插件**:将 `packages/dsh-plugin-browserskill` 从原先约 2.2 万行的自定义 `bsk_*` 工具包装器与 Web UI 前端组件重构为极简 Cordis 插件(彻底移除 `@deepseek-ai/dsh-*` 宿主 `rc` 版本对等依赖锁),直接内嵌主仓唯一的 `crates/bsk-cli/skill` 资源包,自动为宿主补齐 `~/.local/bin`(`$BSK_INSTALL_DIR`)与自定义 `bskPath` 的 `PATH` 解析,并打包跨平台按需 `bsk` CLI 安装器与浏览器扩展连接引导流程(`bin/install-bsk.mjs` + `AGENT_INSTALL.md`),底层与其他所有 Agent 100% 共用标准 `bsk` CLI 与 `DSH_SESSION_ID` 自动作用域机制,同时恢复 `dsh-plugin-v*` 独立发版流水线。 - **Chrome MV3 `registration: "runtime"` 零税双层按需内容脚本注入、SW 启动期孤儿 Agent 窗口兜底回收与共享 `content/dom-primitives.ts`**:将全部 4 个 MV3 内容脚本入口(`content`、`cooperation.content`、`record-frame.content` 与 `long-screenshot-page.content`)以 `noScriptStartedPostMessage: true` 与 `claimIdempotentContentScript` 幂等守卫迁出静态 `manifest.json` `content_scripts`,彻底消除日常浏览普通网页时的全局 `` 内容脚本、`window.postMessage` 广播与 React 19 加载税;通过幂等 `ensureOverlayInjected` 对受控任务标签页、`webNavigation.onCommitted`(含 Speculation Rules 预渲染顶层帧,且严格排在 Tier B `cooperation.js` 激活/补注入之后)、确认前 `tab_borrow` 候选用户标签页、`request_help`、`record` 与活跃 `video` 浮层按 `tabId`/`documentId` 单点注入 Tier A(`content.js` 与 `long-screenshot-page.js`),并在 `hidden` / `CAPTURE_SUPPRESS` / `INPUT_PASSTHROUGH` 时跳过未注入标签页;在活跃会话或录制于 `0` 与 `>= 1` 间切换时以 `persistAcrossSessions: false` 动态启停纯 TS 轻量 Tier B(`cooperation.js` 与 `record-frame.js`),使其在 `document_start` 零 DOM 监听、零启动唤醒 IPC 静默加载并由后台按需点名激活(`COOPERATION_ARM_FRAME` / `RECORD_FRAME_START` 配合单标签页 50ms 子帧合并去抖、`injectRecordFrameAllFrames` 协同注入 `cooperation.js`,以及在 `tab_borrow`、`ack === undefined` 帧与动态插入的普通 DOM 或 open/closed `ShadowRoot` 内同源 `about:blank` / `srcdoc` 子帧上进行无损按帧探测补注入);通过 `chrome.storage.session`(`bsk.orphan_agent_windows`)追踪独占 Agent 窗口创建的标签页集合,在 Service Worker 重启后精准回收无活跃会话的纯孤儿 Agent 窗口(严格保留含用户、借用或人工接管标签页的窗口);并抽离共享 DOM 原语模块 `apps/extension/src/content/dom-primitives.ts`。 - **守护进程单一 `tool_pipeline` 工具管线、`daemon/ipc` 四模块拆解、无锁阻塞持久化与 `cli::orchestration` 客户端共享编排内核**:将 3,489 行的 `daemon/ipc.rs` 拆解为 `daemon/tool_pipeline.rs`(统一收口全部 `tool.*` 执行路径的 RAII `AuditTicketGuard`、`SessionActivityGuard`、`ToolInflightRegistry`、`IntentWal` 与作用域/运行日志结算)、`daemon/ipc_sessions.rs`、`daemon/ipc_transport.rs` 及精简路由层 `daemon/ipc.rs`;在 `bsk-protocol` 与 `daemon/tool_policy.rs` / `daemon/queue.rs` 中建立穷尽式 `OutcomeKind` / `Outcome` 结果与超时归因路由;通过局部状态分离与 `spawn_blocking` 将 `TransferRegistry` 大文件暂存/导入复制/会话销毁目录清理、`AuditStore` JSONL `sync_data()` 刷盘(含 `session.user_interrupt` 标记)与单调序号 `IntentWal` 刷盘及 `sync_data()` 原子压缩彻底移出全局状态 `Mutex` 与 Tokio 工作线程阻塞路径;并提取 `cli/orchestration.rs` 统一 `bsk` CLI 与 `bsk mcp` 的 Canvas 截图坐标点击校验、`open`/`sync`/动作/正数超时边界、显式 `--session` + `--browser` 冲突校验、缓存 VOM 回填以及 2 秒 `CANCEL_RESPONSE_GRACE` 优雅取消结算。 @@ -31,7 +31,7 @@ CLI 和扩展可以分别发布,能否连接由协议兼容性决定。更新 ### 修复 -- 修复 `ToolDispatcher` 持久后台执行租约下失焦或最小化窗口上的 `tool.wheel` 因耦合临时 `restoreFocus` 而跳过合成器渲染就绪检查(`waitForInputPaint`)的问题(将 `ReadyInput.hidden` 拆耦为 `lacksForegroundPresentation` 并在 `ToolDispatcher` 注入 `windowsApi`);修复 `CooperationController` 在 `target` 策略下执行 `--selector` 动作时污染 `session.refStore`、误输出 `used_ref` 或在 `handleSelect` 自定义下拉报错引导中泄漏伪造 `@eN` 引用的问题(改用请求级 `withPinnedSelectorTargets`);在 `target` 策略脚本写 DOM 前于 `CooperationController.beginDomMutation()` 内主动刷新实时帧活动,并在写入前校验以 `effect_state: "none"` 失败时保留 `state.baseline`(`observation_id`);修复 `clearRecordingForSession` 未使用调用方/派发器注入 `deps`(`RECORD_CANCEL` + `bypassOverlay(tabId, false)`)的问题,将 `returnBorrowedTab` 的 `isAgentWindowId` 设为必填契约以防归还借用标签页误落入其他会话的 Agent 窗口,以实例级 `DownloadConcurrencyGate`(`session_busy`、`effect_state: "none"`、`dispatched: false`)替换模块级全局下载锁,并在会话销毁及 `tabs.onRemoved` 时完整清理 `observeIdentityStreaks`、`hoverLatches` 与 `hoverBypassTabs`;补齐 `cooperate sync` 对 Canvas `[visual:screenshot]` `@eN` 引用的对齐,将 `renderDoubleLayer` 底部 `L2 page … occluded by L1` 遮挡汇总行纳入 `maxTokens` 预算核算,并将弹出状态标签 `[expanded]` / `[has-submenu]` 从 `vom.name` 迁至引号后 VOM 标签以消除自定义下拉重复输出 `[expanded]`。 +- 修复 `ToolDispatcher` 持久后台执行租约下失焦或最小化窗口上的 `tool.wheel` 因耦合临时 `restoreFocus` 而跳过合成器渲染就绪检查(`waitForInputPaint`)的问题(将 `ReadyInput.hidden` 拆耦为 `lacksForegroundPresentation` 并在 `ToolDispatcher` 注入 `windowsApi`);修复 `CooperationController` 在 `target` 策略下执行 `--selector` 动作时污染 `session.refStore`、误输出 `used_ref`(同时保留 `--selector` 动作回执的 `used_selector` 且不污染 `tool.drag` 的 `DragResult` 与 `tool.press` 的 `PressResult`)或在 `handleSelect` 自定义下拉报错引导中泄漏伪造 `@eN` 引用的问题(改用请求级 `withPinnedSelectorTargets`);在 `target` 策略脚本写 DOM 前于 `CooperationController.beginDomMutation()` 内主动刷新实时帧活动,并在写入前校验以 `effect_state: "none"` 失败时保留 `state.baseline`(`observation_id`);修复 `clearRecordingForSession` 未使用调用方/派发器注入 `deps`(`RECORD_CANCEL` + `bypassOverlay(tabId, false)`)的问题,将 `returnBorrowedTab` 的 `isAgentWindowId` 设为必填契约以防归还借用标签页误落入其他会话的 Agent 窗口,以实例级 `DownloadConcurrencyGate`(`session_busy`、`effect_state: "none"`、`dispatched: false`)替换模块级全局下载锁,并在会话销毁及 `tabs.onRemoved` 时完整清理 `observeIdentityStreaks`、`hoverLatches` 与 `hoverBypassTabs`;补齐 `cooperate sync` 对 Canvas `[visual:screenshot]` `@eN` 引用的对齐,将 `renderDoubleLayer` 底部 `L2 page … occluded by L1` 遮挡汇总行纳入 `maxTokens` 预算核算,并将弹出状态标签 `[expanded]` / `[has-submenu]` 从 `vom.name` 迁至引号后 VOM 标签以消除自定义下拉重复输出 `[expanded]`。 - 消除 `guardedRunner.runBrowserMutation`(`action-guard.ts`)中重复的连续整标签页帧状态采集、跳过 `handleFullPageScreenshot` 中只读 `inspect` 探测步骤的变更门禁,并将协作帧 IPC 响应超时阈值(`FRAME_REPLY_TIMEOUT_MS` / `OWNER_REPLY_MS`)从 `1200ms` 放宽至 `3000ms`,防止高分屏(2x DPR)/ 软光栅冷启动与按需浮层挂载期间误报 `FrameUnresponsiveError`(#761)。 - 为 `bsk step`、`bsk plan`、MCP `bsk_interact`(搭配 `sync_after` 或 `until_visible`)及 `bsk_plan` 补齐 `--max-tokens`(MCP 为 `max_tokens`)参数并透传至后置 `tool.cooperate` 同步请求,解决此前硬编码 `None` 导致带预算会话在后续动作触发 `token_budget_changed` 并回退为整页无上限重采的问题(#755)。 - 在 `cooperation-capture.ts` 中忽略扩展内部私有 `data-bsk-*` 属性与 ``(`[data-bsk-overlay]`)ShadowHost 变更以消除录制/敏感字段标记对 `FrameState.structure` 的自污染,在 `record-capture.ts`、`record-hover-surface.ts` 与 `cooperation-capture.ts` 中统一过滤扩展自身 ShadowRoot 浮层控件(`HelpRequestOverlay`、`RecordOverlay`、`BorrowConfirmationOverlay`)防止浮层内点击/输入或悬停面被误录为网页操作,并在 `ensureOverlayInjected` 异步注入完成后二次校验标签页受控归属以杜绝注入途中断开控制导致的陈旧浮层闪现。 diff --git a/apps/extension/src/content/dom-primitives.ts b/apps/extension/src/content/dom-primitives.ts index c100842c8..1b8327325 100644 --- a/apps/extension/src/content/dom-primitives.ts +++ b/apps/extension/src/content/dom-primitives.ts @@ -1,9 +1,10 @@ import { - isOverlayHostElementName, - OVERLAY_HOST_MARKER_ATTR, - OVERLAY_HOST_SELECTOR, -} from "@/lib/overlay-bridge"; -import { RECORD_PASSWORD_ATTRIBUTE, sensitiveAutocomplete } from "@/lib/sensitive-autocomplete"; + BSK_OVERLAY_SELECTOR, + OVERLAY_HOST_LEGACY_ATTR, + RECORD_PASSWORD_ATTRIBUTE, + sensitiveAutocomplete, +} from "@/lib/dom-predicates"; +import { isOverlayHostElementName, OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; export const INTERNAL_BSK_ATTR_PREFIX = "data-bsk-"; @@ -13,7 +14,9 @@ export function primaryEventTarget(event: Event): EventTarget | null { export function isExtensionOverlayElement(element: Element): boolean { return ( - isOverlayHostElementName(element.localName) || element.hasAttribute(OVERLAY_HOST_MARKER_ATTR) + isOverlayHostElementName(element.localName) || + element.hasAttribute(OVERLAY_HOST_MARKER_ATTR) || + element.hasAttribute(OVERLAY_HOST_LEGACY_ATTR) ); } @@ -30,7 +33,7 @@ export function isExtensionOverlayNode(target: EventTarget | null): boolean { continue; } if (current instanceof Element) { - if (isExtensionOverlayElement(current) || current.closest(OVERLAY_HOST_SELECTOR)) { + if (isExtensionOverlayElement(current) || current.closest(BSK_OVERLAY_SELECTOR)) { return true; } } diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index b89825f13..eb764bffd 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -2492,11 +2492,20 @@ export class CooperationController implements ControlAuthority { operationSignal, completionSignal, ); + const withUsedSelector = (value: unknown) => + req.method !== "tool.press" && + params.selector && + value && + typeof value === "object" && + !isRpcError(value) && + (value as Record).used_selector === undefined + ? { ...(value as Record), used_selector: params.selector } + : value; // A completed download transfers cleanup ownership to the daemon. Keep its // receipt even if control changed during final cleanup; dropping it would // orphan the file. The consumed context still requires a fresh next step. if (["tool.tab_close", "tool.download"].includes(req.method) && !isRpcError(result)) - return result; + return withUsedSelector(result); // A handler receipt describes completed input, independently of whether its // document can authorize the next action. evaluate reports script exceptions // in-band; a successfully evaluated false value is still a completed script. @@ -2564,18 +2573,7 @@ export class CooperationController implements ControlAuthority { }, }; } - if ( - primaryTarget?.selector && - result && - typeof result === "object" && - (result as Record).used_selector === undefined - ) { - return { - ...(result as Record), - used_selector: primaryTarget.selector, - }; - } - return result; + return withUsedSelector(result); } catch (error) { if (state.stopped) return stopped(sent ? "unknown" : "none"); if (signal.aborted) { diff --git a/apps/extension/src/cooperation/live-text.browser.test.ts b/apps/extension/src/cooperation/live-text.browser.test.ts index 5a2475a95..0433e9a54 100644 --- a/apps/extension/src/cooperation/live-text.browser.test.ts +++ b/apps/extension/src/cooperation/live-text.browser.test.ts @@ -1018,6 +1018,22 @@ describe.skipIf(!process.env.BSK_TEST_CHROME)("target policy in a real browser", }); expect(await page.read("document.activeElement.id")).toBe("other"); expect((await page.inputs()).keys).toEqual([]); + + const resynced = await page.sync(); + const directed = await page.act("tool.press", { + context_id: resynced.context_id, + selector: "#other", + key: "Enter", + }); + expect(directed, JSON.stringify(directed)).toMatchObject({ + key: "Enter", + code: "Enter", + modifiers: [], + }); + expect(directed).not.toHaveProperty("used_selector"); + expect(directed).not.toHaveProperty("used_ref"); + expect(await page.read("document.activeElement.id")).toBe("other"); + expect((await page.inputs()).keys).toEqual(["Enter:true"]); }, { policy: "target" }, ); diff --git a/apps/extension/src/cooperation/target-policy.test.ts b/apps/extension/src/cooperation/target-policy.test.ts index 8be844c37..34eb1b2e9 100644 --- a/apps/extension/src/cooperation/target-policy.test.ts +++ b/apps/extension/src/cooperation/target-policy.test.ts @@ -1900,6 +1900,168 @@ describe("text deltas under target control", () => { expect(okRes).toMatchObject({ tab_id: 7, used_selector: "#save-btn" }); expect(okRes).not.toHaveProperty("used_ref"); expect([...f.session.refStore.entries()]).toEqual(initialRefs); + + // Drag with source.selector pins both endpoints without injecting top-level used_selector onto DragResult. + vi.mocked(resolveActionTarget) + .mockResolvedValueOnce({ + tab: { tabId: 7, windowId: 100, active: true, url: "https://fixture.test" }, + backendNodeId: 79, + handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 79 }, + usedSelector: "#drag-card", + }) + .mockResolvedValueOnce({ + tab: { tabId: 7, windowId: 100, active: true, url: "https://fixture.test" }, + backendNodeId: 80, + handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 80 }, + usedSelector: "#drop-zone", + }); + const dragReq = { + id: "drag-sel", + method: "tool.drag", + params: { + session_id: "s", + tab_id: 7, + context_id: f.first.context_id, + source: { selector: "#drag-card" }, + target: { selector: "#drop-zone" }, + }, + }; + const dragRes = (await f.controller.run(dragReq, f.cdp, f.signal, async (runner) => { + expect(lookupPinnedSelectorTarget(runner, 7, "#drag-card")?.backendNodeId).toBe(79); + expect(lookupPinnedSelectorTarget(runner, 7, "#drop-zone")?.backendNodeId).toBe(80); + await runner.send(7, "Input.dispatchMouseEvent", { + type: "mousePressed", + button: "left", + x: 10, + y: 20, + }); + await runner.send(7, "Input.dispatchMouseEvent", { + type: "mouseReleased", + button: "left", + x: 30, + y: 40, + }); + return { tab_id: 7, source: { x: 10, y: 20 }, target: { x: 30, y: 40 } }; + })) as Record; + expect(dragRes).toEqual({ tab_id: 7, source: { x: 10, y: 20 }, target: { x: 30, y: 40 } }); + expect(dragRes).not.toHaveProperty("used_selector"); + + // Directed press with selector pins focus target without injecting undocumented used_selector onto PressResult. + vi.mocked(resolveActionTarget).mockResolvedValueOnce({ + tab: { tabId: 7, windowId: 100, active: true, url: "https://fixture.test" }, + backendNodeId: 81, + handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 81 }, + usedSelector: "#search-input", + }); + const pressReq = { + id: "press-sel", + method: "tool.press", + params: { + session_id: "s", + tab_id: 7, + context_id: f.first.context_id, + selector: "#search-input", + key: "Enter", + }, + }; + const pressRes = (await f.controller.run(pressReq, f.cdp, f.signal, async (runner) => { + expect(lookupPinnedSelectorTarget(runner, 7, "#search-input")?.backendNodeId).toBe(81); + await runner.send(7, "Input.dispatchKeyEvent", { + type: "keyDown", + key: "Enter", + code: "Enter", + }); + await runner.send(7, "Input.dispatchKeyEvent", { + type: "keyUp", + key: "Enter", + code: "Enter", + }); + return { tab_id: 7, key: "Enter", code: "Enter", modifiers: [] }; + })) as Record; + expect(pressRes).toEqual({ tab_id: 7, key: "Enter", code: "Enter", modifiers: [] }); + expect(pressRes).not.toHaveProperty("used_selector"); + expect(pressRes).not.toHaveProperty("used_ref"); + + // Download with selector preserves baseline on pre-dispatch error and attaches used_selector on completion. + vi.mocked(resolveActionTarget).mockResolvedValueOnce({ + tab: { tabId: 7, windowId: 100, active: true, url: "https://fixture.test" }, + backendNodeId: 82, + handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 82 }, + usedSelector: "#export-csv", + }); + const dlFailRes = await f.controller.run( + { + id: "dl-fail", + method: "tool.download", + params: { + session_id: "s", + tab_id: 7, + context_id: f.first.context_id, + selector: "#export-csv", + }, + }, + f.cdp, + f.signal, + async () => ({ + code: "timeout", + message: "another bsk download is active", + data: { reason: "session_busy", effect_state: "none", dispatched: false }, + }), + ); + expect(dlFailRes).toMatchObject({ + code: "timeout", + data: { reason: "session_busy", effect_state: "none" }, + }); + expect(dlFailRes).not.toHaveProperty("used_selector"); + + vi.mocked(resolveActionTarget).mockResolvedValueOnce({ + tab: { tabId: 7, windowId: 100, active: true, url: "https://fixture.test" }, + backendNodeId: 82, + handle: { tabId: 7, frameId: "", loaderId: "", backendNodeId: 82 }, + usedSelector: "#export-csv", + }); + const dlOkRes = await f.controller.run( + { + id: "dl-ok", + method: "tool.download", + params: { + session_id: "s", + tab_id: 7, + context_id: f.first.context_id, + selector: "#export-csv", + }, + }, + f.cdp, + f.signal, + async (runner) => { + expect(lookupPinnedSelectorTarget(runner, 7, "#export-csv")?.backendNodeId).toBe(82); + await runner.send(7, "Input.dispatchMouseEvent", { + type: "mousePressed", + button: "left", + x: 15, + y: 25, + }); + await runner.send(7, "Input.dispatchMouseEvent", { + type: "mouseReleased", + button: "left", + x: 15, + y: 25, + }); + return { + tab_id: 7, + filename: "report.csv", + mime_type: "text/csv", + byte_size: 42, + browser_path: "/tmp/report.csv", + }; + }, + ); + expect(dlOkRes).toMatchObject({ + tab_id: 7, + filename: "report.csv", + used_selector: "#export-csv", + }); + expect(dlOkRes).not.toHaveProperty("used_ref"); }); it("refreshes live frame activity inside beginDomMutation under target policy and blocks DOM write on human intervention", async () => { diff --git a/apps/extension/src/lib/dom-predicates.ts b/apps/extension/src/lib/dom-predicates.ts index 368f011ec..b90ae3288 100644 --- a/apps/extension/src/lib/dom-predicates.ts +++ b/apps/extension/src/lib/dom-predicates.ts @@ -7,8 +7,10 @@ import { export { RECORD_PASSWORD_ATTRIBUTE, SENSITIVE_AUTOCOMPLETE_SELECTOR, sensitiveAutocomplete }; +export const OVERLAY_HOST_LEGACY_ATTR = "data-bsk-overlay-host"; + /** Selector matching BrowserSkill's injected overlay hosts and Shadow DOM roots. */ -export const BSK_OVERLAY_SELECTOR = `[data-bsk-overlay],[${OVERLAY_HOST_MARKER_ATTR}],${OVERLAY_HOST_NAME}`; +export const BSK_OVERLAY_SELECTOR = `[${OVERLAY_HOST_MARKER_ATTR}],[${OVERLAY_HOST_LEGACY_ATTR}],${OVERLAY_HOST_NAME}`; /** Non-rendered or BrowserSkill-internal ancestor subtrees excluded from passive DOM reads. */ export const READ_REDACTED_ANCESTOR_SELECTOR = `script,style,template,noscript,${BSK_OVERLAY_SELECTOR}`; diff --git a/apps/extension/src/tools/__tests__/evaluate.test.ts b/apps/extension/src/tools/__tests__/evaluate.test.ts index 57d93b02a..51d7fb483 100644 --- a/apps/extension/src/tools/__tests__/evaluate.test.ts +++ b/apps/extension/src/tools/__tests__/evaluate.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; import { type CdpRunner, createStubCdpRunner } from "@/tools/shared"; import { handleEvaluate } from "../evaluate"; @@ -360,22 +360,24 @@ describe("handleEvaluate", () => { const api: CdpDebuggerApi = { attach: vi.fn(async () => {}), detach: vi.fn(async () => {}), - sendCommand: vi.fn(async (target: CdpDebuggee, method: string) => { - if (method === "Runtime.evaluate") { - for (const listener of detachListeners) { - listener({ tabId: target.tabId }, "target_closed"); + sendCommand: vi.fn( + async (target: Parameters[0], method: string) => { + if (method === "Runtime.evaluate") { + for (const listener of detachListeners) { + listener({ tabId: target.tabId }, "target_closed"); + } + return { + result: { + type: "object", + className: "HTMLDivElement", + objectId: "detached-node-1", + description: "div#gone", + }, + }; } - return { - result: { - type: "object", - className: "HTMLDivElement", - objectId: "detached-node-1", - description: "div#gone", - }, - }; - } - return {}; - }), + return {}; + }, + ), // biome-ignore lint/suspicious/noExplicitAny: minimal chrome.events.Event shim onEvent: { addListener: vi.fn(), removeListener: vi.fn() } as any, onDetach: { diff --git a/apps/extension/src/tools/read/read.test.ts b/apps/extension/src/tools/read/read.test.ts index e9a7c6933..7ef0ea5e6 100644 --- a/apps/extension/src/tools/read/read.test.ts +++ b/apps/extension/src/tools/read/read.test.ts @@ -67,6 +67,8 @@ describe("declarative DOM reads", () => { ], }).items[0], ).toMatchObject({ values: { x: null, attr: null }, redacted: ["x", "attr"] }); + document.body.innerHTML = '
overlay-ui
'; + expect(collect().items[0]).toMatchObject({ values: { text: null }, redacted: ["text"] }); }); it("does not quietly redirect a detached node to a replacement", () => { const node = document.querySelector("#row")!; diff --git a/apps/extension/src/vitest-setup.ts b/apps/extension/src/vitest-setup.ts index 9aa5a4395..4e3c42fb8 100644 --- a/apps/extension/src/vitest-setup.ts +++ b/apps/extension/src/vitest-setup.ts @@ -1,3 +1,41 @@ import { i18n } from "@browser-skill/i18n"; +// happy-dom 15.x passes an unreferenced closure to `new WeakRef((record) => this.report(record))` +// inside `MutationObserverListener`, allowing V8 scavenges across async turns to collect active +// observer callbacks before `disconnect()`. Patch the shared `MutationObserverImplementation` +// prototype once per V8 isolate so both top-level and subframe `Window` subclasses pin active +// callbacks while `WeakMap` ephemeron GC still reclaims them upon `disconnect()`. +if (typeof MutationObserver === "function") { + const patchedMark = Symbol.for("bsk.happyDomMutationObserverPinned"); + const baseProto = Object.getPrototypeOf(MutationObserver.prototype) as { + observe?: typeof MutationObserver.prototype.observe; + [key: symbol]: unknown; + } | null; + const targetProto = ( + typeof baseProto?.observe === "function" ? baseProto : MutationObserver.prototype + ) as { observe: typeof MutationObserver.prototype.observe; [key: symbol]: unknown }; + if (!targetProto[patchedMark]) { + targetProto[patchedMark] = true; + const pinnedCallbacks = new WeakMap(); + let mutationListenersSymbol: symbol | undefined; + const originalObserve = targetProto.observe; + targetProto.observe = function (target: Node, options?: MutationObserverInit) { + originalObserve.call(this, target, options); + mutationListenersSymbol ??= Object.getOwnPropertySymbols(target).find( + (sym) => sym.description === "mutationListeners", + ); + const listeners = mutationListenersSymbol + ? (target as unknown as Record)[mutationListenersSymbol] + : undefined; + if (!Array.isArray(listeners)) return; + for (const listener of listeners) { + if (listener && typeof listener === "object" && !pinnedCallbacks.has(listener)) { + const callback = (listener as { callback?: WeakRef }).callback?.deref?.(); + if (callback) pinnedCallbacks.set(listener, callback); + } + } + }; + } +} + await i18n.changeLanguage("zh-CN");