diff --git a/packages/extension/src/lib/actionability.ts b/packages/extension/src/lib/actionability.ts index f0188d8..45b0bc3 100644 --- a/packages/extension/src/lib/actionability.ts +++ b/packages/extension/src/lib/actionability.ts @@ -25,12 +25,26 @@ export type ProbeResult = { state: "hit" } | { state: "none" } | { state: "misse * still receives the press, so the click would "land" and do nothing — and a * one-shot capture listener is armed recording whether the next pointerdown * reaches the element; read it back with `readProbe`. + * + * With `node`, the target is the Jev node cache entry (`reins do` observed it + * by id); `selector` then only names it in messages. */ export async function actionPoint( selector: string, timeoutMs: number, forClick: boolean, + node: number | null = null, ): Promise { + // A Jev node id points at the exact element reins do observed; otherwise + // the CSS selector is re-queried (frameworks may swap nodes while we wait). + const find = (): Element | null => { + if (node === null) return document.querySelector(selector); + const cache = ( + window as unknown as Record } | undefined> + )[Symbol.for("reins.jev")]; + const el = cache?.nodes?.get(node); + return el?.isConnected ? el : null; + }; const describe = (n: Element | null): string => { if (!n) return "nothing (the point is outside the viewport)"; let s = n.tagName.toLowerCase(); @@ -84,7 +98,7 @@ export async function actionPoint( let reason = ""; for (let first = true; ; first = false) { // Re-query every attempt: frameworks may swap the node while we wait. - const el = document.querySelector(selector); + const el = find(); if (!el) { if (first) return { error: "notfound" }; reason = "element was removed from the page"; @@ -119,7 +133,7 @@ export async function actionPoint( const at = e.composedPath()[0]; // Frameworks re-mount nodes on hover; the element the selector // finds now counts as the target too. - const now = document.querySelector(selector); + const now = find(); const path = e.composedPath(); state.result = path.includes(el) || (now !== null && path.includes(now)) diff --git a/packages/extension/src/lib/cdp.test.ts b/packages/extension/src/lib/cdp.test.ts index c9cdf9b..81eb0ae 100644 --- a/packages/extension/src/lib/cdp.test.ts +++ b/packages/extension/src/lib/cdp.test.ts @@ -4,6 +4,7 @@ vi.mock("./monitor.js", () => ({ isMonitored: () => false })); import { __resetDebugSessions, + actionablePoint, cdpClick, cdpOpenTab, cdpType, @@ -310,6 +311,27 @@ describe("cdpClick", () => { }); }); +describe("actionablePoint for a Jev node", () => { + it("passes the node id and the short timeout into the page check", async () => { + const sendCommand = vi.fn(async () => ({ result: { value: { x: 5, y: 6 } } })); + vi.stubGlobal("chrome", { + debugger: { + attach: vi.fn(), + detach: vi.fn(), + onDetach: { addListener: () => {} }, + sendCommand, + }, + }); + await expect( + actionablePoint(7, "node 12", "click", true, { node: 12, timeoutMs: 500 }), + ).resolves.toEqual({ x: 5, y: 6 }); + const expr = ( + sendCommand.mock.calls[0] as unknown as [unknown, string, { expression: string }] + )[2].expression; + expect(expr).toMatch(/\("node 12", 500, true, 12\)$/); + }); +}); + describe("cdpOpenTab", () => { function stubOpenChrome(opts: { id?: number; attachError?: string } = {}) { const order: string[] = []; diff --git a/packages/extension/src/lib/cdp.ts b/packages/extension/src/lib/cdp.ts index d280559..b3bb0ca 100644 --- a/packages/extension/src/lib/cdp.ts +++ b/packages/extension/src/lib/cdp.ts @@ -340,17 +340,22 @@ export async function ensureVisible(tabId: number): Promise { ); } -/** Resolve a point where trusted pointer input will land on the element. */ +/** + * Resolve a point where trusted pointer input will land on the element. + * `opts.node` targets a Jev node id instead of the selector (which then only + * names the element in errors); `opts.timeoutMs` overrides the default wait. + */ export async function actionablePoint( tabId: number, css: string, action: string, forClick: boolean, + opts: { node?: number; timeoutMs?: number } = {}, ): Promise<{ x: number; y: number }> { const { result } = await send<{ result: { value: { x: number; y: number } | { error: string } }; }>(tabId, "Runtime.evaluate", { - expression: `(${actionPoint})(${JSON.stringify(css)}, ${ACTION_TIMEOUT_MS}, ${forClick})`, + expression: `(${actionPoint})(${JSON.stringify(css)}, ${opts.timeoutMs ?? ACTION_TIMEOUT_MS}, ${forClick}, ${opts.node ?? null})`, returnByValue: true, awaitPromise: true, }); @@ -362,50 +367,62 @@ export async function actionablePoint( return point; } +/** + * Press at (x, y) and confirm the press reached the element `actionablePoint` + * armed the probe on. Only a press seen landing elsewhere fails; `what` names + * the target in that error. + * + * `button` and `clickCount` default here because the CLI omits them unless + * flagged, and nothing applies the schema defaults on the way in. They must be + * explicit: CDP's own defaults (button "none", clickCount 0) move the pointer + * but never press. + */ +export async function pressAt( + tabId: number, + x: number, + y: number, + what: string, + button: "left" | "right" | "middle" = "left", + clickCount = 1, +): Promise { + // CDP synthesizes a real click only when the pressed-button bitmask is set + // (button alone isn't enough — the target never sees a `click`). Move the + // pointer first so hit-testing lands on the element under (x, y). + const buttonBit = button === "right" ? 2 : button === "middle" ? 4 : 1; + const base = { x, y, button, clickCount }; + await send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x, y, buttons: 0 }); + await send(tabId, "Input.dispatchMouseEvent", { + type: "mousePressed", + ...base, + buttons: buttonBit, + }); + await send(tabId, "Input.dispatchMouseEvent", { type: "mouseReleased", ...base, buttons: 0 }); + // Confirm the press reached the element — never report a click that missed. + // A throw or a vanished probe means the click navigated: that's success. + const probe = await send<{ result: { value: ProbeResult | null } }>(tabId, "Runtime.evaluate", { + expression: `(${readProbe})()`, + returnByValue: true, + }).then( + (r) => r.result.value, + () => null, + ); + // Only a press seen landing elsewhere is a failure. Seeing nothing proves + // nothing — a page listener may have stopped the event first — and + // failing a click that landed makes the agent click twice. + if (probe?.state === "missed") { + throw new Error( + `click on ${what} landed on ${probe.by} instead — the page changed under the pointer. Re-snapshot and retry.`, + ); + } +} + export async function cdpClick(params: ClickParams): Promise<{ ok: true }> { const tabId = await resolveTabId(params.tabId); const css = selectorFor(params.ref, params.selector); return drivePage(tabId, async () => { await ensureVisible(tabId); const { x, y } = await actionablePoint(tabId, css, "click", true); - // The CLI omits button/count unless flagged, and nothing applies the schema - // defaults on the way in. They must be explicit: CDP's own defaults - // (button "none", clickCount 0) move the pointer but never press. - const button = params.button ?? "left"; - const clickCount = params.clickCount ?? 1; - // CDP synthesizes a real click only when the pressed-button bitmask is set - // (button alone isn't enough — the target never sees a `click`). Move the - // pointer first so hit-testing lands on the element under (x, y). - const buttonBit = button === "right" ? 2 : button === "middle" ? 4 : 1; - const base = { x, y, button, clickCount }; - await send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x, y, buttons: 0 }); - await send(tabId, "Input.dispatchMouseEvent", { - type: "mousePressed", - ...base, - buttons: buttonBit, - }); - await send(tabId, "Input.dispatchMouseEvent", { - type: "mouseReleased", - ...base, - buttons: 0, - }); - // Confirm the press reached the element — never report a click that missed. - // A throw or a vanished probe means the click navigated: that's success. - const probe = await send<{ result: { value: ProbeResult | null } }>(tabId, "Runtime.evaluate", { - expression: `(${readProbe})()`, - returnByValue: true, - }).then( - (r) => r.result.value, - () => null, - ); - // Only a press seen landing elsewhere is a failure. Seeing nothing proves - // nothing — a page listener may have stopped the event first — and - // failing a click that landed makes the agent click twice. - if (probe?.state === "missed") { - throw new Error( - `click on ${css} landed on ${probe.by} instead — the page changed under the pointer. Re-snapshot and retry.`, - ); - } + await pressAt(tabId, x, y, css, params.button ?? "left", params.clickCount ?? 1); return { ok: true }; }); } diff --git a/packages/extension/src/lib/dispatch.test.ts b/packages/extension/src/lib/dispatch.test.ts index 01ba86a..b5b5af3 100644 --- a/packages/extension/src/lib/dispatch.test.ts +++ b/packages/extension/src/lib/dispatch.test.ts @@ -22,6 +22,17 @@ vi.mock("./cdp.js", () => ({ cdpWaitFor: vi.fn(async () => ({ ok: true })), })); +vi.mock("./jev.js", () => ({ + jevObserve: vi.fn(async () => ({ + url: "https://x.com/", + title: "", + text: "", + visible: true, + actions: [], + })), + jevAct: vi.fn(async () => ({ ok: true })), +})); + vi.mock("./reload.js", () => ({ reloadExtension: vi.fn(async () => ({ reloading: true, version: "9.9.9" })), })); @@ -54,6 +65,7 @@ vi.mock("./policy.js", async (importOriginal) => { import { cdpClick } from "./cdp.js"; import { dispatchMethod, dispatchWithMeta } from "./dispatch.js"; +import { jevAct } from "./jev.js"; import { ensureAllowed, PolicyDenied, policy, tightenPolicy } from "./policy.js"; /** chrome stub with enough tabs API for the gate (tabs.get → host). */ @@ -163,6 +175,17 @@ describe("dispatchMethod routing (CDP)", () => { stubTabs(); expect(await dispatchMethod("wait_for", { selector: "#btn" })).toEqual({ ok: true }); }); + it("routes jev_observe to the handler", async () => { + stubTabs(); + expect(await dispatchMethod("jev_observe", {})).toMatchObject({ url: "https://x.com/" }); + }); + it("routes jev_act to the handler with the tab pinned", async () => { + stubTabs(); + const out = await dispatchWithMeta("jev_act", { op: "click", node: 3 }); + expect(out.result).toEqual({ ok: true }); + expect(jevAct).toHaveBeenCalledWith({ op: "click", node: 3, tabId: 1 }); + expect(out.meta).toMatchObject({ tabId: 1, host: "x.com" }); + }); }); describe("dispatchMethod routing (chrome.tabs)", () => { diff --git a/packages/extension/src/lib/dispatch.ts b/packages/extension/src/lib/dispatch.ts index c61b24e..cc56f12 100644 --- a/packages/extension/src/lib/dispatch.ts +++ b/packages/extension/src/lib/dispatch.ts @@ -20,6 +20,7 @@ import { cdpWaitFor, resolveTabId, } from "./cdp.js"; +import { jevAct, jevObserve } from "./jev.js"; import { readConsole, readNetwork } from "./monitor.js"; import { cdpRaw, @@ -196,6 +197,10 @@ async function runHandler(method: GatedMethod, gated: Record): return handleDialog(gated as Parameters[0]); case "cdp": return cdpRaw(gated as Parameters[0]); + case "jev_observe": + return jevObserve(gated as Parameters[0]); + case "jev_act": + return jevAct(gated as Parameters[0]); default: throw new Error(`unknown method: ${method}`); } diff --git a/packages/extension/src/lib/jev-snapshot.ts b/packages/extension/src/lib/jev-snapshot.ts new file mode 100644 index 0000000..4fde858 --- /dev/null +++ b/packages/extension/src/lib/jev-snapshot.ts @@ -0,0 +1,338 @@ +/** + * Page-side halves of `reins do`. Each runs *inside the page* via + * Runtime.evaluate, serialized with Function.prototype.toString, so it must + * stay self-contained: no imports, no references to module scope. + * + * Ported from browser-use/jev-ultrafast `snapshot.js` (MIT). Differences: the + * cache lives under a symbol, not a page global; rects and scroll are left out + * of the result (the daemon's change check must ignore reins' own + * scrollIntoView); guards are kept page-side for jevCheck. + */ + +interface JevCache { + ids: WeakMap; + nodes: Map; + next: number; + guards: Record; + guard: (el: Element | undefined) => string | null; +} + +type Action = { + id?: string; + kind: "click" | "fill" | "select" | "scroll" | "wait"; + node?: number; + role?: string; + label: string; + value?: string; + current_value?: string; + checked?: string; + selected?: string; + expanded?: string; + delta?: number; +}; + +export function jevSnapshot(): { + url: string; + title: string; + text: string; + visible: boolean; + actions: Action[]; +} | null { + if (!document.body) return null; + const KEY = Symbol.for("reins.jev"); + const w = window as unknown as Record; + let cache = w[KEY]; + if (!cache) { + cache = { ids: new WeakMap(), nodes: new Map(), next: 1, guards: {}, guard: () => null }; + w[KEY] = cache; + } + const c = cache; + const identity = (e: Element): number => { + let id = c.ids.get(e); + if (id === undefined) { + id = c.next++; + c.ids.set(e, id); + } + c.nodes.set(id, e); + return id; + }; + for (const [id, e] of c.nodes) if (!e.isConnected) c.nodes.delete(id); + + const safe = (e: Element) => + !["password", "file", "hidden"].includes((e as HTMLInputElement).type); + const visible = (e: Element) => + !e.closest('[aria-hidden="true"],[inert]') && + e.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }); + const name = (e: Element | null, seen = new Set()): string => { + if (!e || seen.has(e)) return ""; + seen.add(e); + const input = e as HTMLInputElement; + const referenced = (e.getAttribute("aria-labelledby") ?? "") + .split(/\s+/) + .map((id) => name(document.getElementById(id), seen)) + .filter(Boolean) + .join(" "); + const raw = + referenced || + e.getAttribute("aria-label") || + [...(input.labels ?? [])] + .map((l) => name(l, seen)) + .filter(Boolean) + .join(" ") || + (["button", "submit", "reset"].includes(input.type) && e.tagName === "INPUT" + ? input.value + : "") || + e.getAttribute("alt") || + (e.tagName === "INPUT" + ? "" + : [...e.childNodes] + .map((n) => + n.nodeType === 3 + ? (n.textContent ?? "") + : n.nodeType === 1 && (n as Element).getAttribute("aria-hidden") !== "true" + ? name(n as Element, seen) + : "", + ) + .join(" ")) || + e.getAttribute("title") || + e.getAttribute("placeholder") || + ""; + return raw.replace(/\s+/g, " ").trim(); + }; + const roles = [ + "button", + "link", + "checkbox", + "radio", + "switch", + "tab", + "menuitem", + "menuitemradio", + "option", + "gridcell", + "combobox", + "textbox", + "searchbox", + "spinbutton", + ]; + const selector = `a[href],button,input,textarea,select,summary,[contenteditable="true"],${roles + .map((r) => `[role="${r}"]`) + .join(",")}`; + const role = (e: Element): string | null => { + const explicit = e.getAttribute("role"); + if (explicit && roles.includes(explicit)) return explicit; + if (e.tagName === "BUTTON" || e.tagName === "SUMMARY") return "button"; + if (e.tagName === "A") return "link"; + if (e.tagName === "SELECT") return "combobox"; + if (e.tagName === "TEXTAREA" || (e as HTMLElement).isContentEditable) return "textbox"; + if (e.tagName === "INPUT") { + const t = (e as HTMLInputElement).type; + if (t === "checkbox" || t === "radio") return t; + if (["button", "submit", "reset", "image"].includes(t)) return "button"; + if (t === "search") return "searchbox"; + if (t === "number") return "spinbutton"; + if (["text", "email", "url", "tel"].includes(t)) return "textbox"; + } + return null; + }; + c.guard = (e) => { + if (!e?.isConnected || !visible(e)) return null; + const f = e as HTMLInputElement; + const scope = + e.closest('form,dialog,[role="dialog"],article,li,tr,[role="row"]') ?? e.parentElement; + return JSON.stringify([ + identity(e), + role(e), + name(e), + f.value ?? null, + f.checked ?? null, + (e as HTMLSelectElement).selectedIndex ?? null, + f.readOnly ?? null, + e.matches(":disabled"), + e.getAttribute("aria-disabled"), + e.getAttribute("aria-expanded"), + e.getAttribute("aria-checked"), + e.getAttribute("aria-selected"), + e.getAttribute("href"), + (scope as HTMLElement | null)?.innerText?.slice(0, 6000) ?? "", + ]); + }; + + const actions: Action[] = []; + for (const e of document.querySelectorAll(selector)) { + if (!safe(e) || !visible(e) || e.matches(":disabled") || e.closest('[aria-disabled="true"]')) + continue; + const r = e.getBoundingClientRect(); + const x = r.x + r.width / 2; + const y = r.y + r.height / 2; + const rname = role(e); + if ( + !rname || + r.width <= 0 || + r.height <= 0 || + x < 0 || + y < 0 || + x >= innerWidth || + y >= innerHeight + ) + continue; + if (rname === "gridcell" && e.querySelector('button,[role="button"]')) continue; + const base: Action = { kind: "click", node: identity(e), role: rname, label: name(e) || rname }; + for (const key of ["checked", "selected", "expanded"] as const) { + const v = e.getAttribute(`aria-${key}`); + if (v !== null) base[key] = v; + } + const input = e as HTMLInputElement; + if (input.type === "checkbox" || input.type === "radio") base.checked = String(input.checked); + if (e instanceof HTMLSelectElement) { + const current = [...e.selectedOptions].map((o) => o.label).join(", "); + for (const o of e.options) { + if (o.selected || o.disabled || o.closest("optgroup[disabled]")) continue; + actions.push({ + ...base, + kind: "select", + value: o.value, + current_value: current, + label: `${base.label} → ${o.label}`, + }); + } + continue; + } + const editable = + !input.readOnly && + e.getAttribute("aria-readonly") !== "true" && + (["textbox", "searchbox", "spinbutton"].includes(rname) || + (rname === "combobox" && ["INPUT", "TEXTAREA"].includes(e.tagName))); + const value = + "value" in e + ? String(input.value) + : (e as HTMLElement).isContentEditable || rname === "combobox" + ? (e as HTMLElement).innerText.trim() + : ""; + actions.push({ ...base, kind: editable ? "fill" : "click", value }); + if (editable) actions.push({ ...base, kind: "click", value, label: `Open ${base.label}` }); + } + + const words: string[] = []; + const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT); + const range = document.createRange(); + let length = 0; + for (let n = walker.nextNode(); n && length < 6000; n = walker.nextNode()) { + const value = (n.textContent ?? "").trim(); + const parent = n.parentElement; + if (!value || !parent || parent.closest("script,style,noscript,template") || !visible(parent)) + continue; + range.selectNodeContents(n); + const r = range.getBoundingClientRect(); + if ( + r.width > 0 && + r.height > 0 && + r.bottom > 0 && + r.top < innerHeight && + r.right > 0 && + r.left < innerWidth + ) { + words.push(value); + length += value.length; + } + } + + c.guards = {}; + for (const a of actions) { + if (a.node !== undefined && !(a.node in c.guards)) + c.guards[a.node] = c.guard(c.nodes.get(a.node)) ?? ""; + } + actions.splice(250); + actions.forEach((a, i) => { + a.id = `e${i + 1}`; + }); + const height = document.documentElement.scrollHeight; + if (scrollY + innerHeight < height - 2) { + actions.push({ id: "scroll_down", kind: "scroll", label: "Scroll down", delta: 560 }); + } + if (scrollY > 0) + actions.push({ id: "scroll_up", kind: "scroll", label: "Scroll up", delta: -560 }); + actions.push({ id: "wait", kind: "wait", label: "Wait for the page to update" }); + return { + url: location.href, + title: document.title, + text: words.join("\n").slice(0, 6000), + visible: document.visibilityState === "visible", + actions, + }; +} + +/** Is `node` still the element Jev saw at the last snapshot? */ +export function jevCheck(node: number): string { + const cache = (window as unknown as Record)[ + Symbol.for("reins.jev") + ]; + const el = cache?.nodes.get(node); + if (!cache || !el?.isConnected) return "the element is gone"; + const now = cache.guard(el); + if (now === null) return "the element is hidden now"; + return now === cache.guards[node] ? "ok" : "the element changed since the page was read"; +} + +/** Choose `value` on a native `; +const JEV_FIXTURE = ` +

Flights

+
+ + + + + + +
+ + +
+ + + +`; + let chromeProc: ChildProcess | undefined; const injected: string[] = []; let server: http.Server | undefined; @@ -79,6 +107,8 @@ let ws: WebSocket | undefined; let url = ""; let nextId = 0; const pending = new Map void>(); +type EventListener = (source: { tabId?: number }, method: string, params?: unknown) => void; +const eventListeners: EventListener[] = []; function cdp(method: string, params: Record = {}): Promise { return new Promise((resolve, reject) => { @@ -100,8 +130,8 @@ async function evaluate(expression: string): Promise { } /** Fresh fixture per test; resolves once the page has loaded. */ -async function load(): Promise { - await cdp("Page.navigate", { url }); +async function load(path = "/"): Promise { + await cdp("Page.navigate", { url: new URL(path, url).href }); for (let i = 0; i < 100; i++) { if ((await evaluate("document.readyState").catch(() => "")) === "complete") return; await new Promise((r) => setTimeout(r, 20)); @@ -116,7 +146,7 @@ describe.skipIf(!CHROME)("pointer + key input in a real browser", () => { server = http.createServer((q, s) => { s.setHeader("content-type", "text/html"); // A login-style page whose password-manager host exists from first parse. - s.end(q.url === "/login" ? LOGIN_FIXTURE : FIXTURE); + s.end(q.url === "/login" ? LOGIN_FIXTURE : q.url === "/jev" ? JEV_FIXTURE : FIXTURE); }); await new Promise((r) => server?.listen(0, "127.0.0.1", r)); url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/`; @@ -155,6 +185,11 @@ describe.skipIf(!CHROME)("pointer + key input in a real browser", () => { ws = sock; sock.on("message", (data) => { const msg = JSON.parse(String(data)); + if (msg.id === undefined && msg.method) { + // A CDP event: deliver it the way chrome.debugger.onEvent would. + for (const listener of eventListeners) listener({ tabId: 1 }, msg.method, msg.params); + return; + } pending.get(msg.id)?.(msg); pending.delete(msg.id); }); @@ -166,18 +201,26 @@ describe.skipIf(!CHROME)("pointer + key input in a real browser", () => { attach: async () => {}, detach: async () => {}, onDetach: { addListener: () => {} }, + onEvent: { addListener: (fn: EventListener) => eventListeners.push(fn) }, sendCommand: async (_target: unknown, method: string, params?: Record) => { const result = await cdp(method, params); // One real page serves every test: remember injected scripts so // afterEach can drop them, as a fresh debugger session would. if (method === "Page.addScriptToEvaluateOnNewDocument") { injected.push((result as { identifier: string }).identifier); + } else if (method === "Page.removeScriptToEvaluateOnNewDocument") { + // A second drivePage in one case re-arms the guard and drops its + // own previous script; don't try to drop it again in afterEach. + const i = injected.indexOf(String(params?.identifier)); + if (i >= 0) injected.splice(i, 1); } return result; }, }, tabs: { update: async () => ({}) }, }); + // jev.ts registered at import against no `chrome`; register on the stub. + initDialogTracking(); }, 20_000); afterEach(async () => { @@ -390,4 +433,90 @@ describe.skipIf(!CHROME)("pointer + key input in a real browser", () => { await pressKey({ tabId: 1, key: "q" }); expect(await evaluate("document.getElementById('field').value")).toBe("q"); }); + + type Snap = NonNullable>; + const snap = () => evaluate(`(${jevSnapshot})()`); + const nodeOf = (s: Snap, label: string) => s.actions.find((a) => a.label === label)?.node; + + it("jev: lists visible controls, including a fixed-position dialog, and skips the rest", async () => { + await load("/jev"); + const s = await snap(); + const labels = s.actions.map((a) => `${a.kind}:${a.label}`); + expect(labels).toContain("fill:Where from?"); + expect(labels).toContain("fill:Where to?"); + expect(labels).toContain("click:Accept cookies"); + expect(labels).toContain("select:Cabin → Business"); + expect(labels.join()).not.toMatch(/Password|Disabled|Hidden|secret/); + expect(s.actions.find((a) => a.label === "Where from?")?.value).toBe("San Francisco"); + expect(s.text).toContain("Flights"); + }); + + it("jev: a node keeps its id across snapshots", async () => { + await load("/jev"); + const a = nodeOf(await snap(), "Search"); + const b = nodeOf(await snap(), "Search"); + expect(a).toBeDefined(); + expect(a).toBe(b); + }); + + it("jev: an observation never carries a password or hidden-field value", async () => { + await load("/jev"); + const wire = JSON.stringify(await jevObserve({ tabId: 1 })); + expect(wire).not.toMatch(/hunter2/); + expect(wire).not.toMatch(/secret/); + }); + + it("jev: types over an existing value", async () => { + await load("/jev"); + const s = await jevObserve({ tabId: 1 }); + const node = s.actions.find((a) => a.label === "Where from?")?.node as number; + expect(await jevAct({ tabId: 1, op: "type", node, text: "Zurich" })).toEqual({ ok: true }); + expect(await evaluate('document.getElementById("from").value')).toBe("Zurich"); + }); + + it("jev: picks a native select option and fires change", async () => { + await load("/jev"); + const s = await jevObserve({ tabId: 1 }); + const opt = s.actions.find((a) => a.label === "Cabin → Business"); + expect( + await jevAct({ tabId: 1, op: "select", node: opt?.node as number, value: "biz" }), + ).toEqual({ ok: true }); + expect(await log()).toContain("change:biz"); + }); + + it("jev: a node replaced after the read comes back stale, not clicked", async () => { + await load("/jev"); + const s = await jevObserve({ tabId: 1 }); + const node = s.actions.find((a) => a.label === "Swap me")?.node as number; + await evaluate( + 'document.getElementById("swap").replaceWith(Object.assign(document.createElement("button"), { textContent: "Swap me" }))', + ); + const r = await jevAct({ tabId: 1, op: "click", node }); + expect(r).toMatchObject({ stale: true, reason: "the element is gone" }); + }); + + it("jev: a covered target comes back stale within about half a second", async () => { + await load("/jev"); + const s = await jevObserve({ tabId: 1 }); + // The snapshot lists it (it is visible); only the act-time hit test sees the overlay. + const node = s.actions.find((a) => a.label === "Covered")?.node as number; + const t0 = Date.now(); + expect(await jevAct({ tabId: 1, op: "click", node })).toMatchObject({ stale: true }); + expect(Date.now() - t0).toBeLessThan(1500); + }); + + it("jev: a click that opens a dialog returns ok, and the next read reports the dialog", async () => { + await load("/jev"); + const s = await jevObserve({ tabId: 1 }); + const node = s.actions.find((a) => a.label === "Ask")?.node as number; + const t0 = Date.now(); + expect(await jevAct({ tabId: 1, op: "click", node })).toEqual({ ok: true }); + expect(Date.now() - t0).toBeLessThan(2000); + const after = await jevObserve({ tabId: 1 }); + expect(after.dialog).toEqual({ type: "confirm", message: "Leave?" }); + // Dismiss it so later cases aren't blocked; Page.javascriptDialogClosed clears the record. + await handleDialog({ tabId: 1, accept: false }); + await new Promise((r) => setTimeout(r, 200)); + expect(openDialog(1)).toBeUndefined(); + }, 10_000); }); diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 3459b12..5fc7dac 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -1,5 +1,6 @@ export * from "./bridge.js"; export * from "./cdp.js"; export * from "./frames.js"; +export * from "./jev.js"; export * from "./policy.js"; export * from "./ports.js"; diff --git a/packages/protocol/src/jev.test.ts b/packages/protocol/src/jev.test.ts new file mode 100644 index 0000000..0830e87 --- /dev/null +++ b/packages/protocol/src/jev.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { JevActParams, JevActResult, JevObservation } from "./jev.js"; + +describe("jev schemas", () => { + it("parses an observation", () => { + const obs = { + url: "https://x.com/", + title: "X", + text: "hello", + visible: true, + actions: [ + { id: "e1", kind: "fill", node: 3, role: "textbox", label: "Where from?", value: "" }, + { id: "wait", kind: "wait", label: "Wait for the page to update" }, + ], + }; + expect(JevObservation.parse(obs)).toEqual(obs); + }); + + it("act params carry typed text under `text` (audit-redacted key)", () => { + expect(JevActParams.parse({ op: "type", node: 3, text: "Zurich" }).text).toBe("Zurich"); + expect(() => JevActParams.parse({ op: "drag" })).toThrow(); + }); + + it("act params carry an optional label for the audit trail", () => { + expect(JevActParams.parse({ op: "click", node: 3, label: "Search" }).label).toBe("Search"); + expect(JevActParams.parse({ op: "click", node: 3 }).label).toBeUndefined(); + }); + + it("act result is ok or stale", () => { + expect(JevActResult.parse({ ok: true })).toEqual({ ok: true }); + expect(JevActResult.parse({ stale: true, reason: "gone" })).toEqual({ + stale: true, + reason: "gone", + }); + expect(() => JevActResult.parse({})).toThrow(); + }); +}); diff --git a/packages/protocol/src/jev.ts b/packages/protocol/src/jev.ts new file mode 100644 index 0000000..91154bf --- /dev/null +++ b/packages/protocol/src/jev.ts @@ -0,0 +1,64 @@ +import { z } from "zod"; + +const tabId = z.number().optional(); +const browserId = z.string().optional(); + +/** One thing Jev may choose: an operation on an observed element, or a page-level control. */ +export const JevActionKind = z.enum(["click", "fill", "select", "scroll", "wait"]); +export type JevActionKind = z.infer; + +export const JevAction = z.object({ + id: z.string(), + kind: JevActionKind, + /** Page-side identity of the element (absent for scroll/wait). */ + node: z.number().optional(), + role: z.string().optional(), + label: z.string(), + value: z.string().optional(), + current_value: z.string().optional(), + checked: z.string().optional(), + selected: z.string().optional(), + expanded: z.string().optional(), + delta: z.number().optional(), +}); +export type JevAction = z.infer; + +export const JevDialog = z.object({ type: z.string(), message: z.string() }); +export type JevDialog = z.infer; + +export const JevObservation = z.object({ + url: z.string(), + title: z.string(), + /** Visible viewport text, capped (~6,000 chars). */ + text: z.string(), + /** document.visibilityState === "visible". */ + visible: z.boolean(), + actions: z.array(JevAction), + /** Set when a JS alert/confirm/prompt blocks the page. */ + dialog: JevDialog.optional(), +}); +export type JevObservation = z.infer; + +export const JevObserveParams = z.object({ browserId, tabId }); +export type JevObserveParams = z.infer; + +export const JevActParams = z.object({ + browserId, + tabId, + op: z.enum(["click", "type", "select", "scroll", "wait"]), + node: z.number().optional(), + /** The chosen element's label, for the audit trail; the extension ignores it. */ + label: z.string().optional(), + /** Typed text. Named `text` so the audit trail redacts it. */ + text: z.string().optional(), + /** Native