diff --git a/.changeset/decision-scroll-targets.md b/.changeset/decision-scroll-targets.md new file mode 100644 index 0000000000..9d9db68bae --- /dev/null +++ b/.changeset/decision-scroll-targets.md @@ -0,0 +1,8 @@ +--- +"e2e": patch +"@e2e-dev/web": patch +"@e2e-dev/mobile": patch +"@e2e-dev/decision": patch +--- + +Semantic observations can now report whether a node owns a scroll surface. The web engine derives this from real overflow geometry and the mobile engine projects agent-device's scroll-container classification. The decision executor uses that evidence to choose between scrolling the viewport and a specific scrollable container instead of always scrolling the page. diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 84dc9ed10c..ee99d8986f 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -1304,12 +1304,14 @@ export interface SemanticNode { readonly inputPurpose?: 'username' | 'password' | 'one-time-code' | 'generic-secret' | 'none'; /** * Boolean states the platform reports. `pressed` is a toggle button's - * pressed state; `secure` marks a field whose value is never observed. + * pressed state; `secure` marks a field whose value is never observed; + * `scrollable` means the engine has evidence that a swipe on this node can + * move its own scroll surface rather than only the viewport. */ readonly states?: Readonly< Partial< Record< - 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure', + 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure' | 'scrollable', boolean > > diff --git a/packages/decision/src/elements.ts b/packages/decision/src/elements.ts index e555d3f4ec..d809c37e60 100644 --- a/packages/decision/src/elements.ts +++ b/packages/decision/src/elements.ts @@ -31,7 +31,8 @@ export interface ActionSpace { readonly elements: readonly Element[]; /** Per operation: target key -> bound target. Only operations with at least one target appear. */ readonly targets: ReadonlyMap>; - readonly controls: ReadonlyMap; + /** Per control: target key -> bound viewport or scroll-container action. */ + readonly controls: ReadonlyMap>; /** Elements left out to stay under the per-question cap; scrolling can bring them into view. */ readonly omitted: number; /** Non-interactive page text from the tree, without node ids, clipped to 6000 chars. */ @@ -73,11 +74,13 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat inViewport: boolean; } const rows: Row[] = []; + const scrollables: ExecutorNode[] = []; const pageText: string[] = []; /** Walks the tree, collecting interactive rows and page text. */ const visit = (node: ExecutorNode, underNativeSelect: boolean): void => { if (node.states?.disabled !== true && node.states?.hidden !== true) { const operations: Operation[] = []; + if (node.states?.scrollable === true && intersects(node, observation.viewport)) scrollables.push(node); const role = node.role ?? ''; const password = node.inputPurpose === 'password' || node.states?.secure === true; const nativeSelect = role === 'combobox' && (node.children ?? []).some((child) => child.role === 'option'); @@ -164,13 +167,29 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat operations: row.operations, }; }); - const controls = new Map(); + const controls = new Map>(); if (verbs.has('scroll')) { - controls.set('scroll_up', { description: 'scroll viewport up', run: () => ctx.actions.scroll('up') }); - controls.set('scroll_down', { description: 'scroll viewport down', run: () => ctx.actions.scroll('down') }); + const scrollGroup = (direction: 'up' | 'down'): Map => { + const group = new Map(); + group.set('viewport', { + description: `scroll viewport ${direction}`, + run: () => ctx.actions.scroll(direction), + }); + // One question must stay inside TypeSafe's choice cap. The viewport is + // always one choice; scrollable nodes are engine-proven candidates. + for (const node of scrollables.slice(0, MAX_CHOICES - 1)) { + group.set(`node:${node.id}`, { + description: `scroll ${scrollTargetLabel(node)} ${direction}`, + run: () => ctx.actions.scroll(direction, { id: node.id }), + }); + } + return group; + }; + controls.set('scroll_up', scrollGroup('up')); + controls.set('scroll_down', scrollGroup('down')); } if (verbs.has('back')) { - controls.set('back', { description: 'back one step in history', run: () => ctx.actions.back() }); + controls.set('back', new Map([['history', { description: 'back one step in history', run: () => ctx.actions.back() }]])); } return { elements, @@ -206,6 +225,27 @@ function nodeLabel(node: ExecutorNode): string { return node.name ?? node.attributes?.['placeholder'] ?? node.text ?? ''; } +/** Human-readable scroll target without platform selectors or node ids. */ +function scrollTargetLabel(node: ExecutorNode): string { + const own = nodeLabel(node).trim(); + if (own !== '') return `${node.role ?? 'container'} ${JSON.stringify(clip(own, 80))}`; + const descendants: string[] = []; + const visit = (current: ExecutorNode): void => { + if (descendants.length >= 2) return; + for (const child of current.children ?? []) { + const label = nodeLabel(child).trim(); + if (label !== '') descendants.push(clip(label, 60)); + if (descendants.length >= 2) return; + visit(child); + if (descendants.length >= 2) return; + } + }; + visit(node); + return descendants.length === 0 + ? (node.role ?? 'scroll container') + : `${node.role ?? 'container'} containing ${descendants.map((label) => JSON.stringify(label)).join(', ')}`; +} + /** * One page-text line for a non-interactive node. A named node keeps its text * when the two differ, as the runner's `formatNode` does: a status named diff --git a/packages/decision/src/executor.ts b/packages/decision/src/executor.ts index ea9ffed65a..c3eedd5c6e 100644 --- a/packages/decision/src/executor.ts +++ b/packages/decision/src/executor.ts @@ -178,9 +178,7 @@ async function run( answers: Record, space: ActionSpace, ): { taken: Taken; targetAnswer?: Decision } { - const control = space.controls.get(choice as Control); - if (control !== undefined) return { taken: { target: control, operation: choice, elementKey: '' } }; - const group = space.targets.get(choice as Operation); + const group = space.controls.get(choice as Control) ?? space.targets.get(choice as Operation); if (group === undefined) throw invalid('The decision model chose an unavailable operation.'); if (group.size === 1) { const only = group.entries().next(); diff --git a/packages/decision/src/questions.ts b/packages/decision/src/questions.ts index aa94dbccea..d3a50ec5bb 100644 --- a/packages/decision/src/questions.ts +++ b/packages/decision/src/questions.ts @@ -86,6 +86,13 @@ export function decisionRequest( } questions[`${operation}_target`] = choice({ operation, rules: TARGET }, options); } + for (const [control, targets] of space.controls) { + if (targets.size < 2) continue; + questions[`${control}_target`] = choice( + { operation: control, rules: TARGET }, + Object.fromEntries([...targets].map(([key, target]) => [key, target.description])), + ); + } if (space.targets.has('typeSecret') && ctx.step.secrets.length >= 2) { const secrets: Record = {}; for (const secret of ctx.step.secrets) secrets[secret.name] = secret.purpose; diff --git a/packages/decision/tests/elements.test.ts b/packages/decision/tests/elements.test.ts index b6ed1bb36c..b1174f62a1 100644 --- a/packages/decision/tests/elements.test.ts +++ b/packages/decision/tests/elements.test.ts @@ -133,6 +133,27 @@ describe('element table', () => { const none = spaceFor([{ id: 'a', role: 'button', name: 'Add' }], ['tap']); expect(none.controls.size).toBe(0); }); + it('offers engine-proven scroll containers beside the viewport', async () => { + const fixture = context({ verbs: ['scroll'] }); + const space = actionSpace(fixture.ctx, { + path: '/feed', + viewport: { width: 800, height: 600 }, + tree: tree([ + { + id: 'feed', + role: 'group', + states: { scrollable: true }, + rect: { x: 20, y: 20, width: 760, height: 480 }, + children: [{ id: 'row', role: 'text', text: 'Item 1' }], + }, + ]), + }, true); + const down = space.controls.get('scroll_down'); + expect([...(down?.keys() ?? [])]).toEqual(['viewport', 'node:feed']); + expect(down?.get('node:feed')?.description).toContain('containing "Item 1"'); + await down?.get('node:feed')?.run(); + expect(fixture.actions.scroll).toHaveBeenCalledWith('down', { id: 'feed' }); + }); }); describe('fingerprint', () => { const page = (): ExecutorNode => ({ id: 'root', children: [ diff --git a/packages/decision/tests/executor.test.ts b/packages/decision/tests/executor.test.ts index 48e51d98bf..9d3e62e376 100644 --- a/packages/decision/tests/executor.test.ts +++ b/packages/decision/tests/executor.test.ts @@ -265,6 +265,31 @@ describe('terminal checks', () => { { element: 'Large', role: 'option' }, ]); }); + it('asks which scroll surface to move and dispatches the chosen container', async () => { + const tree: ExecutorNode = { id: 'root', children: [ + { + id: 'feed', + role: 'group', + states: { scrollable: true }, + rect: { x: 20, y: 20, width: 760, height: 480 }, + children: [{ id: 'row', role: 'text', text: 'Item 1' }], + }, + ] }; + const { model, requests } = scriptedEvaluation((id, keys, call) => { + if (id === 'operation') return { choice: call === 0 ? 'scroll_down' : 'done' }; + if (id === 'scroll_down_target') return { choice: 'node:feed' }; + if (id === 'verdict') return { choice: 'holds' }; + return { choice: keys[0] ?? '' }; + }); + const fixture = context({ tree }); + const verdict = await decisionExecutor({ model }).runStep(fixture.ctx); + expect(verdict).toMatchObject({ status: 'passed' }); + expect(requests[0]?.questions['scroll_down_target']?.criteria).toMatchObject({ + viewport: 'scroll viewport down', + 'node:feed': expect.stringContaining('Item 1'), + }); + expect(fixture.actions.scroll).toHaveBeenCalledWith('down', { id: 'feed' }); + }); }); describe('guard rails', () => { diff --git a/packages/e2e/src/engine/contract.ts b/packages/e2e/src/engine/contract.ts index a511616b7a..cf7ca96a97 100644 --- a/packages/e2e/src/engine/contract.ts +++ b/packages/e2e/src/engine/contract.ts @@ -156,12 +156,14 @@ export interface SemanticNode { readonly inputPurpose?: 'username' | 'password' | 'one-time-code' | 'generic-secret' | 'none'; /** * Boolean states the platform reports. `pressed` is a toggle button's - * pressed state; `secure` marks a field whose value is never observed. + * pressed state; `secure` marks a field whose value is never observed; + * `scrollable` means the engine has evidence that a swipe on this node can + * move its own scroll surface rather than only the viewport. */ readonly states?: Readonly< Partial< Record< - 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure', + 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure' | 'scrollable', boolean > > diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts index 025ab99751..f955c6fd50 100644 --- a/packages/e2e/tests/types/sdk-types.ts +++ b/packages/e2e/tests/types/sdk-types.ts @@ -49,7 +49,7 @@ import { beforeEach, describe, } from '../../src/index.ts'; -import type { Engine, EngineAttemptContext, EngineHandle, EngineObserveOptions, EngineSnapshot } from '../../src/engine/index.ts'; +import type { Engine, EngineAttemptContext, EngineHandle, EngineObserveOptions, EngineSnapshot, SemanticNode } from '../../src/engine/index.ts'; import { createToolLoopExecutor, defineTool } from '../../src/agent/public.ts'; import type { Report } from '../../src/index.ts'; import type { LanguageModelV2, LanguageModelV3, LanguageModelV4 } from '@ai-sdk/provider'; @@ -89,6 +89,8 @@ customSnapshot satisfies EngineSnapshot; customExecutorObservation satisfies ExecutorObservation; declare const engineSnapshot: EngineSnapshot; engineSnapshot.treeUnavailable satisfies true | undefined; +declare const semanticNode: SemanticNode; +semanticNode.states?.scrollable satisfies boolean | undefined; // @ts-expect-error unavailable semantics are explicitly true or absent, never a separate false state. ({ ...engineSnapshot, treeUnavailable: false }) satisfies EngineSnapshot; diff --git a/packages/mobile/src/nodes.ts b/packages/mobile/src/nodes.ts index f66b7d651f..0d227c4ae3 100644 --- a/packages/mobile/src/nodes.ts +++ b/packages/mobile/src/nodes.ts @@ -8,6 +8,7 @@ */ import type { SnapshotNode } from 'agent-device'; +import { isScrollableNodeLike } from 'agent-device/contracts'; import type { SemanticNode, ViewportSize } from 'e2e/engine'; /** @@ -425,6 +426,7 @@ export function projectSnapshot(raw: readonly RawNode[], options: { readonly min ...(source.focused === true ? { focused: true } : {}), ...(source.visibleToUser === false ? { hidden: true } : {}), ...(secure ? { secure: true } : {}), + ...(isScrollableNodeLike(source) ? { scrollable: true } : {}), ...(checked === undefined ? {} : { checked }), ...(spelled?.states.expanded === undefined ? {} : { expanded: spelled.states.expanded }), }; diff --git a/packages/mobile/tests/unit/nodes.test.ts b/packages/mobile/tests/unit/nodes.test.ts index 45089d0a60..e868960f03 100644 --- a/packages/mobile/tests/unit/nodes.test.ts +++ b/packages/mobile/tests/unit/nodes.test.ts @@ -209,6 +209,8 @@ describe('snapshot projection', () => { 'button', 'textbox', ]); + expect(projected.index[3]?.node.states).toMatchObject({ scrollable: true }); + expect(projected.index[13]?.node.states).toMatchObject({ scrollable: true }); expect(projected.index[6]?.node.states).toEqual({ secure: true }); expect(projected.viewport).toEqual({ width: 390, height: 844 }); expect(screenTitle(projected)).toBe('Settings'); @@ -308,6 +310,7 @@ describe('snapshot projection', () => { 'tablist', 'list', ]); + expect(projected.index[11]?.node.states).toMatchObject({ scrollable: true }); }); it('names an Android node by its content description, keeps its text, and reads heading, role description, checked, and placeholder', () => { diff --git a/packages/web/src/in-page/read-semantics.ts b/packages/web/src/in-page/read-semantics.ts index 7cb49686d3..0d97171f54 100644 --- a/packages/web/src/in-page/read-semantics.ts +++ b/packages/web/src/in-page/read-semantics.ts @@ -1132,6 +1132,17 @@ export const readSemanticsFunction = ( return role !== null && ARIA_DISABLED_ROLES.indexOf(role) !== -1 && ariaDisabledInChain(el); }; + /** + * Whether this node can receive a vertical wheel/swipe. The decision + * executor currently offers up/down, not left/right: a horizontal-only + * scroller is not a valid target for those actions. + */ + const isScrollable = (el: Element, style: CSSStyleDeclaration | undefined): boolean => { + if (style === undefined) return false; + const accepts = (value: string): boolean => value === 'auto' || value === 'scroll' || value === 'overlay'; + return accepts(style.overflowY) && el.scrollHeight > el.clientHeight + 1; + }; + const describe = (el: Element, style = styleOf(el)): RawNodeData => { const tag = el.tagName.toLowerCase(); const autocomplete = (el.getAttribute('autocomplete') ?? '').toLowerCase(); @@ -1263,6 +1274,7 @@ export const readSemanticsFunction = ( focused: focusedElement === el, hidden: isHidden(el, style), secure, + scrollable: isScrollable(el, style), }, level, attributes, @@ -1309,7 +1321,8 @@ export const readSemanticsFunction = ( }; /** True when a node carries semantics worth sending to a model. */ - const isInteresting = (el: Element): boolean => { + const isInteresting = (el: Element, style: CSSStyleDeclaration | undefined): boolean => { + if (isScrollable(el, style)) return true; if (el.hasAttribute(options.testIdAttribute)) return true; if (implicitRole(el) !== null && !isPresentational(el)) return true; if (accessibleName(el) !== null) return true; @@ -1351,7 +1364,7 @@ export const readSemanticsFunction = ( let nextParent = parent; // An empty painted rectangle carries no semantics to be "interesting" by and // is still something a person sees and aims at; `roleOf` names it `box`. - if (!hidden && (isInteresting(el) || isVisibleEmptyBox(el))) { + if (!hidden && (isInteresting(el, style) || isVisibleEmptyBox(el))) { if (nodes.length >= maxNodes) { truncated = true; return; diff --git a/packages/web/src/observation.ts b/packages/web/src/observation.ts index 16714ea56a..a666331c05 100644 --- a/packages/web/src/observation.ts +++ b/packages/web/src/observation.ts @@ -343,6 +343,7 @@ export function toSemanticNode( if (raw.states.focused) states['focused'] = true; if (raw.states.hidden) states['hidden'] = true; if (raw.states.secure) states['secure'] = true; + if (raw.states.scrollable) states['scrollable'] = true; return { ref, ...(raw.role !== null ? { role: raw.role } : {}), diff --git a/packages/web/src/read-node.ts b/packages/web/src/read-node.ts index f8c3382940..b069aabd68 100644 --- a/packages/web/src/read-node.ts +++ b/packages/web/src/read-node.ts @@ -46,6 +46,8 @@ export interface RawNodeData { focused: boolean; hidden: boolean; secure: boolean; + /** True when this element owns a scroll surface with content beyond its client box. */ + scrollable?: boolean; }; /** Heading level of a heading: `aria-level`, else the digit of `h1` through `h6`; null for anything else. */ level: number | null; diff --git a/packages/web/tests/integration/read-node-states.test.ts b/packages/web/tests/integration/read-node-states.test.ts index d0a05dd6ab..9007752e4f 100644 --- a/packages/web/tests/integration/read-node-states.test.ts +++ b/packages/web/tests/integration/read-node-states.test.ts @@ -51,6 +51,16 @@ async function disabledByTestId(): Promise> { ); } +/** The `scrollable` state per test id, `false` when the node reports none. */ +async function scrollableByTestId(): Promise> { + const { tree } = await capture(); + return new Map( + flatten(tree) + .filter((node) => node.testId !== undefined) + .map((node) => [node.testId!, node.states?.scrollable === true]), + ); +} + /** Playwright's answer for the same elements, the reference the reader must agree with. */ async function expectAgreesWithPlaywright(states: Map): Promise { for (const [testId, disabled] of states) { @@ -156,3 +166,29 @@ describe('disabled state', () => { await expectAgreesWithPlaywright(states); }); }); + + +describe('scrollable state', () => { + it('comes from real overflow geometry, not the element role or test id', async () => { + await page.setContent(` +
+
long content
+
+
+
long but wheel-clipped
+
+
+
fits
+
+
+
sideways only
+
+ `); + expect(Object.fromEntries(await scrollableByTestId())).toMatchObject({ + scrolls: true, + 'hidden-overflow': false, + fits: false, + 'horizontal-only': false, + }); + }); +});