From 27883085df24de1503232c444d9a4a280abb0d9c Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:54:14 -0500 Subject: [PATCH 01/20] feat(engine): expose scrollable semantic state --- packages/e2e/src/engine/contract.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/e2e/src/engine/contract.ts b/packages/e2e/src/engine/contract.ts index a511616b7a..cf7ca96a97 100644 --- a/packages/e2e/src/engine/contract.ts +++ b/packages/e2e/src/engine/contract.ts @@ -156,12 +156,14 @@ export interface SemanticNode { readonly inputPurpose?: 'username' | 'password' | 'one-time-code' | 'generic-secret' | 'none'; /** * Boolean states the platform reports. `pressed` is a toggle button's - * pressed state; `secure` marks a field whose value is never observed. + * pressed state; `secure` marks a field whose value is never observed; + * `scrollable` means the engine has evidence that a swipe on this node can + * move its own scroll surface rather than only the viewport. */ readonly states?: Readonly< Partial< Record< - 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure', + 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure' | 'scrollable', boolean > > From b930699f55c80df847635c34afa793ee9fa6356e Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:54:25 -0500 Subject: [PATCH 02/20] feat(web): carry scrollable node evidence --- packages/web/src/read-node.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/web/src/read-node.ts b/packages/web/src/read-node.ts index f8c3382940..57c72529c3 100644 --- a/packages/web/src/read-node.ts +++ b/packages/web/src/read-node.ts @@ -46,6 +46,8 @@ export interface RawNodeData { focused: boolean; hidden: boolean; secure: boolean; + /** True when this element owns a scroll surface with content beyond its client box. */ + scrollable: boolean; }; /** Heading level of a heading: `aria-level`, else the digit of `h1` through `h6`; null for anything else. */ level: number | null; From 321ef6a305dc24849a3eafc40f47ed2de7d9e034 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:54:38 -0500 Subject: [PATCH 03/20] feat(web): derive scrollable state from layout --- packages/web/src/in-page/read-semantics.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/web/src/in-page/read-semantics.ts b/packages/web/src/in-page/read-semantics.ts index 7cb49686d3..f5803f4a07 100644 --- a/packages/web/src/in-page/read-semantics.ts +++ b/packages/web/src/in-page/read-semantics.ts @@ -1132,6 +1132,21 @@ export const readSemanticsFunction = ( return role !== null && ARIA_DISABLED_ROLES.indexOf(role) !== -1 && ariaDisabledInChain(el); }; + /** + * Whether a wheel/swipe started inside this element can move the element's + * own scroll surface. This is layout evidence from the platform, not a role + * guess: content must exceed the client box on an axis whose computed + * overflow accepts user scrolling. + */ + const isScrollable = (el: Element, style: CSSStyleDeclaration | undefined): boolean => { + if (style === undefined) return false; + const accepts = (value: string): boolean => value === 'auto' || value === 'scroll' || value === 'overlay'; + return ( + (accepts(style.overflowY) && el.scrollHeight > el.clientHeight + 1) || + (accepts(style.overflowX) && el.scrollWidth > el.clientWidth + 1) + ); + }; + const describe = (el: Element, style = styleOf(el)): RawNodeData => { const tag = el.tagName.toLowerCase(); const autocomplete = (el.getAttribute('autocomplete') ?? '').toLowerCase(); @@ -1263,6 +1278,7 @@ export const readSemanticsFunction = ( focused: focusedElement === el, hidden: isHidden(el, style), secure, + scrollable: isScrollable(el, style), }, level, attributes, From 566ab68d06c623fcd8f3c20f8a0c7508d4a321f2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:54:46 -0500 Subject: [PATCH 04/20] feat(web): publish scrollable semantic state --- packages/web/src/observation.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/web/src/observation.ts b/packages/web/src/observation.ts index 16714ea56a..a666331c05 100644 --- a/packages/web/src/observation.ts +++ b/packages/web/src/observation.ts @@ -343,6 +343,7 @@ export function toSemanticNode( if (raw.states.focused) states['focused'] = true; if (raw.states.hidden) states['hidden'] = true; if (raw.states.secure) states['secure'] = true; + if (raw.states.scrollable) states['scrollable'] = true; return { ref, ...(raw.role !== null ? { role: raw.role } : {}), From 62479b094097434cceaa8500cf52dfd8fed82cc2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:54:56 -0500 Subject: [PATCH 05/20] feat(mobile): publish scrollable semantic state --- packages/mobile/src/nodes.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/mobile/src/nodes.ts b/packages/mobile/src/nodes.ts index f66b7d651f..0d227c4ae3 100644 --- a/packages/mobile/src/nodes.ts +++ b/packages/mobile/src/nodes.ts @@ -8,6 +8,7 @@ */ import type { SnapshotNode } from 'agent-device'; +import { isScrollableNodeLike } from 'agent-device/contracts'; import type { SemanticNode, ViewportSize } from 'e2e/engine'; /** @@ -425,6 +426,7 @@ export function projectSnapshot(raw: readonly RawNode[], options: { readonly min ...(source.focused === true ? { focused: true } : {}), ...(source.visibleToUser === false ? { hidden: true } : {}), ...(secure ? { secure: true } : {}), + ...(isScrollableNodeLike(source) ? { scrollable: true } : {}), ...(checked === undefined ? {} : { checked }), ...(spelled?.states.expanded === undefined ? {} : { expanded: spelled.states.expanded }), }; From c2981d42f152af21d2c42a73c9329afdf0903285 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:55:48 -0500 Subject: [PATCH 06/20] feat(web): keep scroll containers in observations --- packages/web/src/in-page/read-semantics.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/web/src/in-page/read-semantics.ts b/packages/web/src/in-page/read-semantics.ts index f5803f4a07..34e697fd8d 100644 --- a/packages/web/src/in-page/read-semantics.ts +++ b/packages/web/src/in-page/read-semantics.ts @@ -1325,7 +1325,8 @@ export const readSemanticsFunction = ( }; /** True when a node carries semantics worth sending to a model. */ - const isInteresting = (el: Element): boolean => { + const isInteresting = (el: Element, style: CSSStyleDeclaration | undefined): boolean => { + if (isScrollable(el, style)) return true; if (el.hasAttribute(options.testIdAttribute)) return true; if (implicitRole(el) !== null && !isPresentational(el)) return true; if (accessibleName(el) !== null) return true; @@ -1367,7 +1368,7 @@ export const readSemanticsFunction = ( let nextParent = parent; // An empty painted rectangle carries no semantics to be "interesting" by and // is still something a person sees and aims at; `roleOf` names it `box`. - if (!hidden && (isInteresting(el) || isVisibleEmptyBox(el))) { + if (!hidden && (isInteresting(el, style) || isVisibleEmptyBox(el))) { if (nodes.length >= maxNodes) { truncated = true; return; From d88977869d521ee0b6824b7dac602593d6d99947 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:56:22 -0500 Subject: [PATCH 07/20] feat(decision): expose engine-proven scroll targets --- packages/decision/src/elements.ts | 50 +++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/packages/decision/src/elements.ts b/packages/decision/src/elements.ts index e555d3f4ec..d809c37e60 100644 --- a/packages/decision/src/elements.ts +++ b/packages/decision/src/elements.ts @@ -31,7 +31,8 @@ export interface ActionSpace { readonly elements: readonly Element[]; /** Per operation: target key -> bound target. Only operations with at least one target appear. */ readonly targets: ReadonlyMap>; - readonly controls: ReadonlyMap; + /** Per control: target key -> bound viewport or scroll-container action. */ + readonly controls: ReadonlyMap>; /** Elements left out to stay under the per-question cap; scrolling can bring them into view. */ readonly omitted: number; /** Non-interactive page text from the tree, without node ids, clipped to 6000 chars. */ @@ -73,11 +74,13 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat inViewport: boolean; } const rows: Row[] = []; + const scrollables: ExecutorNode[] = []; const pageText: string[] = []; /** Walks the tree, collecting interactive rows and page text. */ const visit = (node: ExecutorNode, underNativeSelect: boolean): void => { if (node.states?.disabled !== true && node.states?.hidden !== true) { const operations: Operation[] = []; + if (node.states?.scrollable === true && intersects(node, observation.viewport)) scrollables.push(node); const role = node.role ?? ''; const password = node.inputPurpose === 'password' || node.states?.secure === true; const nativeSelect = role === 'combobox' && (node.children ?? []).some((child) => child.role === 'option'); @@ -164,13 +167,29 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat operations: row.operations, }; }); - const controls = new Map(); + const controls = new Map>(); if (verbs.has('scroll')) { - controls.set('scroll_up', { description: 'scroll viewport up', run: () => ctx.actions.scroll('up') }); - controls.set('scroll_down', { description: 'scroll viewport down', run: () => ctx.actions.scroll('down') }); + const scrollGroup = (direction: 'up' | 'down'): Map => { + const group = new Map(); + group.set('viewport', { + description: `scroll viewport ${direction}`, + run: () => ctx.actions.scroll(direction), + }); + // One question must stay inside TypeSafe's choice cap. The viewport is + // always one choice; scrollable nodes are engine-proven candidates. + for (const node of scrollables.slice(0, MAX_CHOICES - 1)) { + group.set(`node:${node.id}`, { + description: `scroll ${scrollTargetLabel(node)} ${direction}`, + run: () => ctx.actions.scroll(direction, { id: node.id }), + }); + } + return group; + }; + controls.set('scroll_up', scrollGroup('up')); + controls.set('scroll_down', scrollGroup('down')); } if (verbs.has('back')) { - controls.set('back', { description: 'back one step in history', run: () => ctx.actions.back() }); + controls.set('back', new Map([['history', { description: 'back one step in history', run: () => ctx.actions.back() }]])); } return { elements, @@ -206,6 +225,27 @@ function nodeLabel(node: ExecutorNode): string { return node.name ?? node.attributes?.['placeholder'] ?? node.text ?? ''; } +/** Human-readable scroll target without platform selectors or node ids. */ +function scrollTargetLabel(node: ExecutorNode): string { + const own = nodeLabel(node).trim(); + if (own !== '') return `${node.role ?? 'container'} ${JSON.stringify(clip(own, 80))}`; + const descendants: string[] = []; + const visit = (current: ExecutorNode): void => { + if (descendants.length >= 2) return; + for (const child of current.children ?? []) { + const label = nodeLabel(child).trim(); + if (label !== '') descendants.push(clip(label, 60)); + if (descendants.length >= 2) return; + visit(child); + if (descendants.length >= 2) return; + } + }; + visit(node); + return descendants.length === 0 + ? (node.role ?? 'scroll container') + : `${node.role ?? 'container'} containing ${descendants.map((label) => JSON.stringify(label)).join(', ')}`; +} + /** * One page-text line for a non-interactive node. A named node keeps its text * when the two differ, as the runner's `formatNode` does: a status named From c26ee74d003667e64df74644365ffe721d5db743 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:56:32 -0500 Subject: [PATCH 08/20] feat(decision): ask which scroll surface to move --- packages/decision/src/questions.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/decision/src/questions.ts b/packages/decision/src/questions.ts index aa94dbccea..d3a50ec5bb 100644 --- a/packages/decision/src/questions.ts +++ b/packages/decision/src/questions.ts @@ -86,6 +86,13 @@ export function decisionRequest( } questions[`${operation}_target`] = choice({ operation, rules: TARGET }, options); } + for (const [control, targets] of space.controls) { + if (targets.size < 2) continue; + questions[`${control}_target`] = choice( + { operation: control, rules: TARGET }, + Object.fromEntries([...targets].map(([key, target]) => [key, target.description])), + ); + } if (space.targets.has('typeSecret') && ctx.step.secrets.length >= 2) { const secrets: Record = {}; for (const secret of ctx.step.secrets) secrets[secret.name] = secret.purpose; From 4bb973e8bad42790e5c3966e8a8fcc61bb79bde3 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:56:44 -0500 Subject: [PATCH 09/20] feat(decision): dispatch targeted scroll controls --- packages/decision/src/executor.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/decision/src/executor.ts b/packages/decision/src/executor.ts index ea9ffed65a..c3eedd5c6e 100644 --- a/packages/decision/src/executor.ts +++ b/packages/decision/src/executor.ts @@ -178,9 +178,7 @@ async function run( answers: Record, space: ActionSpace, ): { taken: Taken; targetAnswer?: Decision } { - const control = space.controls.get(choice as Control); - if (control !== undefined) return { taken: { target: control, operation: choice, elementKey: '' } }; - const group = space.targets.get(choice as Operation); + const group = space.controls.get(choice as Control) ?? space.targets.get(choice as Operation); if (group === undefined) throw invalid('The decision model chose an unavailable operation.'); if (group.size === 1) { const only = group.entries().next(); From 565472aafe55fac7fb4b728f858d98fa8b72e53c Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:57:03 -0500 Subject: [PATCH 10/20] test(decision): cover targeted scroll controls --- packages/decision/tests/elements.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/decision/tests/elements.test.ts b/packages/decision/tests/elements.test.ts index b6ed1bb36c..b1174f62a1 100644 --- a/packages/decision/tests/elements.test.ts +++ b/packages/decision/tests/elements.test.ts @@ -133,6 +133,27 @@ describe('element table', () => { const none = spaceFor([{ id: 'a', role: 'button', name: 'Add' }], ['tap']); expect(none.controls.size).toBe(0); }); + it('offers engine-proven scroll containers beside the viewport', async () => { + const fixture = context({ verbs: ['scroll'] }); + const space = actionSpace(fixture.ctx, { + path: '/feed', + viewport: { width: 800, height: 600 }, + tree: tree([ + { + id: 'feed', + role: 'group', + states: { scrollable: true }, + rect: { x: 20, y: 20, width: 760, height: 480 }, + children: [{ id: 'row', role: 'text', text: 'Item 1' }], + }, + ]), + }, true); + const down = space.controls.get('scroll_down'); + expect([...(down?.keys() ?? [])]).toEqual(['viewport', 'node:feed']); + expect(down?.get('node:feed')?.description).toContain('containing "Item 1"'); + await down?.get('node:feed')?.run(); + expect(fixture.actions.scroll).toHaveBeenCalledWith('down', { id: 'feed' }); + }); }); describe('fingerprint', () => { const page = (): ExecutorNode => ({ id: 'root', children: [ From 6ca80df315c78319d2e647b8d97a1980f1633610 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:57:34 -0500 Subject: [PATCH 11/20] test(decision): route scrolls to semantic containers --- packages/decision/tests/executor.test.ts | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/decision/tests/executor.test.ts b/packages/decision/tests/executor.test.ts index 48e51d98bf..9d3e62e376 100644 --- a/packages/decision/tests/executor.test.ts +++ b/packages/decision/tests/executor.test.ts @@ -265,6 +265,31 @@ describe('terminal checks', () => { { element: 'Large', role: 'option' }, ]); }); + it('asks which scroll surface to move and dispatches the chosen container', async () => { + const tree: ExecutorNode = { id: 'root', children: [ + { + id: 'feed', + role: 'group', + states: { scrollable: true }, + rect: { x: 20, y: 20, width: 760, height: 480 }, + children: [{ id: 'row', role: 'text', text: 'Item 1' }], + }, + ] }; + const { model, requests } = scriptedEvaluation((id, keys, call) => { + if (id === 'operation') return { choice: call === 0 ? 'scroll_down' : 'done' }; + if (id === 'scroll_down_target') return { choice: 'node:feed' }; + if (id === 'verdict') return { choice: 'holds' }; + return { choice: keys[0] ?? '' }; + }); + const fixture = context({ tree }); + const verdict = await decisionExecutor({ model }).runStep(fixture.ctx); + expect(verdict).toMatchObject({ status: 'passed' }); + expect(requests[0]?.questions['scroll_down_target']?.criteria).toMatchObject({ + viewport: 'scroll viewport down', + 'node:feed': expect.stringContaining('Item 1'), + }); + expect(fixture.actions.scroll).toHaveBeenCalledWith('down', { id: 'feed' }); + }); }); describe('guard rails', () => { From ce52bb96fe7221eeb4a8118358a6bd4bcf9d8a97 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:58:16 -0500 Subject: [PATCH 12/20] fix(web): keep scrollable wire field additive --- packages/web/src/read-node.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/src/read-node.ts b/packages/web/src/read-node.ts index 57c72529c3..b069aabd68 100644 --- a/packages/web/src/read-node.ts +++ b/packages/web/src/read-node.ts @@ -47,7 +47,7 @@ export interface RawNodeData { hidden: boolean; secure: boolean; /** True when this element owns a scroll surface with content beyond its client box. */ - scrollable: boolean; + scrollable?: boolean; }; /** Heading level of a heading: `aria-level`, else the digit of `h1` through `h6`; null for anything else. */ level: number | null; From be7205be52bafe9ce09a9ab4e66cbe37ccaf7ddf Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:58:27 -0500 Subject: [PATCH 13/20] test(mobile): pin scrollable semantic state --- packages/mobile/tests/unit/nodes.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/mobile/tests/unit/nodes.test.ts b/packages/mobile/tests/unit/nodes.test.ts index 45089d0a60..e868960f03 100644 --- a/packages/mobile/tests/unit/nodes.test.ts +++ b/packages/mobile/tests/unit/nodes.test.ts @@ -209,6 +209,8 @@ describe('snapshot projection', () => { 'button', 'textbox', ]); + expect(projected.index[3]?.node.states).toMatchObject({ scrollable: true }); + expect(projected.index[13]?.node.states).toMatchObject({ scrollable: true }); expect(projected.index[6]?.node.states).toEqual({ secure: true }); expect(projected.viewport).toEqual({ width: 390, height: 844 }); expect(screenTitle(projected)).toBe('Settings'); @@ -308,6 +310,7 @@ describe('snapshot projection', () => { 'tablist', 'list', ]); + expect(projected.index[11]?.node.states).toMatchObject({ scrollable: true }); }); it('names an Android node by its content description, keeps its text, and reads heading, role description, checked, and placeholder', () => { From 20733bb12a27fbc3d40255124d9b2da76cab4b27 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:59:58 -0500 Subject: [PATCH 14/20] test(web): pin scrollable state to overflow geometry --- .../integration/read-node-states.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/web/tests/integration/read-node-states.test.ts b/packages/web/tests/integration/read-node-states.test.ts index d0a05dd6ab..64a9184108 100644 --- a/packages/web/tests/integration/read-node-states.test.ts +++ b/packages/web/tests/integration/read-node-states.test.ts @@ -51,6 +51,16 @@ async function disabledByTestId(): Promise> { ); } +/** The `scrollable` state per test id, `false` when the node reports none. */ +async function scrollableByTestId(): Promise> { + const { tree } = await capture(); + return new Map( + flatten(tree) + .filter((node) => node.testId !== undefined) + .map((node) => [node.testId!, node.states?.scrollable === true]), + ); +} + /** Playwright's answer for the same elements, the reference the reader must agree with. */ async function expectAgreesWithPlaywright(states: Map): Promise { for (const [testId, disabled] of states) { @@ -156,3 +166,25 @@ describe('disabled state', () => { await expectAgreesWithPlaywright(states); }); }); + + +describe('scrollable state', () => { + it('comes from real overflow geometry, not the element role or test id', async () => { + await page.setContent(` +
+
long content
+
+
+
long but wheel-clipped
+
+
+
fits
+
+ `); + expect(Object.fromEntries(await scrollableByTestId())).toMatchObject({ + scrolls: true, + 'hidden-overflow': false, + fits: false, + }); + }); +}); From 085b2df4bd2dc3863e2d3c95cd87936ec368bb90 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:00:29 -0500 Subject: [PATCH 15/20] chore: add targeted scroll changeset --- .changeset/decision-scroll-targets.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/decision-scroll-targets.md diff --git a/.changeset/decision-scroll-targets.md b/.changeset/decision-scroll-targets.md new file mode 100644 index 0000000000..c416eb33d2 --- /dev/null +++ b/.changeset/decision-scroll-targets.md @@ -0,0 +1,8 @@ +--- +"e2e": minor +"@e2e-dev/web": patch +"@e2e-dev/mobile": patch +"@e2e-dev/decision": minor +--- + +Semantic observations can now report whether a node owns a scroll surface. The web engine derives this from real overflow geometry and the mobile engine projects agent-device's scroll-container classification. The decision executor uses that evidence to choose between scrolling the viewport and a specific scrollable container instead of always scrolling the page. From afa81df80b0ba34fb496e2aafc0a06fe76ddf954 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:02:18 -0500 Subject: [PATCH 16/20] test(types): expose scrollable semantic state --- packages/e2e/tests/types/sdk-types.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts index 025ab99751..f955c6fd50 100644 --- a/packages/e2e/tests/types/sdk-types.ts +++ b/packages/e2e/tests/types/sdk-types.ts @@ -49,7 +49,7 @@ import { beforeEach, describe, } from '../../src/index.ts'; -import type { Engine, EngineAttemptContext, EngineHandle, EngineObserveOptions, EngineSnapshot } from '../../src/engine/index.ts'; +import type { Engine, EngineAttemptContext, EngineHandle, EngineObserveOptions, EngineSnapshot, SemanticNode } from '../../src/engine/index.ts'; import { createToolLoopExecutor, defineTool } from '../../src/agent/public.ts'; import type { Report } from '../../src/index.ts'; import type { LanguageModelV2, LanguageModelV3, LanguageModelV4 } from '@ai-sdk/provider'; @@ -89,6 +89,8 @@ customSnapshot satisfies EngineSnapshot; customExecutorObservation satisfies ExecutorObservation; declare const engineSnapshot: EngineSnapshot; engineSnapshot.treeUnavailable satisfies true | undefined; +declare const semanticNode: SemanticNode; +semanticNode.states?.scrollable satisfies boolean | undefined; // @ts-expect-error unavailable semantics are explicitly true or absent, never a separate false state. ({ ...engineSnapshot, treeUnavailable: false }) satisfies EngineSnapshot; From 7c83cfabd8a0e003e71ae0be038df44194183595 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:02:42 -0500 Subject: [PATCH 17/20] docs(engine): document scrollable semantic state --- docs/reference/engine.mdx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 84dc9ed10c..ee99d8986f 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -1304,12 +1304,14 @@ export interface SemanticNode { readonly inputPurpose?: 'username' | 'password' | 'one-time-code' | 'generic-secret' | 'none'; /** * Boolean states the platform reports. `pressed` is a toggle button's - * pressed state; `secure` marks a field whose value is never observed. + * pressed state; `secure` marks a field whose value is never observed; + * `scrollable` means the engine has evidence that a swipe on this node can + * move its own scroll surface rather than only the viewport. */ readonly states?: Readonly< Partial< Record< - 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure', + 'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure' | 'scrollable', boolean > > From a04bf1d1929a9822eddb74cfa8572aacb0bbd11b Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:45:03 -0500 Subject: [PATCH 18/20] fix(changeset): align scroll target feature with 0.x patch policy --- .changeset/decision-scroll-targets.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/decision-scroll-targets.md b/.changeset/decision-scroll-targets.md index c416eb33d2..9d9db68bae 100644 --- a/.changeset/decision-scroll-targets.md +++ b/.changeset/decision-scroll-targets.md @@ -1,8 +1,8 @@ --- -"e2e": minor +"e2e": patch "@e2e-dev/web": patch "@e2e-dev/mobile": patch -"@e2e-dev/decision": minor +"@e2e-dev/decision": patch --- Semantic observations can now report whether a node owns a scroll surface. The web engine derives this from real overflow geometry and the mobile engine projects agent-device's scroll-container classification. The decision executor uses that evidence to choose between scrolling the viewport and a specific scrollable container instead of always scrolling the page. From d1c56857e115fb18849fc71b54301519dc8c5746 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:47:03 -0500 Subject: [PATCH 19/20] fix(web): offer only vertically scrollable nodes to up/down decisions --- packages/web/src/in-page/read-semantics.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/web/src/in-page/read-semantics.ts b/packages/web/src/in-page/read-semantics.ts index 34e697fd8d..0d97171f54 100644 --- a/packages/web/src/in-page/read-semantics.ts +++ b/packages/web/src/in-page/read-semantics.ts @@ -1133,18 +1133,14 @@ export const readSemanticsFunction = ( }; /** - * Whether a wheel/swipe started inside this element can move the element's - * own scroll surface. This is layout evidence from the platform, not a role - * guess: content must exceed the client box on an axis whose computed - * overflow accepts user scrolling. + * Whether this node can receive a vertical wheel/swipe. The decision + * executor currently offers up/down, not left/right: a horizontal-only + * scroller is not a valid target for those actions. */ const isScrollable = (el: Element, style: CSSStyleDeclaration | undefined): boolean => { if (style === undefined) return false; const accepts = (value: string): boolean => value === 'auto' || value === 'scroll' || value === 'overlay'; - return ( - (accepts(style.overflowY) && el.scrollHeight > el.clientHeight + 1) || - (accepts(style.overflowX) && el.scrollWidth > el.clientWidth + 1) - ); + return accepts(style.overflowY) && el.scrollHeight > el.clientHeight + 1; }; const describe = (el: Element, style = styleOf(el)): RawNodeData => { From 5bfc777e44f448d2d0f521feb2cb4db6588f4b40 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:47:21 -0500 Subject: [PATCH 20/20] test(web): exclude horizontal-only targets from vertical scroll choices --- packages/web/tests/integration/read-node-states.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/web/tests/integration/read-node-states.test.ts b/packages/web/tests/integration/read-node-states.test.ts index 64a9184108..9007752e4f 100644 --- a/packages/web/tests/integration/read-node-states.test.ts +++ b/packages/web/tests/integration/read-node-states.test.ts @@ -180,11 +180,15 @@ describe('scrollable state', () => {
fits
+
+
sideways only
+
`); expect(Object.fromEntries(await scrollableByTestId())).toMatchObject({ scrolls: true, 'hidden-overflow': false, fits: false, + 'horizontal-only': false, }); }); });