Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
2788308
feat(engine): expose scrollable semantic state
kvnloo Oct 4, 2026
b930699
feat(web): carry scrollable node evidence
kvnloo Oct 4, 2026
321ef6a
feat(web): derive scrollable state from layout
kvnloo Oct 4, 2026
566ab68
feat(web): publish scrollable semantic state
kvnloo Oct 4, 2026
62479b0
feat(mobile): publish scrollable semantic state
kvnloo Oct 4, 2026
c2981d4
feat(web): keep scroll containers in observations
kvnloo Oct 4, 2026
d889778
feat(decision): expose engine-proven scroll targets
kvnloo Oct 4, 2026
c26ee74
feat(decision): ask which scroll surface to move
kvnloo Oct 4, 2026
4bb973e
feat(decision): dispatch targeted scroll controls
kvnloo Oct 4, 2026
565472a
test(decision): cover targeted scroll controls
kvnloo Oct 4, 2026
6ca80df
test(decision): route scrolls to semantic containers
kvnloo Oct 4, 2026
ce52bb9
fix(web): keep scrollable wire field additive
kvnloo Oct 4, 2026
be7205b
test(mobile): pin scrollable semantic state
kvnloo Oct 4, 2026
20733bb
test(web): pin scrollable state to overflow geometry
kvnloo Oct 4, 2026
085b2df
chore: add targeted scroll changeset
kvnloo Oct 4, 2026
afa81df
test(types): expose scrollable semantic state
kvnloo Oct 4, 2026
7c83cfa
docs(engine): document scrollable semantic state
kvnloo Oct 4, 2026
a04bf1d
fix(changeset): align scroll target feature with 0.x patch policy
kvnloo Oct 17, 2026
d1c5685
fix(web): offer only vertically scrollable nodes to up/down decisions
kvnloo Oct 17, 2026
5bfc777
test(web): exclude horizontal-only targets from vertical scroll choices
kvnloo Oct 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .changeset/decision-scroll-targets.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
"e2e": patch
"@e2e-dev/web": patch
"@e2e-dev/mobile": patch
"@e2e-dev/decision": patch
---

Semantic observations can now report whether a node owns a scroll surface. The web engine derives this from real overflow geometry and the mobile engine projects agent-device's scroll-container classification. The decision executor uses that evidence to choose between scrolling the viewport and a specific scrollable container instead of always scrolling the page.
6 changes: 4 additions & 2 deletions docs/reference/engine.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1304,12 +1304,14 @@ export interface SemanticNode {
readonly inputPurpose?: 'username' | 'password' | 'one-time-code' | 'generic-secret' | 'none';
/**
* Boolean states the platform reports. `pressed` is a toggle button's
* pressed state; `secure` marks a field whose value is never observed.
* pressed state; `secure` marks a field whose value is never observed;
* `scrollable` means the engine has evidence that a swipe on this node can
* move its own scroll surface rather than only the viewport.
*/
readonly states?: Readonly<
Partial<
Record<
'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure',
'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure' | 'scrollable',
boolean
>
>
Expand Down
50 changes: 45 additions & 5 deletions packages/decision/src/elements.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ export interface ActionSpace {
readonly elements: readonly Element[];
/** Per operation: target key -> bound target. Only operations with at least one target appear. */
readonly targets: ReadonlyMap<Operation, ReadonlyMap<string, Target>>;
readonly controls: ReadonlyMap<Control, Target>;
/** Per control: target key -> bound viewport or scroll-container action. */
readonly controls: ReadonlyMap<Control, ReadonlyMap<string, Target>>;
/** Elements left out to stay under the per-question cap; scrolling can bring them into view. */
readonly omitted: number;
/** Non-interactive page text from the tree, without node ids, clipped to 6000 chars. */
Expand Down Expand Up @@ -73,11 +74,13 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
inViewport: boolean;
}
const rows: Row[] = [];
const scrollables: ExecutorNode[] = [];
const pageText: string[] = [];
/** Walks the tree, collecting interactive rows and page text. */
const visit = (node: ExecutorNode, underNativeSelect: boolean): void => {
if (node.states?.disabled !== true && node.states?.hidden !== true) {
const operations: Operation[] = [];
if (node.states?.scrollable === true && intersects(node, observation.viewport)) scrollables.push(node);
const role = node.role ?? '';
const password = node.inputPurpose === 'password' || node.states?.secure === true;
const nativeSelect = role === 'combobox' && (node.children ?? []).some((child) => child.role === 'option');
Expand Down Expand Up @@ -164,13 +167,29 @@ export function actionSpace(ctx: StepExecutorContext, observation: SpaceObservat
operations: row.operations,
};
});
const controls = new Map<Control, Target>();
const controls = new Map<Control, Map<string, Target>>();
if (verbs.has('scroll')) {
controls.set('scroll_up', { description: 'scroll viewport up', run: () => ctx.actions.scroll('up') });
controls.set('scroll_down', { description: 'scroll viewport down', run: () => ctx.actions.scroll('down') });
const scrollGroup = (direction: 'up' | 'down'): Map<string, Target> => {
const group = new Map<string, Target>();
group.set('viewport', {
description: `scroll viewport ${direction}`,
run: () => ctx.actions.scroll(direction),
});
// One question must stay inside TypeSafe's choice cap. The viewport is
// always one choice; scrollable nodes are engine-proven candidates.
for (const node of scrollables.slice(0, MAX_CHOICES - 1)) {
group.set(`node:${node.id}`, {
description: `scroll ${scrollTargetLabel(node)} ${direction}`,
run: () => ctx.actions.scroll(direction, { id: node.id }),
});
}
return group;
};
controls.set('scroll_up', scrollGroup('up'));
controls.set('scroll_down', scrollGroup('down'));
}
if (verbs.has('back')) {
controls.set('back', { description: 'back one step in history', run: () => ctx.actions.back() });
controls.set('back', new Map([['history', { description: 'back one step in history', run: () => ctx.actions.back() }]]));
}
return {
elements,
Expand Down Expand Up @@ -206,6 +225,27 @@ function nodeLabel(node: ExecutorNode): string {
return node.name ?? node.attributes?.['placeholder'] ?? node.text ?? '';
}

/** Human-readable scroll target without platform selectors or node ids. */
function scrollTargetLabel(node: ExecutorNode): string {
const own = nodeLabel(node).trim();
if (own !== '') return `${node.role ?? 'container'} ${JSON.stringify(clip(own, 80))}`;
const descendants: string[] = [];
const visit = (current: ExecutorNode): void => {
if (descendants.length >= 2) return;
for (const child of current.children ?? []) {
const label = nodeLabel(child).trim();
if (label !== '') descendants.push(clip(label, 60));
if (descendants.length >= 2) return;
visit(child);
if (descendants.length >= 2) return;
}
};
visit(node);
return descendants.length === 0
? (node.role ?? 'scroll container')
: `${node.role ?? 'container'} containing ${descendants.map((label) => JSON.stringify(label)).join(', ')}`;
}

/**
* One page-text line for a non-interactive node. A named node keeps its text
* when the two differ, as the runner's `formatNode` does: a status named
Expand Down
4 changes: 1 addition & 3 deletions packages/decision/src/executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -178,9 +178,7 @@ async function run(
answers: Record<string, Decision>,
space: ActionSpace,
): { taken: Taken; targetAnswer?: Decision } {
const control = space.controls.get(choice as Control);
if (control !== undefined) return { taken: { target: control, operation: choice, elementKey: '' } };
const group = space.targets.get(choice as Operation);
const group = space.controls.get(choice as Control) ?? space.targets.get(choice as Operation);
if (group === undefined) throw invalid('The decision model chose an unavailable operation.');
if (group.size === 1) {
const only = group.entries().next();
Expand Down
7 changes: 7 additions & 0 deletions packages/decision/src/questions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,13 @@ export function decisionRequest(
}
questions[`${operation}_target`] = choice({ operation, rules: TARGET }, options);
}
for (const [control, targets] of space.controls) {
if (targets.size < 2) continue;
questions[`${control}_target`] = choice(
{ operation: control, rules: TARGET },
Object.fromEntries([...targets].map(([key, target]) => [key, target.description])),
);
}
if (space.targets.has('typeSecret') && ctx.step.secrets.length >= 2) {
const secrets: Record<string, JsonValue | null> = {};
for (const secret of ctx.step.secrets) secrets[secret.name] = secret.purpose;
Expand Down
21 changes: 21 additions & 0 deletions packages/decision/tests/elements.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,27 @@ describe('element table', () => {
const none = spaceFor([{ id: 'a', role: 'button', name: 'Add' }], ['tap']);
expect(none.controls.size).toBe(0);
});
it('offers engine-proven scroll containers beside the viewport', async () => {
const fixture = context({ verbs: ['scroll'] });
const space = actionSpace(fixture.ctx, {
path: '/feed',
viewport: { width: 800, height: 600 },
tree: tree([
{
id: 'feed',
role: 'group',
states: { scrollable: true },
rect: { x: 20, y: 20, width: 760, height: 480 },
children: [{ id: 'row', role: 'text', text: 'Item 1' }],
},
]),
}, true);
const down = space.controls.get('scroll_down');
expect([...(down?.keys() ?? [])]).toEqual(['viewport', 'node:feed']);
expect(down?.get('node:feed')?.description).toContain('containing "Item 1"');
await down?.get('node:feed')?.run();
expect(fixture.actions.scroll).toHaveBeenCalledWith('down', { id: 'feed' });
});
});
describe('fingerprint', () => {
const page = (): ExecutorNode => ({ id: 'root', children: [
Expand Down
25 changes: 25 additions & 0 deletions packages/decision/tests/executor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,31 @@ describe('terminal checks', () => {
{ element: 'Large', role: 'option' },
]);
});
it('asks which scroll surface to move and dispatches the chosen container', async () => {
const tree: ExecutorNode = { id: 'root', children: [
{
id: 'feed',
role: 'group',
states: { scrollable: true },
rect: { x: 20, y: 20, width: 760, height: 480 },
children: [{ id: 'row', role: 'text', text: 'Item 1' }],
},
] };
const { model, requests } = scriptedEvaluation((id, keys, call) => {
if (id === 'operation') return { choice: call === 0 ? 'scroll_down' : 'done' };
if (id === 'scroll_down_target') return { choice: 'node:feed' };
if (id === 'verdict') return { choice: 'holds' };
return { choice: keys[0] ?? '' };
});
const fixture = context({ tree });
const verdict = await decisionExecutor({ model }).runStep(fixture.ctx);
expect(verdict).toMatchObject({ status: 'passed' });
expect(requests[0]?.questions['scroll_down_target']?.criteria).toMatchObject({
viewport: 'scroll viewport down',
'node:feed': expect.stringContaining('Item 1'),
});
expect(fixture.actions.scroll).toHaveBeenCalledWith('down', { id: 'feed' });
});
});

describe('guard rails', () => {
Expand Down
6 changes: 4 additions & 2 deletions packages/e2e/src/engine/contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,12 +156,14 @@ export interface SemanticNode {
readonly inputPurpose?: 'username' | 'password' | 'one-time-code' | 'generic-secret' | 'none';
/**
* Boolean states the platform reports. `pressed` is a toggle button's
* pressed state; `secure` marks a field whose value is never observed.
* pressed state; `secure` marks a field whose value is never observed;
* `scrollable` means the engine has evidence that a swipe on this node can
* move its own scroll surface rather than only the viewport.
*/
readonly states?: Readonly<
Partial<
Record<
'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure',
'checked' | 'disabled' | 'selected' | 'expanded' | 'pressed' | 'focused' | 'hidden' | 'secure' | 'scrollable',
boolean
>
>
Expand Down
4 changes: 3 additions & 1 deletion packages/e2e/tests/types/sdk-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ import {
beforeEach,
describe,
} from '../../src/index.ts';
import type { Engine, EngineAttemptContext, EngineHandle, EngineObserveOptions, EngineSnapshot } from '../../src/engine/index.ts';
import type { Engine, EngineAttemptContext, EngineHandle, EngineObserveOptions, EngineSnapshot, SemanticNode } from '../../src/engine/index.ts';
import { createToolLoopExecutor, defineTool } from '../../src/agent/public.ts';
import type { Report } from '../../src/index.ts';
import type { LanguageModelV2, LanguageModelV3, LanguageModelV4 } from '@ai-sdk/provider';
Expand Down Expand Up @@ -89,6 +89,8 @@ customSnapshot satisfies EngineSnapshot;
customExecutorObservation satisfies ExecutorObservation;
declare const engineSnapshot: EngineSnapshot;
engineSnapshot.treeUnavailable satisfies true | undefined;
declare const semanticNode: SemanticNode;
semanticNode.states?.scrollable satisfies boolean | undefined;
// @ts-expect-error unavailable semantics are explicitly true or absent, never a separate false state.
({ ...engineSnapshot, treeUnavailable: false }) satisfies EngineSnapshot;

Expand Down
2 changes: 2 additions & 0 deletions packages/mobile/src/nodes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
*/

import type { SnapshotNode } from 'agent-device';
import { isScrollableNodeLike } from 'agent-device/contracts';
import type { SemanticNode, ViewportSize } from 'e2e/engine';

/**
Expand Down Expand Up @@ -425,6 +426,7 @@ export function projectSnapshot(raw: readonly RawNode[], options: { readonly min
...(source.focused === true ? { focused: true } : {}),
...(source.visibleToUser === false ? { hidden: true } : {}),
...(secure ? { secure: true } : {}),
...(isScrollableNodeLike(source) ? { scrollable: true } : {}),
...(checked === undefined ? {} : { checked }),
...(spelled?.states.expanded === undefined ? {} : { expanded: spelled.states.expanded }),
};
Expand Down
3 changes: 3 additions & 0 deletions packages/mobile/tests/unit/nodes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ describe('snapshot projection', () => {
'button',
'textbox',
]);
expect(projected.index[3]?.node.states).toMatchObject({ scrollable: true });
expect(projected.index[13]?.node.states).toMatchObject({ scrollable: true });
expect(projected.index[6]?.node.states).toEqual({ secure: true });
expect(projected.viewport).toEqual({ width: 390, height: 844 });
expect(screenTitle(projected)).toBe('Settings');
Expand Down Expand Up @@ -308,6 +310,7 @@ describe('snapshot projection', () => {
'tablist',
'list',
]);
expect(projected.index[11]?.node.states).toMatchObject({ scrollable: true });
});

it('names an Android node by its content description, keeps its text, and reads heading, role description, checked, and placeholder', () => {
Expand Down
17 changes: 15 additions & 2 deletions packages/web/src/in-page/read-semantics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1132,6 +1132,17 @@ export const readSemanticsFunction = <Mode extends SemanticMode>(
return role !== null && ARIA_DISABLED_ROLES.indexOf(role) !== -1 && ariaDisabledInChain(el);
};

/**
* Whether this node can receive a vertical wheel/swipe. The decision
* executor currently offers up/down, not left/right: a horizontal-only
* scroller is not a valid target for those actions.
*/
const isScrollable = (el: Element, style: CSSStyleDeclaration | undefined): boolean => {
if (style === undefined) return false;
const accepts = (value: string): boolean => value === 'auto' || value === 'scroll' || value === 'overlay';
return accepts(style.overflowY) && el.scrollHeight > el.clientHeight + 1;
};

const describe = (el: Element, style = styleOf(el)): RawNodeData => {
const tag = el.tagName.toLowerCase();
const autocomplete = (el.getAttribute('autocomplete') ?? '').toLowerCase();
Expand Down Expand Up @@ -1263,6 +1274,7 @@ export const readSemanticsFunction = <Mode extends SemanticMode>(
focused: focusedElement === el,
hidden: isHidden(el, style),
secure,
scrollable: isScrollable(el, style),
},
level,
attributes,
Expand Down Expand Up @@ -1309,7 +1321,8 @@ export const readSemanticsFunction = <Mode extends SemanticMode>(
};

/** True when a node carries semantics worth sending to a model. */
const isInteresting = (el: Element): boolean => {
const isInteresting = (el: Element, style: CSSStyleDeclaration | undefined): boolean => {
if (isScrollable(el, style)) return true;
if (el.hasAttribute(options.testIdAttribute)) return true;
if (implicitRole(el) !== null && !isPresentational(el)) return true;
if (accessibleName(el) !== null) return true;
Expand Down Expand Up @@ -1351,7 +1364,7 @@ export const readSemanticsFunction = <Mode extends SemanticMode>(
let nextParent = parent;
// An empty painted rectangle carries no semantics to be "interesting" by and
// is still something a person sees and aims at; `roleOf` names it `box`.
if (!hidden && (isInteresting(el) || isVisibleEmptyBox(el))) {
if (!hidden && (isInteresting(el, style) || isVisibleEmptyBox(el))) {
if (nodes.length >= maxNodes) {
truncated = true;
return;
Expand Down
1 change: 1 addition & 0 deletions packages/web/src/observation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ export function toSemanticNode(
if (raw.states.focused) states['focused'] = true;
if (raw.states.hidden) states['hidden'] = true;
if (raw.states.secure) states['secure'] = true;
if (raw.states.scrollable) states['scrollable'] = true;
return {
ref,
...(raw.role !== null ? { role: raw.role } : {}),
Expand Down
2 changes: 2 additions & 0 deletions packages/web/src/read-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ export interface RawNodeData {
focused: boolean;
hidden: boolean;
secure: boolean;
/** True when this element owns a scroll surface with content beyond its client box. */
scrollable?: boolean;
};
/** Heading level of a heading: `aria-level`, else the digit of `h1` through `h6`; null for anything else. */
level: number | null;
Expand Down
36 changes: 36 additions & 0 deletions packages/web/tests/integration/read-node-states.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,16 @@ async function disabledByTestId(): Promise<Map<string, boolean>> {
);
}

/** The `scrollable` state per test id, `false` when the node reports none. */
async function scrollableByTestId(): Promise<Map<string, boolean>> {
const { tree } = await capture();
return new Map(
flatten(tree)
.filter((node) => node.testId !== undefined)
.map((node) => [node.testId!, node.states?.scrollable === true]),
);
}

/** Playwright's answer for the same elements, the reference the reader must agree with. */
async function expectAgreesWithPlaywright(states: Map<string, boolean>): Promise<void> {
for (const [testId, disabled] of states) {
Expand Down Expand Up @@ -156,3 +166,29 @@ describe('disabled state', () => {
await expectAgreesWithPlaywright(states);
});
});


describe('scrollable state', () => {
it('comes from real overflow geometry, not the element role or test id', async () => {
await page.setContent(`
<div data-testid="scrolls" style="height: 80px; overflow: auto">
<div style="height: 240px">long content</div>
</div>
<div data-testid="hidden-overflow" style="height: 80px; overflow: hidden">
<div style="height: 240px">long but wheel-clipped</div>
</div>
<div data-testid="fits" style="height: 80px; overflow: auto">
<div style="height: 40px">fits</div>
</div>
<div data-testid="horizontal-only" style="width: 80px; height: 80px; overflow-x: auto; overflow-y: hidden">
<div style="width: 240px; height: 40px">sideways only</div>
</div>
`);
expect(Object.fromEntries(await scrollableByTestId())).toMatchObject({
scrolls: true,
'hidden-overflow': false,
fits: false,
'horizontal-only': false,
});
});
});
Loading