From 806e0f2c38b9c1bbb9b26381f8045218a18d10ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 11 Nov 2026 08:43:16 +0100 Subject: [PATCH 1/4] fix: scroll until through virtualized content --- .../contracts/src/snapshot-scroll.test.ts | 2 + packages/contracts/src/snapshot-scroll.ts | 1 + .../daemon/src/execution/scroll-until.test.ts | 40 ++++++++++++++++--- packages/daemon/src/execution/scroll-until.ts | 33 +++------------ src/commands/schema/cli-help-gestures.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 46 insertions(+), 34 deletions(-) diff --git a/packages/contracts/src/snapshot-scroll.test.ts b/packages/contracts/src/snapshot-scroll.test.ts index f4647f2242..88205ee891 100644 --- a/packages/contracts/src/snapshot-scroll.test.ts +++ b/packages/contracts/src/snapshot-scroll.test.ts @@ -12,6 +12,8 @@ test('snapshot scroll semantics classify scrollable types and node-like roles', const scrollView = node({ index: 1, type: 'ScrollView', parentIndex: 0 }); assert.equal(isScrollableType('android.widget.RecyclerView'), true); + assert.equal(isScrollableType('collection'), true); + assert.equal(isScrollableType('collectionitem'), false); assert.equal(isScrollableNodeLike(scrollView), true); assert.equal(isScrollableNodeLike({ role: 'scrollbar' }), true); }); diff --git a/packages/contracts/src/snapshot-scroll.ts b/packages/contracts/src/snapshot-scroll.ts index 116a06ddea..166f1f4fbd 100644 --- a/packages/contracts/src/snapshot-scroll.ts +++ b/packages/contracts/src/snapshot-scroll.ts @@ -6,6 +6,7 @@ export function isScrollableType(type: string | null | undefined): boolean { value.includes('listview') || value.includes('gridview') || value.includes('collectionview') || + value === 'collection' || value === 'table' ); } diff --git a/packages/daemon/src/execution/scroll-until.test.ts b/packages/daemon/src/execution/scroll-until.test.ts index 4a3ae41311..0e7fd44372 100644 --- a/packages/daemon/src/execution/scroll-until.test.ts +++ b/packages/daemon/src/execution/scroll-until.test.ts @@ -110,6 +110,24 @@ test('passes repeat until the selector is on screen, and the last gesture is rep assert.deepEqual(result.result, { pixels: 480 }); }); +test.each(['collection', 'ScrollView'])( + 'virtualized %s reaches rows that appear only after scrolling', + async (type) => { + const frame = (label: string) => { + const nodes = tree(200, label); + nodes[1]!.type = type; + return capture({ nodes }); + }; + let scrolls = 0; + const result = await run({ + captures: [frame('Row 1'), frame('Row 30'), frame('Email')], + onScroll: () => (scrolls += 1), + }); + assert.equal(result.passes, 2); + assert.equal(scrolls, 2); + }, +); + /** * A target below the fold is present but not visible. Stopping on presence would leave the caller * with a row it cannot act on, which is the whole reason the check asks about the viewport. @@ -181,14 +199,14 @@ test('a vertical scroll that moves nothing gives up early instead of spending th assert.ok(error instanceof AppError); assert.equal(error.details?.reason, 'scroll_until_no_progress'); assert.equal(error.details?.passes, 3); - assert.match(String(error.details?.hint), /not reaching this container/); + assert.match(String(error.details?.hint), /may not be reaching this container/); return true; }, ); assert.equal(scrolls, 3); }); -test('running out of content stops before the pass budget does', async () => { +test('unchanged visible content stops after gestures show no progress', async () => { let scrolls = 0; await assert.rejects( () => @@ -199,12 +217,24 @@ test('running out of content stops before the pass budget does', async () => { }), (error: unknown) => { assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'scroll_until_edge_reached'); - assert.match(String(error.details?.hint), /scroll the opposite direction/); + assert.equal(error.details?.reason, 'scroll_until_no_progress'); + assert.equal(error.details?.passes, 3); return true; }, ); - assert.equal(scrolls, 0); + assert.equal(scrolls, 3); +}); + +test('an unchanged collection without a scroll marker reports no progress', async () => { + const nodes = tree(200, 'Other'); + nodes[1]!.type = 'collection'; + await assert.rejects( + () => run({ captures: [capture({ nodes })] }), + (error: unknown) => + error instanceof AppError && + error.details?.reason === 'scroll_until_no_progress' && + error.details?.passes === 3, + ); }); test('a horizontal scroll has no edge signal and is bounded by the budget alone', async () => { diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index f12c9279b6..c6d068a8ad 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -16,7 +16,6 @@ import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pip import { readScrollEdgeState, runScrollPasses, - scrollNoProgressHint, verticalEdgeFor, type ScrollCaptureRefusal, } from '@agent-device/capture-kit/scroll-edge-state'; @@ -39,8 +38,9 @@ import { * first is what keeps a failed read from being reported as end-of-content — coercing it to an empty * tree makes the edge analyzer say "no room below". * - * The end-of-content signal is the one `scroll top`/`scroll bottom` already trust, so both stop in - * the same place. Horizontal scrolls have no such analyzer and are bounded by the pass budget alone. + * Vertical scrolling uses the surface fingerprint to detect a lack of progress. Hidden-content + * hints alone cannot establish an edge because virtualized content may not be in the tree. + * Horizontal scrolls are bounded by the pass budget alone. * The first capture happens before the first gesture, so an already-visible target costs no scroll. */ export async function runScrollUntilVisible(params: { @@ -64,19 +64,15 @@ export async function runScrollUntilVisible(params: { if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); return captured.nodes as SnapshotNode[]; }, - decide: async (nodes, passes) => { + decide: async (nodes) => { if (await isSelectorVisible(nodes, selector, platform)) return { stop: true }; // Horizontal scrolling has no edge analyzer and uses only the pass budget. if (!edge) return { stop: false }; const state = await readScrollEdgeState(nodes, edge); - if (!state.canScroll) { - throw scrollUntilNotFoundError(direction, selector, 'edge-reached', passes); - } return { stop: false, fingerprint: state.fingerprint }; }, passLimit: params.passLimit ?? DEFAULT_SCROLL_UNTIL_PASSES, - passLimitError: (passes) => - scrollUntilNotFoundError(direction, selector, 'pass-limit', passes), + passLimitError: (passes) => scrollUntilNotFoundError(direction, selector, passes), noProgressError: (passes) => scrollUntilNoProgressError(direction, selector, passes), scroll: async () => { scrolled = true; @@ -161,27 +157,11 @@ export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefus return undefined; } -/** The content ran out, or the budget did. Separate messages: the corrective action differs. */ function scrollUntilNotFoundError( direction: ScrollDirection, selector: string, - outcome: 'edge-reached' | 'pass-limit', passes: number, ): AppError { - const spent = `${passes} ${passes === 1 ? 'pass' : 'passes'}`; - if (outcome === 'edge-reached') { - return new AppError( - 'COMMAND_FAILED', - `scroll ${direction} reached the end of the scrollable content after ${spent} without ${selector} becoming visible`, - { - reason: 'scroll_until_edge_reached', - selector, - direction, - passes, - hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, - }, - ); - } return new AppError( 'COMMAND_FAILED', `scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`, @@ -195,7 +175,6 @@ function scrollUntilNotFoundError( ); } -/** The scroll was not reaching the container, not just falling short of a budget — the fix differs. */ function scrollUntilNoProgressError( direction: ScrollDirection, selector: string, @@ -210,7 +189,7 @@ function scrollUntilNoProgressError( selector, direction, passes, - hint: scrollNoProgressHint(), + hint: 'The content may have ended, or the scroll may not be reaching this container. Check the selector and run snapshot -i to inspect the screen. If a field is focused, dismiss the keyboard; if the container is nested, target it directly. Some lists respond to a raw drag: swipe x1 y1 x2 y2 started inside the list.', }, ); } diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index 8beb3dba63..0f63d795b7 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -21,7 +21,7 @@ Shapes: agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 agent-device gesture tap 80 300 200 450 310 600 --count 20 --pause-ms 100 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent; vertical scrolls fail if the visible content stops changing. It is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 3a9e1ece20..98338da266 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -540,7 +540,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' ### Scroll - `scroll [amount]` takes the finger path as a fraction of the viewport (`0.5` is half), or `--pixels ` for a fixed distance. Large distances are clamped. The app's scroll physics decide the final offset, so a scroll does not land on an exact position. -- `scroll --until ` repeats passes until the selector is on screen and fails if the content runs out first. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). +- `scroll --until ` repeats passes until the selector is on screen. A vertical scroll fails if the visible content stops changing. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). - Default snapshot text summarizes off-screen interactive content instead of listing refs. `scroll --settle` (and `back --settle`) waits for the UI to go quiet and returns the diff with fresh refs, so you don't need another `snapshot -i`. Settle is best-effort and never fails the action. - On Android, settle ignores an allowlist of system-bar containers and shade/keyguard status headers; other SystemUI overlays remain content. Appium settle also applies package rules for keyboard and system chrome. - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. From e1488e65eb74bbe632c6d81c0967460846483a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 11 Nov 2026 09:25:09 +0100 Subject: [PATCH 2/4] test: align scroll until provider failures --- test/integration/provider-scenarios/scroll-until.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts index d7d485c517..6939df357b 100644 --- a/test/integration/provider-scenarios/scroll-until.test.ts +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -93,6 +93,7 @@ test.each([ assert.ok(error instanceof AppError); assert.equal(error.details?.reason, 'scroll_until_pass_limit'); assert.equal(error.details?.passes, maxPasses); + assert.match(String(error.message), /without text=Terms becoming visible/); return true; }); } else { @@ -105,7 +106,7 @@ test.each([ }, ); -test('Provider-backed integration scroll --until reports the end of the content as a typed failure', async () => { +test('Provider-backed integration scroll --until reports unchanged content as no progress', async () => { await withProviderScenarioResource( async () => await createAndroidSettingsWorld({ @@ -125,7 +126,9 @@ test('Provider-backed integration scroll --until reports the end of the content }), (error: unknown) => { assert.ok(error instanceof AppError); - assert.match(String(error.message), /without text=NeverPresent becoming visible/); + assert.equal(error.details?.reason, 'scroll_until_no_progress'); + assert.equal(error.details?.passes, 3); + assert.match(String(error.message), /moved nothing across 3 passes/); return true; }, ); From 17d749d935976599ebdb19a4528c3d2f81dee9b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 11 Nov 2026 09:57:31 +0100 Subject: [PATCH 3/4] fix: report scroll edge and detect stalled screens --- .../src/snapshot/scroll-edge-state.ts | 1 + .../snapshot/scroll-edge-state/selection.ts | 18 +++++++ .../daemon/src/execution/scroll-until.test.ts | 47 +++++++++++++++++++ packages/daemon/src/execution/scroll-until.ts | 32 ++++++++++++- src/commands/schema/cli-help-gestures.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 98 insertions(+), 4 deletions(-) diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index e5f9f67f17..bf3c339705 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -18,6 +18,7 @@ export function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefi export type ScrollEdgeState = { canScroll: boolean; emptySnapshot: boolean; + edgeObserved?: boolean; scope?: string; /** * The frame of the container every other field on this record describes, absent when the tree diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts index 5fec0a0727..4415293cdf 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts @@ -32,6 +32,7 @@ export function analyzeScrollEdgeState( return { canScroll: false, emptySnapshot: false, + fingerprint: buildVisibleTreeFingerprint(nodes), }; } @@ -39,6 +40,11 @@ export function analyzeScrollEdgeState( return { canScroll, emptySnapshot: false, + edgeObserved: + !canScroll && + (edge === 'bottom' + ? container.hiddenContentBelow === false + : container.hiddenContentAbove === false), // Every selection path admits only `isUsableRect` containers, so a resolved container always // has a frame to report and its absence always means no container was resolved. ...(container.rect ? { containerRect: container.rect } : {}), @@ -302,6 +308,18 @@ function buildSurfaceFingerprint(container: SnapshotNode, nodes: SnapshotNode[]) return descendants.join(';'); } +function buildVisibleTreeFingerprint(nodes: SnapshotNode[]): string { + const visibility = createSnapshotVisibility(nodes); + return nodes + .filter((node) => isUsableRect(node.rect) && visibility.isVisibleInEffectiveViewport(node)) + .map( + (node) => + `${node.type}#${surfaceIdentity(node)}@${Math.round(node.rect!.x)},${Math.round(node.rect!.y)}`, + ) + .sort() + .join(';'); +} + function surfaceIdentity(node: SnapshotNode): string { // All three content fields, not first-present: a recycled cell keeps its identifier and slot while // its label/value change, and that text change is exactly the progress the signature must register. diff --git a/packages/daemon/src/execution/scroll-until.test.ts b/packages/daemon/src/execution/scroll-until.test.ts index 0e7fd44372..0f5c8b88cc 100644 --- a/packages/daemon/src/execution/scroll-until.test.ts +++ b/packages/daemon/src/execution/scroll-until.test.ts @@ -237,6 +237,53 @@ test('an unchanged collection without a scroll marker reports no progress', asyn ); }); +test('an explicit edge stops before a gesture', async () => { + const nodes = tree(200, 'Other'); + nodes[1]!.hiddenContentBelow = false; + let scrolls = 0; + await assert.rejects( + () => run({ captures: [capture({ nodes })], onScroll: () => (scrolls += 1) }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_edge_reached', + ); + assert.equal(scrolls, 0); +}); + +test('visible hidden content overrides a contradictory explicit edge', async () => { + const nodes = tree(2400); + nodes[1]!.hiddenContentBelow = false; + await assert.rejects( + () => run({ captures: [capture({ nodes })], passLimit: 1 }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_pass_limit', + ); +}); + +test('content that moved and then stops reports the end of content', async () => { + const frames = [tree(400, 'Other'), tree(200, 'Other')].map((nodes) => capture({ nodes })); + await assert.rejects( + () => run({ captures: frames }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_edge_reached'); + assert.match(String(error.details?.hint), /another screen/); + return true; + }, + ); +}); + +test('a non-scrollable screen stops after repeated unchanged observations', async () => { + const nodes = tree(200, 'Other').filter((node) => node.index !== 1); + nodes[1]!.parentIndex = 0; + let scrolls = 0; + await assert.rejects( + () => run({ captures: [capture({ nodes })], onScroll: () => (scrolls += 1) }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_no_progress', + ); + assert.equal(scrolls, 3); +}); + test('a horizontal scroll has no edge signal and is bounded by the budget alone', async () => { let scrolls = 0; await assert.rejects( diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index c6d068a8ad..45327f1450 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -55,6 +55,8 @@ export async function runScrollUntilVisible(params: { const { selector, direction, platform, capture, scroll } = params; const edge = verticalEdgeFor(direction); let scrolled = false; + let previousFingerprint: string | undefined; + let observedMovement = false; try { const { passes, result } = await runScrollPasses({ signal: params.signal, @@ -64,16 +66,24 @@ export async function runScrollUntilVisible(params: { if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); return captured.nodes as SnapshotNode[]; }, - decide: async (nodes) => { + decide: async (nodes, passes) => { if (await isSelectorVisible(nodes, selector, platform)) return { stop: true }; // Horizontal scrolling has no edge analyzer and uses only the pass budget. if (!edge) return { stop: false }; const state = await readScrollEdgeState(nodes, edge); + if (state.edgeObserved) throw scrollUntilEdgeError(direction, selector, passes); + if (state.fingerprint && previousFingerprint && state.fingerprint !== previousFingerprint) { + observedMovement = true; + } + previousFingerprint = state.fingerprint; return { stop: false, fingerprint: state.fingerprint }; }, passLimit: params.passLimit ?? DEFAULT_SCROLL_UNTIL_PASSES, passLimitError: (passes) => scrollUntilNotFoundError(direction, selector, passes), - noProgressError: (passes) => scrollUntilNoProgressError(direction, selector, passes), + noProgressError: (passes) => + observedMovement + ? scrollUntilEdgeError(direction, selector, passes) + : scrollUntilNoProgressError(direction, selector, passes), scroll: async () => { scrolled = true; return await scroll(); @@ -175,6 +185,24 @@ function scrollUntilNotFoundError( ); } +function scrollUntilEdgeError( + direction: ScrollDirection, + selector: string, + passes: number, +): AppError { + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} reached the end of the scrollable content after ${passes} passes without ${selector} becoming visible`, + { + reason: 'scroll_until_edge_reached', + selector, + direction, + passes, + hint: 'The content may end here. Run snapshot -i to inspect the screen, scroll the opposite direction, or check the selector; the element may be on another screen.', + }, + ); +} + function scrollUntilNoProgressError( direction: ScrollDirection, selector: string, diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index 0f63d795b7..f8c7fe4537 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -21,7 +21,7 @@ Shapes: agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 agent-device gesture tap 80 300 200 450 310 600 --count 20 --pause-ms 100 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent; vertical scrolls fail if the visible content stops changing. It is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent; vertical searches compare repeated captures of descendants in the selected scroll container and stop when those captures show no progress. If no container is found, they compare the visible tree. It is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 98338da266..ba6ba517c5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -540,7 +540,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' ### Scroll - `scroll [amount]` takes the finger path as a fraction of the viewport (`0.5` is half), or `--pixels ` for a fixed distance. Large distances are clamped. The app's scroll physics decide the final offset, so a scroll does not land on an exact position. -- `scroll --until ` repeats passes until the selector is on screen. A vertical scroll fails if the visible content stops changing. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). +- `scroll --until ` repeats passes until the selector is on screen. A vertical search stops if repeated captures show no progress in the selected scroll container. If no container is found, it compares the visible tree. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). - Default snapshot text summarizes off-screen interactive content instead of listing refs. `scroll --settle` (and `back --settle`) waits for the UI to go quiet and returns the diff with fresh refs, so you don't need another `snapshot -i`. Settle is best-effort and never fails the action. - On Android, settle ignores an allowlist of system-bar containers and shade/keyguard status headers; other SystemUI overlays remain content. Appium settle also applies package rules for keyboard and system chrome. - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first. From ea5127389ad2b15928553cb8ee5a420bff5d05cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 11 Nov 2026 10:08:23 +0100 Subject: [PATCH 4/4] fix: scope scroll progress evidence --- .../src/snapshot/scroll-edge-state.ts | 1 - .../snapshot/scroll-edge-state/selection.ts | 5 --- .../daemon/src/execution/scroll-until.test.ts | 45 ++++++++++--------- packages/daemon/src/execution/scroll-until.ts | 19 +++++--- src/commands/schema/cli-help-gestures.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state.ts b/packages/capture-kit/src/snapshot/scroll-edge-state.ts index bf3c339705..e5f9f67f17 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state.ts @@ -18,7 +18,6 @@ export function verticalEdgeFor(direction: ScrollDirection): ScrollEdge | undefi export type ScrollEdgeState = { canScroll: boolean; emptySnapshot: boolean; - edgeObserved?: boolean; scope?: string; /** * The frame of the container every other field on this record describes, absent when the tree diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts index 4415293cdf..79d9c77b0b 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts @@ -40,11 +40,6 @@ export function analyzeScrollEdgeState( return { canScroll, emptySnapshot: false, - edgeObserved: - !canScroll && - (edge === 'bottom' - ? container.hiddenContentBelow === false - : container.hiddenContentAbove === false), // Every selection path admits only `isUsableRect` containers, so a resolved container always // has a frame to report and its absence always means no container was resolved. ...(container.rect ? { containerRect: container.rect } : {}), diff --git a/packages/daemon/src/execution/scroll-until.test.ts b/packages/daemon/src/execution/scroll-until.test.ts index 0f5c8b88cc..d967663716 100644 --- a/packages/daemon/src/execution/scroll-until.test.ts +++ b/packages/daemon/src/execution/scroll-until.test.ts @@ -237,28 +237,6 @@ test('an unchanged collection without a scroll marker reports no progress', asyn ); }); -test('an explicit edge stops before a gesture', async () => { - const nodes = tree(200, 'Other'); - nodes[1]!.hiddenContentBelow = false; - let scrolls = 0; - await assert.rejects( - () => run({ captures: [capture({ nodes })], onScroll: () => (scrolls += 1) }), - (error: unknown) => - error instanceof AppError && error.details?.reason === 'scroll_until_edge_reached', - ); - assert.equal(scrolls, 0); -}); - -test('visible hidden content overrides a contradictory explicit edge', async () => { - const nodes = tree(2400); - nodes[1]!.hiddenContentBelow = false; - await assert.rejects( - () => run({ captures: [capture({ nodes })], passLimit: 1 }), - (error: unknown) => - error instanceof AppError && error.details?.reason === 'scroll_until_pass_limit', - ); -}); - test('content that moved and then stops reports the end of content', async () => { const frames = [tree(400, 'Other'), tree(200, 'Other')].map((nodes) => capture({ nodes })); await assert.rejects( @@ -284,6 +262,29 @@ test('a non-scrollable screen stops after repeated unchanged observations', asyn assert.equal(scrolls, 3); }); +test('changes outside a stuck scroll container do not imply scrolling', async () => { + const frame = (label: string) => + capture({ nodes: [...tree(200, 'Other'), { ...row(3, 100, label), parentIndex: 0 }] }); + await assert.rejects( + () => run({ captures: [frame('Clock 1'), frame('Clock 2')] }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_no_progress', + ); +}); + +test('changes in a tree without a scroll container do not imply scrolling', async () => { + const frame = (label: string) => { + const nodes = tree(200, label).filter((node) => node.index !== 1); + nodes[1]!.parentIndex = 0; + return capture({ nodes }); + }; + await assert.rejects( + () => run({ captures: [frame('Clock 1'), frame('Clock 2')] }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_no_progress', + ); +}); + test('a horizontal scroll has no edge signal and is bounded by the budget alone', async () => { let scrolls = 0; await assert.rejects( diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index 45327f1450..a18f3068a4 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -56,6 +56,7 @@ export async function runScrollUntilVisible(params: { const edge = verticalEdgeFor(direction); let scrolled = false; let previousFingerprint: string | undefined; + let previousContainer: string | undefined; let observedMovement = false; try { const { passes, result } = await runScrollPasses({ @@ -66,16 +67,24 @@ export async function runScrollUntilVisible(params: { if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); return captured.nodes as SnapshotNode[]; }, - decide: async (nodes, passes) => { + decide: async (nodes) => { if (await isSelectorVisible(nodes, selector, platform)) return { stop: true }; // Horizontal scrolling has no edge analyzer and uses only the pass budget. if (!edge) return { stop: false }; const state = await readScrollEdgeState(nodes, edge); - if (state.edgeObserved) throw scrollUntilEdgeError(direction, selector, passes); - if (state.fingerprint && previousFingerprint && state.fingerprint !== previousFingerprint) { + const container = state.containerRect && JSON.stringify([state.containerRect, state.scope]); + if (container !== previousContainer) observedMovement = false; + if ( + container && + container === previousContainer && + state.fingerprint && + previousFingerprint && + state.fingerprint !== previousFingerprint + ) { observedMovement = true; } - previousFingerprint = state.fingerprint; + previousContainer = container; + previousFingerprint = container ? state.fingerprint : undefined; return { stop: false, fingerprint: state.fingerprint }; }, passLimit: params.passLimit ?? DEFAULT_SCROLL_UNTIL_PASSES, @@ -192,7 +201,7 @@ function scrollUntilEdgeError( ): AppError { return new AppError( 'COMMAND_FAILED', - `scroll ${direction} reached the end of the scrollable content after ${passes} passes without ${selector} becoming visible`, + `scroll ${direction} reached the end of the scrollable content after ${passes} ${passes === 1 ? 'pass' : 'passes'} without ${selector} becoming visible`, { reason: 'scroll_until_edge_reached', selector, diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index f8c7fe4537..e8debbd501 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -21,7 +21,7 @@ Shapes: agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 agent-device gesture tap 80 300 200 450 310 600 --count 20 --pause-ms 100 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent; vertical searches compare repeated captures of descendants in the selected scroll container and stop when those captures show no progress. If no container is found, they compare the visible tree. It is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent; vertical searches compare repeated captures of descendants in the selected scroll container and fail when those captures show no progress. If no container is found, they compare the visible tree. It is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ba6ba517c5..d9564b0403 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -540,7 +540,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' ### Scroll - `scroll [amount]` takes the finger path as a fraction of the viewport (`0.5` is half), or `--pixels ` for a fixed distance. Large distances are clamped. The app's scroll physics decide the final offset, so a scroll does not land on an exact position. -- `scroll --until ` repeats passes until the selector is on screen. A vertical search stops if repeated captures show no progress in the selected scroll container. If no container is found, it compares the visible tree. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). +- `scroll --until ` repeats passes until the selector is on screen. A vertical search fails with `COMMAND_FAILED` and `reason: scroll_until_no_progress` if repeated captures show no progress in the selected scroll container. If content moved before stopping, the reason is `scroll_until_edge_reached`. If no container is found, it compares the visible tree. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). - Default snapshot text summarizes off-screen interactive content instead of listing refs. `scroll --settle` (and `back --settle`) waits for the UI to go quiet and returns the diff with fresh refs, so you don't need another `snapshot -i`. Settle is best-effort and never fails the action. - On Android, settle ignores an allowlist of system-bar containers and shade/keyguard status headers; other SystemUI overlays remain content. Appium settle also applies package rules for keyboard and system chrome. - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first.