diff --git a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts index 5fec0a0727..79d9c77b0b 100644 --- a/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts +++ b/packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts @@ -32,6 +32,7 @@ export function analyzeScrollEdgeState( return { canScroll: false, emptySnapshot: false, + fingerprint: buildVisibleTreeFingerprint(nodes), }; } @@ -302,6 +303,18 @@ function buildSurfaceFingerprint(container: SnapshotNode, nodes: SnapshotNode[]) return descendants.join(';'); } +function buildVisibleTreeFingerprint(nodes: SnapshotNode[]): string { + const visibility = createSnapshotVisibility(nodes); + return nodes + .filter((node) => isUsableRect(node.rect) && visibility.isVisibleInEffectiveViewport(node)) + .map( + (node) => + `${node.type}#${surfaceIdentity(node)}@${Math.round(node.rect!.x)},${Math.round(node.rect!.y)}`, + ) + .sort() + .join(';'); +} + function surfaceIdentity(node: SnapshotNode): string { // All three content fields, not first-present: a recycled cell keeps its identifier and slot while // its label/value change, and that text change is exactly the progress the signature must register. diff --git a/packages/contracts/src/snapshot-scroll.test.ts b/packages/contracts/src/snapshot-scroll.test.ts index f4647f2242..88205ee891 100644 --- a/packages/contracts/src/snapshot-scroll.test.ts +++ b/packages/contracts/src/snapshot-scroll.test.ts @@ -12,6 +12,8 @@ test('snapshot scroll semantics classify scrollable types and node-like roles', const scrollView = node({ index: 1, type: 'ScrollView', parentIndex: 0 }); assert.equal(isScrollableType('android.widget.RecyclerView'), true); + assert.equal(isScrollableType('collection'), true); + assert.equal(isScrollableType('collectionitem'), false); assert.equal(isScrollableNodeLike(scrollView), true); assert.equal(isScrollableNodeLike({ role: 'scrollbar' }), true); }); diff --git a/packages/contracts/src/snapshot-scroll.ts b/packages/contracts/src/snapshot-scroll.ts index 116a06ddea..166f1f4fbd 100644 --- a/packages/contracts/src/snapshot-scroll.ts +++ b/packages/contracts/src/snapshot-scroll.ts @@ -6,6 +6,7 @@ export function isScrollableType(type: string | null | undefined): boolean { value.includes('listview') || value.includes('gridview') || value.includes('collectionview') || + value === 'collection' || value === 'table' ); } diff --git a/packages/daemon/src/execution/scroll-until.test.ts b/packages/daemon/src/execution/scroll-until.test.ts index 4a3ae41311..d967663716 100644 --- a/packages/daemon/src/execution/scroll-until.test.ts +++ b/packages/daemon/src/execution/scroll-until.test.ts @@ -110,6 +110,24 @@ test('passes repeat until the selector is on screen, and the last gesture is rep assert.deepEqual(result.result, { pixels: 480 }); }); +test.each(['collection', 'ScrollView'])( + 'virtualized %s reaches rows that appear only after scrolling', + async (type) => { + const frame = (label: string) => { + const nodes = tree(200, label); + nodes[1]!.type = type; + return capture({ nodes }); + }; + let scrolls = 0; + const result = await run({ + captures: [frame('Row 1'), frame('Row 30'), frame('Email')], + onScroll: () => (scrolls += 1), + }); + assert.equal(result.passes, 2); + assert.equal(scrolls, 2); + }, +); + /** * A target below the fold is present but not visible. Stopping on presence would leave the caller * with a row it cannot act on, which is the whole reason the check asks about the viewport. @@ -181,14 +199,14 @@ test('a vertical scroll that moves nothing gives up early instead of spending th assert.ok(error instanceof AppError); assert.equal(error.details?.reason, 'scroll_until_no_progress'); assert.equal(error.details?.passes, 3); - assert.match(String(error.details?.hint), /not reaching this container/); + assert.match(String(error.details?.hint), /may not be reaching this container/); return true; }, ); assert.equal(scrolls, 3); }); -test('running out of content stops before the pass budget does', async () => { +test('unchanged visible content stops after gestures show no progress', async () => { let scrolls = 0; await assert.rejects( () => @@ -197,14 +215,74 @@ test('running out of content stops before the pass budget does', async () => { captures: [capture({ nodes: tree(200, 'Other') })], onScroll: () => (scrolls += 1), }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'scroll_until_no_progress'); + assert.equal(error.details?.passes, 3); + return true; + }, + ); + assert.equal(scrolls, 3); +}); + +test('an unchanged collection without a scroll marker reports no progress', async () => { + const nodes = tree(200, 'Other'); + nodes[1]!.type = 'collection'; + await assert.rejects( + () => run({ captures: [capture({ nodes })] }), + (error: unknown) => + error instanceof AppError && + error.details?.reason === 'scroll_until_no_progress' && + error.details?.passes === 3, + ); +}); + +test('content that moved and then stops reports the end of content', async () => { + const frames = [tree(400, 'Other'), tree(200, 'Other')].map((nodes) => capture({ nodes })); + await assert.rejects( + () => run({ captures: frames }), (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.details?.reason, 'scroll_until_edge_reached'); - assert.match(String(error.details?.hint), /scroll the opposite direction/); + assert.match(String(error.details?.hint), /another screen/); return true; }, ); - assert.equal(scrolls, 0); +}); + +test('a non-scrollable screen stops after repeated unchanged observations', async () => { + const nodes = tree(200, 'Other').filter((node) => node.index !== 1); + nodes[1]!.parentIndex = 0; + let scrolls = 0; + await assert.rejects( + () => run({ captures: [capture({ nodes })], onScroll: () => (scrolls += 1) }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_no_progress', + ); + assert.equal(scrolls, 3); +}); + +test('changes outside a stuck scroll container do not imply scrolling', async () => { + const frame = (label: string) => + capture({ nodes: [...tree(200, 'Other'), { ...row(3, 100, label), parentIndex: 0 }] }); + await assert.rejects( + () => run({ captures: [frame('Clock 1'), frame('Clock 2')] }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_no_progress', + ); +}); + +test('changes in a tree without a scroll container do not imply scrolling', async () => { + const frame = (label: string) => { + const nodes = tree(200, label).filter((node) => node.index !== 1); + nodes[1]!.parentIndex = 0; + return capture({ nodes }); + }; + await assert.rejects( + () => run({ captures: [frame('Clock 1'), frame('Clock 2')] }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'scroll_until_no_progress', + ); }); test('a horizontal scroll has no edge signal and is bounded by the budget alone', async () => { diff --git a/packages/daemon/src/execution/scroll-until.ts b/packages/daemon/src/execution/scroll-until.ts index f12c9279b6..a18f3068a4 100644 --- a/packages/daemon/src/execution/scroll-until.ts +++ b/packages/daemon/src/execution/scroll-until.ts @@ -16,7 +16,6 @@ import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pip import { readScrollEdgeState, runScrollPasses, - scrollNoProgressHint, verticalEdgeFor, type ScrollCaptureRefusal, } from '@agent-device/capture-kit/scroll-edge-state'; @@ -39,8 +38,9 @@ import { * first is what keeps a failed read from being reported as end-of-content — coercing it to an empty * tree makes the edge analyzer say "no room below". * - * The end-of-content signal is the one `scroll top`/`scroll bottom` already trust, so both stop in - * the same place. Horizontal scrolls have no such analyzer and are bounded by the pass budget alone. + * Vertical scrolling uses the surface fingerprint to detect a lack of progress. Hidden-content + * hints alone cannot establish an edge because virtualized content may not be in the tree. + * Horizontal scrolls are bounded by the pass budget alone. * The first capture happens before the first gesture, so an already-visible target costs no scroll. */ export async function runScrollUntilVisible(params: { @@ -55,6 +55,9 @@ export async function runScrollUntilVisible(params: { const { selector, direction, platform, capture, scroll } = params; const edge = verticalEdgeFor(direction); let scrolled = false; + let previousFingerprint: string | undefined; + let previousContainer: string | undefined; + let observedMovement = false; try { const { passes, result } = await runScrollPasses({ signal: params.signal, @@ -64,20 +67,32 @@ export async function runScrollUntilVisible(params: { if (refusal) throw scrollUntilCaptureError(direction, selector, refusal); return captured.nodes as SnapshotNode[]; }, - decide: async (nodes, passes) => { + decide: async (nodes) => { if (await isSelectorVisible(nodes, selector, platform)) return { stop: true }; // Horizontal scrolling has no edge analyzer and uses only the pass budget. if (!edge) return { stop: false }; const state = await readScrollEdgeState(nodes, edge); - if (!state.canScroll) { - throw scrollUntilNotFoundError(direction, selector, 'edge-reached', passes); + const container = state.containerRect && JSON.stringify([state.containerRect, state.scope]); + if (container !== previousContainer) observedMovement = false; + if ( + container && + container === previousContainer && + state.fingerprint && + previousFingerprint && + state.fingerprint !== previousFingerprint + ) { + observedMovement = true; } + previousContainer = container; + previousFingerprint = container ? state.fingerprint : undefined; return { stop: false, fingerprint: state.fingerprint }; }, passLimit: params.passLimit ?? DEFAULT_SCROLL_UNTIL_PASSES, - passLimitError: (passes) => - scrollUntilNotFoundError(direction, selector, 'pass-limit', passes), - noProgressError: (passes) => scrollUntilNoProgressError(direction, selector, passes), + passLimitError: (passes) => scrollUntilNotFoundError(direction, selector, passes), + noProgressError: (passes) => + observedMovement + ? scrollUntilEdgeError(direction, selector, passes) + : scrollUntilNoProgressError(direction, selector, passes), scroll: async () => { scrolled = true; return await scroll(); @@ -161,27 +176,11 @@ export function scrollCaptureRefusal(result: SnapshotResult): ScrollCaptureRefus return undefined; } -/** The content ran out, or the budget did. Separate messages: the corrective action differs. */ function scrollUntilNotFoundError( direction: ScrollDirection, selector: string, - outcome: 'edge-reached' | 'pass-limit', passes: number, ): AppError { - const spent = `${passes} ${passes === 1 ? 'pass' : 'passes'}`; - if (outcome === 'edge-reached') { - return new AppError( - 'COMMAND_FAILED', - `scroll ${direction} reached the end of the scrollable content after ${spent} without ${selector} becoming visible`, - { - reason: 'scroll_until_edge_reached', - selector, - direction, - passes, - hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`, - }, - ); - } return new AppError( 'COMMAND_FAILED', `scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`, @@ -195,7 +194,24 @@ function scrollUntilNotFoundError( ); } -/** The scroll was not reaching the container, not just falling short of a budget — the fix differs. */ +function scrollUntilEdgeError( + direction: ScrollDirection, + selector: string, + passes: number, +): AppError { + return new AppError( + 'COMMAND_FAILED', + `scroll ${direction} reached the end of the scrollable content after ${passes} ${passes === 1 ? 'pass' : 'passes'} without ${selector} becoming visible`, + { + reason: 'scroll_until_edge_reached', + selector, + direction, + passes, + hint: 'The content may end here. Run snapshot -i to inspect the screen, scroll the opposite direction, or check the selector; the element may be on another screen.', + }, + ); +} + function scrollUntilNoProgressError( direction: ScrollDirection, selector: string, @@ -210,7 +226,7 @@ function scrollUntilNoProgressError( selector, direction, passes, - hint: scrollNoProgressHint(), + hint: 'The content may have ended, or the scroll may not be reaching this container. Check the selector and run snapshot -i to inspect the screen. If a field is focused, dismiss the keyboard; if the container is nested, target it directly. Some lists respond to a raw drag: swipe x1 y1 x2 y2 started inside the list.', }, ); } diff --git a/src/commands/schema/cli-help-gestures.ts b/src/commands/schema/cli-help-gestures.ts index 8beb3dba63..e8debbd501 100644 --- a/src/commands/schema/cli-help-gestures.ts +++ b/src/commands/schema/cli-help-gestures.ts @@ -21,7 +21,7 @@ Shapes: agent-device gesture rotate 35 200 420 agent-device gesture transform 200 420 80 -40 2 35 700 agent-device gesture tap 80 300 200 450 310 600 --count 20 --pause-ms 100 - scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. + scroll --until repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent; vertical searches compare repeated captures of descendants in the selected scroll container and fail when those captures show no progress. If no container is found, they compare the visible tree. It is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens. longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms. For repeated iOS smoke checks: press --count --jitter-px for tap series, swipe --count for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger. diff --git a/test/integration/provider-scenarios/scroll-until.test.ts b/test/integration/provider-scenarios/scroll-until.test.ts index d7d485c517..6939df357b 100644 --- a/test/integration/provider-scenarios/scroll-until.test.ts +++ b/test/integration/provider-scenarios/scroll-until.test.ts @@ -93,6 +93,7 @@ test.each([ assert.ok(error instanceof AppError); assert.equal(error.details?.reason, 'scroll_until_pass_limit'); assert.equal(error.details?.passes, maxPasses); + assert.match(String(error.message), /without text=Terms becoming visible/); return true; }); } else { @@ -105,7 +106,7 @@ test.each([ }, ); -test('Provider-backed integration scroll --until reports the end of the content as a typed failure', async () => { +test('Provider-backed integration scroll --until reports unchanged content as no progress', async () => { await withProviderScenarioResource( async () => await createAndroidSettingsWorld({ @@ -125,7 +126,9 @@ test('Provider-backed integration scroll --until reports the end of the content }), (error: unknown) => { assert.ok(error instanceof AppError); - assert.match(String(error.message), /without text=NeverPresent becoming visible/); + assert.equal(error.details?.reason, 'scroll_until_no_progress'); + assert.equal(error.details?.passes, 3); + assert.match(String(error.message), /moved nothing across 3 passes/); return true; }, ); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 3a9e1ece20..d9564b0403 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -540,7 +540,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"' ### Scroll - `scroll [amount]` takes the finger path as a fraction of the viewport (`0.5` is half), or `--pixels ` for a fixed distance. Large distances are clamped. The app's scroll physics decide the final offset, so a scroll does not land on an exact position. -- `scroll --until ` repeats passes until the selector is on screen and fails if the content runs out first. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). +- `scroll --until ` repeats passes until the selector is on screen. A vertical search fails with `COMMAND_FAILED` and `reason: scroll_until_no_progress` if repeated captures show no progress in the selected scroll container. If content moved before stopping, the reason is `scroll_until_edge_reached`. If no container is found, it compares the visible tree. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until `) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e). - Default snapshot text summarizes off-screen interactive content instead of listing refs. `scroll --settle` (and `back --settle`) waits for the UI to go quiet and returns the diff with fresh refs, so you don't need another `snapshot -i`. Settle is best-effort and never fails the action. - On Android, settle ignores an allowlist of system-bar containers and shade/keyguard status headers; other SystemUI overlays remain content. Appium settle also applies package rules for keyboard and system chrome. - With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first.