Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions packages/capture-kit/src/snapshot/scroll-edge-state/selection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export function analyzeScrollEdgeState(
return {
canScroll: false,
emptySnapshot: false,
fingerprint: buildVisibleTreeFingerprint(nodes),
};
}

Expand Down Expand Up @@ -302,6 +303,18 @@ function buildSurfaceFingerprint(container: SnapshotNode, nodes: SnapshotNode[])
return descendants.join(';');
}

function buildVisibleTreeFingerprint(nodes: SnapshotNode[]): string {
const visibility = createSnapshotVisibility(nodes);
return nodes
.filter((node) => isUsableRect(node.rect) && visibility.isVisibleInEffectiveViewport(node))
.map(
(node) =>
`${node.type}#${surfaceIdentity(node)}@${Math.round(node.rect!.x)},${Math.round(node.rect!.y)}`,
)
.sort()
.join(';');
}

function surfaceIdentity(node: SnapshotNode): string {
// All three content fields, not first-present: a recycled cell keeps its identifier and slot while
// its label/value change, and that text change is exactly the progress the signature must register.
Expand Down
2 changes: 2 additions & 0 deletions packages/contracts/src/snapshot-scroll.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ test('snapshot scroll semantics classify scrollable types and node-like roles',
const scrollView = node({ index: 1, type: 'ScrollView', parentIndex: 0 });

assert.equal(isScrollableType('android.widget.RecyclerView'), true);
assert.equal(isScrollableType('collection'), true);
assert.equal(isScrollableType('collectionitem'), false);
assert.equal(isScrollableNodeLike(scrollView), true);
assert.equal(isScrollableNodeLike({ role: 'scrollbar' }), true);
});
1 change: 1 addition & 0 deletions packages/contracts/src/snapshot-scroll.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export function isScrollableType(type: string | null | undefined): boolean {
value.includes('listview') ||
value.includes('gridview') ||
value.includes('collectionview') ||
value === 'collection' ||
value === 'table'
);
}
Expand Down
86 changes: 82 additions & 4 deletions packages/daemon/src/execution/scroll-until.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,24 @@ test('passes repeat until the selector is on screen, and the last gesture is rep
assert.deepEqual(result.result, { pixels: 480 });
});

test.each(['collection', 'ScrollView'])(
'virtualized %s reaches rows that appear only after scrolling',
async (type) => {
const frame = (label: string) => {
const nodes = tree(200, label);
nodes[1]!.type = type;
return capture({ nodes });
};
let scrolls = 0;
const result = await run({
captures: [frame('Row 1'), frame('Row 30'), frame('Email')],
onScroll: () => (scrolls += 1),
});
assert.equal(result.passes, 2);
assert.equal(scrolls, 2);
},
);

/**
* A target below the fold is present but not visible. Stopping on presence would leave the caller
* with a row it cannot act on, which is the whole reason the check asks about the viewport.
Expand Down Expand Up @@ -181,14 +199,14 @@ test('a vertical scroll that moves nothing gives up early instead of spending th
assert.ok(error instanceof AppError);
assert.equal(error.details?.reason, 'scroll_until_no_progress');
assert.equal(error.details?.passes, 3);
assert.match(String(error.details?.hint), /not reaching this container/);
assert.match(String(error.details?.hint), /may not be reaching this container/);
return true;
},
);
assert.equal(scrolls, 3);
});

test('running out of content stops before the pass budget does', async () => {
test('unchanged visible content stops after gestures show no progress', async () => {
let scrolls = 0;
await assert.rejects(
() =>
Expand All @@ -197,14 +215,74 @@ test('running out of content stops before the pass budget does', async () => {
captures: [capture({ nodes: tree(200, 'Other') })],
onScroll: () => (scrolls += 1),
}),
(error: unknown) => {
assert.ok(error instanceof AppError);
assert.equal(error.details?.reason, 'scroll_until_no_progress');
assert.equal(error.details?.passes, 3);
return true;
},
);
assert.equal(scrolls, 3);
});

test('an unchanged collection without a scroll marker reports no progress', async () => {
const nodes = tree(200, 'Other');
nodes[1]!.type = 'collection';
await assert.rejects(
() => run({ captures: [capture({ nodes })] }),
(error: unknown) =>
error instanceof AppError &&
error.details?.reason === 'scroll_until_no_progress' &&
error.details?.passes === 3,
);
});

test('content that moved and then stops reports the end of content', async () => {
const frames = [tree(400, 'Other'), tree(200, 'Other')].map((nodes) => capture({ nodes }));
await assert.rejects(
() => run({ captures: frames }),
(error: unknown) => {
assert.ok(error instanceof AppError);
assert.equal(error.details?.reason, 'scroll_until_edge_reached');
assert.match(String(error.details?.hint), /scroll the opposite direction/);
assert.match(String(error.details?.hint), /another screen/);
return true;
},
);
assert.equal(scrolls, 0);
});

test('a non-scrollable screen stops after repeated unchanged observations', async () => {
const nodes = tree(200, 'Other').filter((node) => node.index !== 1);
nodes[1]!.parentIndex = 0;
let scrolls = 0;
await assert.rejects(
() => run({ captures: [capture({ nodes })], onScroll: () => (scrolls += 1) }),
(error: unknown) =>
error instanceof AppError && error.details?.reason === 'scroll_until_no_progress',
);
assert.equal(scrolls, 3);
});

test('changes outside a stuck scroll container do not imply scrolling', async () => {
const frame = (label: string) =>
capture({ nodes: [...tree(200, 'Other'), { ...row(3, 100, label), parentIndex: 0 }] });
await assert.rejects(
() => run({ captures: [frame('Clock 1'), frame('Clock 2')] }),
(error: unknown) =>
error instanceof AppError && error.details?.reason === 'scroll_until_no_progress',
);
});

test('changes in a tree without a scroll container do not imply scrolling', async () => {
const frame = (label: string) => {
const nodes = tree(200, label).filter((node) => node.index !== 1);
nodes[1]!.parentIndex = 0;
return capture({ nodes });
};
await assert.rejects(
() => run({ captures: [frame('Clock 1'), frame('Clock 2')] }),
(error: unknown) =>
error instanceof AppError && error.details?.reason === 'scroll_until_no_progress',
);
});

test('a horizontal scroll has no edge signal and is bounded by the budget alone', async () => {
Expand Down
70 changes: 43 additions & 27 deletions packages/daemon/src/execution/scroll-until.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import {
readScrollEdgeState,
runScrollPasses,
scrollNoProgressHint,
verticalEdgeFor,
type ScrollCaptureRefusal,
} from '@agent-device/capture-kit/scroll-edge-state';
Expand All @@ -39,8 +38,9 @@
* first is what keeps a failed read from being reported as end-of-content — coercing it to an empty
* tree makes the edge analyzer say "no room below".
*
* The end-of-content signal is the one `scroll top`/`scroll bottom` already trust, so both stop in
* the same place. Horizontal scrolls have no such analyzer and are bounded by the pass budget alone.
* Vertical scrolling uses the surface fingerprint to detect a lack of progress. Hidden-content
* hints alone cannot establish an edge because virtualized content may not be in the tree.
* Horizontal scrolls are bounded by the pass budget alone.
* The first capture happens before the first gesture, so an already-visible target costs no scroll.
*/
export async function runScrollUntilVisible<TResult>(params: {
Expand All @@ -55,6 +55,9 @@
const { selector, direction, platform, capture, scroll } = params;
const edge = verticalEdgeFor(direction);
let scrolled = false;
let previousFingerprint: string | undefined;
let previousContainer: string | undefined;
let observedMovement = false;
try {
const { passes, result } = await runScrollPasses({
signal: params.signal,
Expand All @@ -64,20 +67,32 @@
if (refusal) throw scrollUntilCaptureError(direction, selector, refusal);
return captured.nodes as SnapshotNode[];
},
decide: async (nodes, passes) => {
decide: async (nodes) => {
if (await isSelectorVisible(nodes, selector, platform)) return { stop: true };
// Horizontal scrolling has no edge analyzer and uses only the pass budget.
if (!edge) return { stop: false };
const state = await readScrollEdgeState(nodes, edge);

Check warning on line 74 in packages/daemon/src/execution/scroll-until.ts

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

minor: Container identity via JSON.stringify compares key order, not geometry

Two captures of the same container can have different property insertion order (e.g. from an Android hierarchy built from bounds vs. one rewritten by the same-rect ancestor walk), so `JSON.stringify` yields different strings and resets `observedMovement`/`previousFingerprint` every pass—a list that scrolled to its end reports `scroll_until_no_progress` instead of `scroll_until_edge_reached`. Build the key from explicit numeric fields instead of relying on property order. ### Also raised here Also raised at this location by persona `senior-developer`: A container scope that stops being unique erases observed movement — observedMovement is reset whenever the container identity changes (scroll-until.ts:76), and that identity is JSON of `[state.containerRect, state.scope]`. state.scope is not a stable container name: buildScrollContainerScope (selection.ts:222-245) returns the container's identifier or label only while isUniqueScopeValue says exactly one node in the tree contains that string, so a row whose label later contains the container's label (a section header repeating the list title, or a recycled cell) drops the scope from one capture to the next. When that happens on pass 5 of a search that really did scroll 5 screens, observedMovement is cleared and the run ends with scroll\_until\_no\_progress ("the scroll may not be reaching this container", advising the user to target the container directly) instead of edge\_reached — a wrong diagnosis for a search that demonstrably moved. The scope is a capture-refinement hint, not container identity, so it should not gate the movement ledger.
Comment thread
thymikee marked this conversation as resolved.
Resolved
if (!state.canScroll) {
throw scrollUntilNotFoundError(direction, selector, 'edge-reached', passes);
const container = state.containerRect && JSON.stringify([state.containerRect, state.scope]);
if (container !== previousContainer) observedMovement = false;
if (
container &&
container === previousContainer &&
state.fingerprint &&
previousFingerprint &&
state.fingerprint !== previousFingerprint
) {
observedMovement = true;
}
previousContainer = container;
previousFingerprint = container ? state.fingerprint : undefined;
return { stop: false, fingerprint: state.fingerprint };
},
passLimit: params.passLimit ?? DEFAULT_SCROLL_UNTIL_PASSES,
passLimitError: (passes) =>
scrollUntilNotFoundError(direction, selector, 'pass-limit', passes),
noProgressError: (passes) => scrollUntilNoProgressError(direction, selector, passes),
passLimitError: (passes) => scrollUntilNotFoundError(direction, selector, passes),
noProgressError: (passes) =>
observedMovement
? scrollUntilEdgeError(direction, selector, passes)
: scrollUntilNoProgressError(direction, selector, passes),

Check warning on line 95 in packages/daemon/src/execution/scroll-until.ts

View check run for this annotation

apex-sentinel-staging / Apex Sentinel

minor: A bouncing scroll is reported as "end of the scrollable content"

A rubber-banding A,B,A,B pattern trips no-progress after content changed on an earlier pass, so `runScrollUntilVisible` reports `scroll_until_edge_reached` even though content did not end. `observedMovement` is set by any single fingerprint change and only resets when container identity changes, so one bounce suffices. The hint also drops `scrollNoProgressHint()`'s keyboard/nested/raw-drag advice. In the opposite direction, a container that is never resolved (undefined key) can never set `observedMovement`, so a real bottom reports `scroll_until_no_progress`. ### Also raised here Also raised at this location by persona `maintainability`: Content that updates itself inside a stuck container still reports end of content — Re-raising P1 against the rewritten logic: a container that never moves but holds self-updating content — an auto-advancing carousel or a countdown row inside the stuck list — gives every capture the same `containerRect` and a different `fingerprint`, because `surfaceIdentity` counts the recycled cell's changed label/position. That sets `observedMovement`, so after three no-progress passes the user gets `scroll_until_edge_reached` with "The content may end here … scroll the opposite direction", although the container never scrolled and the real fault is a gesture that is not reaching it. Scoping the comparison to the same container (the new "changes outside a stuck scroll container" test) only covers changes outside it; changes inside it still masquerade as scrolling. Gate `observedMovement` on the container geometry or offset actually changing, not on its descendants' content. Also raised at this location by persona `javascript`: A container that moves once then gets stuck is reported as end of content — A scrollable that responds to the first gesture and then jams — rubber-banding back to the same two signatures, or losing scrollability after a keyboard appears — trips no-progress with `observedMovement` already true, so the loop raises \`scrol…
Comment thread
thymikee marked this conversation as resolved.
Resolved
scroll: async () => {
scrolled = true;
return await scroll();
Expand Down Expand Up @@ -161,27 +176,11 @@
return undefined;
}

/** The content ran out, or the budget did. Separate messages: the corrective action differs. */
function scrollUntilNotFoundError(
direction: ScrollDirection,
selector: string,
outcome: 'edge-reached' | 'pass-limit',
passes: number,
): AppError {
const spent = `${passes} ${passes === 1 ? 'pass' : 'passes'}`;
if (outcome === 'edge-reached') {
return new AppError(
'COMMAND_FAILED',
`scroll ${direction} reached the end of the scrollable content after ${spent} without ${selector} becoming visible`,
{
reason: 'scroll_until_edge_reached',
selector,
direction,
passes,
hint: `The content ends here, so no further ${direction} scroll can reveal it. Run snapshot -i to see what is on screen, scroll the opposite direction, or check the selector — the element may be on another screen.`,
},
);
}
return new AppError(
'COMMAND_FAILED',
`scroll ${direction} spent its ${passes}-pass budget without ${selector} becoming visible`,
Expand All @@ -195,7 +194,24 @@
);
}

/** The scroll was not reaching the container, not just falling short of a budget — the fix differs. */
function scrollUntilEdgeError(
direction: ScrollDirection,
selector: string,
passes: number,
): AppError {
return new AppError(
'COMMAND_FAILED',
`scroll ${direction} reached the end of the scrollable content after ${passes} ${passes === 1 ? 'pass' : 'passes'} without ${selector} becoming visible`,
{
reason: 'scroll_until_edge_reached',
selector,
direction,
passes,
hint: 'The content may end here. Run snapshot -i to inspect the screen, scroll the opposite direction, or check the selector; the element may be on another screen.',
},
);
}

function scrollUntilNoProgressError(
direction: ScrollDirection,
selector: string,
Expand All @@ -210,7 +226,7 @@
selector,
direction,
passes,
hint: scrollNoProgressHint(),
hint: 'The content may have ended, or the scroll may not be reaching this container. Check the selector and run snapshot -i to inspect the screen. If a field is focused, dismiss the keyboard; if the container is nested, target it directly. Some lists respond to a raw drag: swipe x1 y1 x2 y2 started inside the list.',
},
);
}
Expand Down
2 changes: 1 addition & 1 deletion src/commands/schema/cli-help-gestures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Shapes:
agent-device gesture rotate 35 200 420
agent-device gesture transform 200 420 80 -40 2 35 700
agent-device gesture tap 80 300 200 450 310 600 --count 20 --pause-ms 100
scroll <dir> --until <selector> repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent, fails when the content runs out first, and is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes <n> (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens.
scroll <dir> --until <selector> repeats scroll-and-check passes until that element is on screen, then stops: one request instead of a scroll-then-snapshot loop, and it stops on the target rather than overshooting it. It reports the passes it spent; vertical searches compare repeated captures of descendants in the selected scroll container and fail when those captures show no progress. If no container is found, they compare the visible tree. It is refused on top/bottom, which already stop themselves. The default limit is ${DEFAULT_SCROLL_UNTIL_PASSES} gestures. For a longer list, add --max-passes <n> (1–${SCROLL_UNTIL_MAX_PASSES}); it requires --until. Large pass budgets can hit the 90-second request timeout; inspect the current screen and continue with another search if that happens.
longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback. Duration and gesture scale/center are positional. gesture pan is one finger by default; add --pointer-count 2 for a parallel two-finger pan. Keep count/pause/pattern on one swipe: --count (cap 200), --pause-ms (cap 10000ms), --pattern ping-pong; the combined swipe/pause schedule is capped at 60000ms.
For repeated iOS smoke checks: press <x> <y> --count <n> --jitter-px <n> for tap series, swipe <x1> <y1> <x2> <y2> --count <n> for drag series. gesture tap touches 2 to 5 distinct points and holds them for --hold-ms (default 50, min 16); --count repeats the same contacts (cap 200) with --pause-ms between, the series must fit 60000ms counting about 500ms per repetition, and a repetition costs about 450ms on a physical iPhone. Use press for one finger.

Expand Down
7 changes: 5 additions & 2 deletions test/integration/provider-scenarios/scroll-until.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ test.each([
assert.ok(error instanceof AppError);
assert.equal(error.details?.reason, 'scroll_until_pass_limit');
assert.equal(error.details?.passes, maxPasses);
assert.match(String(error.message), /without text=Terms becoming visible/);
return true;
});
} else {
Expand All @@ -105,7 +106,7 @@ test.each([
},
);

test('Provider-backed integration scroll --until reports the end of the content as a typed failure', async () => {
test('Provider-backed integration scroll --until reports unchanged content as no progress', async () => {
await withProviderScenarioResource(
async () =>
await createAndroidSettingsWorld({
Expand All @@ -125,7 +126,9 @@ test('Provider-backed integration scroll --until reports the end of the content
}),
(error: unknown) => {
assert.ok(error instanceof AppError);
assert.match(String(error.message), /without text=NeverPresent becoming visible/);
assert.equal(error.details?.reason, 'scroll_until_no_progress');
assert.equal(error.details?.passes, 3);
assert.match(String(error.message), /moved nothing across 3 passes/);
return true;
},
);
Expand Down
2 changes: 1 addition & 1 deletion website/docs/docs/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ agent-device wait 'id="field-phone" value="00 062 91 77"'
### Scroll

- `scroll <direction> [amount]` takes the finger path as a fraction of the viewport (`0.5` is half), or `--pixels <n>` for a fixed distance. Large distances are clamped. The app's scroll physics decide the final offset, so a scroll does not land on an exact position.
- `scroll <direction> --until <selector>` repeats passes until the selector is on screen and fails if the content runs out first. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until <selector>`) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e).
- `scroll <direction> --until <selector>` repeats passes until the selector is on screen. A vertical search fails with `COMMAND_FAILED` and `reason: scroll_until_no_progress` if repeated captures show no progress in the selected scroll container. If content moved before stopping, the reason is `scroll_until_edge_reached`. If no container is found, it compares the visible tree. It stops after at most 12 gestures by default. For a longer list, use `scroll down --until 'id="row-512"' --max-passes 90` (1–200 gestures). The pass limit does not extend the 90-second request timeout. Large pass budgets can hit that timeout; inspect the current screen and continue with another search if that happens. Add an amount (`scroll down 0.8 --until <selector>`) to widen each pass. Use it in `.ad` scripts too, instead of fixed amounts that depend on screen size; see [Replay and E2E testing](/docs/replay-e2e).
- Default snapshot text summarizes off-screen interactive content instead of listing refs. `scroll <direction> --settle` (and `back --settle`) waits for the UI to go quiet and returns the diff with fresh refs, so you don't need another `snapshot -i`. Settle is best-effort and never fails the action.
- On Android, settle ignores an allowlist of system-bar containers and shade/keyguard status headers; other SystemUI overlays remain content. Appium settle also applies package rules for keyboard and system chrome.
- With a keyboard up on iOS or Android, the swipe stays above the keyboard (reported as `keyboardAvoided` and `keyboardMinY`) and the keyboard stays open. If too little room is left, the command refuses with `scroll_keyboard_occludes_surface`; run `keyboard dismiss` first.
Expand Down
Loading