Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions packages/capture-kit/src/snapshot-state.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { expect, test } from 'vitest';
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
import type { SnapshotCaptureProvenance } from '@agent-device/contracts/snapshot-model';
import { buildSnapshotState } from './snapshot-state.ts';

/**
* A Flutter screen on iOS at the top of its content (#4451): the scroll view and its content are
* siblings, and the content below the fold keeps a zero-area frame at the origin.
*/
function scrolledOutButtonTree(): RawSnapshotNode[] {
return [
{ index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'ScrollView',
rect: { x: 0, y: 118, width: 402, height: 756 },
},
{
index: 2,
depth: 1,
parentIndex: 0,
type: 'StaticText',
label: 'Tall block',
rect: { x: 157, y: 546, width: 86, height: 27 },
},
{
index: 3,
depth: 1,
parentIndex: 0,
type: 'Button',
label: 'View all',
rect: { x: 0, y: 0, width: 0, height: 0 },
},
{ index: 4, depth: 1, parentIndex: 0, type: 'StaticText', label: 'No frame' },
];
}

function hintsByNode(nodes: readonly RawSnapshotNode[]) {
return nodes.map((node) => [node.label ?? node.type, node.presentationHints]);
}

const PROVENANCES: SnapshotCaptureProvenance[] = [
{ backend: 'xctest', producer: 'apple-runner' },
{ backend: 'android', producer: 'android-uiautomator' },
];

test.each(PROVENANCES)(
'marks a node published with a zero-area rect as unmeasured ($producer)',
(provenance) => {
const state = buildSnapshotState({ nodes: scrolledOutButtonTree(), ...provenance }, undefined);

expect(hintsByNode(state.nodes)).toEqual([
['Application', undefined],
['ScrollView', undefined],
['Tall block', undefined],
['View all', ['unmeasured']],
// A node that reports no frame at all made no measurement to doubt.
['No frame', undefined],
]);
},
);

test('a raw snapshot keeps the reported nodes without the unmeasured mark', () => {
const state = buildSnapshotState(
{ nodes: scrolledOutButtonTree(), backend: 'xctest', producer: 'apple-runner' },
{ snapshotRaw: true },
);

expect(state.nodes.flatMap((node) => node.presentationHints ?? [])).toEqual([]);
});
24 changes: 23 additions & 1 deletion packages/capture-kit/src/snapshot-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
type SnapshotViewportSize,
type SnapshotQualityNote,
} from '@agent-device/kernel/snapshot';
import { isPositiveFiniteRect } from '@agent-device/kernel/rect';
import {
snapshotRefMinting,
type SnapshotBackend,
Expand Down Expand Up @@ -104,7 +105,9 @@ export function buildSnapshotState(
const nodes = attachRefs(
snapshotRaw
? scopedNodes
: annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)),
: annotateUnmeasuredSnapshotNodes(
annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)),
),
snapshotRefMinting(data),
);
const textInputGeometry = readSnapshotTextInputGeometry(data.nodes);
Expand Down Expand Up @@ -136,6 +139,25 @@ export function buildSnapshotState(
};
}

/**
* The hint a node carries when the rect it is published with has no area. That is not evidence
* that the node is off screen, but nothing confirms that it is on screen either: a scroll view
* keeps its scrolled-out content in the tree with a zero-area frame (#4451), as SpringBoard keeps
* another page's icons (#3841).
*/
const UNMEASURED_PRESENTATION_HINT = 'unmeasured';

function annotateUnmeasuredSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] {
return nodes.map((node) =>
node.rect && !isPositiveFiniteRect(node.rect)
? {
...node,
presentationHints: [...(node.presentationHints ?? []), UNMEASURED_PRESENTATION_HINT],
}
: node,
);
}

function annotateBackendReplacementSurfaces(
owner: object & { backend?: SnapshotBackend },
nodes: RawSnapshotNode[],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import type { Point } from '@agent-device/kernel/snapshot';
import { attachSnapshotNodeInteractionBounds } from '@agent-device/kernel/rect-center';
import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state';
import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
import {
buildUiHierarchySnapshot,
Expand Down Expand Up @@ -120,12 +122,114 @@ test('runtime press refuses a selector match with a zero-area rect and dispatche
assert.equal((error as { code?: string }).code, 'COMMAND_FAILED');
assert.equal(details?.reason, 'target_bounds_invalid');
assert.equal(details?.dispatched, 'no');
// No scroll container holds the icon, so no scroll is offered as the recovery.
assert.equal(details?.scrollContainerRef, undefined);
assert.equal(details?.hint, undefined);
return true;
},
);
assert.deepEqual(taps, []);
});

/**
* A Flutter screen on iOS at the top of its content (#4451), as the Simulator AX bridge presents it:
* the scroll view and its content are siblings, and the content below the fold keeps a 0x0 rect.
*/
function scrolledOutButtonSnapshot(label = 'View all') {
return makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Application',
rect: { x: 0, y: 0, width: 402, height: 874 },
hittable: false,
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'ScrollView',
role: 'FlutterSemanticsScrollView',
rect: { x: 0, y: 118, width: 402, height: 756 },
hittable: true,
},
{
index: 2,
depth: 1,
parentIndex: 0,
type: 'StaticText',
role: 'FlutterSemanticsObject',
label: 'Tall block',
rect: { x: 157, y: 546, width: 86, height: 27 },
hittable: true,
},
{
index: 3,
depth: 1,
parentIndex: 0,
type: 'Button',
role: 'FlutterSemanticsObject',
label,
rect: { x: 0, y: 0, width: 0, height: 0 },
hittable: false,
},
]);
}

test('runtime press on a zero-area target beside a scroll container names the scroll that reveals it', async () => {
const taps: Point[] = [];
const device = createInteractionDevice(scrolledOutButtonSnapshot(), {
tap: async (_context, point) => {
taps.push(point);
return { ok: true };
},
});

await assert.rejects(
() => device.interactions.press(selector('label="View all"'), { session: 'default' }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'target_bounds_invalid');
assert.equal(details?.dispatched, 'no');
assert.equal(details?.scrollContainerRef, 'e2');
assert.match(String(details?.hint), /scroll <direction> --until 'label="View all"'/);
return true;
},
);
await assert.rejects(
() => device.interactions.press(ref('@e4'), { session: 'default' }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'target_bounds_invalid');
assert.equal(details?.scrollContainerRef, 'e2');
// A scroll expires the ref frame, so the ref recovery steers to a selector instead.
assert.match(String(details?.hint), /retry with a selector/);
assert.match(String(details?.hint), /re-run snapshot -i/);
assert.doesNotMatch(String(details?.hint), /--until/);
return true;
},
);
assert.deepEqual(taps, []);
});

test('the scroll a zero-area target is pointed at survives the shell', async () => {
// The hint is copied into a shell: double quotes would expand the `$`, and a bare single-quote
// wrap would end at the apostrophe.
const device = createInteractionDevice(scrolledOutButtonSnapshot("Today's $5 deal"), {
tap: async () => ({ ok: true }),
});

await assert.rejects(
() => device.interactions.press(selector('label="Today\'s $5 deal"'), { session: 'default' }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'target_bounds_invalid');
assert.match(String(details?.hint), /--until 'label="Today'\\''s \$5 deal"'/);
return true;
},
);
});

test('runtime press on a Compose button dispatches its center only once its role node is folded', async () => {
const tree = parseUiHierarchyTree(`<hierarchy>
<node class="android.view.View" resource-id="compose-container" bounds="[53,1856][1027,1982]"
Expand Down Expand Up @@ -166,3 +270,98 @@ test('runtime press on a Compose button dispatches its center only once its role
dispatched: 'no',
});
});

test('runtime press offers no scroll for a zero-area target listed before the only scroll container', async () => {
const state = scrolledOutButtonSnapshot();
const [application, scrollView, , button] = state.nodes;
const device = createInteractionDevice(
makeSnapshotState([application!, { ...button!, index: 1 }, { ...scrollView!, index: 2 }]),
{ tap: async () => ({ ok: true }) },
);

await assert.rejects(
() => device.interactions.press(selector('label="View all"'), { session: 'default' }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'target_bounds_invalid');
assert.equal(details?.scrollContainerRef, undefined);
assert.equal(details?.hint, undefined);
return true;
},
);
});

test('runtime press offers no scroll for a measured target whose input bounds exclude it', async () => {
const root = { x: 0, y: 0, width: 1080, height: 2400 };
// A row with a real frame, wholly under the navigation bar the input bounds stop at.
const row = attachSnapshotNodeInteractionBounds(
{
index: 2,
depth: 2,
parentIndex: 1,
type: 'Button',
identifier: 'row',
label: 'Row',
hittable: true,
rect: { x: 0, y: 2305, width: 1080, height: 95 },
},
{ ...root, height: 2274 },
);
const snapshot = buildSnapshotState(
{
backend: 'android',
producer: 'android-uiautomator',
nodes: [
{ index: 0, depth: 0, type: 'Application', rect: root },
{ index: 1, depth: 1, parentIndex: 0, type: 'ScrollView', rect: root, hittable: true },
row,
],
},
{ snapshotRaw: true },
);
const device = createInteractionDevice(snapshot, {
platform: 'android',
tap: async () => ({ ok: true }),
});

await assert.rejects(
() => device.interactions.press(selector('id=row'), { session: 'default' }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'target_bounds_invalid');
assert.equal(details?.scrollContainerRef, undefined);
assert.equal(details?.hint, undefined);
return true;
},
);
});

test('runtime press names the scroll view, not a scroll bar listed after it, as the scroll container', async () => {
const state = scrolledOutButtonSnapshot();
const [application, scrollView, block, button] = state.nodes;
const device = createInteractionDevice(
makeSnapshotState([
application!,
scrollView!,
{
...block!,
type: 'Other',
role: 'AXScrollBar',
label: 'Vertical scroll bar, 2 pages',
rect: { x: 396, y: 118, width: 6, height: 756 },
},
button!,
]),
{ tap: async () => ({ ok: true }) },
);

await assert.rejects(
() => device.interactions.press(selector('label="View all"'), { session: 'default' }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'target_bounds_invalid');
assert.equal(details?.scrollContainerRef, 'e2');
return true;
},
);
});
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell';
import { AppError } from '@agent-device/kernel/errors';
import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract';
import {
Expand Down Expand Up @@ -162,6 +163,7 @@ async function resolveSelectorInteractionTarget(
invalidMessage: `Selector ${resolved.selector} resolved to invalid bounds`,
blockedTargetLabel: `Selector ${selectorExpression}`,
blockedTargetDetails: { selector: selectorExpression },
shellQuotedSelector: shellQuoteIfNeeded(selectorExpression),
}),
});
return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,41 @@ test('runtime press names a direction for a partial clip whose center is off-scr
assert.equal(taps.length, 0);
});

test('the scroll an off-screen selector target is pointed at survives the shell', async () => {
// The hint is copied into a shell, and a bare single-quote wrap would end at the apostrophe.
const device = createInteractionDevice(
makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Application',
rect: { x: 0, y: 0, width: 400, height: 800 },
hittable: true,
},
{
index: 1,
depth: 2,
parentIndex: 0,
type: 'Button',
label: "Today's cash",
rect: { x: 20, y: 900, width: 200, height: 44 },
hittable: true,
},
]),
{ tap: async () => {} },
);

await assert.rejects(
() => device.interactions.press(selector('label="Today\'s cash"'), { session: 'default' }),
(error: unknown) => {
const details = (error as { details?: Record<string, unknown> }).details;
assert.equal(details?.reason, 'offscreen_selector');
assert.match(String(details?.hint), /--until 'label="Today'\\''s cash"'/);
return true;
},
);
});

test('runtime click rejects refs covered by floating overlays', async () => {
const calls: Point[] = [];
const device = createInteractionDevice(coveredByTabBarSnapshot(), {
Expand Down
Loading
Loading