diff --git a/packages/capture-kit/src/snapshot-state.test.ts b/packages/capture-kit/src/snapshot-state.test.ts new file mode 100644 index 0000000000..8db36fdd27 --- /dev/null +++ b/packages/capture-kit/src/snapshot-state.test.ts @@ -0,0 +1,72 @@ +import { expect, test } from 'vitest'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotCaptureProvenance } from '@agent-device/contracts/snapshot-model'; +import { buildSnapshotState } from './snapshot-state.ts'; + +/** + * A Flutter screen on iOS at the top of its content (#4451): the scroll view and its content are + * siblings, and the content below the fold keeps a zero-area frame at the origin. + */ +function scrolledOutButtonTree(): RawSnapshotNode[] { + return [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'ScrollView', + rect: { x: 0, y: 118, width: 402, height: 756 }, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Tall block', + rect: { x: 157, y: 546, width: 86, height: 27 }, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'View all', + rect: { x: 0, y: 0, width: 0, height: 0 }, + }, + { index: 4, depth: 1, parentIndex: 0, type: 'StaticText', label: 'No frame' }, + ]; +} + +function hintsByNode(nodes: readonly RawSnapshotNode[]) { + return nodes.map((node) => [node.label ?? node.type, node.presentationHints]); +} + +const PROVENANCES: SnapshotCaptureProvenance[] = [ + { backend: 'xctest', producer: 'apple-runner' }, + { backend: 'android', producer: 'android-uiautomator' }, +]; + +test.each(PROVENANCES)( + 'marks a node published with a zero-area rect as unmeasured ($producer)', + (provenance) => { + const state = buildSnapshotState({ nodes: scrolledOutButtonTree(), ...provenance }, undefined); + + expect(hintsByNode(state.nodes)).toEqual([ + ['Application', undefined], + ['ScrollView', undefined], + ['Tall block', undefined], + ['View all', ['unmeasured']], + // A node that reports no frame at all made no measurement to doubt. + ['No frame', undefined], + ]); + }, +); + +test('a raw snapshot keeps the reported nodes without the unmeasured mark', () => { + const state = buildSnapshotState( + { nodes: scrolledOutButtonTree(), backend: 'xctest', producer: 'apple-runner' }, + { snapshotRaw: true }, + ); + + expect(state.nodes.flatMap((node) => node.presentationHints ?? [])).toEqual([]); +}); diff --git a/packages/capture-kit/src/snapshot-state.ts b/packages/capture-kit/src/snapshot-state.ts index e2386e8261..b6300858b7 100644 --- a/packages/capture-kit/src/snapshot-state.ts +++ b/packages/capture-kit/src/snapshot-state.ts @@ -22,6 +22,7 @@ import { type SnapshotViewportSize, type SnapshotQualityNote, } from '@agent-device/kernel/snapshot'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { snapshotRefMinting, type SnapshotBackend, @@ -104,7 +105,9 @@ export function buildSnapshotState( const nodes = attachRefs( snapshotRaw ? scopedNodes - : annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)), + : annotateUnmeasuredSnapshotNodes( + annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)), + ), snapshotRefMinting(data), ); const textInputGeometry = readSnapshotTextInputGeometry(data.nodes); @@ -136,6 +139,25 @@ export function buildSnapshotState( }; } +/** + * The hint a node carries when the rect it is published with has no area. That is not evidence + * that the node is off screen, but nothing confirms that it is on screen either: a scroll view + * keeps its scrolled-out content in the tree with a zero-area frame (#4451), as SpringBoard keeps + * another page's icons (#3841). + */ +const UNMEASURED_PRESENTATION_HINT = 'unmeasured'; + +function annotateUnmeasuredSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { + return nodes.map((node) => + node.rect && !isPositiveFiniteRect(node.rect) + ? { + ...node, + presentationHints: [...(node.presentationHints ?? []), UNMEASURED_PRESENTATION_HINT], + } + : node, + ); +} + function annotateBackendReplacementSurfaces( owner: object & { backend?: SnapshotBackend }, nodes: RawSnapshotNode[], diff --git a/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts b/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts index 47a26f000d..f71878057d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts @@ -1,6 +1,8 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { Point } from '@agent-device/kernel/snapshot'; +import { attachSnapshotNodeInteractionBounds } from '@agent-device/kernel/rect-center'; +import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { buildUiHierarchySnapshot, @@ -120,12 +122,114 @@ test('runtime press refuses a selector match with a zero-area rect and dispatche assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); assert.equal(details?.reason, 'target_bounds_invalid'); assert.equal(details?.dispatched, 'no'); + // No scroll container holds the icon, so no scroll is offered as the recovery. + assert.equal(details?.scrollContainerRef, undefined); + assert.equal(details?.hint, undefined); return true; }, ); assert.deepEqual(taps, []); }); +/** + * A Flutter screen on iOS at the top of its content (#4451), as the Simulator AX bridge presents it: + * the scroll view and its content are siblings, and the content below the fold keeps a 0x0 rect. + */ +function scrolledOutButtonSnapshot(label = 'View all') { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: false, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'ScrollView', + role: 'FlutterSemanticsScrollView', + rect: { x: 0, y: 118, width: 402, height: 756 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + role: 'FlutterSemanticsObject', + label: 'Tall block', + rect: { x: 157, y: 546, width: 86, height: 27 }, + hittable: true, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'Button', + role: 'FlutterSemanticsObject', + label, + rect: { x: 0, y: 0, width: 0, height: 0 }, + hittable: false, + }, + ]); +} + +test('runtime press on a zero-area target beside a scroll container names the scroll that reveals it', async () => { + const taps: Point[] = []; + const device = createInteractionDevice(scrolledOutButtonSnapshot(), { + tap: async (_context, point) => { + taps.push(point); + return { ok: true }; + }, + }); + + await assert.rejects( + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.dispatched, 'no'); + assert.equal(details?.scrollContainerRef, 'e2'); + assert.match(String(details?.hint), /scroll --until 'label="View all"'/); + return true; + }, + ); + await assert.rejects( + () => device.interactions.press(ref('@e4'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.scrollContainerRef, 'e2'); + // A scroll expires the ref frame, so the ref recovery steers to a selector instead. + assert.match(String(details?.hint), /retry with a selector/); + assert.match(String(details?.hint), /re-run snapshot -i/); + assert.doesNotMatch(String(details?.hint), /--until/); + return true; + }, + ); + assert.deepEqual(taps, []); +}); + +test('the scroll a zero-area target is pointed at survives the shell', async () => { + // The hint is copied into a shell: double quotes would expand the `$`, and a bare single-quote + // wrap would end at the apostrophe. + const device = createInteractionDevice(scrolledOutButtonSnapshot("Today's $5 deal"), { + tap: async () => ({ ok: true }), + }); + + await assert.rejects( + () => device.interactions.press(selector('label="Today\'s $5 deal"'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.match(String(details?.hint), /--until 'label="Today'\\''s \$5 deal"'/); + return true; + }, + ); +}); + test('runtime press on a Compose button dispatches its center only once its role node is folded', async () => { const tree = parseUiHierarchyTree(` { + const state = scrolledOutButtonSnapshot(); + const [application, scrollView, , button] = state.nodes; + const device = createInteractionDevice( + makeSnapshotState([application!, { ...button!, index: 1 }, { ...scrollView!, index: 2 }]), + { tap: async () => ({ ok: true }) }, + ); + + await assert.rejects( + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.scrollContainerRef, undefined); + assert.equal(details?.hint, undefined); + return true; + }, + ); +}); + +test('runtime press offers no scroll for a measured target whose input bounds exclude it', async () => { + const root = { x: 0, y: 0, width: 1080, height: 2400 }; + // A row with a real frame, wholly under the navigation bar the input bounds stop at. + const row = attachSnapshotNodeInteractionBounds( + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + identifier: 'row', + label: 'Row', + hittable: true, + rect: { x: 0, y: 2305, width: 1080, height: 95 }, + }, + { ...root, height: 2274 }, + ); + const snapshot = buildSnapshotState( + { + backend: 'android', + producer: 'android-uiautomator', + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: root }, + { index: 1, depth: 1, parentIndex: 0, type: 'ScrollView', rect: root, hittable: true }, + row, + ], + }, + { snapshotRaw: true }, + ); + const device = createInteractionDevice(snapshot, { + platform: 'android', + tap: async () => ({ ok: true }), + }); + + await assert.rejects( + () => device.interactions.press(selector('id=row'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.scrollContainerRef, undefined); + assert.equal(details?.hint, undefined); + return true; + }, + ); +}); + +test('runtime press names the scroll view, not a scroll bar listed after it, as the scroll container', async () => { + const state = scrolledOutButtonSnapshot(); + const [application, scrollView, block, button] = state.nodes; + const device = createInteractionDevice( + makeSnapshotState([ + application!, + scrollView!, + { + ...block!, + type: 'Other', + role: 'AXScrollBar', + label: 'Vertical scroll bar, 2 pages', + rect: { x: 396, y: 118, width: 6, height: 756 }, + }, + button!, + ]), + { tap: async () => ({ ok: true }) }, + ); + + await assert.rejects( + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.scrollContainerRef, 'e2'); + return true; + }, + ); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/resolution.ts b/packages/command-runtime/src/commands/interaction/runtime/resolution.ts index 2eefd79451..51e2365483 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/resolution.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/resolution.ts @@ -1,3 +1,4 @@ +import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell'; import { AppError } from '@agent-device/kernel/errors'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import { @@ -162,6 +163,7 @@ async function resolveSelectorInteractionTarget( invalidMessage: `Selector ${resolved.selector} resolved to invalid bounds`, blockedTargetLabel: `Selector ${selectorExpression}`, blockedTargetDetails: { selector: selectorExpression }, + shellQuotedSelector: shellQuoteIfNeeded(selectorExpression), }), }); return { diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index c434a3ce0f..fcd481d5ff 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -105,6 +105,41 @@ test('runtime press names a direction for a partial clip whose center is off-scr assert.equal(taps.length, 0); }); +test('the scroll an off-screen selector target is pointed at survives the shell', async () => { + // The hint is copied into a shell, and a bare single-quote wrap would end at the apostrophe. + const device = createInteractionDevice( + makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + { + index: 1, + depth: 2, + parentIndex: 0, + type: 'Button', + label: "Today's cash", + rect: { x: 20, y: 900, width: 200, height: 44 }, + hittable: true, + }, + ]), + { tap: async () => {} }, + ); + + await assert.rejects( + () => device.interactions.press(selector('label="Today\'s cash"'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'offscreen_selector'); + assert.match(String(details?.hint), /--until 'label="Today'\\''s cash"'/); + return true; + }, + ); +}); + test('runtime click rejects refs covered by floating overlays', async () => { const calls: Point[] = []; const device = createInteractionDevice(coveredByTabBarSnapshot(), { diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts index c783efb436..a66a95a2fc 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts @@ -1,3 +1,4 @@ +import { shellQuote } from '@agent-device/kernel/device-shell'; import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import { assertSnapshotScreenOrigin } from '@agent-device/contracts/capture'; import type { @@ -175,7 +176,7 @@ function scrollRevealClause( ): string { if (!direction) return 'Scroll toward it,'; if (!selector) return `Scroll ${direction} toward it,`; - return `Run scroll ${direction} --until '${selector}' to bring it on screen,`; + return `Run scroll ${direction} --until ${shellQuote(selector)} to bring it on screen,`; } // Full on-screen visibility (not only the effective-viewport form): items inside an diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index bfdf2f346c..3a920af0d7 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -75,7 +75,7 @@ export const snapshotCommandFacet = defineCommandFacet({ name: SNAPSHOT_COMMAND_NAME, text: { summary: 'Capture or diff the accessibility tree', - cliDetail: `Scoped output (--scope) renders every returned ref with its ref generation pin (@eN~s) and one Refs generation line, so a pasted ref targets the ref frame that issued it; --json keeps plain refs plus the response-level refsGeneration, and --raw keeps byte-for-byte output. Regular Android ADB snapshots clip app rectangles outside measured status and navigation bars, including launcher-owned navigation; fully covered content is omitted and --raw retains reported bounds. Ref and selector taps honor measured visible bounds even for refs from raw snapshots. Regular iOS snapshots clip overflowing elements to their visible area and omit fully outside elements. --json reports clipping counts in snapshotQualityNotes with code regular-geometry-clipped; --raw retains reported frames. --observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation or a readable system alert over the app, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. Android Appium JSON snapshots publish a display viewport only when measurements before and after the tree agree, independent of tree extent or keyboard content resizing. On Android, JSON value carries field content rather than static labels; saved selectors and replay scripts using value= for static labels must use label= instead; a reported showing hint means value is empty, and text equal to the placeholder is preserved when hint state is unknown. HarmonyOS reads the on-screen keyboard's own system window and publishes the keyboard field of --json output: the field is visibility-only — visible or hidden per the keyboard window's z-order column; a failed probe, an unreadable or conflicting z-order, or a table without the panel publishes unmeasurable, never a guessed band. When an iOS snapshot reports a screen viewport, inspect rects with snapshot -i --json and press the intended control's center, then verify with diff snapshot -i or snapshot --diff. When a fallback cannot establish the screen origin, viewport is omitted and coordinate-derived element actions refuse with window-origin-unknown before input; the snapshot may warn about the unavailable origin. Use a plain screenshot to choose screen coordinates. An empty text field with no label or value prints its hint as [placeholder:"..."]; label=, text= and find do not match that hint. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, + cliDetail: `Scoped output (--scope) renders every returned ref with its ref generation pin (@eN~s) and one Refs generation line, so a pasted ref targets the ref frame that issued it; --json keeps plain refs plus the response-level refsGeneration, and --raw keeps byte-for-byte output. Regular Android ADB snapshots clip app rectangles outside measured status and navigation bars, including launcher-owned navigation; fully covered content is omitted and --raw retains reported bounds. Ref and selector taps honor measured visible bounds even for refs from raw snapshots. Regular iOS snapshots clip overflowing elements to their visible area and omit fully outside elements. In a regular (non-raw) snapshot, a listed element whose frame has no area is marked [unmeasured]: nothing confirms it is on screen. If it is inside scrollable content, reach it with scroll --until before acting on it. --json reports clipping counts in snapshotQualityNotes with code regular-geometry-clipped; --raw retains reported frames. --observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation or a readable system alert over the app, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. Android Appium JSON snapshots publish a display viewport only when measurements before and after the tree agree, independent of tree extent or keyboard content resizing. On Android, JSON value carries field content rather than static labels; saved selectors and replay scripts using value= for static labels must use label= instead; a reported showing hint means value is empty, and text equal to the placeholder is preserved when hint state is unknown. HarmonyOS reads the on-screen keyboard's own system window and publishes the keyboard field of --json output: the field is visibility-only — visible or hidden per the keyboard window's z-order column; a failed probe, an unreadable or conflicting z-order, or a table without the panel publishes unmeasurable, never a guessed band. When an iOS snapshot reports a screen viewport, inspect rects with snapshot -i --json and press the intended control's center, then verify with diff snapshot -i or snapshot --diff. When a fallback cannot establish the screen origin, viewport is omitted and coordinate-derived element actions refuse with window-origin-unknown before input; the snapshot may warn about the unavailable origin. Use a plain screenshot to choose screen coordinates. An empty text field with no label or value prints its hint as [placeholder:"..."]; label=, text= and find do not match that hint. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, }, metadata: snapshotCommandMetadata, run: (client, input) => client.capture.snapshot(input), diff --git a/packages/selectors/src/node-touch-point.ts b/packages/selectors/src/node-touch-point.ts index 96f248c05c..dfc0285e08 100644 --- a/packages/selectors/src/node-touch-point.ts +++ b/packages/selectors/src/node-touch-point.ts @@ -3,7 +3,11 @@ import type { Point, SnapshotAddressableNode, SnapshotNode } from '@agent-device import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { normalizeRect } from '@agent-device/kernel/rect-center'; import { normalizeRef } from '@agent-device/kernel/snapshot'; -import { createSnapshotVisibility, isFillableType } from '@agent-device/contracts/snapshot'; +import { + createSnapshotVisibility, + isFillableType, + isScrollableNodeLike, +} from '@agent-device/contracts/snapshot'; import type { Platform } from '@agent-device/kernel/device'; import { INTERACTION_ERROR_REASONS } from './interaction-error.ts'; import { isDescendantOf, resolveInteractionTouchPoint } from './interaction-touch-point.ts'; @@ -104,14 +108,70 @@ function isTextField(node: SnapshotNode, platform: Platform): boolean { ); } +/** + * The scroll container a target with no usable frame may be scrolled out of (#4451): the nearest + * scrollable sibling listed before it. That is the shape Flutter publishes on iOS: its + * accessibility container lists a scroll view and then that scroll view's content, and keeps the + * scrolled-out content with a zero-area frame. A target under a framed scroll ancestor never + * reaches here, because the off-screen stage refuses it first. + */ +function findPrecedingScrollContainer( + node: SnapshotNode, + nodes: SnapshotState['nodes'], +): SnapshotNode | null { + return ( + nodes + .filter( + (candidate) => + candidate.parentIndex === node.parentIndex && + candidate.index < node.index && + hasUsableRect(candidate) && + isScrollableNodeLike(candidate) && + !isScrollIndicator(candidate), + ) + .at(-1) ?? null + ); +} + +/** A scroll bar's kind also says "scroll", and it is not what the target scrolled out of. */ +function isScrollIndicator(node: SnapshotNode): boolean { + const kind = `${node.type ?? ''} ${node.role ?? ''} ${node.subrole ?? ''}`.toLowerCase(); + return /scroll\s?(bar|indicator)/.test(kind); +} + +/** + * The recovery for a target with no usable frame beside a scroll container. The frame that + * refuses the touch point is also the one that would have named the direction, so the hint names + * the command and leaves the direction open; the off-screen stage owns the directed form. + */ +function scrollContainerRecovery( + node: SnapshotNode, + nodes: SnapshotState['nodes'], + shellQuotedSelector: string | undefined, +): { scrollContainerRef?: string; hint?: string } { + if (hasUsableRect(node)) return {}; + const container = findPrecedingScrollContainer(node, nodes); + if (!container) return {}; + const recovery = shellQuotedSelector + ? `run scroll --until ${shellQuotedSelector} to bring it on screen, then retry with the same selector.` + : 'scroll toward it, then retry with a selector (e.g. text=/id=) rather than an @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.'; + return { + ...(container.ref ? { scrollContainerRef: container.ref } : {}), + hint: `The target reports no frame, so nothing confirms it is on screen, and the frame names no direction. It may be scrolled out of a scrollable container: ${recovery}`, + }; +} + export function resolveNodeTouchPoint( node: SnapshotNode, nodes: SnapshotState['nodes'], - failure: { - invalidMessage: string; - blockedTargetLabel: string; - blockedTargetDetails: { ref: string } | { selector: string }; - }, + failure: { invalidMessage: string; blockedTargetLabel: string } & ( + | { blockedTargetDetails: { ref: string }; shellQuotedSelector?: never } + /** + * The recovery names a command that takes the selector, so the caller hands it over as one + * shell word: this entry's eager closure stops short of the kernel's `shellQuote`. + */ + | { blockedTargetDetails: { selector: string }; shellQuotedSelector: string } + ), ): Point { const visibility = createSnapshotVisibility(nodes); const effectiveViewport = visibility.resolveEffectiveViewport(node); @@ -125,6 +185,7 @@ export function resolveNodeTouchPoint( new AppError('COMMAND_FAILED', failure.invalidMessage, { reason: INTERACTION_ERROR_REASONS.targetBoundsInvalid, ...bareTargetDetails(failure.blockedTargetDetails), + ...scrollContainerRecovery(node, nodes, failure.shellQuotedSelector), }), 'no', ); diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index b2801462be..29f41940ef 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -84,6 +84,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`. - If your target only appears in an off-screen summary, run `scroll --until ` to stop on it, or `scroll --settle` to get the diff with fresh refs. See [Interactions](/docs/commands#interactions). - When agent-device knows which container holds the hidden content, it shows the summary inside that scroll or list container, for example `[content above scroll-area hidden]` or `[content below list hidden]`. +- In a regular (non-`--raw`) snapshot, a listed element whose frame has no area is marked `[unmeasured]`: nothing confirms it is on screen. On iOS, a Flutter scroll view keeps its scrolled-out content this way. If the element is inside scrollable content, run `scroll --until ` before acting on it. - Add `-s "