From 5aabbd726a0ee7c18e90d5b86937f9b74355f8bb Mon Sep 17 00:00:00 2001 From: Emanuel Ruiz <31832843+ruizemanuel@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:43:08 -0300 Subject: [PATCH 1/5] fix(ios): mark zero-area nodes unmeasured and point their press refusal at scroll --until A scroll view can keep its scrolled-out content in the tree with a zero-area frame (Flutter does on iOS). The Simulator AX bridge fold kept such a node as if it were on screen, and pressing one was refused with no recovery. The fold now marks such a node [unmeasured]. A target with no usable frame is still refused; when a scrollable sibling is listed before it, which is how Flutter publishes a scroll view and its content, the refusal names scroll --until with no direction, since the frame gives none. Fixes #4451 --- .../geometry-policy.test.ts | 55 +++++++ .../src/ios-snapshot-engine/geometry.ts | 17 ++ .../runtime/resolution-touch-point.test.ts | 150 ++++++++++++++++++ .../command-surface/src/capture/snapshot.ts | 2 +- packages/selectors/src/node-touch-point.ts | 54 ++++++- website/docs/docs/snapshots.md | 1 + 6 files changed, 277 insertions(+), 2 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts index 9afe2542b1..58da2be2ed 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry-policy.test.ts @@ -257,3 +257,58 @@ test('fails closed when the producer did not read whether the dimming view takes ]); expect(collectModalContainedIndexes(nodes).size).toBe(0); }); + +/** + * A Flutter screen at the top of its content, as the Simulator AX bridge acquires it (#4451): one + * container lists the scroll view and then its content, and the content below the fold keeps a + * zero-area frame at the origin. + */ +function flutterScrollViewAtTop(): RawSnapshotNode[] { + const unmeasured: Rect = { x: 0, y: 0, width: 0, height: 0 }; + return windowWith([ + snapshotNode(1, 1, 0, 'SemanticsObjectContainer', 'Other'), + snapshotNode(2, 2, 1, 'FlutterSemanticsScrollView', 'ScrollView', undefined, { + x: 0, + y: 118, + width: 402, + height: 756, + }), + snapshotNode(3, 2, 1, 'FlutterSemanticsObject', 'StaticText', 'Tall block', { + x: 157, + y: 546, + width: 86, + height: 27, + }), + snapshotNode(4, 2, 1, 'FlutterSemanticsObject', 'Button', 'View all', unmeasured), + ]); +} + +test('marks a kept node whose producer reported a zero-area frame as unmeasured', () => { + const folded = foldIosSnapshot(flutterScrollViewAtTop(), VIEWPORT, false, 'cursor-projected'); + expect( + folded.nodes.map((node) => [node.raw.label ?? node.raw.type, node.raw.presentationHints]), + ).toEqual([ + ['Window', undefined], + ['Other', undefined], + ['ScrollView', undefined], + ['Tall block', undefined], + ['View all', ['unmeasured']], + ]); +}); + +test('leaves a zero-area root unmarked, since nothing measures a root against a parent', () => { + const folded = foldIosSnapshot( + [ + snapshotNode(0, 0, undefined, 'UIWindow', 'Window', undefined, { + x: 0, + y: 0, + width: 0, + height: 0, + }), + ], + VIEWPORT, + false, + 'cursor-projected', + ); + expect(folded.nodes.map((node) => node.raw.presentationHints)).toEqual([undefined]); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 8eae1dfa7a..39ed484e83 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -15,6 +15,15 @@ import type { IosSnapshotPresentationStats, } from './types.ts'; +/** + * The hint a presented node carries when its producer reported a frame that measures nothing. The + * cursor-projected fold keeps such a node, because a missing measurement is not evidence that it + * is off screen, but nothing confirms that it is on screen either: a scroll view keeps its + * scrolled-out content in the tree this way (#4451), as SpringBoard keeps another page's icons + * (#3841). + */ +const UNMEASURED_PRESENTATION_HINT = 'unmeasured'; + export function foldIosSnapshot( nodes: readonly RawSnapshotNode[], viewport: Rect, @@ -142,6 +151,7 @@ function appendFoldedNode( index, depth: keptDepth, parentIndex: keptIndex, + ...unmeasuredHint(node), ...foldedHittability( sourceHittable, node.parentIndex !== undefined, @@ -158,6 +168,13 @@ function appendFoldedNode( return { keptIndex, keptDepth }; } +function unmeasuredHint( + node: RawSnapshotNode, +): Partial> { + if (node.parentIndex === undefined || !node.rect || isPositiveFiniteRect(node.rect)) return {}; + return { presentationHints: [UNMEASURED_PRESENTATION_HINT] }; +} + function foldedHittability( sourceHittable: RawSnapshotNode['hittable'], hasParent: boolean, diff --git a/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts b/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts index 47a26f000d..755bd962bd 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/resolution-touch-point.test.ts @@ -1,6 +1,8 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { Point } from '@agent-device/kernel/snapshot'; +import { attachSnapshotNodeInteractionBounds } from '@agent-device/kernel/rect-center'; +import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { buildUiHierarchySnapshot, @@ -120,6 +122,89 @@ test('runtime press refuses a selector match with a zero-area rect and dispatche assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); assert.equal(details?.reason, 'target_bounds_invalid'); assert.equal(details?.dispatched, 'no'); + // No scroll container holds the icon, so no scroll is offered as the recovery. + assert.equal(details?.scrollContainerRef, undefined); + assert.equal(details?.hint, undefined); + return true; + }, + ); + assert.deepEqual(taps, []); +}); + +/** + * A Flutter screen on iOS at the top of its content (#4451), as the Simulator AX bridge presents it: + * the scroll view and its content are siblings, and the content below the fold keeps a 0x0 rect. + */ +function scrolledOutButtonSnapshot() { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: false, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'ScrollView', + role: 'FlutterSemanticsScrollView', + rect: { x: 0, y: 118, width: 402, height: 756 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + role: 'FlutterSemanticsObject', + label: 'Tall block', + rect: { x: 157, y: 546, width: 86, height: 27 }, + hittable: true, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'Button', + role: 'FlutterSemanticsObject', + label: 'View all', + rect: { x: 0, y: 0, width: 0, height: 0 }, + hittable: false, + }, + ]); +} + +test('runtime press on a zero-area target beside a scroll container names the scroll that reveals it', async () => { + const taps: Point[] = []; + const device = createInteractionDevice(scrolledOutButtonSnapshot(), { + tap: async (_context, point) => { + taps.push(point); + return { ok: true }; + }, + }); + + await assert.rejects( + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.dispatched, 'no'); + assert.equal(details?.scrollContainerRef, '@e2'); + assert.match(String(details?.hint), /scroll --until 'label="View all"'/); + return true; + }, + ); + await assert.rejects( + () => device.interactions.press(ref('@e4'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.scrollContainerRef, '@e2'); + // A scroll expires the ref frame, so the ref recovery steers to a selector instead. + assert.match(String(details?.hint), /retry with a selector/); + assert.doesNotMatch(String(details?.hint), /--until/); return true; }, ); @@ -166,3 +251,68 @@ test('runtime press on a Compose button dispatches its center only once its role dispatched: 'no', }); }); + +test('runtime press offers no scroll for a zero-area target listed before the only scroll container', async () => { + const state = scrolledOutButtonSnapshot(); + const [application, scrollView, , button] = state.nodes; + const device = createInteractionDevice( + makeSnapshotState([application!, { ...button!, index: 1 }, { ...scrollView!, index: 2 }]), + { tap: async () => ({ ok: true }) }, + ); + + await assert.rejects( + () => device.interactions.press(selector('label="View all"'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.scrollContainerRef, undefined); + assert.equal(details?.hint, undefined); + return true; + }, + ); +}); + +test('runtime press offers no scroll for a measured target whose input bounds exclude it', async () => { + const root = { x: 0, y: 0, width: 1080, height: 2400 }; + // A row with a real frame, wholly under the navigation bar the input bounds stop at. + const row = attachSnapshotNodeInteractionBounds( + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + identifier: 'row', + label: 'Row', + hittable: true, + rect: { x: 0, y: 2305, width: 1080, height: 95 }, + }, + { ...root, height: 2274 }, + ); + const snapshot = buildSnapshotState( + { + backend: 'android', + producer: 'android-uiautomator', + nodes: [ + { index: 0, depth: 0, type: 'Application', rect: root }, + { index: 1, depth: 1, parentIndex: 0, type: 'ScrollView', rect: root, hittable: true }, + row, + ], + }, + { snapshotRaw: true }, + ); + const device = createInteractionDevice(snapshot, { + platform: 'android', + tap: async () => ({ ok: true }), + }); + + await assert.rejects( + () => device.interactions.press(selector('id=row'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_bounds_invalid'); + assert.equal(details?.scrollContainerRef, undefined); + assert.equal(details?.hint, undefined); + return true; + }, + ); +}); diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index bfdf2f346c..61de65453b 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -75,7 +75,7 @@ export const snapshotCommandFacet = defineCommandFacet({ name: SNAPSHOT_COMMAND_NAME, text: { summary: 'Capture or diff the accessibility tree', - cliDetail: `Scoped output (--scope) renders every returned ref with its ref generation pin (@eN~s) and one Refs generation line, so a pasted ref targets the ref frame that issued it; --json keeps plain refs plus the response-level refsGeneration, and --raw keeps byte-for-byte output. Regular Android ADB snapshots clip app rectangles outside measured status and navigation bars, including launcher-owned navigation; fully covered content is omitted and --raw retains reported bounds. Ref and selector taps honor measured visible bounds even for refs from raw snapshots. Regular iOS snapshots clip overflowing elements to their visible area and omit fully outside elements. --json reports clipping counts in snapshotQualityNotes with code regular-geometry-clipped; --raw retains reported frames. --observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation or a readable system alert over the app, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. Android Appium JSON snapshots publish a display viewport only when measurements before and after the tree agree, independent of tree extent or keyboard content resizing. On Android, JSON value carries field content rather than static labels; saved selectors and replay scripts using value= for static labels must use label= instead; a reported showing hint means value is empty, and text equal to the placeholder is preserved when hint state is unknown. HarmonyOS reads the on-screen keyboard's own system window and publishes the keyboard field of --json output: the field is visibility-only — visible or hidden per the keyboard window's z-order column; a failed probe, an unreadable or conflicting z-order, or a table without the panel publishes unmeasurable, never a guessed band. When an iOS snapshot reports a screen viewport, inspect rects with snapshot -i --json and press the intended control's center, then verify with diff snapshot -i or snapshot --diff. When a fallback cannot establish the screen origin, viewport is omitted and coordinate-derived element actions refuse with window-origin-unknown before input; the snapshot may warn about the unavailable origin. Use a plain screenshot to choose screen coordinates. An empty text field with no label or value prints its hint as [placeholder:"..."]; label=, text= and find do not match that hint. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, + cliDetail: `Scoped output (--scope) renders every returned ref with its ref generation pin (@eN~s) and one Refs generation line, so a pasted ref targets the ref frame that issued it; --json keeps plain refs plus the response-level refsGeneration, and --raw keeps byte-for-byte output. Regular Android ADB snapshots clip app rectangles outside measured status and navigation bars, including launcher-owned navigation; fully covered content is omitted and --raw retains reported bounds. Ref and selector taps honor measured visible bounds even for refs from raw snapshots. Regular iOS snapshots clip overflowing elements to their visible area and omit fully outside elements. On the Simulator AX bridge, an element that reports a frame measuring nothing is kept and marked [unmeasured]: nothing confirms it is on screen. If it is inside scrollable content, reach it with scroll --until before acting on it. --json reports clipping counts in snapshotQualityNotes with code regular-geometry-clipped; --raw retains reported frames. --observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation or a readable system alert over the app, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. Android Appium JSON snapshots publish a display viewport only when measurements before and after the tree agree, independent of tree extent or keyboard content resizing. On Android, JSON value carries field content rather than static labels; saved selectors and replay scripts using value= for static labels must use label= instead; a reported showing hint means value is empty, and text equal to the placeholder is preserved when hint state is unknown. HarmonyOS reads the on-screen keyboard's own system window and publishes the keyboard field of --json output: the field is visibility-only — visible or hidden per the keyboard window's z-order column; a failed probe, an unreadable or conflicting z-order, or a table without the panel publishes unmeasurable, never a guessed band. When an iOS snapshot reports a screen viewport, inspect rects with snapshot -i --json and press the intended control's center, then verify with diff snapshot -i or snapshot --diff. When a fallback cannot establish the screen origin, viewport is omitted and coordinate-derived element actions refuse with window-origin-unknown before input; the snapshot may warn about the unavailable origin. Use a plain screenshot to choose screen coordinates. An empty text field with no label or value prints its hint as [placeholder:"..."]; label=, text= and find do not match that hint. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, }, metadata: snapshotCommandMetadata, run: (client, input) => client.capture.snapshot(input), diff --git a/packages/selectors/src/node-touch-point.ts b/packages/selectors/src/node-touch-point.ts index 96f248c05c..b2bd5ad83b 100644 --- a/packages/selectors/src/node-touch-point.ts +++ b/packages/selectors/src/node-touch-point.ts @@ -3,7 +3,11 @@ import type { Point, SnapshotAddressableNode, SnapshotNode } from '@agent-device import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { normalizeRect } from '@agent-device/kernel/rect-center'; import { normalizeRef } from '@agent-device/kernel/snapshot'; -import { createSnapshotVisibility, isFillableType } from '@agent-device/contracts/snapshot'; +import { + createSnapshotVisibility, + isFillableType, + isScrollableNodeLike, +} from '@agent-device/contracts/snapshot'; import type { Platform } from '@agent-device/kernel/device'; import { INTERACTION_ERROR_REASONS } from './interaction-error.ts'; import { isDescendantOf, resolveInteractionTouchPoint } from './interaction-touch-point.ts'; @@ -104,6 +108,53 @@ function isTextField(node: SnapshotNode, platform: Platform): boolean { ); } +/** + * The scroll container a target with no usable frame may be scrolled out of (#4451): the nearest + * scrollable sibling listed before it. That is the shape Flutter publishes on iOS: its + * accessibility container lists a scroll view and then that scroll view's content, and keeps the + * scrolled-out content with a zero-area frame. A target under a framed scroll ancestor never + * reaches here, because the off-screen stage refuses it first. + */ +function findPrecedingScrollContainer( + node: SnapshotNode, + nodes: SnapshotState['nodes'], +): SnapshotNode | null { + return ( + nodes + .filter( + (candidate) => + candidate.parentIndex === node.parentIndex && + candidate.index < node.index && + hasUsableRect(candidate) && + isScrollableNodeLike(candidate), + ) + .at(-1) ?? null + ); +} + +/** + * The recovery for a target with no usable frame beside a scroll container. The frame that + * refuses the touch point is also the one that would have named the direction, so the hint names + * the command and leaves the direction open; the off-screen stage owns the directed form. + */ +function scrollContainerRecovery( + node: SnapshotNode, + nodes: SnapshotState['nodes'], + target: { ref: string } | { selector: string }, +): { scrollContainerRef?: string; hint?: string } { + if (hasUsableRect(node)) return {}; + const container = findPrecedingScrollContainer(node, nodes); + if (!container) return {}; + const recovery = + 'selector' in target + ? `run scroll --until '${target.selector}' to bring it on screen, then retry with the same selector.` + : 'scroll toward it, then retry with a selector (e.g. text=/id=) rather than an @ref — the scroll expires the ref frame.'; + return { + ...(container.ref ? { scrollContainerRef: `@${container.ref}` } : {}), + hint: `The target reports no frame, so nothing confirms it is on screen, and the frame names no direction. It may be scrolled out of a scrollable container: ${recovery}`, + }; +} + export function resolveNodeTouchPoint( node: SnapshotNode, nodes: SnapshotState['nodes'], @@ -125,6 +176,7 @@ export function resolveNodeTouchPoint( new AppError('COMMAND_FAILED', failure.invalidMessage, { reason: INTERACTION_ERROR_REASONS.targetBoundsInvalid, ...bareTargetDetails(failure.blockedTargetDetails), + ...scrollContainerRecovery(node, nodes, failure.blockedTargetDetails), }), 'no', ); diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index b2801462be..d4f1896d70 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -84,6 +84,7 @@ and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issue - Off-screen interactive content is collapsed into summaries that show only a few labels, such as `[off-screen below] 3 interactive items: "Privacy", "Battery", "About"`. - If your target only appears in an off-screen summary, run `scroll --until ` to stop on it, or `scroll --settle` to get the diff with fresh refs. See [Interactions](/docs/commands#interactions). - When agent-device knows which container holds the hidden content, it shows the summary inside that scroll or list container, for example `[content above scroll-area hidden]` or `[content below list hidden]`. +- On the iOS Simulator AX bridge, an element that reports a frame measuring nothing is kept and marked `[unmeasured]`: nothing confirms it is on screen. A Flutter scroll view keeps its scrolled-out content this way. If the element is inside scrollable content, run `scroll --until ` before acting on it. XCTest runner snapshots do not add this mark. - Add `-s "