From a8995d3b4e1bbdf66b18ac2da2bb796db70c0aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:09:06 +0100 Subject: [PATCH 01/14] fix(settle): never diff a settle against a full-tree snapshot After a full snapshot (no -i), the settle diff compared that tree with the interactive view the settle loop reads, so every wrapper node the view omits read as removed while nothing changed. The baseline now carries whether it is a full view, and such a settle diffs from its own settled capture and says so in its hint. Later settles diff interactive against interactive. Fixes #4486 Co-Authored-By: Claude Opus 5.5 --- packages/capture-kit/src/snapshot-baseline.ts | 8 ++- .../runtime/settle-view-baseline.test.ts | 61 +++++++++++++++++++ .../commands/interaction/runtime/settle.ts | 15 ++++- packages/contracts/src/interaction.ts | 6 ++ website/docs/docs/commands.md | 2 +- 5 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts diff --git a/packages/capture-kit/src/snapshot-baseline.ts b/packages/capture-kit/src/snapshot-baseline.ts index 93fe201d17..ce907a47f0 100644 --- a/packages/capture-kit/src/snapshot-baseline.ts +++ b/packages/capture-kit/src/snapshot-baseline.ts @@ -16,7 +16,10 @@ import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; */ export function surfaceScopedNodes( snapshot: - | Pick + | Pick< + SnapshotState, + 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation' + > | undefined, ): SurfaceScopedNodes { return { @@ -25,5 +28,8 @@ export function surfaceScopedNodes( ? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId } : {}), ...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}), + ...(snapshot?.presentation && snapshot.presentation.interactiveOnly !== true + ? { fullView: true as const } + : {}), }; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts new file mode 100644 index 0000000000..9398c41458 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createSettleDevice } from './__tests__/settle-device-fixtures.ts'; + +// #4486: a full `snapshot` keeps wrapper nodes that the interactive view the settle loop reads +// omits. A press that changes nothing must not report those wrappers as removed. +// +// Budgets are injected (fake clock) — no real waiting. + +const DIALOG = ['Device name', 'Cancel', 'OK'].map((label, index) => ({ + index: index + 1, + depth: 1, + parentIndex: 0, + type: index === 0 ? 'StaticText' : 'Button', + label, + rect: { x: 10, y: 100 + index * 60, width: 200, height: 40 }, + hittable: true, +})); + +function dialogWithWrapper(interactiveOnly: boolean): SnapshotState { + return { + ...makeSnapshotState([ + { index: 0, depth: 0, type: 'Group', rect: { x: 0, y: 0, width: 400, height: 800 } }, + ...DIALOG, + ]), + presentation: interactiveOnly ? { interactiveOnly: true } : {}, + }; +} + +async function pressAfterSnapshot(stored: SnapshotState) { + const device = createSettleDevice({ + stored, + // The interactive view of the unchanged dialog: the wrapper is not in it. + captureSnapshot: () => ({ + snapshot: makeSnapshotState(DIALOG.map((node, index) => ({ ...node, index, depth: 0 }))), + }), + }); + const result = await device.interactions.press(ref('@e3'), { + session: 'default', + settle: {}, + }); + assert.ok(result.settle?.settled); + return result.settle; +} + +test('a settle after a full snapshot reports no removals the view change caused', async () => { + const settle = await pressAfterSnapshot(dialogWithWrapper(false)); + + assert.equal(settle?.diff?.summary.removals, 0); + assert.match(settle?.hint ?? '', /previous snapshot was a full tree/); +}); + +test('a settle after an interactive snapshot still reports what left the screen', async () => { + const settle = await pressAfterSnapshot(dialogWithWrapper(true)); + + assert.equal(settle?.diff?.summary.removals, 1); + assert.doesNotMatch(settle?.hint ?? '', /full tree/); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 0a0f17d5a5..8517d6ff14 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -79,6 +79,9 @@ const MAX_SETTLE_TAIL_ENTRIES = 20; export const NEVER_SETTLED_HINT = 'The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.'; +const FULL_VIEW_BASELINE_HINT = + 'The previous snapshot was a full tree and settle reads the interactive view, so this diff starts from the settled screen instead of listing changes. Later settles diff against it.'; + const SETTLE_CAPTURE_STALLED_HINT = 'A snapshot capture stalled past the settle budget, so no settled diff is shown. The action itself succeeded; observe with wait stable or snapshot.'; @@ -192,6 +195,10 @@ async function readSettledOutcome( // replacement presented as change within one surface, refs included, so it is refused and the // transition is disclosed instead. const surfaceChange = resolvePostActionSurfaceChange(params.baseline, settledCapture); + // A full-tree baseline holds nodes the interactive settle view never reads. The settled view + // becomes the baseline instead, so the diff cannot report those nodes as removed. + const diffBaseline = params.baseline.fullView ? settledNodes : params.baseline.nodes; + const diffShown = outcome.settled && stored && !surfaceChange; return { observation: { ...observation, @@ -203,15 +210,17 @@ async function readSettledOutcome( // captures are intentionally diff-less: they are not a stable // observation, so surfacing refs would invite agents to act on // advisory state. - ...(outcome.settled && stored && !surfaceChange + ...(diffShown ? buildSettleDiffAndTail( - params.baseline.nodes, + diffBaseline, settledNodes, params.actionPoint, session?.appBundleId, ) : {}), - ...resolveSettleHint(outcome, stored, settledNodes.length, surfaceChange), + ...(diffShown && params.baseline.fullView + ? { hint: FULL_VIEW_BASELINE_HINT } + : resolveSettleHint(outcome, stored, settledNodes.length, surfaceChange)), }, settledCapture, }; diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 27cec69e84..9753ffd004 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -160,6 +160,12 @@ export type SurfaceScopedNodes = { * Absent means the capture's producer published no fact. */ presentedSurface?: PresentedSurface; + /** + * The nodes come from a full-tree `snapshot` (no `-i`). A settle reads the interactive view, so a + * diff against these nodes would report every node that view omits as removed. Absent means an + * interactive view, or a capture that recorded no presentation. + */ + fullView?: true; }; /** Which tree a resolved `@ref`'s node was read from (ADR 0014, "Mutation admission and read behavior"). */ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 034da43e81..3d36d6fc44 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -494,7 +494,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), the next settle has no interactive baseline, so it lists no changes, says so in its hint, and later settles diff against its view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 7181fe94e3627a87df34ed5ca0c24335ff0a40c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:19:56 +0100 Subject: [PATCH 02/14] fix(settle): keep loop hints beside the full-view baseline hint Address review: the full-view hint replaced the loop hint (such as the tiny-tree readiness hint) instead of joining it, the wording could read as diffing against the full view, and the interactive-baseline test used a tree that held the full-view wrapper. Co-Authored-By: Claude Opus 5.5 --- .../runtime/settle-view-baseline.test.ts | 57 ++++++++++--------- .../commands/interaction/runtime/settle.ts | 19 +++++-- website/docs/docs/commands.md | 2 +- 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 9398c41458..8553b9bd2d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -10,33 +10,24 @@ import { createSettleDevice } from './__tests__/settle-device-fixtures.ts'; // // Budgets are injected (fake clock) — no real waiting. -const DIALOG = ['Device name', 'Cancel', 'OK'].map((label, index) => ({ - index: index + 1, - depth: 1, - parentIndex: 0, - type: index === 0 ? 'StaticText' : 'Button', - label, - rect: { x: 10, y: 100 + index * 60, width: 200, height: 40 }, - hittable: true, -})); - -function dialogWithWrapper(interactiveOnly: boolean): SnapshotState { - return { - ...makeSnapshotState([ - { index: 0, depth: 0, type: 'Group', rect: { x: 0, y: 0, width: 400, height: 800 } }, - ...DIALOG, - ]), - presentation: interactiveOnly ? { interactiveOnly: true } : {}, - }; +const DIALOG_LABELS = ['Device name', 'Cancel', 'OK']; + +function dialogNodes(labels: readonly string[], firstIndex = 0) { + return labels.map((label, index) => ({ + index: firstIndex + index, + depth: firstIndex === 0 ? 0 : 1, + ...(firstIndex === 0 ? {} : { parentIndex: 0 }), + type: label === 'Device name' ? 'StaticText' : 'Button', + label, + rect: { x: 10, y: 100 + DIALOG_LABELS.indexOf(label) * 60, width: 200, height: 40 }, + hittable: true, + })); } -async function pressAfterSnapshot(stored: SnapshotState) { +async function pressAfterSnapshot(stored: SnapshotState, settledLabels: readonly string[]) { const device = createSettleDevice({ stored, - // The interactive view of the unchanged dialog: the wrapper is not in it. - captureSnapshot: () => ({ - snapshot: makeSnapshotState(DIALOG.map((node, index) => ({ ...node, index, depth: 0 }))), - }), + captureSnapshot: () => ({ snapshot: makeSnapshotState(dialogNodes(settledLabels)) }), }); const result = await device.interactions.press(ref('@e3'), { session: 'default', @@ -47,14 +38,28 @@ async function pressAfterSnapshot(stored: SnapshotState) { } test('a settle after a full snapshot reports no removals the view change caused', async () => { - const settle = await pressAfterSnapshot(dialogWithWrapper(false)); + const fullTree: SnapshotState = { + ...makeSnapshotState([ + { index: 0, depth: 0, type: 'Group', rect: { x: 0, y: 0, width: 400, height: 800 } }, + ...dialogNodes(DIALOG_LABELS, 1), + ]), + presentation: {}, + }; + + const settle = await pressAfterSnapshot(fullTree, DIALOG_LABELS); assert.equal(settle?.diff?.summary.removals, 0); - assert.match(settle?.hint ?? '', /previous snapshot was a full tree/); + // The tiny-tree readiness hint the settled screen earns is kept beside the view-change hint. + assert.match(settle?.hint ?? '', /previous snapshot was a full tree.*nearly-empty tree/); }); test('a settle after an interactive snapshot still reports what left the screen', async () => { - const settle = await pressAfterSnapshot(dialogWithWrapper(true)); + const interactiveTree: SnapshotState = { + ...makeSnapshotState(dialogNodes(DIALOG_LABELS)), + presentation: { interactiveOnly: true }, + }; + + const settle = await pressAfterSnapshot(interactiveTree, ['Cancel', 'OK']); assert.equal(settle?.diff?.summary.removals, 1); assert.doesNotMatch(settle?.hint ?? '', /full tree/); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 8517d6ff14..e664db4cca 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -80,7 +80,7 @@ export const NEVER_SETTLED_HINT = 'The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.'; const FULL_VIEW_BASELINE_HINT = - 'The previous snapshot was a full tree and settle reads the interactive view, so this diff starts from the settled screen instead of listing changes. Later settles diff against it.'; + 'The previous snapshot was a full tree and settle reads the interactive view, so this diff starts from the settled screen instead of listing changes. Later settles diff against that settled interactive view.'; const SETTLE_CAPTURE_STALLED_HINT = 'A snapshot capture stalled past the settle budget, so no settled diff is shown. The action itself succeeded; observe with wait stable or snapshot.'; @@ -218,9 +218,10 @@ async function readSettledOutcome( session?.appBundleId, ) : {}), - ...(diffShown && params.baseline.fullView - ? { hint: FULL_VIEW_BASELINE_HINT } - : resolveSettleHint(outcome, stored, settledNodes.length, surfaceChange)), + ...withFullViewBaselineHint( + resolveSettleHint(outcome, stored, settledNodes.length, surfaceChange), + diffShown && params.baseline.fullView === true, + ), }, settledCapture, }; @@ -454,6 +455,16 @@ function capSettleDiffLines(changed: T[]): T[] { * itself reports rather than in place of it: the transition explains the missing diff, and the * loop's own verdict (stalled, never settled, sparse, tiny tree) still explains the capture. */ +function withFullViewBaselineHint( + resolved: { hint?: string }, + fullViewBaselineReset: boolean, +): { hint?: string } { + if (!fullViewBaselineReset) return resolved; + return { + hint: resolved.hint ? `${FULL_VIEW_BASELINE_HINT} ${resolved.hint}` : FULL_VIEW_BASELINE_HINT, + }; +} + function resolveSettleHint( outcome: { settled: boolean; stalled: boolean }, stored: boolean, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 3d36d6fc44..47ff0c27c6 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -494,7 +494,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), the next settle has no interactive baseline, so it lists no changes, says so in its hint, and later settles diff against its view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), the next settle has no interactive baseline, so it lists no changes, says so in its hint, and later settles diff against that settled interactive view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 3d1ae0e10675e385150b593d5e0f9d8c5012d688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:25:21 +0100 Subject: [PATCH 03/14] fix(settle): mark only a full-view ref frame, not selector captures Address review: selector resolution captures are full trees and fed the settle baseline through preAction, so marking every non-interactive presentation reset every selector-targeted settle to +0 -0. Only the ref frame the agent read as a full snapshot is marked now; selector and generic-route baselines keep their diff. Co-Authored-By: Claude Opus 5.5 --- packages/capture-kit/src/snapshot-baseline.ts | 8 +- .../runtime/settle-view-baseline.test.ts | 97 +++++++++++++------ .../commands/interaction/runtime/settle.ts | 21 +++- packages/contracts/src/interaction.ts | 6 +- website/docs/docs/commands.md | 2 +- 5 files changed, 91 insertions(+), 43 deletions(-) diff --git a/packages/capture-kit/src/snapshot-baseline.ts b/packages/capture-kit/src/snapshot-baseline.ts index ce907a47f0..93fe201d17 100644 --- a/packages/capture-kit/src/snapshot-baseline.ts +++ b/packages/capture-kit/src/snapshot-baseline.ts @@ -16,10 +16,7 @@ import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; */ export function surfaceScopedNodes( snapshot: - | Pick< - SnapshotState, - 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation' - > + | Pick | undefined, ): SurfaceScopedNodes { return { @@ -28,8 +25,5 @@ export function surfaceScopedNodes( ? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId } : {}), ...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}), - ...(snapshot?.presentation && snapshot.presentation.interactiveOnly !== true - ? { fullView: true as const } - : {}), }; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 8553b9bd2d..30a8eee2b5 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -1,9 +1,14 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { ref } from './selector-read-utils.ts'; -import { createSettleDevice } from './__tests__/settle-device-fixtures.ts'; +import { createAgentDevice } from '../../../runtime.ts'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { ref, selector } from './selector-read-utils.ts'; +import { createFakeClock } from './__tests__/settle-device-fixtures.ts'; // #4486: a full `snapshot` keeps wrapper nodes that the interactive view the settle loop reads // omits. A press that changes nothing must not report those wrappers as removed. @@ -24,43 +29,79 @@ function dialogNodes(labels: readonly string[], firstIndex = 0) { })); } -async function pressAfterSnapshot(stored: SnapshotState, settledLabels: readonly string[]) { - const device = createSettleDevice({ - stored, - captureSnapshot: () => ({ snapshot: makeSnapshotState(dialogNodes(settledLabels)) }), - }); - const result = await device.interactions.press(ref('@e3'), { - session: 'default', - settle: {}, - }); - assert.ok(result.settle?.settled); - return result.settle; -} - -test('a settle after a full snapshot reports no removals the view change caused', async () => { - const fullTree: SnapshotState = { +/** The dialog as a full `snapshot` reads it: the wrapper the interactive view omits, then the dialog. */ +function fullDialogTree(): SnapshotState { + return { ...makeSnapshotState([ { index: 0, depth: 0, type: 'Group', rect: { x: 0, y: 0, width: 400, height: 800 } }, ...dialogNodes(DIALOG_LABELS, 1), ]), presentation: {}, }; +} - const settle = await pressAfterSnapshot(fullTree, DIALOG_LABELS); +function interactiveTree(labels: readonly string[]): SnapshotState { + return { ...makeSnapshotState(dialogNodes(labels)), presentation: { interactiveOnly: true } }; +} + +function createDevice(params: { + refFrame: SnapshotState; + capture: (call: number) => SnapshotState; +}) { + let calls = 0; + return createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async () => ({ snapshot: params.capture((calls += 1)) }), + tap: async () => ({ ok: true }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([ + { name: 'default', snapshot: params.refFrame, refFrameSnapshot: params.refFrame }, + ]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); +} - assert.equal(settle?.diff?.summary.removals, 0); +test('a ref press after a full snapshot reports no removals the view change caused', async () => { + const device = createDevice({ + refFrame: fullDialogTree(), + capture: () => interactiveTree(DIALOG_LABELS), + }); + + const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); + + assert.equal(result.settle?.diff?.summary.removals, 0); // The tiny-tree readiness hint the settled screen earns is kept beside the view-change hint. - assert.match(settle?.hint ?? '', /previous snapshot was a full tree.*nearly-empty tree/); + assert.match(result.settle?.hint ?? '', /previous snapshot was a full tree.*nearly-empty tree/); }); -test('a settle after an interactive snapshot still reports what left the screen', async () => { - const interactiveTree: SnapshotState = { - ...makeSnapshotState(dialogNodes(DIALOG_LABELS)), - presentation: { interactiveOnly: true }, - }; +test('a ref press after an interactive snapshot still reports what left the screen', async () => { + const device = createDevice({ + refFrame: interactiveTree(DIALOG_LABELS), + capture: () => interactiveTree(['Cancel', 'OK']), + }); + + const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); + + assert.equal(result.settle?.diff?.summary.removals, 1); + assert.doesNotMatch(result.settle?.hint ?? '', /full tree/); +}); + +test('a selector press keeps its diff although its resolution read a full tree', async () => { + const device = createDevice({ + refFrame: interactiveTree(DIALOG_LABELS), + // Capture 1 resolves the selector on a full tree; the settle loop then reads the interactive + // view, where the dialog title is gone after the press. + capture: (call) => (call === 1 ? fullDialogTree() : interactiveTree(['Cancel', 'OK'])), + }); - const settle = await pressAfterSnapshot(interactiveTree, ['Cancel', 'OK']); + const result = await device.interactions.press(selector('label=OK'), { + session: 'default', + settle: {}, + }); - assert.equal(settle?.diff?.summary.removals, 1); - assert.doesNotMatch(settle?.hint ?? '', /full tree/); + assert.ok((result.settle?.diff?.summary.removals ?? 0) > 0); + assert.doesNotMatch(result.settle?.hint ?? '', /full tree/); }); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index e664db4cca..aa7cfb4afc 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -21,6 +21,7 @@ import type { SettleTailEntry, SurfaceScopedNodes, } from '@agent-device/contracts/interaction'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { RuntimeCommand } from '../../runtime-types.ts'; import type { CapturedSnapshot } from './selector-read-shared.ts'; import { @@ -265,7 +266,19 @@ function authorizedRefBaseline( session: CommandSessionRecord | undefined, ): SurfaceScopedNodes | undefined { if (resolved.kind !== 'ref') return undefined; - return nonEmptyBaseline(surfaceScopedNodes(session?.refFrameSnapshot)); + return nonEmptyBaseline(refFrameBaseline(session?.refFrameSnapshot)); +} + +/** + * The ref frame the agent last read. A frame read as a full tree (`snapshot` without `-i`) is marked + * `fullView`, because the settle loop reads the interactive view. Selector captures are full trees + * too, but no agent chose that view, so only the ref frame carries the mark. + */ +function refFrameBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNodes { + const baseline = surfaceScopedNodes(snapshot); + return snapshot?.presentation && snapshot.presentation.interactiveOnly !== true + ? { ...baseline, fullView: true } + : baseline; } /** @@ -275,9 +288,9 @@ function authorizedRefBaseline( * dismissal (#3565). */ function sessionBaseline(session: CommandSessionRecord | undefined): SurfaceScopedNodes { - const { presentedSurface: _unverified, ...baseline } = surfaceScopedNodes( - session?.refFrameSnapshot ?? session?.snapshot, - ); + const { presentedSurface: _unverified, ...baseline } = session?.refFrameSnapshot + ? refFrameBaseline(session.refFrameSnapshot) + : surfaceScopedNodes(session?.snapshot); return baseline; } diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 9753ffd004..c97b9dca79 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -161,9 +161,9 @@ export type SurfaceScopedNodes = { */ presentedSurface?: PresentedSurface; /** - * The nodes come from a full-tree `snapshot` (no `-i`). A settle reads the interactive view, so a - * diff against these nodes would report every node that view omits as removed. Absent means an - * interactive view, or a capture that recorded no presentation. + * The nodes are a ref frame the agent read as a full tree (`snapshot` without `-i`). A settle reads + * the interactive view, so a diff against these nodes would report every node that view omits as + * removed. Only the settle baseline sets it. */ fullView?: true; }; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 47ff0c27c6..01c19a607c 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -494,7 +494,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), the next settle has no interactive baseline, so it lists no changes, says so in its hint, and later settles diff against that settled interactive view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s has no interactive baseline. That settle lists no changes, says so in its hint, and later settles diff against that settled interactive view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From a79152d9bec63ab47605a790b2f62bda28c70432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:32:34 +0100 Subject: [PATCH 04/14] refactor(settle): keep resolveSettleHint beside its doc comment Address review: the new hint helper sat between resolveSettleHint and its JSDoc. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/interaction/runtime/settle.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index aa7cfb4afc..7aba855a01 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -463,11 +463,6 @@ function capSettleDiffLines(changed: T[]): T[] { return kept; } -/** - * The settled observation's hint. A surface change (#2438) is stated alongside whatever the loop - * itself reports rather than in place of it: the transition explains the missing diff, and the - * loop's own verdict (stalled, never settled, sparse, tiny tree) still explains the capture. - */ function withFullViewBaselineHint( resolved: { hint?: string }, fullViewBaselineReset: boolean, @@ -478,6 +473,11 @@ function withFullViewBaselineHint( }; } +/** + * The settled observation's hint. A surface change (#2438) is stated alongside whatever the loop + * itself reports rather than in place of it: the transition explains the missing diff, and the + * loop's own verdict (stalled, never settled, sparse, tiny tree) still explains the capture. + */ function resolveSettleHint( outcome: { settled: boolean; stalled: boolean }, stored: boolean, From 9a623dc60db0122a4c053ba27b3479f3dc62be4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:47:48 +0100 Subject: [PATCH 05/14] fix(settle): list additions against a full-view ref frame Address review: resetting the baseline to the settled capture made every settle after a full snapshot report +0 -0, even when the press navigated. The diff now runs against the full ref frame and lists only added lines: the interactive view adds no node of its own, so additions are evidence, while removals of nodes that view never reads are not. Co-Authored-By: Claude Opus 5.5 --- .../runtime/settle-view-baseline.test.ts | 14 +++++++++++ .../commands/interaction/runtime/settle.ts | 23 ++++++++++--------- website/docs/docs/commands.md | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 30a8eee2b5..3edcbffcfd 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -77,6 +77,20 @@ test('a ref press after a full snapshot reports no removals the view change caus assert.match(result.settle?.hint ?? '', /previous snapshot was a full tree.*nearly-empty tree/); }); +test('a ref press after a full snapshot still lists what the press added', async () => { + const device = createDevice({ + refFrame: fullDialogTree(), + // The press opens another screen: its controls are new, the dialog is gone. + capture: () => interactiveTree(['Next', 'Back', 'Home', 'Menu', 'Help']), + }); + + const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); + + assert.equal(result.settle?.diff?.summary.additions, 5); + assert.equal(result.settle?.diff?.summary.removals, 0); + assert.ok(result.settle?.diff?.lines.every((line) => line.kind === 'added' && line.ref)); +}); + test('a ref press after an interactive snapshot still reports what left the screen', async () => { const device = createDevice({ refFrame: interactiveTree(DIALOG_LABELS), diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 7aba855a01..a3279c4dbb 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -81,7 +81,7 @@ export const NEVER_SETTLED_HINT = 'The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.'; const FULL_VIEW_BASELINE_HINT = - 'The previous snapshot was a full tree and settle reads the interactive view, so this diff starts from the settled screen instead of listing changes. Later settles diff against that settled interactive view.'; + 'The previous snapshot was a full tree and settle reads the interactive view, so this diff lists only added lines; removals are not listed. Later settles diff against that settled interactive view.'; const SETTLE_CAPTURE_STALLED_HINT = 'A snapshot capture stalled past the settle budget, so no settled diff is shown. The action itself succeeded; observe with wait stable or snapshot.'; @@ -196,9 +196,6 @@ async function readSettledOutcome( // replacement presented as change within one surface, refs included, so it is refused and the // transition is disclosed instead. const surfaceChange = resolvePostActionSurfaceChange(params.baseline, settledCapture); - // A full-tree baseline holds nodes the interactive settle view never reads. The settled view - // becomes the baseline instead, so the diff cannot report those nodes as removed. - const diffBaseline = params.baseline.fullView ? settledNodes : params.baseline.nodes; const diffShown = outcome.settled && stored && !surfaceChange; return { observation: { @@ -213,7 +210,7 @@ async function readSettledOutcome( // advisory state. ...(diffShown ? buildSettleDiffAndTail( - diffBaseline, + params.baseline, settledNodes, params.actionPoint, session?.appBundleId, @@ -301,7 +298,7 @@ function nonEmptyBaseline( } function buildSettleDiff( - baselineNodes: SnapshotNode[], + baseline: SurfaceScopedNodes, settledNodes: SnapshotNode[], appBundleId: string | undefined, ): NonNullable { @@ -311,30 +308,34 @@ function buildSettleDiff( // snapshot), extra baseline-only lines surface as removals — advisory noise, // the same baseline caveat --verify's changedFromBefore already accepts. const diff = buildSnapshotDiff( - withoutSettleChrome(baselineNodes, appBundleId), + withoutSettleChrome(baseline.nodes, appBundleId), withoutSettleChrome(settledNodes, appBundleId), { flatten: true, withRefs: true }, ); - const changed = diff.lines.filter((line) => line.kind !== 'unchanged'); + // A full-tree baseline holds nodes the interactive settle view never reads, so its removals are + // not evidence and are left out. An added line is: the interactive view adds no node of its own. + const changed = diff.lines.filter( + (line) => line.kind === 'added' || (line.kind === 'removed' && !baseline.fullView), + ); const lines = capSettleDiffLines(changed).map((line) => ({ kind: line.kind as 'added' | 'removed', text: line.text, ...(line.ref ? { ref: line.ref } : {}), })); return { - summary: diff.summary, + summary: baseline.fullView ? { ...diff.summary, removals: 0 } : diff.summary, lines, ...(changed.length > lines.length ? { truncated: true as const } : {}), }; } function buildSettleDiffAndTail( - baselineNodes: SnapshotNode[], + baseline: SurfaceScopedNodes, settledNodes: SnapshotNode[], actionPoint: Point | undefined, appBundleId: string | undefined, ): Pick { - const diff = buildSettleDiff(baselineNodes, settledNodes, appBundleId); + const diff = buildSettleDiff(baseline, settledNodes, appBundleId); return { diff, ...buildSettleTail(diff, settledNodes, actionPoint, appBundleId) }; } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 01c19a607c..3f4d88cbe8 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -494,7 +494,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s has no interactive baseline. That settle lists no changes, says so in its hint, and later settles diff against that settled interactive view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s has no interactive baseline. That settle lists only added lines, not removals, says so in its hint, and later settles diff against that settled interactive view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 6e5ada80cd40bac60301d76874388f981ebf1eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 19:58:27 +0100 Subject: [PATCH 06/14] test(settle): pin the added-line count after a full-view baseline Address review: the every() check passed on an empty line list. The settle comment now states that only Android's interactive view adds no node of its own; an iOS row rewrite can still read as added, as before. Co-Authored-By: Claude Opus 5.5 --- .../commands/interaction/runtime/settle-view-baseline.test.ts | 1 + .../command-runtime/src/commands/interaction/runtime/settle.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 3edcbffcfd..19cc5e1e8c 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -88,6 +88,7 @@ test('a ref press after a full snapshot still lists what the press added', async assert.equal(result.settle?.diff?.summary.additions, 5); assert.equal(result.settle?.diff?.summary.removals, 0); + assert.equal(result.settle?.diff?.lines.length, 5); assert.ok(result.settle?.diff?.lines.every((line) => line.kind === 'added' && line.ref)); }); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index a3279c4dbb..412b098603 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -313,7 +313,8 @@ function buildSettleDiff( { flatten: true, withRefs: true }, ); // A full-tree baseline holds nodes the interactive settle view never reads, so its removals are - // not evidence and are left out. An added line is: the interactive view adds no node of its own. + // not evidence and are left out. Added lines stay: on Android the interactive view adds no node + // of its own, while an iOS row rewrite can also read as added, as it did before #4486. const changed = diff.lines.filter( (line) => line.kind === 'added' || (line.kind === 'removed' && !baseline.fullView), ); From 54438311cbfb68336f929462788ee8dd07eac384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 21:28:25 +0100 Subject: [PATCH 07/14] fix(settle): mark omitted removals with a typed flag Address review: a forced removals count of 0 read as a fact. The settle diff now carries removalsOmitted, the text verdict prints "removals not listed", the hint no longer promises a baseline the ref frame may not advance to, and the docs note the iOS row-rewrite limit. Co-Authored-By: Claude Opus 5.5 --- .../command-output/src/settle-output.test.ts | 27 +++++++++++++++++++ packages/command-output/src/settle-output.ts | 5 +++- .../runtime/settle-view-baseline.test.ts | 9 ++++++- .../commands/interaction/runtime/settle.ts | 7 ++--- packages/contracts/src/interaction.ts | 5 ++++ website/docs/docs/commands.md | 2 +- 6 files changed, 49 insertions(+), 6 deletions(-) diff --git a/packages/command-output/src/settle-output.test.ts b/packages/command-output/src/settle-output.test.ts index dd1be26a30..f28340949f 100644 --- a/packages/command-output/src/settle-output.test.ts +++ b/packages/command-output/src/settle-output.test.ts @@ -49,3 +49,30 @@ test('post-action observation CLI output formatters follow descriptor traits', a ); } }); + +test('a settle that omitted removals does not print a removal count', async () => { + const format = listCliOutputFormatters()[PUBLIC_COMMANDS.press]!; + const settle = (removalsOmitted: boolean) => ({ + settled: true, + waitedMs: 12, + diff: { + summary: { additions: 0, removals: 0, unchanged: 6 }, + lines: [], + ...(removalsOmitted ? { removalsOmitted: true } : {}), + }, + }); + const text = async (removalsOmitted: boolean) => + String( + ( + await format({ + env: process.env, + stderrIsTTY: false, + input: {}, + result: { message: 'ok', settle: settle(removalsOmitted) }, + }) + ).text, + ); + + assert.match(await text(true), /settled after 12ms: \+0 removals not listed \(~6 unchanged\)/); + assert.match(await text(false), /settled after 12ms: \+0 -0 \(~6 unchanged\)/); +}); diff --git a/packages/command-output/src/settle-output.ts b/packages/command-output/src/settle-output.ts index 97fde2a3b2..bd4221b76e 100644 --- a/packages/command-output/src/settle-output.ts +++ b/packages/command-output/src/settle-output.ts @@ -19,6 +19,7 @@ type SettleTextView = { summary?: { additions?: number; removals?: number; unchanged?: number }; lines?: Array<{ kind?: string; text?: string; ref?: string }>; truncated?: boolean; + removalsOmitted?: boolean; }; tail?: Array<{ ref?: string; role?: string; label?: string }>; tailTruncated?: boolean; @@ -140,5 +141,7 @@ function formatSettleVerdict(view: SettleTextView): string { const verdict = view.settled === true ? 'settled' : 'not settled'; const summary = view.diff?.summary; if (!summary) return `${verdict} after ${view.waitedMs ?? 0}ms`; - return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} -${summary.removals ?? 0} (~${summary.unchanged ?? 0} unchanged)`; + const removals = + view.diff?.removalsOmitted === true ? 'removals not listed' : `-${summary.removals ?? 0}`; + return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} ${removals} (~${summary.unchanged ?? 0} unchanged)`; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 19cc5e1e8c..92e89b1f12 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -73,6 +73,7 @@ test('a ref press after a full snapshot reports no removals the view change caus const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); assert.equal(result.settle?.diff?.summary.removals, 0); + assert.equal(result.settle?.diff?.removalsOmitted, true); // The tiny-tree readiness hint the settled screen earns is kept beside the view-change hint. assert.match(result.settle?.hint ?? '', /previous snapshot was a full tree.*nearly-empty tree/); }); @@ -117,6 +118,12 @@ test('a selector press keeps its diff although its resolution read a full tree', settle: {}, }); - assert.ok((result.settle?.diff?.summary.removals ?? 0) > 0); + // The genuine removal is listed; this route keeps comparing removals (#4486 leaves it open). + assert.ok( + result.settle?.diff?.lines.some( + (line) => line.kind === 'removed' && line.text.includes('Device name'), + ), + ); + assert.equal(result.settle?.diff?.removalsOmitted, undefined); assert.doesNotMatch(result.settle?.hint ?? '', /full tree/); }); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 412b098603..110d91ddc6 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -81,7 +81,7 @@ export const NEVER_SETTLED_HINT = 'The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.'; const FULL_VIEW_BASELINE_HINT = - 'The previous snapshot was a full tree and settle reads the interactive view, so this diff lists only added lines; removals are not listed. Later settles diff against that settled interactive view.'; + 'The previous snapshot was a full tree and settle reads the interactive view, so this diff lists only added lines; removals are not listed. A settle on refs from this response compares interactive views.'; const SETTLE_CAPTURE_STALLED_HINT = 'A snapshot capture stalled past the settle budget, so no settled diff is shown. The action itself succeeded; observe with wait stable or snapshot.'; @@ -316,7 +316,7 @@ function buildSettleDiff( // not evidence and are left out. Added lines stay: on Android the interactive view adds no node // of its own, while an iOS row rewrite can also read as added, as it did before #4486. const changed = diff.lines.filter( - (line) => line.kind === 'added' || (line.kind === 'removed' && !baseline.fullView), + (line) => line.kind === 'added' || (line.kind === 'removed' && baseline.fullView !== true), ); const lines = capSettleDiffLines(changed).map((line) => ({ kind: line.kind as 'added' | 'removed', @@ -324,8 +324,9 @@ function buildSettleDiff( ...(line.ref ? { ref: line.ref } : {}), })); return { - summary: baseline.fullView ? { ...diff.summary, removals: 0 } : diff.summary, + summary: baseline.fullView === true ? { ...diff.summary, removals: 0 } : diff.summary, lines, + ...(baseline.fullView === true ? { removalsOmitted: true as const } : {}), ...(changed.length > lines.length ? { truncated: true as const } : {}), }; } diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index c97b9dca79..7e3851ea8b 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -350,6 +350,11 @@ export type SettleObservation = { lines: SettleDiffLine[]; /** Present (true) when lines were capped to the response bound. */ truncated?: boolean; + /** + * Present (true) when the baseline was a ref frame read as a full tree: removals were not + * compared, so `summary.removals` is 0 and `lines` holds only additions. + */ + removalsOmitted?: true; }; /** * Unchanged interactive refs tail: benchmarks (July 2026) showed 27% of diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 3f4d88cbe8..000e990f31 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -494,7 +494,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s has no interactive baseline. That settle lists only added lines, not removals, says so in its hint, and later settles diff against that settled interactive view. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s has no interactive baseline. That settle lists only added lines, sets `diff.removalsOmitted`, and says so in its hint. A settle on refs from that response compares interactive views. On iOS, a row the interactive view rewrites can still be listed as added. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 3b6a9f7d5aa1c02954430fb6f586c718d1162248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 21:31:54 +0100 Subject: [PATCH 08/14] fix(settle): settle on the full tree after a full snapshot Address review: listing only additions against a full-tree baseline still reported the keyboard window as added, because the interactive view reads windows the full tree does not. A settle on refs from a full snapshot now captures the full tree too, so its diff compares one view and issues full-tree refs. The removals flag, hint and formatter change are gone. Co-Authored-By: Claude Opus 5.5 --- .../command-output/src/settle-output.test.ts | 27 ---- packages/command-output/src/settle-output.ts | 5 +- .../runtime/settle-view-baseline.test.ts | 116 +++++++++++------- .../commands/interaction/runtime/settle.ts | 42 ++----- .../interaction/runtime/stable-capture.ts | 9 +- packages/contracts/src/interaction.ts | 11 +- website/docs/docs/commands.md | 2 +- 7 files changed, 91 insertions(+), 121 deletions(-) diff --git a/packages/command-output/src/settle-output.test.ts b/packages/command-output/src/settle-output.test.ts index f28340949f..dd1be26a30 100644 --- a/packages/command-output/src/settle-output.test.ts +++ b/packages/command-output/src/settle-output.test.ts @@ -49,30 +49,3 @@ test('post-action observation CLI output formatters follow descriptor traits', a ); } }); - -test('a settle that omitted removals does not print a removal count', async () => { - const format = listCliOutputFormatters()[PUBLIC_COMMANDS.press]!; - const settle = (removalsOmitted: boolean) => ({ - settled: true, - waitedMs: 12, - diff: { - summary: { additions: 0, removals: 0, unchanged: 6 }, - lines: [], - ...(removalsOmitted ? { removalsOmitted: true } : {}), - }, - }); - const text = async (removalsOmitted: boolean) => - String( - ( - await format({ - env: process.env, - stderrIsTTY: false, - input: {}, - result: { message: 'ok', settle: settle(removalsOmitted) }, - }) - ).text, - ); - - assert.match(await text(true), /settled after 12ms: \+0 removals not listed \(~6 unchanged\)/); - assert.match(await text(false), /settled after 12ms: \+0 -0 \(~6 unchanged\)/); -}); diff --git a/packages/command-output/src/settle-output.ts b/packages/command-output/src/settle-output.ts index bd4221b76e..97fde2a3b2 100644 --- a/packages/command-output/src/settle-output.ts +++ b/packages/command-output/src/settle-output.ts @@ -19,7 +19,6 @@ type SettleTextView = { summary?: { additions?: number; removals?: number; unchanged?: number }; lines?: Array<{ kind?: string; text?: string; ref?: string }>; truncated?: boolean; - removalsOmitted?: boolean; }; tail?: Array<{ ref?: string; role?: string; label?: string }>; tailTruncated?: boolean; @@ -141,7 +140,5 @@ function formatSettleVerdict(view: SettleTextView): string { const verdict = view.settled === true ? 'settled' : 'not settled'; const summary = view.diff?.summary; if (!summary) return `${verdict} after ${view.waitedMs ?? 0}ms`; - const removals = - view.diff?.removalsOmitted === true ? 'removals not listed' : `-${summary.removals ?? 0}`; - return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} ${removals} (~${summary.unchanged ?? 0} unchanged)`; + return `${verdict} after ${view.waitedMs ?? 0}ms: +${summary.additions ?? 0} -${summary.removals ?? 0} (~${summary.unchanged ?? 0} unchanged)`; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 92e89b1f12..c5f075028f 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -10,8 +10,8 @@ import { createMemorySessionStore } from '../../../command-runtime/runtime-facto import { ref, selector } from './selector-read-utils.ts'; import { createFakeClock } from './__tests__/settle-device-fixtures.ts'; -// #4486: a full `snapshot` keeps wrapper nodes that the interactive view the settle loop reads -// omits. A press that changes nothing must not report those wrappers as removed. +// #4486: a full `snapshot` keeps wrapper nodes the interactive view omits. A settle on its refs reads +// the full tree too, so an unchanged screen reports no change and wrappers are never false removals. // // Budgets are injected (fake clock) — no real waiting. @@ -44,16 +44,29 @@ function interactiveTree(labels: readonly string[]): SnapshotState { return { ...makeSnapshotState(dialogNodes(labels)), presentation: { interactiveOnly: true } }; } -function createDevice(params: { - refFrame: SnapshotState; - capture: (call: number) => SnapshotState; -}) { - let calls = 0; - return createAgentDevice({ +type Screen = { full: SnapshotState; interactive: SnapshotState }; + +const DIALOG: Screen = { full: fullDialogTree(), interactive: interactiveTree(DIALOG_LABELS) }; + +/** + * A backend that answers each capture in the view it was asked for. `screens` is the screen before + * the press and after it; `views` records the view of every capture taken after the press. + */ +function createDevice(params: { refFrame: SnapshotState; before: Screen; after: Screen }) { + let pressed = false; + const views: Array<'full' | 'interactive'> = []; + const device = createAgentDevice({ backend: { platform: 'android', - captureSnapshot: async () => ({ snapshot: params.capture((calls += 1)) }), - tap: async () => ({ ok: true }), + captureSnapshot: async (_context, options) => { + const screen = pressed ? params.after : params.before; + if (pressed) views.push(options?.interactiveOnly === true ? 'interactive' : 'full'); + return { snapshot: options?.interactiveOnly === true ? screen.interactive : screen.full }; + }, + tap: async () => { + pressed = true; + return { ok: true }; + }, } satisfies AgentDeviceBackend, artifacts: createLocalArtifactAdapter(), sessions: createMemorySessionStore([ @@ -62,55 +75,70 @@ function createDevice(params: { policy: localCommandPolicy(), clock: createFakeClock(), }); + return { device, views }; +} + +function removedLabels(lines: Array<{ kind: string; text: string }> | undefined): string[] { + return (lines ?? []).filter((line) => line.kind === 'removed').map((line) => line.text); } -test('a ref press after a full snapshot reports no removals the view change caused', async () => { - const device = createDevice({ - refFrame: fullDialogTree(), - capture: () => interactiveTree(DIALOG_LABELS), +test('a ref press after a full snapshot settles on the full tree and reports no false removals', async () => { + const { device, views } = createDevice({ + refFrame: DIALOG.full, + before: DIALOG, + after: DIALOG, }); const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); - assert.equal(result.settle?.diff?.summary.removals, 0); - assert.equal(result.settle?.diff?.removalsOmitted, true); - // The tiny-tree readiness hint the settled screen earns is kept beside the view-change hint. - assert.match(result.settle?.hint ?? '', /previous snapshot was a full tree.*nearly-empty tree/); + assert.ok(views.length > 0 && views.every((view) => view === 'full')); + assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); }); -test('a ref press after a full snapshot still lists what the press added', async () => { - const device = createDevice({ - refFrame: fullDialogTree(), - // The press opens another screen: its controls are new, the dialog is gone. - capture: () => interactiveTree(['Next', 'Back', 'Home', 'Menu', 'Help']), +test('a ref press after a full snapshot still reports what changed on the full tree', async () => { + const next = ['Next', 'Back']; + const { device } = createDevice({ + refFrame: DIALOG.full, + before: DIALOG, + after: { + full: { + ...makeSnapshotState([ + { index: 0, depth: 0, type: 'Group', rect: { x: 0, y: 0, width: 400, height: 800 } }, + ...dialogNodes(next, 1), + ]), + presentation: {}, + }, + interactive: interactiveTree(next), + }, }); const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); - assert.equal(result.settle?.diff?.summary.additions, 5); - assert.equal(result.settle?.diff?.summary.removals, 0); - assert.equal(result.settle?.diff?.lines.length, 5); - assert.ok(result.settle?.diff?.lines.every((line) => line.kind === 'added' && line.ref)); + assert.equal(result.settle?.diff?.summary.additions, 2); + assert.equal(result.settle?.diff?.summary.removals, 3); + assert.ok(removedLabels(result.settle?.diff?.lines).some((text) => text.includes('Device name'))); }); -test('a ref press after an interactive snapshot still reports what left the screen', async () => { - const device = createDevice({ - refFrame: interactiveTree(DIALOG_LABELS), - capture: () => interactiveTree(['Cancel', 'OK']), +test('a ref press after an interactive snapshot settles on the interactive view', async () => { + const titleGone = interactiveTree(['Cancel', 'OK']); + const { device, views } = createDevice({ + refFrame: DIALOG.interactive, + before: DIALOG, + after: { full: DIALOG.full, interactive: titleGone }, }); const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); - assert.equal(result.settle?.diff?.summary.removals, 1); - assert.doesNotMatch(result.settle?.hint ?? '', /full tree/); + assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); + assert.deepEqual(removedLabels(result.settle?.diff?.lines).length, 1); }); -test('a selector press keeps its diff although its resolution read a full tree', async () => { - const device = createDevice({ - refFrame: interactiveTree(DIALOG_LABELS), - // Capture 1 resolves the selector on a full tree; the settle loop then reads the interactive - // view, where the dialog title is gone after the press. - capture: (call) => (call === 1 ? fullDialogTree() : interactiveTree(['Cancel', 'OK'])), +test('a selector press settles on the interactive view although its resolution read a full tree', async () => { + const titleGone = interactiveTree(['Cancel', 'OK']); + const { device, views } = createDevice({ + refFrame: DIALOG.interactive, + before: DIALOG, + after: { full: DIALOG.full, interactive: titleGone }, }); const result = await device.interactions.press(selector('label=OK'), { @@ -118,12 +146,6 @@ test('a selector press keeps its diff although its resolution read a full tree', settle: {}, }); - // The genuine removal is listed; this route keeps comparing removals (#4486 leaves it open). - assert.ok( - result.settle?.diff?.lines.some( - (line) => line.kind === 'removed' && line.text.includes('Device name'), - ), - ); - assert.equal(result.settle?.diff?.removalsOmitted, undefined); - assert.doesNotMatch(result.settle?.hint ?? '', /full tree/); + assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); + assert.ok(removedLabels(result.settle?.diff?.lines).some((text) => text.includes('Device name'))); }); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 110d91ddc6..22ab84d4e9 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -80,9 +80,6 @@ const MAX_SETTLE_TAIL_ENTRIES = 20; export const NEVER_SETTLED_HINT = 'The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.'; -const FULL_VIEW_BASELINE_HINT = - 'The previous snapshot was a full tree and settle reads the interactive view, so this diff lists only added lines; removals are not listed. A settle on refs from this response compares interactive views.'; - const SETTLE_CAPTURE_STALLED_HINT = 'A snapshot capture stalled past the settle budget, so no settled diff is shown. The action itself succeeded; observe with wait stable or snapshot.'; @@ -210,16 +207,13 @@ async function readSettledOutcome( // advisory state. ...(diffShown ? buildSettleDiffAndTail( - params.baseline, + params.baseline.nodes, settledNodes, params.actionPoint, session?.appBundleId, ) : {}), - ...withFullViewBaselineHint( - resolveSettleHint(outcome, stored, settledNodes.length, surfaceChange), - diffShown && params.baseline.fullView === true, - ), + ...resolveSettleHint(outcome, stored, settledNodes.length, surfaceChange), }, settledCapture, }; @@ -268,8 +262,8 @@ function authorizedRefBaseline( /** * The ref frame the agent last read. A frame read as a full tree (`snapshot` without `-i`) is marked - * `fullView`, because the settle loop reads the interactive view. Selector captures are full trees - * too, but no agent chose that view, so only the ref frame carries the mark. + * `fullView`, so the settle loop reads the full tree too and the diff compares one view. Selector + * captures are full trees as well, but no agent chose that view, so only the ref frame carries it. */ function refFrameBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNodes { const baseline = surfaceScopedNodes(snapshot); @@ -298,7 +292,7 @@ function nonEmptyBaseline( } function buildSettleDiff( - baseline: SurfaceScopedNodes, + baselineNodes: SnapshotNode[], settledNodes: SnapshotNode[], appBundleId: string | undefined, ): NonNullable { @@ -308,36 +302,30 @@ function buildSettleDiff( // snapshot), extra baseline-only lines surface as removals — advisory noise, // the same baseline caveat --verify's changedFromBefore already accepts. const diff = buildSnapshotDiff( - withoutSettleChrome(baseline.nodes, appBundleId), + withoutSettleChrome(baselineNodes, appBundleId), withoutSettleChrome(settledNodes, appBundleId), { flatten: true, withRefs: true }, ); - // A full-tree baseline holds nodes the interactive settle view never reads, so its removals are - // not evidence and are left out. Added lines stay: on Android the interactive view adds no node - // of its own, while an iOS row rewrite can also read as added, as it did before #4486. - const changed = diff.lines.filter( - (line) => line.kind === 'added' || (line.kind === 'removed' && baseline.fullView !== true), - ); + const changed = diff.lines.filter((line) => line.kind !== 'unchanged'); const lines = capSettleDiffLines(changed).map((line) => ({ kind: line.kind as 'added' | 'removed', text: line.text, ...(line.ref ? { ref: line.ref } : {}), })); return { - summary: baseline.fullView === true ? { ...diff.summary, removals: 0 } : diff.summary, + summary: diff.summary, lines, - ...(baseline.fullView === true ? { removalsOmitted: true as const } : {}), ...(changed.length > lines.length ? { truncated: true as const } : {}), }; } function buildSettleDiffAndTail( - baseline: SurfaceScopedNodes, + baselineNodes: SnapshotNode[], settledNodes: SnapshotNode[], actionPoint: Point | undefined, appBundleId: string | undefined, ): Pick { - const diff = buildSettleDiff(baseline, settledNodes, appBundleId); + const diff = buildSettleDiff(baselineNodes, settledNodes, appBundleId); return { diff, ...buildSettleTail(diff, settledNodes, actionPoint, appBundleId) }; } @@ -466,16 +454,6 @@ function capSettleDiffLines(changed: T[]): T[] { return kept; } -function withFullViewBaselineHint( - resolved: { hint?: string }, - fullViewBaselineReset: boolean, -): { hint?: string } { - if (!fullViewBaselineReset) return resolved; - return { - hint: resolved.hint ? `${FULL_VIEW_BASELINE_HINT} ${resolved.hint}` : FULL_VIEW_BASELINE_HINT, - }; -} - /** * The settled observation's hint. A surface change (#2438) is stated alongside whatever the loop * itself reports rather than in place of it: the transition explains the missing diff, and the diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index d0f5c86b5c..b22e06dd4b 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -103,7 +103,10 @@ export async function runStableCaptureLoop( quietMs?: number; timeoutMs: number; resetBudgetOnPrivateAxRecovery?: boolean; - /** Immutable pre-action projection; the stored session may advance before settle begins. */ + /** + * Immutable pre-action projection; the stored session may advance before settle begins. The loop + * reads the view the baseline was read in, so a full-view baseline is diffed full against full. + */ broadTransitionBaseline?: SurfaceScopedNodes; }, ): Promise { @@ -154,6 +157,7 @@ export async function runStableCaptureLoop( runtime, options, deadlineMs - now(runtime), + params.broadTransitionBaseline?.fullView !== true, preferredBackend, ); if (read.kind !== 'captured') return unfinished(read); @@ -543,6 +547,7 @@ async function captureStableSignalWithinDeadline( runtime: AgentDeviceRuntime, options: CommandContext & SelectorSnapshotOptions, remainingMs: number, + interactiveOnly: boolean, preferredBackend?: SnapshotPreferredBackend, ): Promise { try { @@ -552,7 +557,7 @@ async function captureStableSignalWithinDeadline( { ...options, signal }, { updateSession: false, - interactiveOnly: true, + interactiveOnly, preferredBackend, }, ); diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 7e3851ea8b..d72b3b995c 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -161,9 +161,9 @@ export type SurfaceScopedNodes = { */ presentedSurface?: PresentedSurface; /** - * The nodes are a ref frame the agent read as a full tree (`snapshot` without `-i`). A settle reads - * the interactive view, so a diff against these nodes would report every node that view omits as - * removed. Only the settle baseline sets it. + * The nodes are a ref frame the agent read as a full tree (`snapshot` without `-i`). The settle + * loop then reads the full tree too, because diffing these nodes against the interactive view + * reports every node that view omits as removed. Only the settle baseline sets it. */ fullView?: true; }; @@ -350,11 +350,6 @@ export type SettleObservation = { lines: SettleDiffLine[]; /** Present (true) when lines were capped to the response bound. */ truncated?: boolean; - /** - * Present (true) when the baseline was a ref frame read as a full tree: removals were not - * compared, so `summary.removals` is 0 and `lines` holds only additions. - */ - removalsOmitted?: true; }; /** * Unchanged interactive refs tail: benchmarks (July 2026) showed 27% of diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 000e990f31..d731f07a10 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -494,7 +494,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. The settled diff compares interactive views. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s has no interactive baseline. That settle lists only added lines, sets `diff.removalsOmitted`, and says so in its hint. A settle on refs from that response compares interactive views. On iOS, a row the interactive view rewrites can still be listed as added. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s reads the full tree too, so its diff compares one view and its refs are full-tree refs. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 937ae6223edbb46cbba9280e63b2809724319884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 21:51:26 +0100 Subject: [PATCH 09/14] fix(settle): mark only a whole full-tree ref frame, without a tail Address review: a scoped, depth-limited or raw frame is a projection the settle capture does not reproduce, so only an unprojected full snapshot is marked fullView. Point taps keep the session baseline unmarked. A full-tree settle carries no tail, because the tail names interactive targets. Comments now describe the full-tree path. Co-Authored-By: Claude Opus 5.5 --- .../runtime/settle-view-baseline.test.ts | 17 ++++++++- .../commands/interaction/runtime/settle.ts | 37 ++++++++++++------- .../interaction/runtime/stable-capture.ts | 4 +- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index c5f075028f..86afa1dbef 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -93,6 +93,21 @@ test('a ref press after a full snapshot settles on the full tree and reports no assert.ok(views.length > 0 && views.every((view) => view === 'full')); assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); + // The tail names interactive targets, which a full-tree capture does not single out. + assert.equal(result.settle?.tail, undefined); +}); + +test('a ref press after a scoped full snapshot keeps the interactive settle view', async () => { + const { device, views } = createDevice({ + refFrame: { ...DIALOG.full, presentation: { scope: 'Device name' } }, + before: DIALOG, + after: DIALOG, + }); + + await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); + + // The settle capture cannot reproduce a scoped projection, so the frame is not a full-view baseline. + assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); }); test('a ref press after a full snapshot still reports what changed on the full tree', async () => { @@ -133,7 +148,7 @@ test('a ref press after an interactive snapshot settles on the interactive view' assert.deepEqual(removedLabels(result.settle?.diff?.lines).length, 1); }); -test('a selector press settles on the interactive view although its resolution read a full tree', async () => { +test('a selector press settles on the interactive view', async () => { const titleGone = interactiveTree(['Cancel', 'OK']); const { device, views } = createDevice({ refFrame: DIALOG.interactive, diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 22ab84d4e9..c27ec4b7e4 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -207,7 +207,7 @@ async function readSettledOutcome( // advisory state. ...(diffShown ? buildSettleDiffAndTail( - params.baseline.nodes, + params.baseline, settledNodes, params.actionPoint, session?.appBundleId, @@ -261,15 +261,22 @@ function authorizedRefBaseline( } /** - * The ref frame the agent last read. A frame read as a full tree (`snapshot` without `-i`) is marked - * `fullView`, so the settle loop reads the full tree too and the diff compares one view. Selector - * captures are full trees as well, but no agent chose that view, so only the ref frame carries it. + * The ref frame the agent last read. A frame read as the whole full tree (`snapshot` without `-i`, + * `-s`, `-d` or `--raw`) is marked `fullView`, so the settle loop reads the full tree too and the + * diff compares one view. A scoped, depth-limited or raw frame is a projection the settle capture + * does not reproduce, so it stays unmarked. Selector captures are full trees as well, but no agent + * chose that view, so only the ref frame carries the mark. */ function refFrameBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNodes { const baseline = surfaceScopedNodes(snapshot); - return snapshot?.presentation && snapshot.presentation.interactiveOnly !== true - ? { ...baseline, fullView: true } - : baseline; + const presentation = snapshot?.presentation; + const wholeFullTree = + presentation !== undefined && + presentation.interactiveOnly !== true && + presentation.scope === undefined && + presentation.depth === undefined && + presentation.raw !== true; + return wholeFullTree ? { ...baseline, fullView: true } : baseline; } /** @@ -279,9 +286,9 @@ function refFrameBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNod * dismissal (#3565). */ function sessionBaseline(session: CommandSessionRecord | undefined): SurfaceScopedNodes { - const { presentedSurface: _unverified, ...baseline } = session?.refFrameSnapshot - ? refFrameBaseline(session.refFrameSnapshot) - : surfaceScopedNodes(session?.snapshot); + const { presentedSurface: _unverified, ...baseline } = surfaceScopedNodes( + session?.refFrameSnapshot ?? session?.snapshot, + ); return baseline; } @@ -296,8 +303,8 @@ function buildSettleDiff( settledNodes: SnapshotNode[], appBundleId: string | undefined, ): NonNullable { - // Flattened compare, like `diff -i`: both sides are interactive-flavored - // captures and depth jitter across captures should not read as change. When + // Flattened compare, like `diff -i`: both sides are captures in one view (interactive, or the full + // tree for a `fullView` baseline), and depth jitter across captures should not read as change. When // the baseline came from a richer stored tree (ref targets reuse the session // snapshot), extra baseline-only lines surface as removals — advisory noise, // the same baseline caveat --verify's changedFromBefore already accepts. @@ -320,12 +327,14 @@ function buildSettleDiff( } function buildSettleDiffAndTail( - baselineNodes: SnapshotNode[], + baseline: SurfaceScopedNodes, settledNodes: SnapshotNode[], actionPoint: Point | undefined, appBundleId: string | undefined, ): Pick { - const diff = buildSettleDiff(baselineNodes, settledNodes, appBundleId); + const diff = buildSettleDiff(baseline.nodes, settledNodes, appBundleId); + // The tail names next interactive targets, which a full-tree settle capture does not single out. + if (baseline.fullView === true) return { diff }; return { diff, ...buildSettleTail(diff, settledNodes, actionPoint, appBundleId) }; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index b22e06dd4b..082b103a38 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -533,8 +533,8 @@ function stableCaptureDelayMs(params: { } // Intentionally does not update the session snapshot: the stable loop captures -// an interactive-only tree purely as a settle signal, and overwriting the -// session's richer cached snapshot with the filtered tree would degrade +// an interactive-only tree (the full tree for a `fullView` baseline) purely as a +// settle signal, and overwriting the session's richer cached snapshot with it would degrade // subsequent ref/get/find lookups against the same session. (`--settle` DOES // store its final capture, but explicitly, in one place, after the loop.) // From 2e941c2aa5e9aec026a4f20419aa09bad27c48f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 21:52:41 +0100 Subject: [PATCH 10/14] fix(settle): keep the tail on a full-tree settle An unchanged full-tree settle without a tail handed back no refs, so the next @ref press hit an expired frame. The tail stays, listing addressable refs from the full-tree view the agent chose. Co-Authored-By: Claude Opus 5.5 --- .../interaction/runtime/settle-view-baseline.test.ts | 4 ++-- .../src/commands/interaction/runtime/settle.ts | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 86afa1dbef..aa3a8c1065 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -93,8 +93,8 @@ test('a ref press after a full snapshot settles on the full tree and reports no assert.ok(views.length > 0 && views.every((view) => view === 'full')); assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); - // The tail names interactive targets, which a full-tree capture does not single out. - assert.equal(result.settle?.tail, undefined); + // An unchanged screen still hands back refs to act on next, from the full-tree view. + assert.ok(result.settle?.tail?.some((entry) => entry.label === 'OK')); }); test('a ref press after a scoped full snapshot keeps the interactive settle view', async () => { diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index c27ec4b7e4..20212924a2 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -207,7 +207,7 @@ async function readSettledOutcome( // advisory state. ...(diffShown ? buildSettleDiffAndTail( - params.baseline, + params.baseline.nodes, settledNodes, params.actionPoint, session?.appBundleId, @@ -327,14 +327,12 @@ function buildSettleDiff( } function buildSettleDiffAndTail( - baseline: SurfaceScopedNodes, + baselineNodes: SnapshotNode[], settledNodes: SnapshotNode[], actionPoint: Point | undefined, appBundleId: string | undefined, ): Pick { - const diff = buildSettleDiff(baseline.nodes, settledNodes, appBundleId); - // The tail names next interactive targets, which a full-tree settle capture does not single out. - if (baseline.fullView === true) return { diff }; + const diff = buildSettleDiff(baselineNodes, settledNodes, appBundleId); return { diff, ...buildSettleTail(diff, settledNodes, actionPoint, appBundleId) }; } From 0149a53b7c4b198e7943a94b90dbbb3d14611625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 12 Nov 2026 22:02:20 +0100 Subject: [PATCH 11/14] fix(settle): leave --actions ref frames on the interactive settle view Address review: custom actions are another projection the settle capture does not reproduce, like scope, depth and raw. Co-Authored-By: Claude Opus 5.5 --- .../runtime/settle-view-baseline.test.ts | 27 +++++++++++-------- .../commands/interaction/runtime/settle.ts | 9 ++++--- website/docs/docs/commands.md | 2 +- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index aa3a8c1065..7474d9a177 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -97,18 +97,23 @@ test('a ref press after a full snapshot settles on the full tree and reports no assert.ok(result.settle?.tail?.some((entry) => entry.label === 'OK')); }); -test('a ref press after a scoped full snapshot keeps the interactive settle view', async () => { - const { device, views } = createDevice({ - refFrame: { ...DIALOG.full, presentation: { scope: 'Device name' } }, - before: DIALOG, - after: DIALOG, +for (const [flag, presentation] of [ + ['-s', { scope: 'Device name' }], + ['--actions', { customActions: true }], +] as const) { + test(`a ref press after a full snapshot with ${flag} keeps the interactive settle view`, async () => { + const { device, views } = createDevice({ + refFrame: { ...DIALOG.full, presentation }, + before: DIALOG, + after: DIALOG, + }); + + await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); + + // The settle capture cannot reproduce that projection, so the frame is not a full-view baseline. + assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); }); - - await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); - - // The settle capture cannot reproduce a scoped projection, so the frame is not a full-view baseline. - assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); -}); +} test('a ref press after a full snapshot still reports what changed on the full tree', async () => { const next = ['Next', 'Back']; diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 20212924a2..f1e2865b89 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -262,9 +262,9 @@ function authorizedRefBaseline( /** * The ref frame the agent last read. A frame read as the whole full tree (`snapshot` without `-i`, - * `-s`, `-d` or `--raw`) is marked `fullView`, so the settle loop reads the full tree too and the - * diff compares one view. A scoped, depth-limited or raw frame is a projection the settle capture - * does not reproduce, so it stays unmarked. Selector captures are full trees as well, but no agent + * `-s`, `-d`, `--raw` or `--actions`) is marked `fullView`, so the settle loop reads the full tree + * too and the diff compares one view. Any of those flags makes a projection the settle capture + * does not reproduce, so such a frame stays unmarked. Selector captures are full trees as well, but no agent * chose that view, so only the ref frame carries the mark. */ function refFrameBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNodes { @@ -275,7 +275,8 @@ function refFrameBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNod presentation.interactiveOnly !== true && presentation.scope === undefined && presentation.depth === undefined && - presentation.raw !== true; + presentation.raw !== true && + presentation.customActions !== true; return wholeFullTree ? { ...baseline, fullView: true } : baseline; } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d731f07a10..bc3b784f48 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -494,7 +494,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. After a full `snapshot` (no `-i`), a `--settle` action on one of its `@ref`s reads the full tree too, so its diff compares one view and its refs are full-tree refs. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. After a full `snapshot` (no `-i`, `-s`, `-d`, `--raw` or `--actions`), a `--settle` action on one of its `@ref`s reads the full tree too, so its diff compares one view and its refs are full-tree refs. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 1f15eb0b4bf06c514370aa419bbdab1dd637b3eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 10:41:21 +0100 Subject: [PATCH 12/14] fix(settle): compare post-action trees in one view across every route A settle diff compared whatever pre-action tree it found with an interactive settle capture, so a full snapshot, a selector fallback or a full tree a selector read stored showed up as false removals (#4486). One rule now owns every post-action comparison: two trees are compared only when they were read in the same presentation. capture-kit's postActionBaseline picks the baseline in the agent's view, the presentation of the ref frame it last read, and SurfaceScopedNodes carries that presentation. The settle loop captures in it for press, click, fill, longpress, scroll and back, on refs, selectors and coordinates. --verify captures in the pre-action tree's view, and diff snapshot starts a new baseline when the stored tree was read in another view. Fixes #4486 Co-Authored-By: Claude Opus 5.5 --- .../capture-kit/src/snapshot-baseline.test.ts | 51 +++- packages/capture-kit/src/snapshot-baseline.ts | 44 +++- .../src/commands/capture/runtime/snapshot.ts | 9 +- .../runtime/interaction-snapshot-capture.ts | 3 +- .../runtime/post-action-observation.ts | 32 ++- .../runtime/selector-read-shared.ts | 25 +- .../runtime/settle-view-baseline.test.ts | 218 +++++++++--------- .../commands/interaction/runtime/settle.ts | 79 ++----- .../interaction/runtime/stable-capture.ts | 14 +- packages/contracts/src/interaction.ts | 15 +- .../src/execution/generic-settle.test.ts | 29 +++ .../daemon/src/execution/generic-settle.ts | 17 +- packages/kernel/src/snapshot.ts | 17 ++ website/docs/docs/commands.md | 4 +- 14 files changed, 355 insertions(+), 202 deletions(-) diff --git a/packages/capture-kit/src/snapshot-baseline.test.ts b/packages/capture-kit/src/snapshot-baseline.test.ts index 5e2f97e577..7418905e23 100644 --- a/packages/capture-kit/src/snapshot-baseline.test.ts +++ b/packages/capture-kit/src/snapshot-baseline.test.ts @@ -1,6 +1,7 @@ import { expect, test } from 'vitest'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; -import { surfaceScopedNodes } from './snapshot-baseline.ts'; +import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; +import { postActionBaseline, surfaceScopedNodes } from './snapshot-baseline.ts'; const nodes = [{ index: 0, type: 'Application', ref: 'e1' }]; @@ -33,6 +34,54 @@ test('a session with no stored snapshot still mints an empty baseline', () => { expect(surfaceScopedNodes(undefined)).toEqual({ nodes: [] }); }); +test('the mint carries the presentation the capture read', () => { + expect(surfaceScopedNodes(snapshot({ presentation: { interactiveOnly: true } }))).toEqual({ + nodes, + presentation: { interactiveOnly: true }, + }); +}); + +const INTERACTIVE = { interactiveOnly: true }; +const FULL = {}; +const frame = (presentation: SurfaceScopedNodes['presentation']) => tree('frame', presentation); + +// #4486: a settle diff compares one view, and the agent's view is its ref frame's presentation. +test.each([ + ['interactive frame, interactive candidate', INTERACTIVE, INTERACTIVE, 'candidate'], + ['interactive frame, full candidate', INTERACTIVE, FULL, 'frame'], + ['full frame, interactive candidate', FULL, INTERACTIVE, 'frame'], + ['full frame, full candidate', FULL, FULL, 'candidate'], + [ + 'interactive frame, scoped interactive candidate', + INTERACTIVE, + { interactiveOnly: true, scope: 'Composer' }, + 'frame', + ], + ['interactive frame, candidate with no recorded view', INTERACTIVE, undefined, 'candidate'], +] as const)('%s: the %s answers', (_case, framePresentation, candidatePresentation, expected) => { + const baseline = postActionBaseline({ + frame: frame(framePresentation), + candidates: [tree('candidate', candidatePresentation)], + }); + expect(baseline.nodes[0]?.label).toBe(expected); +}); + +test('without a frame the first non-empty candidate answers, in its own view', () => { + const baseline = postActionBaseline({ + frame: undefined, + candidates: [undefined, { nodes: [] }, tree('candidate', FULL)], + }); + expect(baseline).toEqual(tree('candidate', FULL)); + expect(postActionBaseline({ frame: undefined, candidates: [] })).toEqual({ nodes: [] }); +}); + +function tree(label: string, presentation: SurfaceScopedNodes['presentation']): SurfaceScopedNodes { + return { + nodes: [{ index: 0, type: 'Button', label, ref: 'e1' }], + ...(presentation ? { presentation } : {}), + }; +} + function snapshot(fields: Partial): SnapshotState { return { nodes, createdAt: 0, presentationKey: '', ...fields } as SnapshotState; } diff --git a/packages/capture-kit/src/snapshot-baseline.ts b/packages/capture-kit/src/snapshot-baseline.ts index 93fe201d17..6cc34e4d96 100644 --- a/packages/capture-kit/src/snapshot-baseline.ts +++ b/packages/capture-kit/src/snapshot-baseline.ts @@ -1,9 +1,17 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; +import { + isSameSnapshotPresentation, + type SnapshotPresentation, +} from '@agent-device/kernel/snapshot'; + +/** The view a post-action read captures in when its baseline recorded none. */ +export const DEFAULT_POST_ACTION_PRESENTATION: SnapshotPresentation = { interactiveOnly: true }; /** * Mints the one baseline value a post-action comparison carries: the captured nodes together with the - * surface they describe and the presentation the capture read (#2438, #3565). + * surface they describe, the presentation the capture read (#2438, #3565), and the view it was read + * in (#4486). * * Both settle routes take their baseline from here — the element-targeted interaction route and the * generic daemon route (`scroll`/`back --settle`) — so a fact a capture established cannot be carried @@ -16,7 +24,10 @@ import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; */ export function surfaceScopedNodes( snapshot: - | Pick + | Pick< + SnapshotState, + 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation' + > | undefined, ): SurfaceScopedNodes { return { @@ -25,5 +36,34 @@ export function surfaceScopedNodes( ? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId } : {}), ...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}), + ...(snapshot?.presentation ? { presentation: snapshot.presentation } : {}), }; } + +/** + * The pre-action tree a settle diff runs against, and through its `presentation` the view the settle + * captures in (#4486). Two trees compare only within one presentation, and the agent's view is the + * presentation of the ref frame it last read: `snapshot -i`, a full `snapshot`, a scoped one, or the + * refs a settle issued. So the first candidate read in that view answers. A candidate read in another + * view, such as a selector resolution that fell back to the full tree or a full tree a selector read + * stored, is skipped, and the frame itself answers when no candidate qualifies. Without a frame, the + * first candidate answers. + * + * Every settle route picks its baseline here, so none can compare an interactive read with a full one. + */ +export function postActionBaseline(params: { + /** The ref frame, as the caller admits it as a baseline. */ + frame: SurfaceScopedNodes | undefined; + /** Pre-action trees in the caller's order of preference. */ + candidates: readonly (SurfaceScopedNodes | undefined)[]; +}): SurfaceScopedNodes { + const { frame } = params; + const view = frame && frame.nodes.length > 0 ? frame.presentation : undefined; + const candidate = params.candidates.find( + (baseline) => + baseline !== undefined && + baseline.nodes.length > 0 && + isSameSnapshotPresentation(baseline.presentation, view), + ); + return candidate ?? frame ?? { nodes: [] }; +} diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index d99479a72c..64ad9bdd55 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -8,6 +8,7 @@ import { type SnapshotDiagnosticsSummary, } from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; +import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers'; import type { @@ -135,7 +136,13 @@ export const diffSnapshotCommand: RuntimeCommand< } const capture = await captureRuntimeSnapshot(runtime, options); const flattenForDiff = options.interactiveOnly === true; - const previousSnapshot = capture.session?.snapshot; + // A diff compares one view: the stored tree, else the ref frame, read in the requested + // presentation. With neither, the capture initializes the baseline (#4486). + const previousSnapshot = [capture.session?.snapshot, capture.session?.refFrameSnapshot].find( + (stored) => + stored !== undefined && + isSameSnapshotPresentation(stored.presentation, capture.snapshot.presentation), + ); const nextSession = nextSnapshotSession(options.session, capture); if (!previousSnapshot) { diff --git a/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts index 58864db20e..645e7be06a 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts @@ -6,8 +6,7 @@ import { toBackendContext } from '../../runtime-common.ts'; /** * A leaf shared by every interaction runtime module that needs a fresh (or session-cached) tree: * `resolution.ts`'s ref/selector resolution, `selector-target-resolution.ts`'s acting selector lookup, - * `moving-target-gate.ts`'s re-observation, `gestures.ts`'s viewport resolution, and - * `post-action-observation.ts`'s `--verify` baseline. + * `moving-target-gate.ts`'s re-observation, and `gestures.ts`'s viewport resolution. * Deliberately self-contained — it imports no sibling of this directory, so those siblings can * import it without ever risking a cycle back. */ diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts index 1bcf72e9e1..d05eca5eb3 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts @@ -5,9 +5,13 @@ import type { SettleObservation, SettleParams, } from '@agent-device/contracts/interaction'; -import { captureInteractionSnapshot } from './interaction-snapshot-capture.ts'; +import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot'; import { summarizePostActionEvidence } from './post-action-surface.ts'; -import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline'; +import { + DEFAULT_POST_ACTION_PRESENTATION, + surfaceScopedNodes, +} from '@agent-device/capture-kit/snapshot-baseline'; +import { captureSelectorSnapshot } from './selector-read-shared.ts'; import { settleAfterInteraction, settleEvidence } from './settle.ts'; type ObservedResult = T & { @@ -81,9 +85,17 @@ async function observeAfterInteraction( ...params.settle, resolved, }); - const evidence = params.verify - ? settleEvidence(outcome.settledCapture, resolved.preAction) - : undefined; + // The evidence compares one view: a pre-action tree read in another view than the settle + // capture makes no changedFromBefore claim (#4486). + const baseline = + outcome.settledCapture && + isSameSnapshotPresentation( + resolved.preAction?.presentation, + outcome.settledCapture.presentation, + ) + ? resolved.preAction + : undefined; + const evidence = params.verify ? settleEvidence(outcome.settledCapture, baseline) : undefined; return { settle: outcome.observation, ...(evidence ? { evidence } : {}) }; } if (!params.verify) return {}; @@ -92,8 +104,8 @@ async function observeAfterInteraction( } /** - * Post-action side of `--verify` (#1047): one interactive-only capture through - * the same capture helper the resolution path already uses, digested and then + * Post-action side of `--verify` (#1047): one capture in the pre-action tree's presentation + * (interactive-only by default), so the comparison reads one view (#4486), digested and then * discarded. The node tree itself is never attached to the result, only the * cheap summary. A missing baseline yields no `changedFromBefore` claim — * no baseline, no comparison. @@ -104,7 +116,11 @@ async function captureVerifyEvidence( resolved: ResolvedInteractionTarget, ): Promise { try { - const capture = await captureInteractionSnapshot(runtime, options, true); + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + includeRects: true, + presentation: resolved.preAction?.presentation ?? DEFAULT_POST_ACTION_PRESENTATION, + }); return summarizePostActionEvidence(surfaceScopedNodes(capture.snapshot), resolved.preAction); } catch { return undefined; diff --git a/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts b/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts index 3d93336f25..b4331d4d92 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts @@ -15,6 +15,7 @@ import { type SnapshotCommandOptionFields, type SnapshotNode, type SnapshotPreferredBackend, + type SnapshotPresentation, } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { @@ -72,6 +73,11 @@ export async function captureSelectorSnapshot( interactiveOnly?: boolean; includeHiddenContentHints?: boolean; preferredBackend?: SnapshotPreferredBackend; + /** + * The whole presentation to capture in, overriding `interactiveOnly`, `scope` and the command's + * depth and raw flags: a post-action read reproduces its baseline's view (#4486). + */ + presentation?: SnapshotPresentation; } = { updateSession: true, }, @@ -82,11 +88,22 @@ export async function captureSelectorSnapshot( } const sessionName = options.session ?? 'default'; const session = await runtime.sessions.get(sessionName); + const view = captureOptions.presentation; const result = await captureSnapshot(toBackendContext(runtime, options), { - interactiveOnly: captureOptions.interactiveOnly ?? false, - depth: options.depth, - scope: captureOptions.scope ?? options.scope, - raw: options.raw, + ...(view + ? { + interactiveOnly: view.interactiveOnly === true, + depth: view.depth, + scope: view.scope, + raw: view.raw, + ...(view.customActions === true ? { customActions: true } : {}), + } + : { + interactiveOnly: captureOptions.interactiveOnly ?? false, + depth: options.depth, + scope: captureOptions.scope ?? options.scope, + raw: options.raw, + }), includeRects: captureOptions.includeRects, ...(captureOptions.preferredBackend ? { preferredBackend: captureOptions.preferredBackend } diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 7474d9a177..0770a29fe5 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -1,7 +1,8 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import type { AgentDeviceBackend, BackendSnapshotOptions } from '@agent-device/contracts/backend'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { SnapshotPresentation } from '@agent-device/kernel/snapshot'; import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; @@ -10,67 +11,84 @@ import { createMemorySessionStore } from '../../../command-runtime/runtime-facto import { ref, selector } from './selector-read-utils.ts'; import { createFakeClock } from './__tests__/settle-device-fixtures.ts'; -// #4486: a full `snapshot` keeps wrapper nodes the interactive view omits. A settle on its refs reads -// the full tree too, so an unchanged screen reports no change and wrappers are never false removals. -// -// Budgets are injected (fake clock) — no real waiting. - -const DIALOG_LABELS = ['Device name', 'Cancel', 'OK']; - -function dialogNodes(labels: readonly string[], firstIndex = 0) { - return labels.map((label, index) => ({ - index: firstIndex + index, - depth: firstIndex === 0 ? 0 : 1, - ...(firstIndex === 0 ? {} : { parentIndex: 0 }), - type: label === 'Device name' ? 'StaticText' : 'Button', - label, - rect: { x: 10, y: 100 + DIALOG_LABELS.indexOf(label) * 60, width: 200, height: 40 }, - hittable: true, - })); +// #4486: a post-action comparison reads one view. A settle captures in its baseline's presentation, +// the baseline is read in the agent's view (its ref frame's presentation), and verify and +// `diff snapshot` compare only trees read in one presentation. Budgets are injected (fake clock). + +const DIALOG = ['Device name', 'Cancel', 'OK']; + +function dialogNodes(labels: readonly string[], wrapper: boolean) { + const offset = wrapper ? 1 : 0; + return [ + ...(wrapper + ? [ + { + index: 0, + depth: 0, + type: 'Group', + label: 'Dialog', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + ] + : []), + ...labels.map((label, index) => ({ + index: offset + index, + depth: offset, + ...(wrapper ? { parentIndex: 0 } : {}), + type: label === 'Device name' ? 'StaticText' : 'Button', + label, + rect: { x: 10, y: 100 + index * 60, width: 200, height: 40 }, + hittable: true, + })), + ]; } -/** The dialog as a full `snapshot` reads it: the wrapper the interactive view omits, then the dialog. */ -function fullDialogTree(): SnapshotState { - return { - ...makeSnapshotState([ - { index: 0, depth: 0, type: 'Group', rect: { x: 0, y: 0, width: 400, height: 800 } }, - ...dialogNodes(DIALOG_LABELS, 1), - ]), - presentation: {}, - }; -} - -function interactiveTree(labels: readonly string[]): SnapshotState { - return { ...makeSnapshotState(dialogNodes(labels)), presentation: { interactiveOnly: true } }; +/** The screen as each view reads it: the full tree keeps a wrapper the interactive view omits. */ +function screen(labels: readonly string[], presentation: SnapshotPresentation): SnapshotState { + const full = presentation.interactiveOnly !== true; + return { ...makeSnapshotState(dialogNodes(labels, full)), presentation }; } -type Screen = { full: SnapshotState; interactive: SnapshotState }; +const FULL: SnapshotPresentation = {}; +const INTERACTIVE: SnapshotPresentation = { interactiveOnly: true }; -const DIALOG: Screen = { full: fullDialogTree(), interactive: interactiveTree(DIALOG_LABELS) }; +function viewOf(options: BackendSnapshotOptions | undefined): SnapshotPresentation { + return { + ...(options?.interactiveOnly === true ? { interactiveOnly: true } : {}), + ...(options?.scope ? { scope: options.scope } : {}), + }; +} /** - * A backend that answers each capture in the view it was asked for. `screens` is the screen before - * the press and after it; `views` records the view of every capture taken after the press. + * A backend that answers every capture in the view it was asked for, as the daemon stamps it. + * `before` and `after` are the dialog labels on screen before and after the action; `views` + * records the view of every capture taken after it. */ -function createDevice(params: { refFrame: SnapshotState; before: Screen; after: Screen }) { - let pressed = false; - const views: Array<'full' | 'interactive'> = []; +function createDevice(params: { + frame: SnapshotState; + before?: readonly string[]; + after?: readonly string[]; +}) { + let acted = false; + const views: SnapshotPresentation[] = []; const device = createAgentDevice({ backend: { platform: 'android', captureSnapshot: async (_context, options) => { - const screen = pressed ? params.after : params.before; - if (pressed) views.push(options?.interactiveOnly === true ? 'interactive' : 'full'); - return { snapshot: options?.interactiveOnly === true ? screen.interactive : screen.full }; + const view = viewOf(options); + if (acted) views.push(view); + return { + snapshot: screen(acted ? (params.after ?? DIALOG) : (params.before ?? DIALOG), view), + }; }, tap: async () => { - pressed = true; + acted = true; return { ok: true }; }, } satisfies AgentDeviceBackend, artifacts: createLocalArtifactAdapter(), sessions: createMemorySessionStore([ - { name: 'default', snapshot: params.refFrame, refFrameSnapshot: params.refFrame }, + { name: 'default', snapshot: params.frame, refFrameSnapshot: params.frame }, ]), policy: localCommandPolicy(), clock: createFakeClock(), @@ -78,94 +96,74 @@ function createDevice(params: { refFrame: SnapshotState; before: Screen; after: return { device, views }; } -function removedLabels(lines: Array<{ kind: string; text: string }> | undefined): string[] { - return (lines ?? []).filter((line) => line.kind === 'removed').map((line) => line.text); -} +const removed = (lines: Array<{ kind: string; text: string }> | undefined) => + (lines ?? []).filter((line) => line.kind === 'removed').map((line) => line.text); test('a ref press after a full snapshot settles on the full tree and reports no false removals', async () => { - const { device, views } = createDevice({ - refFrame: DIALOG.full, - before: DIALOG, - after: DIALOG, - }); + const { device, views } = createDevice({ frame: screen(DIALOG, FULL) }); const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); - assert.ok(views.length > 0 && views.every((view) => view === 'full')); - assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); - // An unchanged screen still hands back refs to act on next, from the full-tree view. - assert.ok(result.settle?.tail?.some((entry) => entry.label === 'OK')); + assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly !== true)); + assert.equal(result.settle?.diff?.summary.removals, 0); + assert.equal(result.settle?.diff?.summary.additions, 0); }); -for (const [flag, presentation] of [ - ['-s', { scope: 'Device name' }], - ['--actions', { customActions: true }], -] as const) { - test(`a ref press after a full snapshot with ${flag} keeps the interactive settle view`, async () => { - const { device, views } = createDevice({ - refFrame: { ...DIALOG.full, presentation }, - before: DIALOG, - after: DIALOG, - }); - - await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); - - // The settle capture cannot reproduce that projection, so the frame is not a full-view baseline. - assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); - }); -} - -test('a ref press after a full snapshot still reports what changed on the full tree', async () => { - const next = ['Next', 'Back']; - const { device } = createDevice({ - refFrame: DIALOG.full, - before: DIALOG, - after: { - full: { - ...makeSnapshotState([ - { index: 0, depth: 0, type: 'Group', rect: { x: 0, y: 0, width: 400, height: 800 } }, - ...dialogNodes(next, 1), - ]), - presentation: {}, - }, - interactive: interactiveTree(next), - }, +test('a ref press after snapshot -i settles on the interactive view and lists what left', async () => { + const { device, views } = createDevice({ + frame: screen(DIALOG, INTERACTIVE), + after: ['Cancel', 'OK'], }); - const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); + const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); - assert.equal(result.settle?.diff?.summary.additions, 2); - assert.equal(result.settle?.diff?.summary.removals, 3); - assert.ok(removedLabels(result.settle?.diff?.lines).some((text) => text.includes('Device name'))); + assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly === true)); + assert.deepEqual(removed(result.settle?.diff?.lines).length, 1); }); -test('a ref press after an interactive snapshot settles on the interactive view', async () => { - const titleGone = interactiveTree(['Cancel', 'OK']); +test('a ref press after a scoped snapshot settles in that scope', async () => { const { device, views } = createDevice({ - refFrame: DIALOG.interactive, - before: DIALOG, - after: { full: DIALOG.full, interactive: titleGone }, + frame: screen(DIALOG, { interactiveOnly: true, scope: 'Device name' }), }); - const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); + await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); - assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); - assert.deepEqual(removedLabels(result.settle?.diff?.lines).length, 1); + assert.ok(views.length > 0 && views.every((view) => view.scope === 'Device name')); }); -test('a selector press settles on the interactive view', async () => { - const titleGone = interactiveTree(['Cancel', 'OK']); - const { device, views } = createDevice({ - refFrame: DIALOG.interactive, - before: DIALOG, - after: { full: DIALOG.full, interactive: titleGone }, - }); +test('a selector press resolved on the full tree still diffs in the ref frame view', async () => { + // `label=Dialog` names the wrapper only the full tree holds, so resolution falls back to it. + const { device, views } = createDevice({ frame: screen(DIALOG, INTERACTIVE) }); - const result = await device.interactions.press(selector('label=OK'), { + const result = await device.interactions.press(selector('label=Dialog'), { session: 'default', settle: {}, }); - assert.ok(views.length > 0 && views.every((view) => view === 'interactive')); - assert.ok(removedLabels(result.settle?.diff?.lines).some((text) => text.includes('Device name'))); + assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly === true)); + assert.deepEqual(removed(result.settle?.diff?.lines), []); +}); + +test('verify after a full snapshot reads the full tree, so an unchanged screen reads unchanged', async () => { + const { device, views } = createDevice({ frame: screen(DIALOG, FULL) }); + + const result = await device.interactions.press(ref('@e3'), { session: 'default', verify: true }); + + assert.deepEqual(views, [FULL]); + assert.equal(result.evidence?.changedFromBefore, false); +}); + +test('diff snapshot -i after a full snapshot starts a baseline instead of listing the view change', async () => { + const { device } = createDevice({ frame: screen(DIALOG, FULL) }); + + const interactive = await device.capture.diffSnapshot({ + session: 'default', + interactiveOnly: true, + }); + assert.equal(interactive.baselineInitialized, true); + assert.deepEqual(interactive.summary, { additions: 0, removals: 0, unchanged: 3 }); + + const again = await device.capture.diffSnapshot({ session: 'default', interactiveOnly: true }); + assert.equal(again.baselineInitialized, false); + assert.deepEqual(again.summary, { additions: 0, removals: 0, unchanged: 3 }); }); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index f1e2865b89..dd0c64a38d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -5,7 +5,10 @@ import type { CommandSessionRecord, } from '@agent-device/contracts/runtime-contract'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; -import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline'; +import { + postActionBaseline, + surfaceScopedNodes, +} from '@agent-device/capture-kit/snapshot-baseline'; import { buildSnapshotDiff } from '@agent-device/capture-kit/snapshot-diff'; import { displayLabel, formatRole } from '@agent-device/capture-kit/snapshot-lines'; import { @@ -21,7 +24,6 @@ import type { SettleTailEntry, SurfaceScopedNodes, } from '@agent-device/contracts/interaction'; -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { RuntimeCommand } from '../../runtime-types.ts'; import type { CapturedSnapshot } from './selector-read-shared.ts'; import { @@ -193,7 +195,6 @@ async function readSettledOutcome( // replacement presented as change within one surface, refs included, so it is refused and the // transition is disclosed instead. const surfaceChange = resolvePostActionSurfaceChange(params.baseline, settledCapture); - const diffShown = outcome.settled && stored && !surfaceChange; return { observation: { ...observation, @@ -205,7 +206,7 @@ async function readSettledOutcome( // captures are intentionally diff-less: they are not a stable // observation, so surfacing refs would invite agents to act on // advisory state. - ...(diffShown + ...(outcome.settled && stored && !surfaceChange ? buildSettleDiffAndTail( params.baseline.nodes, settledNodes, @@ -238,65 +239,35 @@ async function resolveSettleBaseline( resolved: ResolvedInteractionTarget, ): Promise { const session = await runtime.sessions.get(options.session ?? 'default'); + const frame = surfaceScopedNodes(session?.refFrameSnapshot); // A ref is authorized against the stored ref frame. Keep that visible presentation as the // transition baseline: a best-effort evidence recapture can recover through private AX and see // covered background controls that were not actionable when the ref was issued. // Resolved-target evidence is best-effort at the contracts boundary. The session still owns the // authoritative ref frame, so reuse it rather than silently turning a missing optional field // into an empty transition/diff baseline. Fall back to the latest observation for point targets - // and pre-frame sessions. - return ( - authorizedRefBaseline(resolved, session) ?? - nonEmptyBaseline(resolved.preAction) ?? - sessionBaseline(session) - ); -} - -function authorizedRefBaseline( - resolved: ResolvedInteractionTarget, - session: CommandSessionRecord | undefined, -): SurfaceScopedNodes | undefined { - if (resolved.kind !== 'ref') return undefined; - return nonEmptyBaseline(refFrameBaseline(session?.refFrameSnapshot)); -} - -/** - * The ref frame the agent last read. A frame read as the whole full tree (`snapshot` without `-i`, - * `-s`, `-d`, `--raw` or `--actions`) is marked `fullView`, so the settle loop reads the full tree - * too and the diff compares one view. Any of those flags makes a projection the settle capture - * does not reproduce, so such a frame stays unmarked. Selector captures are full trees as well, but no agent - * chose that view, so only the ref frame carries the mark. - */ -function refFrameBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNodes { - const baseline = surfaceScopedNodes(snapshot); - const presentation = snapshot?.presentation; - const wholeFullTree = - presentation !== undefined && - presentation.interactiveOnly !== true && - presentation.scope === undefined && - presentation.depth === undefined && - presentation.raw !== true && - presentation.customActions !== true; - return wholeFullTree ? { ...baseline, fullView: true } : baseline; + // and pre-frame sessions. A candidate read in another view than the frame is skipped (#4486). + return postActionBaseline({ + frame: withoutPresentedSurface(frame), + candidates: [ + resolved.kind === 'ref' ? frame : undefined, + resolved.preAction, + session?.refFrameSnapshot + ? undefined + : withoutPresentedSurface(surfaceScopedNodes(session?.snapshot)), + ], + }); } /** - * The latest stored tree, for a target no capture resolved. The diff can use its nodes, but its + * A stored tree, for a target no capture resolved. The diff can use its nodes, but its * presented-surface fact may predate a side effect this layer cannot see, and an `absent` read * taken before a sheet was presented must not rule out the confirmation owed to that sheet's * dismissal (#3565). */ -function sessionBaseline(session: CommandSessionRecord | undefined): SurfaceScopedNodes { - const { presentedSurface: _unverified, ...baseline } = surfaceScopedNodes( - session?.refFrameSnapshot ?? session?.snapshot, - ); - return baseline; -} - -function nonEmptyBaseline( - baseline: SurfaceScopedNodes | undefined, -): SurfaceScopedNodes | undefined { - return baseline?.nodes.length ? baseline : undefined; +function withoutPresentedSurface(baseline: SurfaceScopedNodes): SurfaceScopedNodes { + const { presentedSurface: _unverified, ...stored } = baseline; + return stored; } function buildSettleDiff( @@ -304,11 +275,9 @@ function buildSettleDiff( settledNodes: SnapshotNode[], appBundleId: string | undefined, ): NonNullable { - // Flattened compare, like `diff -i`: both sides are captures in one view (interactive, or the full - // tree for a `fullView` baseline), and depth jitter across captures should not read as change. When - // the baseline came from a richer stored tree (ref targets reuse the session - // snapshot), extra baseline-only lines surface as removals — advisory noise, - // the same baseline caveat --verify's changedFromBefore already accepts. + // Flattened compare, like `diff -i`: both sides are captures in one presentation + // (postActionBaseline picks the baseline, and the settle loop captures in its view), and + // depth jitter across captures should not read as change. const diff = buildSnapshotDiff( withoutSettleChrome(baselineNodes, appBundleId), withoutSettleChrome(settledNodes, appBundleId), diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index 082b103a38..fcf0177eda 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -1,8 +1,10 @@ import type { SnapshotNode, SnapshotPreferredBackend, + SnapshotPresentation, SnapshotQualityVerdict, } from '@agent-device/kernel/snapshot'; +import { DEFAULT_POST_ACTION_PRESENTATION } from '@agent-device/capture-kit/snapshot-baseline'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers'; @@ -105,7 +107,7 @@ export async function runStableCaptureLoop( resetBudgetOnPrivateAxRecovery?: boolean; /** * Immutable pre-action projection; the stored session may advance before settle begins. The loop - * reads the view the baseline was read in, so a full-view baseline is diffed full against full. + * captures in its presentation, so the settled tree and the baseline share one view (#4486). */ broadTransitionBaseline?: SurfaceScopedNodes; }, @@ -157,7 +159,7 @@ export async function runStableCaptureLoop( runtime, options, deadlineMs - now(runtime), - params.broadTransitionBaseline?.fullView !== true, + params.broadTransitionBaseline?.presentation ?? DEFAULT_POST_ACTION_PRESENTATION, preferredBackend, ); if (read.kind !== 'captured') return unfinished(read); @@ -533,8 +535,8 @@ function stableCaptureDelayMs(params: { } // Intentionally does not update the session snapshot: the stable loop captures -// an interactive-only tree (the full tree for a `fullView` baseline) purely as a -// settle signal, and overwriting the session's richer cached snapshot with it would degrade +// in its baseline's presentation (interactive-only by default) purely as a settle +// signal, and overwriting the session's richer cached snapshot with it would degrade // subsequent ref/get/find lookups against the same session. (`--settle` DOES // store its final capture, but explicitly, in one place, after the loop.) // @@ -547,7 +549,7 @@ async function captureStableSignalWithinDeadline( runtime: AgentDeviceRuntime, options: CommandContext & SelectorSnapshotOptions, remainingMs: number, - interactiveOnly: boolean, + presentation: SnapshotPresentation, preferredBackend?: SnapshotPreferredBackend, ): Promise { try { @@ -557,7 +559,7 @@ async function captureStableSignalWithinDeadline( { ...options, signal }, { updateSession: false, - interactiveOnly, + presentation, preferredBackend, }, ); diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index d72b3b995c..a2b95e65bc 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -3,6 +3,7 @@ import type { Point, SnapshotKeyboardBandFact, SnapshotNode, + SnapshotPresentation, } from '@agent-device/kernel/snapshot'; import type { ResponseCost } from '@agent-device/kernel/contracts'; import type { ClickButton } from './click-button.ts'; @@ -161,11 +162,10 @@ export type SurfaceScopedNodes = { */ presentedSurface?: PresentedSurface; /** - * The nodes are a ref frame the agent read as a full tree (`snapshot` without `-i`). The settle - * loop then reads the full tree too, because diffing these nodes against the interactive view - * reports every node that view omits as removed. Only the settle baseline sets it. + * The presentation the capture read (`-i`, `-s`, `-d`, `--raw`, `--actions`). A post-action read + * captures in it, so the diff compares one view (#4486). Absent when the producer recorded none. */ - fullView?: true; + presentation?: SnapshotPresentation; }; /** Which tree a resolved `@ref`'s node was read from (ADR 0014, "Mutation admission and read behavior"). */ @@ -234,9 +234,10 @@ export type PostActionSurfaceChange = { /** * Opt-in (`--verify`) cheap post-condition evidence for mutating interaction * commands (#1047). `digest`/`nodeCount`/`interactiveNodeCount` describe a single - * interactive-only capture taken right after the action; `changedFromBefore` - * compares that digest against the pre-action capture the resolution path already - * held, so no extra device round trip is spent beyond the one verify capture. + * capture taken right after the action in the pre-action capture's view (interactive-only + * by default, #4486); `changedFromBefore` compares that digest against the pre-action + * capture the resolution path already held, so no extra device round trip is spent beyond + * the one verify capture. * `changedFromBefore: false` is evidence, not failure — the command still * succeeded. * diff --git a/packages/daemon/src/execution/generic-settle.test.ts b/packages/daemon/src/execution/generic-settle.test.ts index 419318de7c..0c8d3297d4 100644 --- a/packages/daemon/src/execution/generic-settle.test.ts +++ b/packages/daemon/src/execution/generic-settle.test.ts @@ -312,6 +312,35 @@ test('scroll --settle passes the stored baseline and attaches the observation wi expect(settle.refsGeneration).toBe(stored.snapshotGeneration); }); +test('scroll --settle keeps the ref frame view when the stored tree was read in another view', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-view'; + // The agent read refs from snapshot -i; a later selector read stored a full tree. + const session = makeIosSession(sessionName); + setSessionSnapshot( + session, + buildSnapshotState(snapshotPayload({ nodes: BEFORE_NODES }), { snapshotInteractiveOnly: true }), + ); + activateCompleteRefFrame(session); + setSessionSnapshot(session, buildSnapshotState(snapshotPayload({ nodes: AFTER_NODES }), {})); + sessionStore.publish(sessionName, session); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + // #4486: the diff compares one view, so the interactive frame answers, not the full stored tree. + expect(settleObservationCalls[0]?.baseline.presentation?.interactiveOnly).toBe(true); + expect(settleObservationCalls[0]?.baseline.nodes).toEqual( + expect.arrayContaining([expect.objectContaining({ label: 'Continue' })]), + ); +}); + test('back --settle attaches the observation alongside the command result', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back'; diff --git a/packages/daemon/src/execution/generic-settle.ts b/packages/daemon/src/execution/generic-settle.ts index 7439d9ab09..075c4ef325 100644 --- a/packages/daemon/src/execution/generic-settle.ts +++ b/packages/daemon/src/execution/generic-settle.ts @@ -1,7 +1,10 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { CommandFlags } from '@agent-device/contracts/command'; import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; -import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline'; +import { + postActionBaseline, + surfaceScopedNodes, +} from '@agent-device/capture-kit/snapshot-baseline'; import type { SettleObservation, SettleParams, @@ -18,7 +21,7 @@ import type { BoundContextFromFlags } from '../core/context.ts'; import type { RuntimeAdmissionBindings } from '../core/request-runtime-binding.ts'; import type { DaemonPlatformServices } from '../core/platform-services.ts'; import { resolveSettleObservationCapture } from '../core/settle-observation-capture.ts'; -import { isOutdatedObservation } from '../core/ref-frame.ts'; +import { isOutdatedObservation, refFrameTree } from '../core/ref-frame.ts'; import { issueSettleRefs } from './session-snapshot.ts'; import type { SessionStore } from '../core/session-store.ts'; import type { DaemonRequest } from '../core/daemon-request.ts'; @@ -45,7 +48,9 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; * * 1. {@link planGenericSettleObservation} runs BEFORE the command dispatches and * freezes the diff baseline. The baseline is the session's STORED pre-action - * tree, so the diff reads "settled tree vs the last tree you observed" — it + * tree, or the ref frame when that tree was read in another view than the + * frame (`postActionBaseline`), so the diff reads "settled tree vs the last + * tree you observed" in one view — it * may be several commands old, unlike press's freshly resolved pre-action * capture. That is the honest #1101 contract for a route with no resolution * step, and the settle loop's own captures overwrite `session.snapshot`, so @@ -87,7 +92,11 @@ export function planGenericSettleObservation( if (invalidSettleFlags) return { response: invalidSettleFlags }; const settle = readSettleRequest(params.flags); if (!settle) return {}; - const baseline = storedTreeBaseline(params.session.snapshot); + // The stored tree answers when it shares the ref frame's view; otherwise the frame does (#4486). + const baseline = postActionBaseline({ + frame: storedTreeBaseline(refFrameTree(params.session)), + candidates: [storedTreeBaseline(params.session.snapshot)], + }); return { observe: async () => await observeSettled(params, settle, baseline) }; } diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index a42ce47550..8480f1aa96 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -786,6 +786,23 @@ export function buildSnapshotPresentationKey(flags: SnapshotOptions | undefined) }); } +/** + * Whether two captures read one presentation, so a diff between them compares like with like + * (#4486): a full tree holds nodes an interactive read omits, and a scoped, depth-limited, raw or + * custom-action read holds a different node set than a plain one. A capture whose producer + * recorded no presentation compares with any. + */ +export function isSameSnapshotPresentation( + a: SnapshotPresentation | undefined, + b: SnapshotPresentation | undefined, +): boolean { + return ( + a === undefined || + b === undefined || + buildSnapshotPresentationKey(a) === buildSnapshotPresentationKey(b) + ); +} + export function centerOfRect(rect: Rect): Point { return { x: Math.round(rect.x + rect.width / 2), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index eb36e50d72..5e43469b24 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -416,7 +416,7 @@ agent-device get attrs @e1 - `viewport: { width, height }` names the box node rects are measured in; see [Coordinates and viewport](/docs/snapshots#coordinates-and-viewport). - `--scope ` re-roots the snapshot at the first matching node; see [Scope a snapshot](/docs/snapshots#scope-a-snapshot). - `--actions` lists custom accessibility actions an element merged away (iOS simulator only, rejected with `--raw`); see [Snapshots](/docs/snapshots). -- `diff snapshot` (alias `snapshot --diff`) compares with the previous session baseline, then updates it. +- `diff snapshot` (alias `snapshot --diff`) compares with the previous session baseline, then updates it. It compares only trees read with the same `-i`, `-d`, `-s`, `--raw` and `--actions`: when the stored snapshot was read in another view, the capture starts a new baseline (`baselineInitialized: true`) instead of listing the view change as changes. ## Wait and alerts @@ -499,7 +499,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. After a full `snapshot` (no `-i`, `-s`, `-d`, `--raw` or `--actions`), a `--settle` action on one of its `@ref`s reads the full tree too, so its diff compares one view and its refs are full-tree refs. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. `--settle` and `--verify` compare trees read in one view: the view of the refs you last read (`snapshot -i`, a full `snapshot`, or a scoped one), interactive when you have none. After a full `snapshot`, a settle reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, on refs, selectors and coordinates. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From bba617261edbe701d586b3712647028dd7d7eb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 10:56:45 +0100 Subject: [PATCH 13/14] fix(snapshot): preserve view options and lazy baseline imports --- .../capture-kit/src/snapshot-baseline.test.ts | 23 +++++----- packages/capture-kit/src/snapshot-baseline.ts | 10 ++--- .../src/commands/capture/runtime/snapshot.ts | 12 +++-- .../runtime/settle-view-baseline.test.ts | 44 +++++++++++++++---- .../interaction/runtime/stable-capture.ts | 7 ++- .../interaction/runtime/wait-stable.test.ts | 41 ++++++++++++++++- .../src/execution/generic-settle.test.ts | 6 ++- .../daemon/src/execution/generic-settle.ts | 12 ++--- .../android-lifecycle.test.ts | 2 +- .../remote-proxy-parity.test.ts | 2 +- 10 files changed, 117 insertions(+), 42 deletions(-) diff --git a/packages/capture-kit/src/snapshot-baseline.test.ts b/packages/capture-kit/src/snapshot-baseline.test.ts index 7418905e23..f6958f88b8 100644 --- a/packages/capture-kit/src/snapshot-baseline.test.ts +++ b/packages/capture-kit/src/snapshot-baseline.test.ts @@ -58,21 +58,24 @@ test.each([ 'frame', ], ['interactive frame, candidate with no recorded view', INTERACTIVE, undefined, 'candidate'], -] as const)('%s: the %s answers', (_case, framePresentation, candidatePresentation, expected) => { - const baseline = postActionBaseline({ - frame: frame(framePresentation), - candidates: [tree('candidate', candidatePresentation)], - }); - expect(baseline.nodes[0]?.label).toBe(expected); -}); +] as const)( + '%s: the %s answers', + async (_case, framePresentation, candidatePresentation, expected) => { + const baseline = await postActionBaseline({ + frame: frame(framePresentation), + candidates: [tree('candidate', candidatePresentation)], + }); + expect(baseline.nodes[0]?.label).toBe(expected); + }, +); -test('without a frame the first non-empty candidate answers, in its own view', () => { - const baseline = postActionBaseline({ +test('without a frame the first non-empty candidate answers, in its own view', async () => { + const baseline = await postActionBaseline({ frame: undefined, candidates: [undefined, { nodes: [] }, tree('candidate', FULL)], }); expect(baseline).toEqual(tree('candidate', FULL)); - expect(postActionBaseline({ frame: undefined, candidates: [] })).toEqual({ nodes: [] }); + expect(await postActionBaseline({ frame: undefined, candidates: [] })).toEqual({ nodes: [] }); }); function tree(label: string, presentation: SurfaceScopedNodes['presentation']): SurfaceScopedNodes { diff --git a/packages/capture-kit/src/snapshot-baseline.ts b/packages/capture-kit/src/snapshot-baseline.ts index 6cc34e4d96..ecc3dc97c8 100644 --- a/packages/capture-kit/src/snapshot-baseline.ts +++ b/packages/capture-kit/src/snapshot-baseline.ts @@ -1,9 +1,6 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; -import { - isSameSnapshotPresentation, - type SnapshotPresentation, -} from '@agent-device/kernel/snapshot'; +import type { SnapshotPresentation } from '@agent-device/kernel/snapshot'; /** The view a post-action read captures in when its baseline recorded none. */ export const DEFAULT_POST_ACTION_PRESENTATION: SnapshotPresentation = { interactiveOnly: true }; @@ -51,12 +48,13 @@ export function surfaceScopedNodes( * * Every settle route picks its baseline here, so none can compare an interactive read with a full one. */ -export function postActionBaseline(params: { +export async function postActionBaseline(params: { /** The ref frame, as the caller admits it as a baseline. */ frame: SurfaceScopedNodes | undefined; /** Pre-action trees in the caller's order of preference. */ candidates: readonly (SurfaceScopedNodes | undefined)[]; -}): SurfaceScopedNodes { +}): Promise { + const { isSameSnapshotPresentation } = await import('@agent-device/kernel/snapshot'); const { frame } = params; const view = frame && frame.nodes.length > 0 ? frame.presentation : undefined; const candidate = params.candidates.find( diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 64ad9bdd55..6ecd70a477 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -136,13 +136,11 @@ export const diffSnapshotCommand: RuntimeCommand< } const capture = await captureRuntimeSnapshot(runtime, options); const flattenForDiff = options.interactiveOnly === true; - // A diff compares one view: the stored tree, else the ref frame, read in the requested - // presentation. With neither, the capture initializes the baseline (#4486). - const previousSnapshot = [capture.session?.snapshot, capture.session?.refFrameSnapshot].find( - (stored) => - stored !== undefined && - isSameSnapshotPresentation(stored.presentation, capture.snapshot.presentation), - ); + const stored = capture.session?.snapshot; + const previousSnapshot = + stored && isSameSnapshotPresentation(stored.presentation, capture.snapshot.presentation) + ? stored + : undefined; const nextSession = nextSnapshotSession(options.session, capture); if (!previousSnapshot) { diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index 0770a29fe5..de294600cb 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -55,7 +55,10 @@ const INTERACTIVE: SnapshotPresentation = { interactiveOnly: true }; function viewOf(options: BackendSnapshotOptions | undefined): SnapshotPresentation { return { ...(options?.interactiveOnly === true ? { interactiveOnly: true } : {}), - ...(options?.scope ? { scope: options.scope } : {}), + ...(options?.scope !== undefined ? { scope: options.scope } : {}), + ...(options?.depth !== undefined ? { depth: options.depth } : {}), + ...(options?.raw !== undefined ? { raw: options.raw } : {}), + ...(options?.customActions !== undefined ? { customActions: options.customActions } : {}), }; } @@ -105,8 +108,7 @@ test('a ref press after a full snapshot settles on the full tree and reports no const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly !== true)); - assert.equal(result.settle?.diff?.summary.removals, 0); - assert.equal(result.settle?.diff?.summary.additions, 0); + assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); }); test('a ref press after snapshot -i settles on the interactive view and lists what left', async () => { @@ -121,14 +123,23 @@ test('a ref press after snapshot -i settles on the interactive view and lists wh assert.deepEqual(removed(result.settle?.diff?.lines).length, 1); }); -test('a ref press after a scoped snapshot settles in that scope', async () => { - const { device, views } = createDevice({ - frame: screen(DIALOG, { interactiveOnly: true, scope: 'Device name' }), - }); +test.each([ + { interactiveOnly: true, scope: 'Device name' }, + { depth: 2 }, + { raw: true }, + { customActions: true }, +])('a ref press preserves snapshot presentation %j when settling', async (presentation) => { + const { device, views } = createDevice({ frame: screen(DIALOG, presentation) }); - await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); + const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); - assert.ok(views.length > 0 && views.every((view) => view.scope === 'Device name')); + assert.ok(views.length > 0); + for (const view of views) assert.deepEqual(view, presentation); + assert.deepEqual(result.settle?.diff?.summary, { + additions: 0, + removals: 0, + unchanged: presentation.interactiveOnly ? 3 : 4, + }); }); test('a selector press resolved on the full tree still diffs in the ref frame view', async () => { @@ -167,3 +178,18 @@ test('diff snapshot -i after a full snapshot starts a baseline instead of listin assert.equal(again.baselineInitialized, false); assert.deepEqual(again.summary, { additions: 0, removals: 0, unchanged: 3 }); }); + +test('diff snapshot starts a baseline when only an older ref frame matches the requested view', async () => { + const { device } = createDevice({ frame: screen(DIALOG, INTERACTIVE) }); + // Selector reads replace the operational snapshot without issuing a new ref frame. + await device.selectors.get({ + session: 'default', + target: selector('label=Dialog'), + property: 'text', + }); + + const result = await device.capture.diffSnapshot({ session: 'default', interactiveOnly: true }); + + assert.equal(result.baselineInitialized, true); + assert.deepEqual(result.summary, { additions: 0, removals: 0, unchanged: 3 }); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index fcf0177eda..7bd3cdb2ec 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -159,7 +159,12 @@ export async function runStableCaptureLoop( runtime, options, deadlineMs - now(runtime), - params.broadTransitionBaseline?.presentation ?? DEFAULT_POST_ACTION_PRESENTATION, + params.broadTransitionBaseline?.presentation ?? { + ...DEFAULT_POST_ACTION_PRESENTATION, + depth: options.depth, + scope: options.scope, + raw: options.raw, + }, preferredBackend, ); if (read.kind !== 'captured') return unfinished(read); diff --git a/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts b/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts index 0d1c8bfd06..6f7fdcd79c 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { AgentDeviceBackend, BackendSnapshotResult } from '@agent-device/contracts/backend'; +import type { + AgentDeviceBackend, + BackendSnapshotOptions, + BackendSnapshotResult, +} from '@agent-device/contracts/backend'; import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; import { createAgentDevice } from '../../../runtime.ts'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; @@ -8,6 +12,41 @@ import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fix import { createFakeClock, selectorReadSnapshot } from './__tests__/test-utils/index.ts'; import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +test('runtime wait stable preserves the requested scope, depth and raw view', async () => { + const captures: Array = []; + const snapshot = selectorReadSnapshot(); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captures.push(options); + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.selectors.wait({ + session: 'default', + scope: 'Contact details', + depth: 2, + raw: true, + target: { kind: 'stable', quietMs: 25, timeoutMs: 1_000 }, + }); + + assert.equal(result.kind, 'stable'); + assert.ok(captures.length >= 2); + for (const capture of captures) { + assert.equal(capture?.interactiveOnly, true); + assert.equal(capture?.scope, 'Contact details'); + assert.equal(capture?.depth, 2); + assert.equal(capture?.raw, true); + } +}); + test('runtime wait stable settles after two unchanged captures', async () => { const snapshot = selectorReadSnapshot(); const clock = createFakeClock(); diff --git a/packages/daemon/src/execution/generic-settle.test.ts b/packages/daemon/src/execution/generic-settle.test.ts index 0c8d3297d4..f301e83e36 100644 --- a/packages/daemon/src/execution/generic-settle.test.ts +++ b/packages/daemon/src/execution/generic-settle.test.ts @@ -319,10 +319,13 @@ test('scroll --settle keeps the ref frame view when the stored tree was read in const session = makeIosSession(sessionName); setSessionSnapshot( session, - buildSnapshotState(snapshotPayload({ nodes: BEFORE_NODES }), { snapshotInteractiveOnly: true }), + buildSnapshotState(snapshotPayload({ nodes: BEFORE_NODES, presentedSurface: 'absent' }), { + snapshotInteractiveOnly: true, + }), ); activateCompleteRefFrame(session); setSessionSnapshot(session, buildSnapshotState(snapshotPayload({ nodes: AFTER_NODES }), {})); + markSessionSnapshotOutdated(session); sessionStore.publish(sessionName, session); mockCommandDispatch([AFTER_NODES]); @@ -339,6 +342,7 @@ test('scroll --settle keeps the ref frame view when the stored tree was read in expect(settleObservationCalls[0]?.baseline.nodes).toEqual( expect.arrayContaining([expect.objectContaining({ label: 'Continue' })]), ); + expect(settleObservationCalls[0]?.baseline.presentedSurface).toBeUndefined(); }); test('back --settle attaches the observation alongside the command result', async () => { diff --git a/packages/daemon/src/execution/generic-settle.ts b/packages/daemon/src/execution/generic-settle.ts index 075c4ef325..f2accd1537 100644 --- a/packages/daemon/src/execution/generic-settle.ts +++ b/packages/daemon/src/execution/generic-settle.ts @@ -84,17 +84,19 @@ type GenericSettleContext = { * otherwise returns the observer this dispatch owes — with the pre-action * baseline already captured. */ -export function planGenericSettleObservation( +export async function planGenericSettleObservation( params: GenericSettleContext & { command: string; flags: CommandFlags | undefined }, -): GenericSettlePlan { +): Promise { if (!commandSupportsSettleObservation(params.command)) return {}; const invalidSettleFlags = settleFlagGuardResponse(params.command, params.flags); if (invalidSettleFlags) return { response: invalidSettleFlags }; const settle = readSettleRequest(params.flags); if (!settle) return {}; - // The stored tree answers when it shares the ref frame's view; otherwise the frame does (#4486). - const baseline = postActionBaseline({ - frame: storedTreeBaseline(refFrameTree(params.session)), + const { presentedSurface: _unverified, ...frame } = surfaceScopedNodes( + refFrameTree(params.session), + ); + const baseline = await postActionBaseline({ + frame, candidates: [storedTreeBaseline(params.session.snapshot)], }); return { observe: async () => await observeSettled(params, settle, baseline) }; diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index cb63852342..928db61851 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -748,7 +748,7 @@ async function runAndroidCaptureInteractionAndReplayWorkflow( ...selection, }); assert.equal(diff.mode, 'snapshot'); - assert.equal(diff.baselineInitialized, false); + assert.equal(diff.baselineInitialized, true); assert.deepEqual(diff.summary, { additions: 0, removals: 0, unchanged: 3 }); const rotate = await client.command.orientation({ diff --git a/test/integration/provider-scenarios/remote-proxy-parity.test.ts b/test/integration/provider-scenarios/remote-proxy-parity.test.ts index 1263d230ca..c75187839d 100644 --- a/test/integration/provider-scenarios/remote-proxy-parity.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-parity.test.ts @@ -374,7 +374,7 @@ function assertScriptOutcomes(responses: DaemonResponse[]): void { assert.equal(open?.ok, true); assert.equal(snapshot?.ok, true); assert.equal(raw?.ok, true); - assert.equal(baselineInitialized(diff), false); + assert.equal(baselineInitialized(diff), true); assert.equal(missingRef?.ok, false); assert.equal( baselineInitialized(freshDiff), From 5b2c4cd685cf2e297149e34457b7a40c941874cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 11:00:27 +0100 Subject: [PATCH 14/14] fix(snapshot): retain default views and surface transition evidence --- .../capture-kit/src/snapshot-baseline.test.ts | 3 +- packages/capture-kit/src/snapshot-baseline.ts | 10 +++-- .../runtime/post-action-observation.ts | 21 +++++----- .../runtime/post-action-surface.test.ts | 9 +++-- .../runtime/settle-view-baseline.test.ts | 40 ++++++++++++------- .../interaction/runtime/stable-capture.ts | 16 ++++---- packages/contracts/src/interaction.ts | 2 +- packages/kernel/src/snapshot.ts | 10 ++--- website/docs/docs/commands.md | 6 ++- 9 files changed, 70 insertions(+), 47 deletions(-) diff --git a/packages/capture-kit/src/snapshot-baseline.test.ts b/packages/capture-kit/src/snapshot-baseline.test.ts index f6958f88b8..dbb529e0cc 100644 --- a/packages/capture-kit/src/snapshot-baseline.test.ts +++ b/packages/capture-kit/src/snapshot-baseline.test.ts @@ -57,7 +57,8 @@ test.each([ { interactiveOnly: true, scope: 'Composer' }, 'frame', ], - ['interactive frame, candidate with no recorded view', INTERACTIVE, undefined, 'candidate'], + ['interactive frame, candidate with default full view', INTERACTIVE, undefined, 'frame'], + ['full frame, candidate with default full view', FULL, undefined, 'candidate'], ] as const)( '%s: the %s answers', async (_case, framePresentation, candidatePresentation, expected) => { diff --git a/packages/capture-kit/src/snapshot-baseline.ts b/packages/capture-kit/src/snapshot-baseline.ts index ecc3dc97c8..50ce2a05bd 100644 --- a/packages/capture-kit/src/snapshot-baseline.ts +++ b/packages/capture-kit/src/snapshot-baseline.ts @@ -2,8 +2,12 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; import type { SnapshotPresentation } from '@agent-device/kernel/snapshot'; -/** The view a post-action read captures in when its baseline recorded none. */ -export const DEFAULT_POST_ACTION_PRESENTATION: SnapshotPresentation = { interactiveOnly: true }; +/** Reproduce a populated baseline's view; without one, capture the interactive view. */ +export function postActionPresentation( + baseline: SurfaceScopedNodes | undefined, +): SnapshotPresentation { + return baseline?.presentation ?? (baseline?.nodes.length ? {} : { interactiveOnly: true }); +} /** * Mints the one baseline value a post-action comparison carries: the captured nodes together with the @@ -61,7 +65,7 @@ export async function postActionBaseline(params: { (baseline) => baseline !== undefined && baseline.nodes.length > 0 && - isSameSnapshotPresentation(baseline.presentation, view), + (!frame?.nodes.length || isSameSnapshotPresentation(baseline.presentation, view)), ); return candidate ?? frame ?? { nodes: [] }; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts index d05eca5eb3..396ff5d52e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts @@ -6,9 +6,12 @@ import type { SettleParams, } from '@agent-device/contracts/interaction'; import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot'; -import { summarizePostActionEvidence } from './post-action-surface.ts'; import { - DEFAULT_POST_ACTION_PRESENTATION, + resolvePostActionSurfaceChange, + summarizePostActionEvidence, +} from './post-action-surface.ts'; +import { + postActionPresentation, surfaceScopedNodes, } from '@agent-device/capture-kit/snapshot-baseline'; import { captureSelectorSnapshot } from './selector-read-shared.ts'; @@ -85,14 +88,14 @@ async function observeAfterInteraction( ...params.settle, resolved, }); - // The evidence compares one view: a pre-action tree read in another view than the settle - // capture makes no changedFromBefore claim (#4486). + // A surface transition is evidence in every view; node digests compare only within one view. const baseline = outcome.settledCapture && - isSameSnapshotPresentation( - resolved.preAction?.presentation, - outcome.settledCapture.presentation, - ) + (resolvePostActionSurfaceChange(resolved.preAction, outcome.settledCapture) || + isSameSnapshotPresentation( + resolved.preAction?.presentation, + outcome.settledCapture.presentation, + )) ? resolved.preAction : undefined; const evidence = params.verify ? settleEvidence(outcome.settledCapture, baseline) : undefined; @@ -119,7 +122,7 @@ async function captureVerifyEvidence( const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true, includeRects: true, - presentation: resolved.preAction?.presentation ?? DEFAULT_POST_ACTION_PRESENTATION, + presentation: postActionPresentation(resolved.preAction), }); return summarizePostActionEvidence(surfaceScopedNodes(capture.snapshot), resolved.preAction); } catch { diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts index 21c4023cae..ab868bb842 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts @@ -117,12 +117,13 @@ test('press --settle attaches no diff across a sheet-to-app surface change and d assert.match(settle.hint ?? '', /different surfaces/); }); -test('press --settle --verify reports one app-to-sheet surface change on both payloads', async () => { - const before = buttonSnapshot(); - const sheet = webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS); +test('press --settle --verify reports a surface change even when the selector and frame views differ', async () => { + const before = { ...buttonSnapshot(), presentation: { interactiveOnly: true } }; + const frame = { ...before, presentation: {} }; + const sheet = { ...webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS), presentation: {} }; let captures = 0; const device = createSettleDevice({ - stored: before, + stored: frame, captureSnapshot: () => { captures += 1; return { snapshot: captures === 1 ? before : sheet }; diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts index de294600cb..04f5506323 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -44,8 +44,11 @@ function dialogNodes(labels: readonly string[], wrapper: boolean) { } /** The screen as each view reads it: the full tree keeps a wrapper the interactive view omits. */ -function screen(labels: readonly string[], presentation: SnapshotPresentation): SnapshotState { - const full = presentation.interactiveOnly !== true; +function screen( + labels: readonly string[], + presentation: SnapshotPresentation | undefined, +): SnapshotState { + const full = presentation?.interactiveOnly !== true; return { ...makeSnapshotState(dialogNodes(labels, full)), presentation }; } @@ -102,14 +105,17 @@ function createDevice(params: { const removed = (lines: Array<{ kind: string; text: string }> | undefined) => (lines ?? []).filter((line) => line.kind === 'removed').map((line) => line.text); -test('a ref press after a full snapshot settles on the full tree and reports no false removals', async () => { - const { device, views } = createDevice({ frame: screen(DIALOG, FULL) }); +test.each([FULL, undefined])( + 'a ref press after a full snapshot (%j) settles without false removals', + async (presentation) => { + const { device, views } = createDevice({ frame: screen(DIALOG, presentation) }); - const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); + const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); - assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly !== true)); - assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); -}); + assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly !== true)); + assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); + }, +); test('a ref press after snapshot -i settles on the interactive view and lists what left', async () => { const { device, views } = createDevice({ @@ -155,14 +161,20 @@ test('a selector press resolved on the full tree still diffs in the ref frame vi assert.deepEqual(removed(result.settle?.diff?.lines), []); }); -test('verify after a full snapshot reads the full tree, so an unchanged screen reads unchanged', async () => { - const { device, views } = createDevice({ frame: screen(DIALOG, FULL) }); +test.each([FULL, undefined])( + 'verify after a full snapshot (%j) reads unchanged content in the full view', + async (presentation) => { + const { device, views } = createDevice({ frame: screen(DIALOG, presentation) }); - const result = await device.interactions.press(ref('@e3'), { session: 'default', verify: true }); + const result = await device.interactions.press(ref('@e3'), { + session: 'default', + verify: true, + }); - assert.deepEqual(views, [FULL]); - assert.equal(result.evidence?.changedFromBefore, false); -}); + assert.deepEqual(views, [FULL]); + assert.equal(result.evidence?.changedFromBefore, false); + }, +); test('diff snapshot -i after a full snapshot starts a baseline instead of listing the view change', async () => { const { device } = createDevice({ frame: screen(DIALOG, FULL) }); diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index 7bd3cdb2ec..0060264cd2 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -4,7 +4,7 @@ import type { SnapshotPresentation, SnapshotQualityVerdict, } from '@agent-device/kernel/snapshot'; -import { DEFAULT_POST_ACTION_PRESENTATION } from '@agent-device/capture-kit/snapshot-baseline'; +import { postActionPresentation } from '@agent-device/capture-kit/snapshot-baseline'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers'; @@ -159,12 +159,14 @@ export async function runStableCaptureLoop( runtime, options, deadlineMs - now(runtime), - params.broadTransitionBaseline?.presentation ?? { - ...DEFAULT_POST_ACTION_PRESENTATION, - depth: options.depth, - scope: options.scope, - raw: options.raw, - }, + params.broadTransitionBaseline + ? postActionPresentation(params.broadTransitionBaseline) + : { + ...postActionPresentation(undefined), + depth: options.depth, + scope: options.scope, + raw: options.raw, + }, preferredBackend, ); if (read.kind !== 'captured') return unfinished(read); diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index a2b95e65bc..fd30cea822 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -163,7 +163,7 @@ export type SurfaceScopedNodes = { presentedSurface?: PresentedSurface; /** * The presentation the capture read (`-i`, `-s`, `-d`, `--raw`, `--actions`). A post-action read - * captures in it, so the diff compares one view (#4486). Absent when the producer recorded none. + * captures in it, so the diff compares one view (#4486). Absent means the default full tree. */ presentation?: SnapshotPresentation; }; diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 8480f1aa96..dab8618c9d 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -789,18 +789,14 @@ export function buildSnapshotPresentationKey(flags: SnapshotOptions | undefined) /** * Whether two captures read one presentation, so a diff between them compares like with like * (#4486): a full tree holds nodes an interactive read omits, and a scoped, depth-limited, raw or - * custom-action read holds a different node set than a plain one. A capture whose producer - * recorded no presentation compares with any. + * custom-action read holds a different node set than a plain one. An absent presentation is + * the default full tree, as on SnapshotState. */ export function isSameSnapshotPresentation( a: SnapshotPresentation | undefined, b: SnapshotPresentation | undefined, ): boolean { - return ( - a === undefined || - b === undefined || - buildSnapshotPresentationKey(a) === buildSnapshotPresentationKey(b) - ); + return buildSnapshotPresentationKey(a) === buildSnapshotPresentationKey(b); } export function centerOfRect(rect: Rect): Point { diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 5e43469b24..d68250fd82 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -499,7 +499,11 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. `--settle` and `--verify` compare trees read in one view: the view of the refs you last read (`snapshot -i`, a full `snapshot`, or a scoped one), interactive when you have none. After a full `snapshot`, a settle reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, on refs, selectors and coordinates. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). + +`--settle` compares trees in the view of the refs you last read. After a full `snapshot`, it reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. It also preserves `-d`, `-s`, `--raw` and `--actions`. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, including ref, selector and coordinate targets. + +Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`.