diff --git a/packages/capture-kit/src/snapshot-baseline.test.ts b/packages/capture-kit/src/snapshot-baseline.test.ts index 5e2f97e577..dbb529e0cc 100644 --- a/packages/capture-kit/src/snapshot-baseline.test.ts +++ b/packages/capture-kit/src/snapshot-baseline.test.ts @@ -1,6 +1,7 @@ import { expect, test } from 'vitest'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; -import { surfaceScopedNodes } from './snapshot-baseline.ts'; +import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; +import { postActionBaseline, surfaceScopedNodes } from './snapshot-baseline.ts'; const nodes = [{ index: 0, type: 'Application', ref: 'e1' }]; @@ -33,6 +34,58 @@ test('a session with no stored snapshot still mints an empty baseline', () => { expect(surfaceScopedNodes(undefined)).toEqual({ nodes: [] }); }); +test('the mint carries the presentation the capture read', () => { + expect(surfaceScopedNodes(snapshot({ presentation: { interactiveOnly: true } }))).toEqual({ + nodes, + presentation: { interactiveOnly: true }, + }); +}); + +const INTERACTIVE = { interactiveOnly: true }; +const FULL = {}; +const frame = (presentation: SurfaceScopedNodes['presentation']) => tree('frame', presentation); + +// #4486: a settle diff compares one view, and the agent's view is its ref frame's presentation. +test.each([ + ['interactive frame, interactive candidate', INTERACTIVE, INTERACTIVE, 'candidate'], + ['interactive frame, full candidate', INTERACTIVE, FULL, 'frame'], + ['full frame, interactive candidate', FULL, INTERACTIVE, 'frame'], + ['full frame, full candidate', FULL, FULL, 'candidate'], + [ + 'interactive frame, scoped interactive candidate', + INTERACTIVE, + { interactiveOnly: true, scope: 'Composer' }, + 'frame', + ], + ['interactive frame, candidate with default full view', INTERACTIVE, undefined, 'frame'], + ['full frame, candidate with default full view', FULL, undefined, 'candidate'], +] as const)( + '%s: the %s answers', + async (_case, framePresentation, candidatePresentation, expected) => { + const baseline = await postActionBaseline({ + frame: frame(framePresentation), + candidates: [tree('candidate', candidatePresentation)], + }); + expect(baseline.nodes[0]?.label).toBe(expected); + }, +); + +test('without a frame the first non-empty candidate answers, in its own view', async () => { + const baseline = await postActionBaseline({ + frame: undefined, + candidates: [undefined, { nodes: [] }, tree('candidate', FULL)], + }); + expect(baseline).toEqual(tree('candidate', FULL)); + expect(await postActionBaseline({ frame: undefined, candidates: [] })).toEqual({ nodes: [] }); +}); + +function tree(label: string, presentation: SurfaceScopedNodes['presentation']): SurfaceScopedNodes { + return { + nodes: [{ index: 0, type: 'Button', label, ref: 'e1' }], + ...(presentation ? { presentation } : {}), + }; +} + function snapshot(fields: Partial): SnapshotState { return { nodes, createdAt: 0, presentationKey: '', ...fields } as SnapshotState; } diff --git a/packages/capture-kit/src/snapshot-baseline.ts b/packages/capture-kit/src/snapshot-baseline.ts index 93fe201d17..50ce2a05bd 100644 --- a/packages/capture-kit/src/snapshot-baseline.ts +++ b/packages/capture-kit/src/snapshot-baseline.ts @@ -1,9 +1,18 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; +import type { SnapshotPresentation } from '@agent-device/kernel/snapshot'; + +/** Reproduce a populated baseline's view; without one, capture the interactive view. */ +export function postActionPresentation( + baseline: SurfaceScopedNodes | undefined, +): SnapshotPresentation { + return baseline?.presentation ?? (baseline?.nodes.length ? {} : { interactiveOnly: true }); +} /** * Mints the one baseline value a post-action comparison carries: the captured nodes together with the - * surface they describe and the presentation the capture read (#2438, #3565). + * surface they describe, the presentation the capture read (#2438, #3565), and the view it was read + * in (#4486). * * Both settle routes take their baseline from here — the element-targeted interaction route and the * generic daemon route (`scroll`/`back --settle`) — so a fact a capture established cannot be carried @@ -16,7 +25,10 @@ import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; */ export function surfaceScopedNodes( snapshot: - | Pick + | Pick< + SnapshotState, + 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation' + > | undefined, ): SurfaceScopedNodes { return { @@ -25,5 +37,35 @@ export function surfaceScopedNodes( ? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId } : {}), ...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}), + ...(snapshot?.presentation ? { presentation: snapshot.presentation } : {}), }; } + +/** + * The pre-action tree a settle diff runs against, and through its `presentation` the view the settle + * captures in (#4486). Two trees compare only within one presentation, and the agent's view is the + * presentation of the ref frame it last read: `snapshot -i`, a full `snapshot`, a scoped one, or the + * refs a settle issued. So the first candidate read in that view answers. A candidate read in another + * view, such as a selector resolution that fell back to the full tree or a full tree a selector read + * stored, is skipped, and the frame itself answers when no candidate qualifies. Without a frame, the + * first candidate answers. + * + * Every settle route picks its baseline here, so none can compare an interactive read with a full one. + */ +export async function postActionBaseline(params: { + /** The ref frame, as the caller admits it as a baseline. */ + frame: SurfaceScopedNodes | undefined; + /** Pre-action trees in the caller's order of preference. */ + candidates: readonly (SurfaceScopedNodes | undefined)[]; +}): Promise { + const { isSameSnapshotPresentation } = await import('@agent-device/kernel/snapshot'); + const { frame } = params; + const view = frame && frame.nodes.length > 0 ? frame.presentation : undefined; + const candidate = params.candidates.find( + (baseline) => + baseline !== undefined && + baseline.nodes.length > 0 && + (!frame?.nodes.length || isSameSnapshotPresentation(baseline.presentation, view)), + ); + return candidate ?? frame ?? { nodes: [] }; +} diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index d99479a72c..6ecd70a477 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -8,6 +8,7 @@ import { type SnapshotDiagnosticsSummary, } from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; +import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers'; import type { @@ -135,7 +136,11 @@ export const diffSnapshotCommand: RuntimeCommand< } const capture = await captureRuntimeSnapshot(runtime, options); const flattenForDiff = options.interactiveOnly === true; - const previousSnapshot = capture.session?.snapshot; + const stored = capture.session?.snapshot; + const previousSnapshot = + stored && isSameSnapshotPresentation(stored.presentation, capture.snapshot.presentation) + ? stored + : undefined; const nextSession = nextSnapshotSession(options.session, capture); if (!previousSnapshot) { diff --git a/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts index 58864db20e..645e7be06a 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interaction-snapshot-capture.ts @@ -6,8 +6,7 @@ import { toBackendContext } from '../../runtime-common.ts'; /** * A leaf shared by every interaction runtime module that needs a fresh (or session-cached) tree: * `resolution.ts`'s ref/selector resolution, `selector-target-resolution.ts`'s acting selector lookup, - * `moving-target-gate.ts`'s re-observation, `gestures.ts`'s viewport resolution, and - * `post-action-observation.ts`'s `--verify` baseline. + * `moving-target-gate.ts`'s re-observation, and `gestures.ts`'s viewport resolution. * Deliberately self-contained — it imports no sibling of this directory, so those siblings can * import it without ever risking a cycle back. */ diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts index 1bcf72e9e1..396ff5d52e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts @@ -5,9 +5,16 @@ import type { SettleObservation, SettleParams, } from '@agent-device/contracts/interaction'; -import { captureInteractionSnapshot } from './interaction-snapshot-capture.ts'; -import { summarizePostActionEvidence } from './post-action-surface.ts'; -import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline'; +import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot'; +import { + resolvePostActionSurfaceChange, + summarizePostActionEvidence, +} from './post-action-surface.ts'; +import { + postActionPresentation, + surfaceScopedNodes, +} from '@agent-device/capture-kit/snapshot-baseline'; +import { captureSelectorSnapshot } from './selector-read-shared.ts'; import { settleAfterInteraction, settleEvidence } from './settle.ts'; type ObservedResult = T & { @@ -81,9 +88,17 @@ async function observeAfterInteraction( ...params.settle, resolved, }); - const evidence = params.verify - ? settleEvidence(outcome.settledCapture, resolved.preAction) - : undefined; + // A surface transition is evidence in every view; node digests compare only within one view. + const baseline = + outcome.settledCapture && + (resolvePostActionSurfaceChange(resolved.preAction, outcome.settledCapture) || + isSameSnapshotPresentation( + resolved.preAction?.presentation, + outcome.settledCapture.presentation, + )) + ? resolved.preAction + : undefined; + const evidence = params.verify ? settleEvidence(outcome.settledCapture, baseline) : undefined; return { settle: outcome.observation, ...(evidence ? { evidence } : {}) }; } if (!params.verify) return {}; @@ -92,8 +107,8 @@ async function observeAfterInteraction( } /** - * Post-action side of `--verify` (#1047): one interactive-only capture through - * the same capture helper the resolution path already uses, digested and then + * Post-action side of `--verify` (#1047): one capture in the pre-action tree's presentation + * (interactive-only by default), so the comparison reads one view (#4486), digested and then * discarded. The node tree itself is never attached to the result, only the * cheap summary. A missing baseline yields no `changedFromBefore` claim — * no baseline, no comparison. @@ -104,7 +119,11 @@ async function captureVerifyEvidence( resolved: ResolvedInteractionTarget, ): Promise { try { - const capture = await captureInteractionSnapshot(runtime, options, true); + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + includeRects: true, + presentation: postActionPresentation(resolved.preAction), + }); return summarizePostActionEvidence(surfaceScopedNodes(capture.snapshot), resolved.preAction); } catch { return undefined; diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts index 21c4023cae..ab868bb842 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts @@ -117,12 +117,13 @@ test('press --settle attaches no diff across a sheet-to-app surface change and d assert.match(settle.hint ?? '', /different surfaces/); }); -test('press --settle --verify reports one app-to-sheet surface change on both payloads', async () => { - const before = buttonSnapshot(); - const sheet = webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS); +test('press --settle --verify reports a surface change even when the selector and frame views differ', async () => { + const before = { ...buttonSnapshot(), presentation: { interactiveOnly: true } }; + const frame = { ...before, presentation: {} }; + const sheet = { ...webSignInSheetSnapshot(WEB_SIGN_IN_SHEET_LABELS), presentation: {} }; let captures = 0; const device = createSettleDevice({ - stored: before, + stored: frame, captureSnapshot: () => { captures += 1; return { snapshot: captures === 1 ? before : sheet }; diff --git a/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts b/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts index 3d93336f25..b4331d4d92 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/selector-read-shared.ts @@ -15,6 +15,7 @@ import { type SnapshotCommandOptionFields, type SnapshotNode, type SnapshotPreferredBackend, + type SnapshotPresentation, } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { @@ -72,6 +73,11 @@ export async function captureSelectorSnapshot( interactiveOnly?: boolean; includeHiddenContentHints?: boolean; preferredBackend?: SnapshotPreferredBackend; + /** + * The whole presentation to capture in, overriding `interactiveOnly`, `scope` and the command's + * depth and raw flags: a post-action read reproduces its baseline's view (#4486). + */ + presentation?: SnapshotPresentation; } = { updateSession: true, }, @@ -82,11 +88,22 @@ export async function captureSelectorSnapshot( } const sessionName = options.session ?? 'default'; const session = await runtime.sessions.get(sessionName); + const view = captureOptions.presentation; const result = await captureSnapshot(toBackendContext(runtime, options), { - interactiveOnly: captureOptions.interactiveOnly ?? false, - depth: options.depth, - scope: captureOptions.scope ?? options.scope, - raw: options.raw, + ...(view + ? { + interactiveOnly: view.interactiveOnly === true, + depth: view.depth, + scope: view.scope, + raw: view.raw, + ...(view.customActions === true ? { customActions: true } : {}), + } + : { + interactiveOnly: captureOptions.interactiveOnly ?? false, + depth: options.depth, + scope: captureOptions.scope ?? options.scope, + raw: options.raw, + }), includeRects: captureOptions.includeRects, ...(captureOptions.preferredBackend ? { preferredBackend: captureOptions.preferredBackend } diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts new file mode 100644 index 0000000000..04f5506323 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-view-baseline.test.ts @@ -0,0 +1,207 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend, BackendSnapshotOptions } from '@agent-device/contracts/backend'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { SnapshotPresentation } from '@agent-device/kernel/snapshot'; +import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { createAgentDevice } from '../../../runtime.ts'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { ref, selector } from './selector-read-utils.ts'; +import { createFakeClock } from './__tests__/settle-device-fixtures.ts'; + +// #4486: a post-action comparison reads one view. A settle captures in its baseline's presentation, +// the baseline is read in the agent's view (its ref frame's presentation), and verify and +// `diff snapshot` compare only trees read in one presentation. Budgets are injected (fake clock). + +const DIALOG = ['Device name', 'Cancel', 'OK']; + +function dialogNodes(labels: readonly string[], wrapper: boolean) { + const offset = wrapper ? 1 : 0; + return [ + ...(wrapper + ? [ + { + index: 0, + depth: 0, + type: 'Group', + label: 'Dialog', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + ] + : []), + ...labels.map((label, index) => ({ + index: offset + index, + depth: offset, + ...(wrapper ? { parentIndex: 0 } : {}), + type: label === 'Device name' ? 'StaticText' : 'Button', + label, + rect: { x: 10, y: 100 + index * 60, width: 200, height: 40 }, + hittable: true, + })), + ]; +} + +/** The screen as each view reads it: the full tree keeps a wrapper the interactive view omits. */ +function screen( + labels: readonly string[], + presentation: SnapshotPresentation | undefined, +): SnapshotState { + const full = presentation?.interactiveOnly !== true; + return { ...makeSnapshotState(dialogNodes(labels, full)), presentation }; +} + +const FULL: SnapshotPresentation = {}; +const INTERACTIVE: SnapshotPresentation = { interactiveOnly: true }; + +function viewOf(options: BackendSnapshotOptions | undefined): SnapshotPresentation { + return { + ...(options?.interactiveOnly === true ? { interactiveOnly: true } : {}), + ...(options?.scope !== undefined ? { scope: options.scope } : {}), + ...(options?.depth !== undefined ? { depth: options.depth } : {}), + ...(options?.raw !== undefined ? { raw: options.raw } : {}), + ...(options?.customActions !== undefined ? { customActions: options.customActions } : {}), + }; +} + +/** + * A backend that answers every capture in the view it was asked for, as the daemon stamps it. + * `before` and `after` are the dialog labels on screen before and after the action; `views` + * records the view of every capture taken after it. + */ +function createDevice(params: { + frame: SnapshotState; + before?: readonly string[]; + after?: readonly string[]; +}) { + let acted = false; + const views: SnapshotPresentation[] = []; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async (_context, options) => { + const view = viewOf(options); + if (acted) views.push(view); + return { + snapshot: screen(acted ? (params.after ?? DIALOG) : (params.before ?? DIALOG), view), + }; + }, + tap: async () => { + acted = true; + return { ok: true }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([ + { name: 'default', snapshot: params.frame, refFrameSnapshot: params.frame }, + ]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + return { device, views }; +} + +const removed = (lines: Array<{ kind: string; text: string }> | undefined) => + (lines ?? []).filter((line) => line.kind === 'removed').map((line) => line.text); + +test.each([FULL, undefined])( + 'a ref press after a full snapshot (%j) settles without false removals', + async (presentation) => { + const { device, views } = createDevice({ frame: screen(DIALOG, presentation) }); + + const result = await device.interactions.press(ref('@e3'), { session: 'default', settle: {} }); + + assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly !== true)); + assert.deepEqual(result.settle?.diff?.summary, { additions: 0, removals: 0, unchanged: 4 }); + }, +); + +test('a ref press after snapshot -i settles on the interactive view and lists what left', async () => { + const { device, views } = createDevice({ + frame: screen(DIALOG, INTERACTIVE), + after: ['Cancel', 'OK'], + }); + + const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); + + assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly === true)); + assert.deepEqual(removed(result.settle?.diff?.lines).length, 1); +}); + +test.each([ + { interactiveOnly: true, scope: 'Device name' }, + { depth: 2 }, + { raw: true }, + { customActions: true }, +])('a ref press preserves snapshot presentation %j when settling', async (presentation) => { + const { device, views } = createDevice({ frame: screen(DIALOG, presentation) }); + + const result = await device.interactions.press(ref('@e1'), { session: 'default', settle: {} }); + + assert.ok(views.length > 0); + for (const view of views) assert.deepEqual(view, presentation); + assert.deepEqual(result.settle?.diff?.summary, { + additions: 0, + removals: 0, + unchanged: presentation.interactiveOnly ? 3 : 4, + }); +}); + +test('a selector press resolved on the full tree still diffs in the ref frame view', async () => { + // `label=Dialog` names the wrapper only the full tree holds, so resolution falls back to it. + const { device, views } = createDevice({ frame: screen(DIALOG, INTERACTIVE) }); + + const result = await device.interactions.press(selector('label=Dialog'), { + session: 'default', + settle: {}, + }); + + assert.ok(views.length > 0 && views.every((view) => view.interactiveOnly === true)); + assert.deepEqual(removed(result.settle?.diff?.lines), []); +}); + +test.each([FULL, undefined])( + 'verify after a full snapshot (%j) reads unchanged content in the full view', + async (presentation) => { + const { device, views } = createDevice({ frame: screen(DIALOG, presentation) }); + + const result = await device.interactions.press(ref('@e3'), { + session: 'default', + verify: true, + }); + + assert.deepEqual(views, [FULL]); + assert.equal(result.evidence?.changedFromBefore, false); + }, +); + +test('diff snapshot -i after a full snapshot starts a baseline instead of listing the view change', async () => { + const { device } = createDevice({ frame: screen(DIALOG, FULL) }); + + const interactive = await device.capture.diffSnapshot({ + session: 'default', + interactiveOnly: true, + }); + assert.equal(interactive.baselineInitialized, true); + assert.deepEqual(interactive.summary, { additions: 0, removals: 0, unchanged: 3 }); + + const again = await device.capture.diffSnapshot({ session: 'default', interactiveOnly: true }); + assert.equal(again.baselineInitialized, false); + assert.deepEqual(again.summary, { additions: 0, removals: 0, unchanged: 3 }); +}); + +test('diff snapshot starts a baseline when only an older ref frame matches the requested view', async () => { + const { device } = createDevice({ frame: screen(DIALOG, INTERACTIVE) }); + // Selector reads replace the operational snapshot without issuing a new ref frame. + await device.selectors.get({ + session: 'default', + target: selector('label=Dialog'), + property: 'text', + }); + + const result = await device.capture.diffSnapshot({ session: 'default', interactiveOnly: true }); + + assert.equal(result.baselineInitialized, true); + assert.deepEqual(result.summary, { additions: 0, removals: 0, unchanged: 3 }); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index 0a0f17d5a5..dd0c64a38d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -5,7 +5,10 @@ import type { CommandSessionRecord, } from '@agent-device/contracts/runtime-contract'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; -import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline'; +import { + postActionBaseline, + surfaceScopedNodes, +} from '@agent-device/capture-kit/snapshot-baseline'; import { buildSnapshotDiff } from '@agent-device/capture-kit/snapshot-diff'; import { displayLabel, formatRole } from '@agent-device/capture-kit/snapshot-lines'; import { @@ -236,45 +239,35 @@ async function resolveSettleBaseline( resolved: ResolvedInteractionTarget, ): Promise { const session = await runtime.sessions.get(options.session ?? 'default'); + const frame = surfaceScopedNodes(session?.refFrameSnapshot); // A ref is authorized against the stored ref frame. Keep that visible presentation as the // transition baseline: a best-effort evidence recapture can recover through private AX and see // covered background controls that were not actionable when the ref was issued. // Resolved-target evidence is best-effort at the contracts boundary. The session still owns the // authoritative ref frame, so reuse it rather than silently turning a missing optional field // into an empty transition/diff baseline. Fall back to the latest observation for point targets - // and pre-frame sessions. - return ( - authorizedRefBaseline(resolved, session) ?? - nonEmptyBaseline(resolved.preAction) ?? - sessionBaseline(session) - ); -} - -function authorizedRefBaseline( - resolved: ResolvedInteractionTarget, - session: CommandSessionRecord | undefined, -): SurfaceScopedNodes | undefined { - if (resolved.kind !== 'ref') return undefined; - return nonEmptyBaseline(surfaceScopedNodes(session?.refFrameSnapshot)); + // and pre-frame sessions. A candidate read in another view than the frame is skipped (#4486). + return postActionBaseline({ + frame: withoutPresentedSurface(frame), + candidates: [ + resolved.kind === 'ref' ? frame : undefined, + resolved.preAction, + session?.refFrameSnapshot + ? undefined + : withoutPresentedSurface(surfaceScopedNodes(session?.snapshot)), + ], + }); } /** - * The latest stored tree, for a target no capture resolved. The diff can use its nodes, but its + * A stored tree, for a target no capture resolved. The diff can use its nodes, but its * presented-surface fact may predate a side effect this layer cannot see, and an `absent` read * taken before a sheet was presented must not rule out the confirmation owed to that sheet's * dismissal (#3565). */ -function sessionBaseline(session: CommandSessionRecord | undefined): SurfaceScopedNodes { - const { presentedSurface: _unverified, ...baseline } = surfaceScopedNodes( - session?.refFrameSnapshot ?? session?.snapshot, - ); - return baseline; -} - -function nonEmptyBaseline( - baseline: SurfaceScopedNodes | undefined, -): SurfaceScopedNodes | undefined { - return baseline?.nodes.length ? baseline : undefined; +function withoutPresentedSurface(baseline: SurfaceScopedNodes): SurfaceScopedNodes { + const { presentedSurface: _unverified, ...stored } = baseline; + return stored; } function buildSettleDiff( @@ -282,11 +275,9 @@ function buildSettleDiff( settledNodes: SnapshotNode[], appBundleId: string | undefined, ): NonNullable { - // Flattened compare, like `diff -i`: both sides are interactive-flavored - // captures and depth jitter across captures should not read as change. When - // the baseline came from a richer stored tree (ref targets reuse the session - // snapshot), extra baseline-only lines surface as removals — advisory noise, - // the same baseline caveat --verify's changedFromBefore already accepts. + // Flattened compare, like `diff -i`: both sides are captures in one presentation + // (postActionBaseline picks the baseline, and the settle loop captures in its view), and + // depth jitter across captures should not read as change. const diff = buildSnapshotDiff( withoutSettleChrome(baselineNodes, appBundleId), withoutSettleChrome(settledNodes, appBundleId), diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index d0f5c86b5c..0060264cd2 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -1,8 +1,10 @@ import type { SnapshotNode, SnapshotPreferredBackend, + SnapshotPresentation, SnapshotQualityVerdict, } from '@agent-device/kernel/snapshot'; +import { postActionPresentation } from '@agent-device/capture-kit/snapshot-baseline'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers'; @@ -103,7 +105,10 @@ export async function runStableCaptureLoop( quietMs?: number; timeoutMs: number; resetBudgetOnPrivateAxRecovery?: boolean; - /** Immutable pre-action projection; the stored session may advance before settle begins. */ + /** + * Immutable pre-action projection; the stored session may advance before settle begins. The loop + * captures in its presentation, so the settled tree and the baseline share one view (#4486). + */ broadTransitionBaseline?: SurfaceScopedNodes; }, ): Promise { @@ -154,6 +159,14 @@ export async function runStableCaptureLoop( runtime, options, deadlineMs - now(runtime), + params.broadTransitionBaseline + ? postActionPresentation(params.broadTransitionBaseline) + : { + ...postActionPresentation(undefined), + depth: options.depth, + scope: options.scope, + raw: options.raw, + }, preferredBackend, ); if (read.kind !== 'captured') return unfinished(read); @@ -529,8 +542,8 @@ function stableCaptureDelayMs(params: { } // Intentionally does not update the session snapshot: the stable loop captures -// an interactive-only tree purely as a settle signal, and overwriting the -// session's richer cached snapshot with the filtered tree would degrade +// in its baseline's presentation (interactive-only by default) purely as a settle +// signal, and overwriting the session's richer cached snapshot with it would degrade // subsequent ref/get/find lookups against the same session. (`--settle` DOES // store its final capture, but explicitly, in one place, after the loop.) // @@ -543,6 +556,7 @@ async function captureStableSignalWithinDeadline( runtime: AgentDeviceRuntime, options: CommandContext & SelectorSnapshotOptions, remainingMs: number, + presentation: SnapshotPresentation, preferredBackend?: SnapshotPreferredBackend, ): Promise { try { @@ -552,7 +566,7 @@ async function captureStableSignalWithinDeadline( { ...options, signal }, { updateSession: false, - interactiveOnly: true, + presentation, preferredBackend, }, ); diff --git a/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts b/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts index 0d1c8bfd06..6f7fdcd79c 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/wait-stable.test.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { AgentDeviceBackend, BackendSnapshotResult } from '@agent-device/contracts/backend'; +import type { + AgentDeviceBackend, + BackendSnapshotOptions, + BackendSnapshotResult, +} from '@agent-device/contracts/backend'; import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; import { createAgentDevice } from '../../../runtime.ts'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; @@ -8,6 +12,41 @@ import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fix import { createFakeClock, selectorReadSnapshot } from './__tests__/test-utils/index.ts'; import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +test('runtime wait stable preserves the requested scope, depth and raw view', async () => { + const captures: Array = []; + const snapshot = selectorReadSnapshot(); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captures.push(options); + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.selectors.wait({ + session: 'default', + scope: 'Contact details', + depth: 2, + raw: true, + target: { kind: 'stable', quietMs: 25, timeoutMs: 1_000 }, + }); + + assert.equal(result.kind, 'stable'); + assert.ok(captures.length >= 2); + for (const capture of captures) { + assert.equal(capture?.interactiveOnly, true); + assert.equal(capture?.scope, 'Contact details'); + assert.equal(capture?.depth, 2); + assert.equal(capture?.raw, true); + } +}); + test('runtime wait stable settles after two unchanged captures', async () => { const snapshot = selectorReadSnapshot(); const clock = createFakeClock(); diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 27cec69e84..fd30cea822 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -3,6 +3,7 @@ import type { Point, SnapshotKeyboardBandFact, SnapshotNode, + SnapshotPresentation, } from '@agent-device/kernel/snapshot'; import type { ResponseCost } from '@agent-device/kernel/contracts'; import type { ClickButton } from './click-button.ts'; @@ -160,6 +161,11 @@ export type SurfaceScopedNodes = { * Absent means the capture's producer published no fact. */ presentedSurface?: PresentedSurface; + /** + * The presentation the capture read (`-i`, `-s`, `-d`, `--raw`, `--actions`). A post-action read + * captures in it, so the diff compares one view (#4486). Absent means the default full tree. + */ + presentation?: SnapshotPresentation; }; /** Which tree a resolved `@ref`'s node was read from (ADR 0014, "Mutation admission and read behavior"). */ @@ -228,9 +234,10 @@ export type PostActionSurfaceChange = { /** * Opt-in (`--verify`) cheap post-condition evidence for mutating interaction * commands (#1047). `digest`/`nodeCount`/`interactiveNodeCount` describe a single - * interactive-only capture taken right after the action; `changedFromBefore` - * compares that digest against the pre-action capture the resolution path already - * held, so no extra device round trip is spent beyond the one verify capture. + * capture taken right after the action in the pre-action capture's view (interactive-only + * by default, #4486); `changedFromBefore` compares that digest against the pre-action + * capture the resolution path already held, so no extra device round trip is spent beyond + * the one verify capture. * `changedFromBefore: false` is evidence, not failure — the command still * succeeded. * diff --git a/packages/daemon/src/execution/generic-settle.test.ts b/packages/daemon/src/execution/generic-settle.test.ts index 419318de7c..f301e83e36 100644 --- a/packages/daemon/src/execution/generic-settle.test.ts +++ b/packages/daemon/src/execution/generic-settle.test.ts @@ -312,6 +312,39 @@ test('scroll --settle passes the stored baseline and attaches the observation wi expect(settle.refsGeneration).toBe(stored.snapshotGeneration); }); +test('scroll --settle keeps the ref frame view when the stored tree was read in another view', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-view'; + // The agent read refs from snapshot -i; a later selector read stored a full tree. + const session = makeIosSession(sessionName); + setSessionSnapshot( + session, + buildSnapshotState(snapshotPayload({ nodes: BEFORE_NODES, presentedSurface: 'absent' }), { + snapshotInteractiveOnly: true, + }), + ); + activateCompleteRefFrame(session); + setSessionSnapshot(session, buildSnapshotState(snapshotPayload({ nodes: AFTER_NODES }), {})); + markSessionSnapshotOutdated(session); + sessionStore.publish(sessionName, session); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ + sessionName, + sessionStore, + command: 'scroll', + positionals: ['down'], + flags: { ...SETTLE_FLAGS }, + }); + + // #4486: the diff compares one view, so the interactive frame answers, not the full stored tree. + expect(settleObservationCalls[0]?.baseline.presentation?.interactiveOnly).toBe(true); + expect(settleObservationCalls[0]?.baseline.nodes).toEqual( + expect.arrayContaining([expect.objectContaining({ label: 'Continue' })]), + ); + expect(settleObservationCalls[0]?.baseline.presentedSurface).toBeUndefined(); +}); + test('back --settle attaches the observation alongside the command result', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-back'; diff --git a/packages/daemon/src/execution/generic-settle.ts b/packages/daemon/src/execution/generic-settle.ts index 7439d9ab09..f2accd1537 100644 --- a/packages/daemon/src/execution/generic-settle.ts +++ b/packages/daemon/src/execution/generic-settle.ts @@ -1,7 +1,10 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { CommandFlags } from '@agent-device/contracts/command'; import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; -import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline'; +import { + postActionBaseline, + surfaceScopedNodes, +} from '@agent-device/capture-kit/snapshot-baseline'; import type { SettleObservation, SettleParams, @@ -18,7 +21,7 @@ import type { BoundContextFromFlags } from '../core/context.ts'; import type { RuntimeAdmissionBindings } from '../core/request-runtime-binding.ts'; import type { DaemonPlatformServices } from '../core/platform-services.ts'; import { resolveSettleObservationCapture } from '../core/settle-observation-capture.ts'; -import { isOutdatedObservation } from '../core/ref-frame.ts'; +import { isOutdatedObservation, refFrameTree } from '../core/ref-frame.ts'; import { issueSettleRefs } from './session-snapshot.ts'; import type { SessionStore } from '../core/session-store.ts'; import type { DaemonRequest } from '../core/daemon-request.ts'; @@ -45,7 +48,9 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; * * 1. {@link planGenericSettleObservation} runs BEFORE the command dispatches and * freezes the diff baseline. The baseline is the session's STORED pre-action - * tree, so the diff reads "settled tree vs the last tree you observed" — it + * tree, or the ref frame when that tree was read in another view than the + * frame (`postActionBaseline`), so the diff reads "settled tree vs the last + * tree you observed" in one view — it * may be several commands old, unlike press's freshly resolved pre-action * capture. That is the honest #1101 contract for a route with no resolution * step, and the settle loop's own captures overwrite `session.snapshot`, so @@ -79,15 +84,21 @@ type GenericSettleContext = { * otherwise returns the observer this dispatch owes — with the pre-action * baseline already captured. */ -export function planGenericSettleObservation( +export async function planGenericSettleObservation( params: GenericSettleContext & { command: string; flags: CommandFlags | undefined }, -): GenericSettlePlan { +): Promise { if (!commandSupportsSettleObservation(params.command)) return {}; const invalidSettleFlags = settleFlagGuardResponse(params.command, params.flags); if (invalidSettleFlags) return { response: invalidSettleFlags }; const settle = readSettleRequest(params.flags); if (!settle) return {}; - const baseline = storedTreeBaseline(params.session.snapshot); + const { presentedSurface: _unverified, ...frame } = surfaceScopedNodes( + refFrameTree(params.session), + ); + const baseline = await postActionBaseline({ + frame, + candidates: [storedTreeBaseline(params.session.snapshot)], + }); return { observe: async () => await observeSettled(params, settle, baseline) }; } diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index a42ce47550..dab8618c9d 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -786,6 +786,19 @@ export function buildSnapshotPresentationKey(flags: SnapshotOptions | undefined) }); } +/** + * Whether two captures read one presentation, so a diff between them compares like with like + * (#4486): a full tree holds nodes an interactive read omits, and a scoped, depth-limited, raw or + * custom-action read holds a different node set than a plain one. An absent presentation is + * the default full tree, as on SnapshotState. + */ +export function isSameSnapshotPresentation( + a: SnapshotPresentation | undefined, + b: SnapshotPresentation | undefined, +): boolean { + return buildSnapshotPresentationKey(a) === buildSnapshotPresentationKey(b); +} + export function centerOfRect(rect: Rect): Point { return { x: Math.round(rect.x + rect.width / 2), diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index cb63852342..928db61851 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -748,7 +748,7 @@ async function runAndroidCaptureInteractionAndReplayWorkflow( ...selection, }); assert.equal(diff.mode, 'snapshot'); - assert.equal(diff.baselineInitialized, false); + assert.equal(diff.baselineInitialized, true); assert.deepEqual(diff.summary, { additions: 0, removals: 0, unchanged: 3 }); const rotate = await client.command.orientation({ diff --git a/test/integration/provider-scenarios/remote-proxy-parity.test.ts b/test/integration/provider-scenarios/remote-proxy-parity.test.ts index 1263d230ca..c75187839d 100644 --- a/test/integration/provider-scenarios/remote-proxy-parity.test.ts +++ b/test/integration/provider-scenarios/remote-proxy-parity.test.ts @@ -374,7 +374,7 @@ function assertScriptOutcomes(responses: DaemonResponse[]): void { assert.equal(open?.ok, true); assert.equal(snapshot?.ok, true); assert.equal(raw?.ok, true); - assert.equal(baselineInitialized(diff), false); + assert.equal(baselineInitialized(diff), true); assert.equal(missingRef?.ok, false); assert.equal( baselineInitialized(freshDiff), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 85ec3793ac..d68250fd82 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -416,7 +416,7 @@ agent-device get attrs @e1 - `viewport: { width, height }` names the box node rects are measured in; see [Coordinates and viewport](/docs/snapshots#coordinates-and-viewport). - `--scope ` re-roots the snapshot at the first matching node; see [Scope a snapshot](/docs/snapshots#scope-a-snapshot). - `--actions` lists custom accessibility actions an element merged away (iOS simulator only, rejected with `--raw`); see [Snapshots](/docs/snapshots). -- `diff snapshot` (alias `snapshot --diff`) compares with the previous session baseline, then updates it. +- `diff snapshot` (alias `snapshot --diff`) compares with the previous session baseline, then updates it. It compares only trees read with the same `-i`, `-d`, `-s`, `--raw` and `--actions`: when the stored snapshot was read in another view, the capture starts a new baseline (`baselineInitialized: true`) instead of listing the view change as changes. ## Wait and alerts @@ -501,6 +501,10 @@ If rotation stops repeated iOS presses, inspect the screen before continuing wit A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +`--settle` compares trees in the view of the refs you last read. After a full `snapshot`, it reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. It also preserves `-d`, `-s`, `--raw` and `--actions`. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, including ref, selector and coordinate targets. + +Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. + A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. ### Text entry