From e0f4723879a03b5a405335fa965e745ba256742a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 01:38:45 +0800 Subject: [PATCH 01/16] feat(interaction): press/click --expect-screenshot-diff out-of-tree readback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A coordinate press on a surface the accessibility tree cannot describe — a PIN keypad whose keys never reach the tree — dispatches fine, leaves --verify's node digest flat whether or not the digit was entered, and reports success. Repeated attempts walk toward a device lockout with no signal in any response. The flag brackets the dispatch with two lightweight screenshots and compares them on pixels instead of nodes: screenshotDiff carries changedFromBefore and the changed-pixel ratio. A matching pair adds a response warning, because the dispatch may have had no visible effect; the action itself still succeeds. Best-effort like --verify evidence: a failed capture or a screenshot-less backend drops the field and claims nothing about the screen. The flag routes press/click through the runtime path (the native ref fast path and the fused direct-iOS route never reach the bracket), and the decode stack loads behind function-scoped dynamic imports so the flag-less fast path pays nothing. Co-Authored-By: Claude Code --- packages/capture-kit/package.json | 4 + .../src/flag-definitions-action.ts | 10 + .../runtime/__tests__/test-utils/index.ts | 4 + .../interaction/runtime/interactions.ts | 38 ++- .../runtime/screenshot-diff-evidence.test.ts | 249 ++++++++++++++++++ .../runtime/screenshot-diff-evidence.ts | 113 ++++++++ packages/command-surface/src/command-flags.ts | 1 + .../command-surface/src/interaction/index.ts | 21 ++ .../src/interaction/interactions.ts | 2 + .../src/interaction/metadata.ts | 7 + packages/contracts/src/cli-flags.ts | 7 + packages/contracts/src/client-gesture.ts | 10 + packages/contracts/src/interaction.ts | 21 ++ packages/contracts/src/request-envelope.ts | 1 + ...nteraction-touch-direct-ios-eligibility.ts | 3 + .../internal/interaction-touch-press.ts | 1 + .../internal/interaction-touch-targets.ts | 7 +- scripts/integration-progress-model.ts | 9 + ...-descriptor-expect-screenshot-diff.test.ts | 164 ++++++++++++ src/commands/schema/cli-help.ts | 2 +- 20 files changed, 669 insertions(+), 5 deletions(-) create mode 100644 packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts create mode 100644 packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts create mode 100644 src/__tests__/command-descriptor-expect-screenshot-diff.test.ts diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 581081453f..d87876b127 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -150,6 +150,10 @@ "types": "./src/png-rgb-difference.ts", "default": "./src/png-rgb-difference.ts" }, + "./png-changed-pixel-ratio": { + "types": "./src/png-changed-pixel-ratio.ts", + "default": "./src/png-changed-pixel-ratio.ts" + }, "./png-size": { "types": "./src/png-size.ts", "default": "./src/png-size.ts" diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index e03b62befb..81be2fc7c1 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -206,6 +206,16 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ projectConfig: true, recorded: false, }, + { + key: 'expectScreenshotDiff', + names: ['--expect-screenshot-diff'], + type: 'boolean', + usageLabel: '--expect-screenshot-diff', + usageDescription: + 'Press/click: compare lightweight screenshots taken right before and after the dispatch and report screenshotDiff in the response; use it when the target lives outside the accessibility tree (a PIN keypad) and --verify would compare empty trees. A matching pair adds a warning, because the dispatch may have had no visible effect', + projectConfig: true, + recorded: false, + }, { key: 'settle', names: ['--settle'], diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index b646a045bc..5b6c9d8167 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -343,6 +343,7 @@ export function createInteractionDevice( Pick< AgentDeviceBackend, | 'captureSnapshot' + | 'captureScreenshot' | 'resolveGestureViewport' | 'tap' | 'tapTarget' @@ -366,6 +367,9 @@ export function createInteractionDevice( platform: overrides.platform ?? 'ios', captureSnapshot: async (...args) => overrides.captureSnapshot ? await overrides.captureSnapshot(...args) : { snapshot }, + captureScreenshot: overrides.captureScreenshot + ? async (...args) => (await overrides.captureScreenshot?.(...args)) ?? {} + : undefined, resolveGestureViewport: overrides.resolveGestureViewport ? async (...args) => await overrides.resolveGestureViewport?.(...args) : undefined, diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 6c19cdc82f..29f59c5bfa 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -14,6 +14,7 @@ import { isFillableType } from '@agent-device/contracts/snapshot'; import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type RuntimeCommand } from '../../runtime-types.ts'; +import { capturePairScreenshotFrame, compareScreenshotPair } from './screenshot-diff-evidence.ts'; import { applyPostActionObservation, planPostActionObservation, @@ -41,6 +42,14 @@ export type PressCommandOptions = CommandContext & RepeatedInput & { target: InteractionTarget; button?: ClickButton; + /** + * Opt-in: bracket the dispatch with two lightweight screenshots and attach + * `screenshotDiff` to the result, so a tap on a surface the accessibility + * tree cannot describe (a PIN keypad) still answers "did the screen react + * at all". Best-effort — a capture failure drops the evidence; a matching + * pair adds a warning instead of failing the action. + */ + expectScreenshotDiff?: boolean; /** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */ expectedResolvedTarget?: ExpectedResolvedTarget; /** #1654: a mutating `find`'s already-resolved node; see resolution.ts. */ @@ -140,9 +149,12 @@ async function tapCommand( action: 'click' | 'press', ): Promise { const observation = planPostActionObservation(options); - const nativeRefTap = observation.needsPreActionBaseline - ? null - : await maybeTapRefTarget(runtime, options, action); + // --expect-screenshot-diff brackets the dispatch below, which the native ref + // fast path never reaches — route through the runtime path like verify/settle. + const nativeRefTap = + observation.needsPreActionBaseline || options.expectScreenshotDiff === true + ? null + : await maybeTapRefTarget(runtime, options, action); if (nativeRefTap) return nativeRefTap; // One request object for the resolution and for the looks admission may spend, so a second look is @@ -162,6 +174,12 @@ async function tapCommand( const atRest = await admitResolvedTarget(runtime, options, resolutionRequest, resolved); const point = requireResolvedPoint(atRest); const pointObservation = pointObservationOf(atRest); + // The before-frame is taken after every pre-dispatch guard and immediately + // before the dispatch, so the pair brackets the tap and nothing else. + const screenshotBaseline = + options.expectScreenshotDiff === true + ? await capturePairScreenshotFrame(runtime, options, 'before') + : undefined; let backendResult; try { backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, { @@ -178,8 +196,13 @@ async function tapCommand( // band so a daemon-level failure corroboration can still record the same // target identity if the backend reports an ambiguous tap outcome. attachResolvedInteractionTarget(error, atRest); + if (screenshotBaseline) await screenshotBaseline.cleanup(); throw error; } + const screenshotOutcome = screenshotBaseline + ? await compareScreenshotPair(runtime, options, screenshotBaseline) + : undefined; + const screenshotWarning = screenshotOutcome?.unchangedWarning; const formattedBackendResult = toBackendResult(backendResult); return await applyPostActionObservation( runtime, @@ -188,6 +211,15 @@ async function tapCommand( { ...atRest, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...(screenshotOutcome?.evidence ? { screenshotDiff: screenshotOutcome.evidence } : {}), + ...(screenshotWarning + ? { + warning: + 'warning' in atRest && atRest.warning + ? `${atRest.warning} ${screenshotWarning}` + : screenshotWarning, + } + : {}), }, observation, ); diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts new file mode 100644 index 0000000000..b1230a9132 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts @@ -0,0 +1,249 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { PNG } from '@agent-device/capture-kit/png'; +import { test } from 'vitest'; +import type { BackendScreenshotOptions } from '@agent-device/contracts/backend'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref } from './selector-read-utils.ts'; +import { createInteractionDevice, selectorSnapshot } from './__tests__/test-utils/index.ts'; + +// `--expect-screenshot-diff` (#press-expect): the out-of-tree readback for the +// surfaces --verify cannot answer for — a PIN keypad whose keys never reach the +// accessibility tree digests identically whether or not the digit was entered, +// so the pair compares pixels instead of nodes. + +function solidPngBuffer( + width: number, + height: number, + color: { r: number; g: number; b: number }, +): Buffer { + const png = new PNG({ width, height }); + for (let i = 0; i < png.data.length; i += 4) { + png.data[i] = color.r; + png.data[i + 1] = color.g; + png.data[i + 2] = color.b; + png.data[i + 3] = 255; + } + return PNG.sync.write(png); +} + +type FramePlan = Array<{ bytes: Buffer; options?: BackendScreenshotOptions }>; + +function framePlanBackend(frames: FramePlan) { + let frameIndex = 0; + const captured: BackendScreenshotOptions[] = []; + return { + captured, + captureScreenshot: async ( + _context: unknown, + outPath: string, + options?: BackendScreenshotOptions, + ) => { + const frame = frames[Math.min(frameIndex, frames.length - 1)]; + frameIndex += 1; + if (options) captured.push(options); + assert.ok(frame, 'the frame plan must cover every capture the pair takes'); + fs.writeFileSync(outPath, frame.bytes); + return { path: outPath }; + }, + }; +} + +test('press with expectScreenshotDiff brackets the dispatch and reports a changed screen', async () => { + const backend = framePlanBackend([ + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + { bytes: solidPngBuffer(8, 8, { r: 255, g: 255, b: 255 }) }, + ]); + const taps: Array<{ x: number; y: number }> = []; + const device = createInteractionDevice(selectorSnapshot(), { + ...backend, + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { + session: 'default', + expectScreenshotDiff: true, + }, + ); + + assert.deepEqual(taps, [{ x: 10, y: 20 }]); + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, true); + assert.equal(result.screenshotDiff?.changedPixelRatio, 1); + assert.equal('warning' in result ? result.warning : undefined, undefined); + // The status-bar clock must not read as the reaction the caller asked about. + assert.deepEqual(backend.captured, [{ normalizeStatusBar: true }, { normalizeStatusBar: true }]); +}); + +test('press with expectScreenshotDiff warns when the frames match', async () => { + const backend = framePlanBackend([ + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + ]); + const device = createInteractionDevice(selectorSnapshot(), { + ...backend, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { + session: 'default', + expectScreenshotDiff: true, + }, + ); + + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, false); + assert.equal(result.screenshotDiff?.changedPixelRatio, 0); + // The silent no-op must not read as success: the response says so next to + // the evidence, and the action itself still succeeds. + assert.match( + ('warning' in result ? result.warning : undefined) ?? '', + /may have had no visible effect/, + ); +}); + +test('press with expectScreenshotDiff keeps a resolution warning beside its own', async () => { + const backend = framePlanBackend([ + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + ]); + // A point outside the viewport the stored tree captured earns the + // outside-viewport warning; the pair warning composes after it. + const viewportSnapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + ]); + const device = createInteractionDevice(viewportSnapshot, { + ...backend, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 100000, y: 100000 }, + { + session: 'default', + expectScreenshotDiff: true, + }, + ); + + const warning = 'warning' in result ? result.warning : undefined; + assert.match(warning ?? '', /outside the last-known viewport/); + assert.match(warning ?? '', /may have had no visible effect/); +}); + +test('press with expectScreenshotDiff reports a shape change without a ratio', async () => { + const backend = framePlanBackend([ + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + { bytes: solidPngBuffer(4, 16, { r: 0, g: 0, b: 0 }) }, + ]); + const device = createInteractionDevice(selectorSnapshot(), { + ...backend, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { + session: 'default', + expectScreenshotDiff: true, + }, + ); + + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, true); + assert.equal(result.screenshotDiff?.changedPixelRatio, undefined); + assert.equal('warning' in result ? result.warning : undefined, undefined); +}); + +test('press with expectScreenshotDiff drops the evidence when a capture fails', async () => { + let captureCount = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureScreenshot: async () => { + captureCount += 1; + if (captureCount === 1) throw new Error('screen capture unavailable'); + return { path: '/unused/after.png' }; + }, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { + session: 'default', + expectScreenshotDiff: true, + }, + ); + + // No pair, no claim — and the dispatch itself still succeeds. + assert.equal('screenshotDiff' in result, false); + assert.equal('warning' in result ? result.warning : undefined, undefined); + assert.deepEqual(captureCount, 1); +}); + +test('press without a screenshot backend drops the evidence and still taps', async () => { + const taps: Array<{ x: number; y: number }> = []; + const device = createInteractionDevice(selectorSnapshot(), { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { + session: 'default', + expectScreenshotDiff: true, + }, + ); + + assert.deepEqual(taps, [{ x: 10, y: 20 }]); + assert.equal('screenshotDiff' in result, false); +}); + +test('press with expectScreenshotDiff routes a ref target through the runtime path', async () => { + const backend = framePlanBackend([ + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + { bytes: solidPngBuffer(8, 8, { r: 255, g: 255, b: 255 }) }, + ]); + const refTaps: string[] = []; + const device = createInteractionDevice(selectorSnapshot(), { + ...backend, + platform: 'web', + tapTarget: async (_context, target) => { + refTaps.push(target.ref); + return { ref: target.ref.replace(/^@/, '') }; + }, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(ref('@e1'), { + session: 'default', + expectScreenshotDiff: true, + }); + + // The native ref fast path would skip the pair entirely; the flag forces the + // runtime path so the readback brackets the dispatch it names. + assert.deepEqual(refTaps, []); + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, true); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts new file mode 100644 index 0000000000..d801771788 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -0,0 +1,113 @@ +import { promises as fs } from 'node:fs'; +import type { TemporaryFile } from '@agent-device/contracts/artifact-adapter'; +import type { ScreenshotDiffEvidence } from '@agent-device/contracts/interaction'; +import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; +import { toBackendContext } from '../../runtime-common.ts'; +import { createCommandTempFile } from '../../io-policy.ts'; + +/** + * `--expect-screenshot-diff` for press/click: the out-of-tree readback + * primitive. The screenshots bracket the dispatch, and the comparison runs on + * pixels, because the surfaces this answers for — a PIN keypad whose keys never + * reach the accessibility tree, a custom-drawn keyboard — leave `--verify`'s + * node digest flat no matter what the tap did. + * + * Best-effort by the same contract as `--verify` evidence: a capture that fails + * or a backend without screenshots yields no evidence, which claims nothing + * about the screen. The comparison never fails the action. + * + * The PNG decoder and the pixel ratio load behind function-scoped dynamic + * imports: an eager edge from every press/click into the decode stack would + * tax the flag-less fast path, and only the opt-in readback ever decodes a + * frame (the eager-closure budget). + */ + +export type ScreenshotPairOutcome = { + evidence?: ScreenshotDiffEvidence; + /** Present when the caller expected a diff and the frames matched. */ + unchangedWarning?: string; +}; + +/** + * One frame of the pair. A backend without `captureScreenshot` returns no + * frame, and the whole readback silently does not run — the flag is advisory, + * so a screenshot-less platform keeps the dispatch untouched. + */ +export async function capturePairScreenshotFrame( + runtime: AgentDeviceRuntime, + options: CommandContext, + label: 'before' | 'after', +): Promise { + if (!runtime.backend.captureScreenshot) return undefined; + let frame: TemporaryFile; + try { + frame = await createCommandTempFile(runtime, { + prefix: `agent-device-press-${label}`, + ext: '.png', + }); + } catch { + return undefined; + } + try { + await runtime.backend.captureScreenshot(toBackendContext(runtime, options), frame.path, { + // The status-bar clock ticks between the frames; normalizing it keeps a + // minute rollover from reading as the reaction the caller asked about. + normalizeStatusBar: true, + }); + } catch { + await frame.cleanup(); + return undefined; + } + return frame; +} + +/** + * Closes the pair: one after-frame, then the pixel comparison. A frame that + * changed shape (rotation, split-screen) changed the screen by construction — + * the ratio is omitted because the two frames describe different viewports. + * Any decode or capture failure drops the evidence rather than guessing. + */ +export async function compareScreenshotPair( + runtime: AgentDeviceRuntime, + options: CommandContext, + before: TemporaryFile, +): Promise { + try { + const after = await capturePairScreenshotFrame(runtime, options, 'after'); + if (!after) return {}; + try { + const [{ computePngChangedPixelRatio }, { decodePng }] = await Promise.all([ + import('@agent-device/capture-kit/png-changed-pixel-ratio'), + import('@agent-device/capture-kit/png'), + ]); + const [beforeBytes, afterBytes] = await Promise.all([ + fs.readFile(before.path), + fs.readFile(after.path), + ]); + const ratio = computePngChangedPixelRatio( + decodePng(beforeBytes, 'the press before-screenshot'), + decodePng(afterBytes, 'the press after-screenshot'), + ); + if (ratio.status === 'dimension_mismatch') { + return { evidence: { changedFromBefore: true } }; + } + const evidence: ScreenshotDiffEvidence = { + changedFromBefore: ratio.changedPixelRatio > 0, + changedPixelRatio: ratio.changedPixelRatio, + }; + return { + evidence, + ...(evidence.changedFromBefore + ? {} + : { + unchangedWarning: + 'Expected a screen change, but the screenshots taken before and after the dispatch match. The dispatch may have had no visible effect: check the target coordinates against a fresh snapshot.', + }), + }; + } finally { + await after.cleanup(); + } + } finally { + await before.cleanup(); + } +} diff --git a/packages/command-surface/src/command-flags.ts b/packages/command-surface/src/command-flags.ts index fc2be6a74f..9ae10afc59 100644 --- a/packages/command-surface/src/command-flags.ts +++ b/packages/command-surface/src/command-flags.ts @@ -100,6 +100,7 @@ function buildFlags(options: InternalRequestOptions): CommandFlags { maxPasses: options.maxPasses, doubleTap: options.doubleTap, verify: options.verify, + expectScreenshotDiff: options.expectScreenshotDiff, settle: options.settle, settleQuietMs: options.settleQuietMs, clickButton: options.clickButton, diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index 25bebab3b1..ab5e653a21 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -82,6 +82,21 @@ const interactionEvidenceSchema: JsonSchema = objectSchema( ['nodeCount', 'interactiveNodeCount', 'digest'], ); +// ScreenshotDiffEvidence (packages/contracts/src/interaction.ts) — opt-in +// `--expect-screenshot-diff` out-of-tree readback: pixels instead of nodes. +const screenshotDiffSchema: JsonSchema = objectSchema( + { + changedFromBefore: booleanSchema( + 'Whether any pixel moved between the screenshots taken right before and after the dispatch. false means the frames match, which reads as "the dispatch may have had no visible effect" next to the response warning.', + ), + changedPixelRatio: numberSchema( + 'Share of pixels whose color moved between the two frames. Absent when the frames could not be compared, for example a shape change.', + ), + }, + ['changedFromBefore'], + 'Pixel comparison between the screenshots taken right before and after the dispatch. Absent when the pair was not taken, which claims nothing about the screen.', +); + const responseCostSchema: JsonSchema = objectSchema( { wallClockMs: numberSchema('Total wall-clock time for the request in milliseconds.'), @@ -213,6 +228,9 @@ function interactionResponseDataSchema(extra: InteractionExtra = {}): JsonSchema const tapInteractionResponseDataSchema = interactionResponseDataSchema({ properties: { evidence: interactionEvidenceSchema, + // Press/click only: the pixel pair brackets the dispatch, which fill's + // value-level verification and the hold/hover results never take. + screenshotDiff: screenshotDiffSchema, button: enumSchema(['secondary', 'middle']), count: numberSchema('Number of press/click repetitions.'), intervalMs: numberSchema('Delay between repeated press/click actions.'), @@ -369,6 +387,7 @@ const interactionCliSchemas = { allowedFlags: [ ...REPEATED_TOUCH_FLAGS, 'clickButton', + 'expectScreenshotDiff', ...postActionObservationCliFlags('click'), ...SELECTOR_SNAPSHOT_FLAGS, ], @@ -380,6 +399,7 @@ const interactionCliSchemas = { allowsExtraPositionals: true, allowedFlags: [ ...REPEATED_TOUCH_FLAGS, + 'expectScreenshotDiff', ...postActionObservationCliFlags('press'), ...SELECTOR_SNAPSHOT_FLAGS, ], @@ -666,6 +686,7 @@ function toPressOptions(input: PressInput): PressOptions { ...toSelectorSnapshotOptions(input), ...toRepeatedOptions(input), verify: input.verify, + expectScreenshotDiff: input.expectScreenshotDiff, ...toSettleOptions(input), }; } diff --git a/packages/command-surface/src/interaction/interactions.ts b/packages/command-surface/src/interaction/interactions.ts index 298ee246aa..6e3316b9e0 100644 --- a/packages/command-surface/src/interaction/interactions.ts +++ b/packages/command-surface/src/interaction/interactions.ts @@ -42,6 +42,7 @@ export const interactionCliReaders = { ...readInteractionTargetFromPositionals(positionals), button: flags.clickButton, verify: flags.verify, + expectScreenshotDiff: flags.expectScreenshotDiff, }), press: (positionals, flags) => ({ ...commonInputFromFlags(flags), @@ -49,6 +50,7 @@ export const interactionCliReaders = { ...repeatedInputFromFlags(flags), ...readInteractionTargetFromPositionals(positionals), verify: flags.verify, + expectScreenshotDiff: flags.expectScreenshotDiff, }), longpress: (positionals, flags) => { const decoded = readLongPressTargetFromPositionals(positionals); diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index b386cc2d1e..e03e59ba98 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -88,9 +88,15 @@ const interactionCommandDescriptions = { type InteractionCommandName = keyof typeof interactionCommandDescriptions; +const expectScreenshotDiffField = () => + booleanField( + 'Compare lightweight screenshots taken right before and after the dispatch, and report the comparison as screenshotDiff in the response. Use it when the target lives outside the accessibility tree, such as a PIN keypad, where verify would compare empty trees. A matching pair adds a warning, because the dispatch may have had no visible effect.', + ); + const clickFields = { ...interactionTargetFields(), button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'), + expectScreenshotDiff: expectScreenshotDiffField(), ...selectorSnapshotFields(), ...repeatedFields(), ...postActionObservationFields('click'), @@ -98,6 +104,7 @@ const clickFields = { const pressFields = { ...interactionTargetFields(), + expectScreenshotDiff: expectScreenshotDiffField(), ...selectorSnapshotFields(), ...repeatedFields(), ...postActionObservationFields('press'), diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index a30fe6ae91..4259035ef4 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -125,6 +125,13 @@ export type CliFlags = CloudProviderProfileFields & maxPasses?: number; doubleTap?: boolean; verify?: boolean; + /** + * Press/click opt-in (#1047 sibling): bracket the dispatch with two + * lightweight screenshots and compare them, so a tap on a surface the + * accessibility tree cannot describe (a PIN keypad) still answers "did the + * screen react at all". Advisory; never fails the action. + */ + expectScreenshotDiff?: boolean; settle?: boolean; settleQuietMs?: number; postGestureStabilization?: boolean; diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index c053d52158..040171cdb2 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -63,6 +63,14 @@ export type ClickOptions = DeviceCommandBaseOptions & * snapshot to confirm the action had an effect. */ verify?: boolean; + /** + * Opt-in: bracket the dispatch with two lightweight screenshots and return + * the pixel comparison as `screenshotDiff` in the response. For targets the + * accessibility tree cannot describe (a PIN keypad), where `verify` would + * compare empty trees. Advisory — never fails the action; a matching pair + * adds a response warning instead. + */ + expectScreenshotDiff?: boolean; }; export type PressOptions = DeviceCommandBaseOptions & @@ -71,6 +79,8 @@ export type PressOptions = DeviceCommandBaseOptions & RepeatedPressOptions & SettleCommandOptions & { verify?: boolean; + /** See {@link ClickOptions.expectScreenshotDiff}. */ + expectScreenshotDiff?: boolean; }; export type LongPressOptions = DeviceCommandBaseOptions & diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 27cec69e84..23542ea04b 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -252,6 +252,25 @@ export type InteractionEvidence = { surfaceChange?: PostActionSurfaceChange; }; +/** + * Opt-in (`--expect-screenshot-diff`) out-of-tree post-action readback for + * press/click: the pixel comparison between the screenshots taken immediately + * before and after the dispatch. Answers "did the screen react at all" for + * surfaces the accessibility tree cannot describe — a PIN keypad whose keys + * never reach the tree leaves `--verify`'s digest flat no matter what the tap + * did — so the comparison runs on pixels instead of nodes. + * + * Advisory by contract: the comparison never fails the action. Absent means the + * pair was not taken (capture failed or the platform cannot screenshot), which + * is no claim about the screen. `changedFromBefore` compares pixels, not the + * `--verify` digest; `changedPixelRatio` is the share of pixels whose color + * moved, absent when the two frames could not be compared (shape change). + */ +export type ScreenshotDiffEvidence = { + changedFromBefore: boolean; + changedPixelRatio?: number; +}; + export type SettleDiffLine = { kind: 'added' | 'removed'; text: string; @@ -382,6 +401,7 @@ type TouchResponseDataBase = { referenceWidth?: number; referenceHeight?: number; evidence?: InteractionEvidence; + screenshotDiff?: ScreenshotDiffEvidence; settle?: SettleObservation; resolution?: ResolutionDisclosure; cost?: ResponseCost; @@ -478,6 +498,7 @@ export type PressCommandResult = ResolvedInteractionTarget & { message?: string; warning?: string; evidence?: InteractionEvidence; + screenshotDiff?: ScreenshotDiffEvidence; settle?: SettleObservation; }; diff --git a/packages/contracts/src/request-envelope.ts b/packages/contracts/src/request-envelope.ts index 8882de1551..cba23c197c 100644 --- a/packages/contracts/src/request-envelope.ts +++ b/packages/contracts/src/request-envelope.ts @@ -45,6 +45,7 @@ export type CommandExecutionOptions = Partial & maxPasses?: number; doubleTap?: boolean; verify?: boolean; + expectScreenshotDiff?: boolean; settle?: boolean; settleQuietMs?: number; clickButton?: ClickButton; diff --git a/packages/daemon/src/core/interaction/internal/interaction-touch-direct-ios-eligibility.ts b/packages/daemon/src/core/interaction/internal/interaction-touch-direct-ios-eligibility.ts index 66f4843e83..a40a77a874 100644 --- a/packages/daemon/src/core/interaction/internal/interaction-touch-direct-ios-eligibility.ts +++ b/packages/daemon/src/core/interaction/internal/interaction-touch-direct-ios-eligibility.ts @@ -33,6 +33,9 @@ export function readDirectIosSelectorTapTarget(params: { if (hasNonDefaultClickOptions(flags)) return null; if (commandSupportsVerifyEvidence(commandLabel) && flags?.verify === true) return null; if (commandSupportsSettleObservation(commandLabel) && flags?.settle === true) return null; + // The screenshot pair brackets the runtime dispatch; this fused route never + // reaches it, so the flag cannot silently no-op here. + if (flags?.expectScreenshotDiff === true) return null; return readDirectSelectorWithMaestroFallback(session, target.selector, flags); } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index ec0127dfbc..c103a640d6 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -214,6 +214,7 @@ function pressRuntimeOptions( jitterPx: flags?.jitterPx, doubleTap: flags?.doubleTap, verify: flags?.verify, + expectScreenshotDiff: flags?.expectScreenshotDiff, // Only click/press take it: `find` dispatches click and fill, never // longpress or hover, so declaring it on their options would be an // unconsumed claim (#1649 review). diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-targets.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-targets.ts index 7e87b66394..9dc905994d 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-targets.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-targets.ts @@ -162,7 +162,10 @@ export function interactionResultExtra( // LongPressCommandResult has no evidence field, so it reads as undefined // (and gets dropped by the response layer) for longpress. `settle` (#1101, // opt-in via --settle) is additive on all four touch commands. + // `screenshotDiff` (--expect-screenshot-diff) is press/click only: fill's + // value-level verification and longpress/hover carry no pixel pair. const evidence = 'evidence' in result ? result.evidence : undefined; + const screenshotDiff = 'screenshotDiff' in result ? result.screenshotDiff : undefined; const settle = result.settle; if (result.kind === 'ref') { return { @@ -172,6 +175,7 @@ export function interactionResultExtra( targetHittable: result.targetHittable, hint: result.hint, evidence, + screenshotDiff, settle, resolution: result.resolution, }; @@ -184,11 +188,12 @@ export function interactionResultExtra( targetHittable: result.targetHittable, hint: result.hint, evidence, + screenshotDiff, settle, resolution: result.resolution, }; } - return { evidence, settle }; + return { evidence, screenshotDiff, settle }; } export function formatTouchTargetLabel( diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index f019844d62..e92204ae9c 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -430,6 +430,15 @@ function summarizeProviderScenarioFlagExclusions() { owner: 'daemon screenshot-crop unit and live device verification', keys: ['screenshotCropOn'], }, + { + // The pair brackets a real dispatch and the comparison reads real screen pixels: a + // provider scenario fixture cannot fabricate a before/after frame pair whose changed-pixel + // ratio means anything. Covered by the command-runtime unit tests that drive the bracket + // through a fake screenshot backend. + name: 'press screenshot-diff readback', + owner: 'command-runtime press screenshot-pair unit tests', + keys: ['expectScreenshotDiff'], + }, ]; } diff --git a/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts b/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts new file mode 100644 index 0000000000..5d9eed8517 --- /dev/null +++ b/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts @@ -0,0 +1,164 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { CliFlags } from '@agent-device/contracts/command'; +import { + COMMAND_OUTPUT_SCHEMAS, + findCommandMetadata, + getCliCommandSchema, + readInputFromCli, + runCommand, + type AgentDeviceClient, + type CommandName, +} from '@agent-device/command-surface'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; + +/** + * The descriptor gate for `--expect-screenshot-diff` (press/click only), in the + * style of the `--settle`/`--verify` gate beside this directory: every + * caller-facing surface is pinned to the press/click scope, so a forgotten or + * over-broad cell fails here instead of silently no-op'ing the flag — or + * leaking it onto a command that never reads it. + */ + +const SCREENSHOT_DIFF_COMMANDS = [PUBLIC_COMMANDS.click, PUBLIC_COMMANDS.press] as const; + +const OUT_OF_SCOPE_COMMANDS = [ + PUBLIC_COMMANDS.longPress, + PUBLIC_COMMANDS.hover, + PUBLIC_COMMANDS.fill, + PUBLIC_COMMANDS.scroll, + PUBLIC_COMMANDS.back, + PUBLIC_COMMANDS.swipe, + PUBLIC_COMMANDS.type, +] as const; + +const READER_POSITIONALS: Record = { + [PUBLIC_COMMANDS.click]: ['10', '20'], + [PUBLIC_COMMANDS.press]: ['10', '20'], +}; + +const FLAG_SNAPSHOT = { expectScreenshotDiff: true } as CliFlags; + +test('expectScreenshotDiff CLI flags are scoped to press and click', () => { + for (const command of SCREENSHOT_DIFF_COMMANDS) { + const schema = getCliCommandSchema(command); + assert.equal( + (schema.allowedFlags ?? []).includes('expectScreenshotDiff'), + true, + `${command}: missing --expect-screenshot-diff`, + ); + } + for (const command of OUT_OF_SCOPE_COMMANDS) { + const schema = getCliCommandSchema(command); + assert.equal( + (schema.allowedFlags ?? []).includes('expectScreenshotDiff'), + false, + `${command}: --expect-screenshot-diff must stay scoped to press/click`, + ); + } +}); + +test('expectScreenshotDiff metadata fields are scoped to press and click', () => { + for (const command of [...SCREENSHOT_DIFF_COMMANDS, ...OUT_OF_SCOPE_COMMANDS]) { + const metadata = findCommandMetadata(command); + if (!metadata) continue; + const properties = metadata.inputSchema.properties ?? {}; + assert.equal( + Object.hasOwn(properties, 'expectScreenshotDiff'), + command === PUBLIC_COMMANDS.click || command === PUBLIC_COMMANDS.press, + `${command}: expectScreenshotDiff field presence must match the press/click scope`, + ); + } +}); + +test('expectScreenshotDiff CLI reader input reaches only the press and click readers', () => { + for (const [command, positionals] of Object.entries(READER_POSITIONALS)) { + const input = readInputFromCli(command as CommandName, positionals, FLAG_SNAPSHOT); + assert.equal( + input.expectScreenshotDiff, + true, + `${command}: --expect-screenshot-diff must reach daemon input`, + ); + } + for (const command of OUT_OF_SCOPE_COMMANDS) { + // Targeted readers demand their target positionals; the flag must drop out + // of the input regardless of the shape the reader accepts. + const positionals = + command === PUBLIC_COMMANDS.scroll + ? ['down'] + : command === PUBLIC_COMMANDS.fill + ? ['10', '20', 'hello'] + : command === PUBLIC_COMMANDS.longPress || + command === PUBLIC_COMMANDS.hover || + command === PUBLIC_COMMANDS.swipe + ? command === PUBLIC_COMMANDS.swipe + ? ['30', '40', '50', '60'] + : ['10', '20'] + : []; + const input = readInputFromCli(command as CommandName, positionals, FLAG_SNAPSHOT); + assert.equal( + 'expectScreenshotDiff' in input, + false, + `${command}: out-of-scope reader must ignore --expect-screenshot-diff`, + ); + } +}); + +/** A client proxy that records the options of every method call at any depth. */ +function recordingClient(calls: Array>): AgentDeviceClient { + const surface: unknown = new Proxy(function () {}, { + apply: (_target, _thisArg, args) => { + calls.push(args[0] as Record); + return undefined; + }, + get: () => surface, + }); + return surface as AgentDeviceClient; +} + +test('expectScreenshotDiff client option survives the press and click projections', async () => { + for (const [command, positionals] of Object.entries(READER_POSITIONALS)) { + const calls: Array> = []; + await runCommand( + recordingClient(calls), + command as CommandName, + readInputFromCli(command as CommandName, positionals, FLAG_SNAPSHOT), + ); + assert.equal(calls.length, 1, `${command}: expected exactly one client call`); + assert.equal( + calls[0]?.expectScreenshotDiff, + true, + `${command}: expectScreenshotDiff option must survive projection`, + ); + } +}); + +type SchemaView = { + properties?: Record; + oneOf?: readonly SchemaView[]; +}; + +function* propertyGroups(schema: SchemaView): Generator> { + for (const branch of schema.oneOf ?? []) yield* propertyGroups(branch); + if (schema.properties) yield schema.properties; +} + +test('expectScreenshotDiff output schema covers press and click and no other command', () => { + for (const [command, schema] of Object.entries(COMMAND_OUTPUT_SCHEMAS)) { + const expected = command === PUBLIC_COMMANDS.click || command === PUBLIC_COMMANDS.press; + let groups = 0; + let hits = 0; + for (const properties of propertyGroups(schema as SchemaView)) { + groups += 1; + if (Object.hasOwn(properties, 'screenshotDiff')) hits += 1; + } + assert.equal( + hits > 0, + expected, + `${command}: screenshotDiff advertisement must match the press/click scope`, + ); + if (expected) { + assert.ok(groups > 0, `${command}: press/click schema must have object groups`); + } + } +}); diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 73d95483eb..b4910ee147 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -123,7 +123,7 @@ Navigation: Validation and evidence: Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing. Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. - When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. + When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. A coordinate press on such a surface can dispatch and still enter nothing: press/click --expect-screenshot-diff compares frames taken before and after the tap and warns when they match, so a silent no-op shows in the response instead of surfacing as repeated failed attempts (for example PIN retries). iOS merged: child ref => press it; else press parent @ref --settle. Names are not selectors. Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting. From 565160ef4e84bcd466a9cc2d4a62705043a5af78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 01:41:27 +0800 Subject: [PATCH 02/16] docs(interaction): drop the workflow-card sentence for --expect-screenshot-diff The compact workflow card has a byte ratchet (cli-help.test.ts) with no headroom for a new sentence, and the flag's owning help already carries the full guidance: the flag definition's usage text renders in "Command flags:" for press/click, and the metadata field description documents the PIN-keypad case for the Node SDK and MCP schemas. Co-Authored-By: Claude Code --- src/commands/schema/cli-help.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index b4910ee147..73d95483eb 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -123,7 +123,7 @@ Navigation: Validation and evidence: Nearby mutation diff: diff snapshot -i; with no prior snapshot it initializes the baseline (zero changes) instead of failing. Named expectations need the exact text/selector via wait/is/get/find -- a bare screenshot/snapshot is not verification. Before declaring a task done, confirm the requested end state is actually visible on the current screen, scrolling it into view if needed; get text alone, or stopping one screen early, is not enough. - When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. A coordinate press on such a surface can dispatch and still enter nothing: press/click --expect-screenshot-diff compares frames taken before and after the tap and warns when they match, so a silent no-op shows in the response instead of surfacing as repeated failed attempts (for example PIN retries). + When an action only reveals or reaches a target, verify the exact target named, not just the action. Prefer testIDs/ids/selectors over visible text. Icon/tappable proof: screenshot --overlay-refs; if snapshot is sparse/AX-unavailable, use plain screenshot and coordinates, then retry snapshot -i on another screen. iOS merged: child ref => press it; else press parent @ref --settle. Names are not selectors. Perf/memory/log/network/trace/crash: help debugging. Recording, save-script, batch, replay repair: help scripting. From b6b63bd2e88932a1c92a14298846091de51d05fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 01:44:05 +0800 Subject: [PATCH 03/16] refactor(interaction): move the screenshot bracket lifecycle out of tapCommand The before-frame capture, its dispatch-failure cleanup, and the pair outcome composition each added a branch to a function the complexity gate already watches. dispatchWithScreenshotPair owns the bracket lifecycle and attachScreenshotPairOutcome owns the result composition, so tapCommand reads as dispatch-plus-observation again; the reader gate's positional table flattens the same way. Co-Authored-By: Claude Code --- .../interaction/runtime/interactions.ts | 60 ++++++++----------- .../runtime/screenshot-diff-evidence.ts | 60 +++++++++++++++++++ ...-descriptor-expect-screenshot-diff.test.ts | 27 ++++----- 3 files changed, 99 insertions(+), 48 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 29f59c5bfa..80baf479cc 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -14,7 +14,11 @@ import { isFillableType } from '@agent-device/contracts/snapshot'; import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type RuntimeCommand } from '../../runtime-types.ts'; -import { capturePairScreenshotFrame, compareScreenshotPair } from './screenshot-diff-evidence.ts'; +import { + attachScreenshotPairOutcome, + dispatchWithScreenshotPair, + type ScreenshotPairOutcome, +} from './screenshot-diff-evidence.ts'; import { applyPostActionObservation, planPostActionObservation, @@ -174,53 +178,41 @@ async function tapCommand( const atRest = await admitResolvedTarget(runtime, options, resolutionRequest, resolved); const point = requireResolvedPoint(atRest); const pointObservation = pointObservationOf(atRest); - // The before-frame is taken after every pre-dispatch guard and immediately - // before the dispatch, so the pair brackets the tap and nothing else. - const screenshotBaseline = - options.expectScreenshotDiff === true - ? await capturePairScreenshotFrame(runtime, options, 'before') - : undefined; + const dispatchTap = runtime.backend.tap; let backendResult; + let screenshotOutcome: ScreenshotPairOutcome | undefined; try { - backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, { - button: options.button, - count: options.count, - intervalMs: options.intervalMs, - holdMs: options.holdMs, - jitterPx: options.jitterPx, - doubleTap: options.doubleTap, - ...(pointObservation ? { pointObservation } : {}), - }); + ({ result: backendResult, outcome: screenshotOutcome } = await dispatchWithScreenshotPair( + runtime, + options, + options.expectScreenshotDiff === true, + async () => + await dispatchTap(toBackendContext(runtime, options), point, { + button: options.button, + count: options.count, + intervalMs: options.intervalMs, + holdMs: options.holdMs, + jitterPx: options.jitterPx, + doubleTap: options.doubleTap, + ...(pointObservation ? { pointObservation } : {}), + }), + )); } catch (error) { // Resolution is complete before the backend call. Preserve it out of // band so a daemon-level failure corroboration can still record the same // target identity if the backend reports an ambiguous tap outcome. attachResolvedInteractionTarget(error, atRest); - if (screenshotBaseline) await screenshotBaseline.cleanup(); throw error; } - const screenshotOutcome = screenshotBaseline - ? await compareScreenshotPair(runtime, options, screenshotBaseline) - : undefined; - const screenshotWarning = screenshotOutcome?.unchangedWarning; const formattedBackendResult = toBackendResult(backendResult); return await applyPostActionObservation( runtime, options, atRest, - { - ...atRest, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...(screenshotOutcome?.evidence ? { screenshotDiff: screenshotOutcome.evidence } : {}), - ...(screenshotWarning - ? { - warning: - 'warning' in atRest && atRest.warning - ? `${atRest.warning} ${screenshotWarning}` - : screenshotWarning, - } - : {}), - }, + attachScreenshotPairOutcome( + { ...atRest, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}) }, + screenshotOutcome, + ), observation, ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts index d801771788..c240a56f0b 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -28,6 +28,66 @@ export type ScreenshotPairOutcome = { unchangedWarning?: string; }; +/** + * Runs the dispatch inside the screenshot bracket. Unrequested, this is the + * dispatch alone. Requested, the before-frame lands after every pre-dispatch + * guard and immediately before the dispatch, so the pair brackets the tap and + * nothing else; a dispatch failure cleans the frame up and rethrows, because a + * failed action owes no post-action comparison. + */ +export async function dispatchWithScreenshotPair( + runtime: AgentDeviceRuntime, + options: CommandContext, + requested: boolean, + dispatch: () => Promise, +): Promise<{ result: T; outcome?: ScreenshotPairOutcome }> { + const before = requested + ? await capturePairScreenshotFrame(runtime, options, 'before') + : undefined; + let result: T; + try { + result = await dispatch(); + } catch (error) { + if (before) await before.cleanup(); + throw error; + } + return { + result, + ...(before ? { outcome: await compareScreenshotPair(runtime, options, before) } : {}), + }; +} + +/** + * Folds the pair outcome into the runtime result: the evidence rides as + * `screenshotDiff`, and the unmet expectation composes its warning after any + * resolution warning the target already carries, so neither disclosure clobbers + * the other. The warning is read structurally: only the point/ref/selector + * variants carry one, and the point variant's is a conditional spread. + */ +export function attachScreenshotPairOutcome( + result: T, + outcome: ScreenshotPairOutcome | undefined, +): T { + if (!outcome) return result; + const { evidence, unchangedWarning } = outcome; + return { + ...result, + ...(evidence ? { screenshotDiff: evidence } : {}), + ...(unchangedWarning + ? { warning: composeWarnings(readResolutionWarning(result), unchangedWarning) } + : {}), + }; +} + +function readResolutionWarning(result: object): string | undefined { + const warning = (result as { warning?: unknown }).warning; + return typeof warning === 'string' && warning.length > 0 ? warning : undefined; +} + +function composeWarnings(existing: string | undefined, added: string): string { + return existing ? `${existing} ${added}` : added; +} + /** * One frame of the pair. A backend without `captureScreenshot` returns no * frame, and the whole readback silently does not run — the flag is advisory, diff --git a/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts b/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts index 5d9eed8517..9bf878ab8b 100644 --- a/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts +++ b/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts @@ -37,6 +37,18 @@ const READER_POSITIONALS: Record = { [PUBLIC_COMMANDS.press]: ['10', '20'], }; +// Targeted readers demand their target positionals, so each out-of-scope +// command names the positionals its own reader accepts. +const OUT_OF_SCOPE_READER_POSITIONALS: Record = { + [PUBLIC_COMMANDS.longPress]: ['10', '20'], + [PUBLIC_COMMANDS.hover]: ['10', '20'], + [PUBLIC_COMMANDS.fill]: ['10', '20', 'hello'], + [PUBLIC_COMMANDS.scroll]: ['down'], + [PUBLIC_COMMANDS.swipe]: ['30', '40', '50', '60'], + [PUBLIC_COMMANDS.type]: ['hi'], + [PUBLIC_COMMANDS.back]: [], +}; + const FLAG_SNAPSHOT = { expectScreenshotDiff: true } as CliFlags; test('expectScreenshotDiff CLI flags are scoped to press and click', () => { @@ -81,20 +93,7 @@ test('expectScreenshotDiff CLI reader input reaches only the press and click rea ); } for (const command of OUT_OF_SCOPE_COMMANDS) { - // Targeted readers demand their target positionals; the flag must drop out - // of the input regardless of the shape the reader accepts. - const positionals = - command === PUBLIC_COMMANDS.scroll - ? ['down'] - : command === PUBLIC_COMMANDS.fill - ? ['10', '20', 'hello'] - : command === PUBLIC_COMMANDS.longPress || - command === PUBLIC_COMMANDS.hover || - command === PUBLIC_COMMANDS.swipe - ? command === PUBLIC_COMMANDS.swipe - ? ['30', '40', '50', '60'] - : ['10', '20'] - : []; + const positionals = OUT_OF_SCOPE_READER_POSITIONALS[command] ?? []; const input = readInputFromCli(command as CommandName, positionals, FLAG_SNAPSHOT); assert.equal( 'expectScreenshotDiff' in input, From 70043f937cac084b2caea1ac9fe2337bd77703d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 01:49:16 +0800 Subject: [PATCH 04/16] refactor(interaction): keep the screenshot frame helpers module-private The bracket lifecycle moved behind dispatchWithScreenshotPair, so the two frame helpers no longer need exports; the dead-export gate reads them as unused module surface. Co-Authored-By: Claude Code --- .../commands/interaction/runtime/screenshot-diff-evidence.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts index c240a56f0b..8f13ef1db9 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -93,7 +93,7 @@ function composeWarnings(existing: string | undefined, added: string): string { * frame, and the whole readback silently does not run — the flag is advisory, * so a screenshot-less platform keeps the dispatch untouched. */ -export async function capturePairScreenshotFrame( +async function capturePairScreenshotFrame( runtime: AgentDeviceRuntime, options: CommandContext, label: 'before' | 'after', @@ -127,7 +127,7 @@ export async function capturePairScreenshotFrame( * the ratio is omitted because the two frames describe different viewports. * Any decode or capture failure drops the evidence rather than guessing. */ -export async function compareScreenshotPair( +async function compareScreenshotPair( runtime: AgentDeviceRuntime, options: CommandContext, before: TemporaryFile, From c6f226daef148046e1e6437e0a3d7e2c2957126f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 03:14:21 +0800 Subject: [PATCH 05/16] fix(interaction): register --expect-screenshot-diff in the guarantee matrix, settle the after-frame, and document the out-of-tree readback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes for the press/click --expect-screenshot-diff PR: - The guarantee matrix (ADR 0011) gains the screenshotDiffEvidence row: runtime cells on runtime-selector/runtime-ref/coordinate scoped to press/click, a delegated cell on native-ref, and inapplicable cells on the paths the flag cannot reach. The three waivers that named only --verify/--settle as the opt-in post-action evidence routes now name this flag too. Each enforced cell carries a contract scenario, and the shared harness replays a solid-color frame plan. - The after-frame waits 150ms past the dispatch reply before it is captured: the reaction the flag asks about can render frames after the transport round trip returns, and an immediate frame would read an effective tap as unchanged. The flag help text, field descriptions, and schema prose state the timing instead of leaving it blank. - commands.md names the flag where it teaches that a successful press is not proof of effect: a PIN keypad target is the case --verify cannot answer, since it compares empty trees. - Gate drift the PR introduced: the new capture-kit export joins the layering manifest baseline, and the flag descriptor gate moves to test/repo-guards/ (a repo-level guard on command descriptors, not a module's unit test — R84 test-colocation). Co-Authored-By: Claude Code --- .../src/flag-definitions-action.ts | 2 +- .../runtime/screenshot-diff-evidence.ts | 30 +++++++++--- .../command-surface/src/interaction/index.ts | 4 +- .../src/interaction/metadata.ts | 2 +- .../contracts/src/interaction-guarantees.ts | 49 +++++++++++++++++-- packages/contracts/src/interaction.ts | 8 +-- scripts/layering/package-boundaries.test.ts | 1 + .../coordinate.contract.test.ts | 23 ++++++++- .../coordinate.coverage.ts | 2 + .../native-ref.contract.test.ts | 28 ++++++++++- .../native-ref.coverage.ts | 2 + .../interaction-contract/runtime-harness.ts | 44 ++++++++++++++++- .../runtime-ref.contract.test.ts | 24 ++++++++- .../runtime-ref.coverage.ts | 2 + .../runtime-selector.contract.test.ts | 25 +++++++++- .../runtime-selector.coverage.ts | 2 + ...-descriptor-expect-screenshot-diff.test.ts | 6 ++- website/docs/docs/commands.md | 2 +- 18 files changed, 228 insertions(+), 28 deletions(-) rename {src/__tests__ => test/repo-guards}/command-descriptor-expect-screenshot-diff.test.ts (95%) diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 81be2fc7c1..09ca08bd61 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -212,7 +212,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--expect-screenshot-diff', usageDescription: - 'Press/click: compare lightweight screenshots taken right before and after the dispatch and report screenshotDiff in the response; use it when the target lives outside the accessibility tree (a PIN keypad) and --verify would compare empty trees. A matching pair adds a warning, because the dispatch may have had no visible effect', + 'Press/click: compare lightweight screenshots taken right before the dispatch and after it settles briefly, and report screenshotDiff in the response; use it when the target lives outside the accessibility tree (a PIN keypad) and --verify would compare empty trees. A matching pair adds a warning, because the dispatch may have had no visible effect', projectConfig: true, recorded: false, }, diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts index 8f13ef1db9..68d62aacdf 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'; import type { TemporaryFile } from '@agent-device/contracts/artifact-adapter'; import type { ScreenshotDiffEvidence } from '@agent-device/contracts/interaction'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import { toBackendContext } from '../../runtime-common.ts'; +import { sleep, toBackendContext } from '../../runtime-common.ts'; import { createCommandTempFile } from '../../io-policy.ts'; /** @@ -22,6 +22,17 @@ import { createCommandTempFile } from '../../io-policy.ts'; * frame (the eager-closure budget). */ +/** + * The after-frame waits this long past the dispatch reply before it is + * captured. The reaction the caller asked about (a key highlight, a PIN dot) + * can reach the screen several frames after the transport round trip returns — + * exactly on the low-end or loaded devices this flag exists for — and a frame + * that raced the rendering would read an effective tap as unchanged, which is + * the one direction this primitive cannot cheaply retry. Long enough to cover + * that render delay; short enough to keep the flag's added latency negligible. + */ +const AFTER_FRAME_SETTLE_MS = 150; + export type ScreenshotPairOutcome = { evidence?: ScreenshotDiffEvidence; /** Present when the caller expected a diff and the frames matched. */ @@ -31,9 +42,10 @@ export type ScreenshotPairOutcome = { /** * Runs the dispatch inside the screenshot bracket. Unrequested, this is the * dispatch alone. Requested, the before-frame lands after every pre-dispatch - * guard and immediately before the dispatch, so the pair brackets the tap and - * nothing else; a dispatch failure cleans the frame up and rethrows, because a - * failed action owes no post-action comparison. + * guard and immediately before the dispatch, and the after-frame lands once + * the dispatch has settled briefly (AFTER_FRAME_SETTLE_MS), so the pair + * brackets the tap and its rendering; a dispatch failure cleans the frame up + * and rethrows, because a failed action owes no post-action comparison. */ export async function dispatchWithScreenshotPair( runtime: AgentDeviceRuntime, @@ -122,10 +134,11 @@ async function capturePairScreenshotFrame( } /** - * Closes the pair: one after-frame, then the pixel comparison. A frame that - * changed shape (rotation, split-screen) changed the screen by construction — - * the ratio is omitted because the two frames describe different viewports. - * Any decode or capture failure drops the evidence rather than guessing. + * Closes the pair: one after-frame after a brief settle, then the pixel + * comparison. A frame that changed shape (rotation, split-screen) changed the + * screen by construction — the ratio is omitted because the two frames + * describe different viewports. Any decode or capture failure drops the + * evidence rather than guessing. */ async function compareScreenshotPair( runtime: AgentDeviceRuntime, @@ -133,6 +146,7 @@ async function compareScreenshotPair( before: TemporaryFile, ): Promise { try { + await sleep(runtime, AFTER_FRAME_SETTLE_MS); const after = await capturePairScreenshotFrame(runtime, options, 'after'); if (!after) return {}; try { diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index ab5e653a21..64b0cee3fd 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -87,14 +87,14 @@ const interactionEvidenceSchema: JsonSchema = objectSchema( const screenshotDiffSchema: JsonSchema = objectSchema( { changedFromBefore: booleanSchema( - 'Whether any pixel moved between the screenshots taken right before and after the dispatch. false means the frames match, which reads as "the dispatch may have had no visible effect" next to the response warning.', + 'Whether any pixel moved between the screenshots taken right before the dispatch and after it settles briefly. false means the frames match, which reads as "the dispatch may have had no visible effect" next to the response warning.', ), changedPixelRatio: numberSchema( 'Share of pixels whose color moved between the two frames. Absent when the frames could not be compared, for example a shape change.', ), }, ['changedFromBefore'], - 'Pixel comparison between the screenshots taken right before and after the dispatch. Absent when the pair was not taken, which claims nothing about the screen.', + 'Pixel comparison between the screenshots taken right before the dispatch and after it settles briefly. Absent when the pair was not taken, which claims nothing about the screen.', ); const responseCostSchema: JsonSchema = objectSchema( diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index e03e59ba98..efcd909b98 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -90,7 +90,7 @@ type InteractionCommandName = keyof typeof interactionCommandDescriptions; const expectScreenshotDiffField = () => booleanField( - 'Compare lightweight screenshots taken right before and after the dispatch, and report the comparison as screenshotDiff in the response. Use it when the target lives outside the accessibility tree, such as a PIN keypad, where verify would compare empty trees. A matching pair adds a warning, because the dispatch may have had no visible effect.', + 'Compare lightweight screenshots taken right before the dispatch and after it settles briefly, and report the comparison as screenshotDiff in the response. Use it when the target lives outside the accessibility tree, such as a PIN keypad, where verify would compare empty trees. A matching pair adds a warning, because the dispatch may have had no visible effect.', ); const clickFields = { diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 30dc392063..f8d20a127a 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -63,6 +63,13 @@ export const INTERACTION_GUARANTEES = [ // settled tree was stored) in the same response. Best-effort: never fails // the action. 'settleObservation', + // --expect-screenshot-diff brackets the dispatch with two lightweight + // screenshots and compares pixels: the out-of-tree readback for surfaces the + // accessibility tree cannot describe (a PIN keypad), where --verify's node + // digest stays flat no matter what the tap did. Advisory by the same contract + // as settleObservation — the comparison never fails the action, and a + // matching pair adds a warning instead. + 'screenshotDiffEvidence', // Failures use the shared codes/messages/hints (no-match diagnostics, // ambiguous shape, offscreen reasons). NOTE: expected to split into // errorCodes (stable codes / fallback classification) vs errorDiagnostics @@ -186,7 +193,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS Simulator open that did not read the launched app (any postOpenObservation other than observable) marks the next capture in that session to wait for a quiet surface (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only the opt-in --verify, --settle, and --expect-screenshot-diff features capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS Simulator open that did not read the launched app (any postOpenObservation other than observable) marks the next capture in that session to wait for a quiet surface (markPostOpenStabilization); that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; @@ -199,7 +206,7 @@ const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { const DIRECT_IOS_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - "gap: this replay-only route observes no outcome beyond the runner's own report; --verify/--settle do not apply here (see the inapplicable cells on this row), and the shared ambiguous-failure corroboration only reconsiders a thrown error, never a successful dispatch.", + "gap: this replay-only route observes no outcome beyond the runner's own report; --verify/--settle/--expect-screenshot-diff do not apply here (see the inapplicable cells on this row), and the shared ambiguous-failure corroboration only reconsiders a thrown error, never a successful dispatch.", trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; @@ -280,6 +287,11 @@ const RUNTIME_TREE_SHARED_GUARANTEES = { kind: 'runtime', via: 'packages/command-runtime/src/commands/interaction/runtime/settle.ts#settleAfterInteraction', }, + screenshotDiffEvidence: { + kind: 'runtime', + via: 'packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts#dispatchWithScreenshotPair', + appliesTo: ['press', 'click'], + }, targetStability: RUNTIME_TREE_TARGET_STABILITY, } satisfies Partial>; @@ -379,6 +391,11 @@ export const INTERACTION_DISPATCH_PATHS: Record { '@agent-device/capture-kit/perf-capture-session-resource', '@agent-device/capture-kit/perf-runtime-plan', '@agent-device/capture-kit/png', + '@agent-device/capture-kit/png-changed-pixel-ratio', '@agent-device/capture-kit/png-crop', '@agent-device/capture-kit/png-resize', '@agent-device/capture-kit/png-rgb-difference', diff --git a/test/integration/interaction-contract/coordinate.contract.test.ts b/test/integration/interaction-contract/coordinate.contract.test.ts index 6ae749819d..ae3f3d23fc 100644 --- a/test/integration/interaction-contract/coordinate.contract.test.ts +++ b/test/integration/interaction-contract/coordinate.contract.test.ts @@ -6,7 +6,7 @@ import { assertRpcError, assertRpcOk } from '../provider-scenarios/assertions.ts import { scenarioName } from './coverage-manifest.ts'; import { COORDINATE_COVERAGE } from './coordinate.coverage.ts'; import { settledWelcomeSnapshot, viewportOnlySnapshot } from './fixtures.ts'; -import { createContractDevice } from './runtime-harness.ts'; +import { createContractDevice, screenshotFramePlanBackend } from './runtime-harness.ts'; import { runnerTapEntry, runnerTapErrorEntry, withIosContractDaemon } from './daemon-harness.ts'; // ADR 0011 Layer 3, coordinate path: raw x/y tap, intentionally minimal @@ -77,6 +77,27 @@ test(scenario('settleObservation'), async () => { assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); }); +test(scenario('screenshotDiffEvidence'), async () => { + const device = createContractDevice(viewportOnlySnapshot(), { + ...screenshotFramePlanBackend([{ color: [0, 0, 0] }, { color: [255, 255, 255] }]), + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { + session: 'default', + expectScreenshotDiff: true, + }, + ); + + // The escape hatch gets the same opt-in readback: pixels answer where the + // tree-less point target gives --verify nothing to digest. + assert.equal(result.kind, 'point'); + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, true); +}); + test(scenario('errorTaxonomy'), async () => { await withIosContractDaemon( [runnerTapErrorEntry(new Error('runner tap crashed'))], diff --git a/test/integration/interaction-contract/coordinate.coverage.ts b/test/integration/interaction-contract/coordinate.coverage.ts index efdce51bae..de9f8b3dbd 100644 --- a/test/integration/interaction-contract/coordinate.coverage.ts +++ b/test/integration/interaction-contract/coordinate.coverage.ts @@ -8,5 +8,7 @@ export const COORDINATE_COVERAGE = definePathCoverage('coordinate', { 'coordinate verifyEvidence: point click --verify returns a digest with change detection', settleObservation: 'coordinate settleObservation: point press --settle captures a baseline and returns the settled diff', + screenshotDiffEvidence: + 'coordinate screenshotDiffEvidence: point press --expect-screenshot-diff returns the pixel comparison', errorTaxonomy: 'coordinate errorTaxonomy: backend failure surfaces as a normalized daemon error', }); diff --git a/test/integration/interaction-contract/native-ref.contract.test.ts b/test/integration/interaction-contract/native-ref.contract.test.ts index e3218e75d9..6a2af1a6d6 100644 --- a/test/integration/interaction-contract/native-ref.contract.test.ts +++ b/test/integration/interaction-contract/native-ref.contract.test.ts @@ -1,4 +1,4 @@ -import { ref, createContractDevice } from './runtime-harness.ts'; +import { ref, createContractDevice, screenshotFramePlanBackend } from './runtime-harness.ts'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { InteractionGuarantee } from '@agent-device/contracts/interaction-guarantees'; @@ -162,6 +162,32 @@ test(scenario('settleObservation'), async () => { assert.deepEqual(settle.diff?.summary, { additions: 1, removals: 1, unchanged: 0 }); }); +test(scenario('screenshotDiffEvidence'), async () => { + const calls: string[] = []; + const device = createContractDevice(continueButtonSnapshot(), { + platform: 'web', + ...screenshotFramePlanBackend([{ color: [0, 0, 0] }, { color: [255, 255, 255] }]), + captureSnapshot: async () => ({ snapshot: continueButtonSnapshot() }), + tap: async () => ({ ok: true }), + tapTarget: async (_context, target) => { + calls.push(target.ref); + return {}; + }, + }); + + const result = await device.interactions.click(ref('@e1'), { + session: 'default', + expectScreenshotDiff: true, + }); + + // --expect-screenshot-diff delegates to the runtime-ref path: the fast path + // is skipped so the pixel pair brackets the dispatch it names. + assert.deepEqual(calls, []); + assert.equal(result.kind, 'ref'); + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, true); +}); + test(scenario('responseIdentity'), async () => { const calls: string[] = []; const device = createNativeRefDevice(continueButtonSnapshot(), calls); diff --git a/test/integration/interaction-contract/native-ref.coverage.ts b/test/integration/interaction-contract/native-ref.coverage.ts index 72e80853b1..5c311e2b75 100644 --- a/test/integration/interaction-contract/native-ref.coverage.ts +++ b/test/integration/interaction-contract/native-ref.coverage.ts @@ -14,6 +14,8 @@ export const NATIVE_REF_COVERAGE = definePathCoverage('native-ref', { verifyEvidence: 'native-ref verifyEvidence: --verify skips the fast path and returns evidence', settleObservation: 'native-ref settleObservation: --settle skips the fast path and returns the settled diff', + screenshotDiffEvidence: + 'native-ref screenshotDiffEvidence: --expect-screenshot-diff skips the fast path and returns the pixel comparison', // The preflight raises the runtime path's exact offscreen_ref shape (code, // reason, hint), which is the shared taxonomy on this path. errorTaxonomy: diff --git a/test/integration/interaction-contract/runtime-harness.ts b/test/integration/interaction-contract/runtime-harness.ts index 4edfc4bd33..4816f80a61 100644 --- a/test/integration/interaction-contract/runtime-harness.ts +++ b/test/integration/interaction-contract/runtime-harness.ts @@ -1,6 +1,8 @@ -import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import fs from 'node:fs'; +import type { AgentDeviceBackend, BackendScreenshotOptions } from '@agent-device/contracts/backend'; import type { CommandClock, CommandSessionRecord } from '@agent-device/contracts/runtime-contract'; -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import { PNG } from '@agent-device/capture-kit/png'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { createLocalArtifactAdapter } from '../../../src/io.ts'; import { createCommandSurfaceAgentDevice } from '@agent-device/command-runtime'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; @@ -9,6 +11,7 @@ type ContractBackendOverrides = Partial< Pick< AgentDeviceBackend, | 'captureSnapshot' + | 'captureScreenshot' | 'tap' | 'tapTarget' | 'fill' @@ -46,6 +49,9 @@ export function createContractDevice( platform: overrides.platform ?? 'ios', captureSnapshot: async (...args) => overrides.captureSnapshot ? await overrides.captureSnapshot(...args) : { snapshot }, + captureScreenshot: overrides.captureScreenshot + ? async (...args) => (await overrides.captureScreenshot?.(...args)) ?? {} + : undefined, tap: async (...args) => await overrides.tap?.(...args), tapTarget: overrides.tapTarget, fill: async (...args) => await overrides.fill?.(...args), @@ -97,3 +103,37 @@ export function selector( ): import('@agent-device/contracts/interaction').SelectorTarget { return { kind: 'selector', selector: value }; } + +/** + * The screenshot-diff evidence scenarios pair frames by capture order (before, + * then after): the backend replays one solid-color PNG per planned frame and + * repeats the last one if the runtime captures more. The recorded capture + * options ride along for the scenario to assert on. + */ +export function screenshotFramePlanBackend( + frames: ReadonlyArray<{ + color: readonly [number, number, number]; + width?: number; + height?: number; + }>, +): Pick & { captured: BackendScreenshotOptions[] } { + let frameIndex = 0; + const captured: BackendScreenshotOptions[] = []; + return { + captured, + captureScreenshot: async (_context, outPath, options) => { + const frame = frames[Math.min(frameIndex, frames.length - 1)]!; + frameIndex += 1; + if (options) captured.push(options); + const png = new PNG({ width: frame.width ?? 8, height: frame.height ?? 8 }); + for (let i = 0; i < png.data.length; i += 4) { + png.data[i] = frame.color[0]; + png.data[i + 1] = frame.color[1]; + png.data[i + 2] = frame.color[2]; + png.data[i + 3] = 255; + } + fs.writeFileSync(outPath, PNG.sync.write(png)); + return { path: outPath }; + }, + }; +} diff --git a/test/integration/interaction-contract/runtime-ref.contract.test.ts b/test/integration/interaction-contract/runtime-ref.contract.test.ts index 4db849a7ae..f7e201623a 100644 --- a/test/integration/interaction-contract/runtime-ref.contract.test.ts +++ b/test/integration/interaction-contract/runtime-ref.contract.test.ts @@ -1,4 +1,9 @@ -import { ref, createContractClock, createContractDevice } from './runtime-harness.ts'; +import { + ref, + createContractClock, + createContractDevice, + screenshotFramePlanBackend, +} from './runtime-harness.ts'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { InteractionGuarantee } from '@agent-device/contracts/interaction-guarantees'; @@ -227,6 +232,23 @@ test(scenario('settleObservation'), async () => { assert.equal(settle.diff?.lines.find((line) => line.kind === 'added')?.ref, 'e1'); }); +test(scenario('screenshotDiffEvidence'), async () => { + const device = createContractDevice(continueButtonSnapshot(), { + ...screenshotFramePlanBackend([{ color: [0, 0, 0] }, { color: [255, 255, 255] }]), + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.click(ref('@e1'), { + session: 'default', + expectScreenshotDiff: true, + }); + + // The pixel pair brackets the runtime dispatch the ref resolved for. + assert.equal(result.kind, 'ref'); + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, true); +}); + test(scenario('errorTaxonomy'), async () => { const device = createContractDevice(continueButtonSnapshot(), { tap: async () => ({ ok: true }), diff --git a/test/integration/interaction-contract/runtime-ref.coverage.ts b/test/integration/interaction-contract/runtime-ref.coverage.ts index 50ef4698c6..e4379e9fe3 100644 --- a/test/integration/interaction-contract/runtime-ref.coverage.ts +++ b/test/integration/interaction-contract/runtime-ref.coverage.ts @@ -15,6 +15,8 @@ export const RUNTIME_REF_COVERAGE = definePathCoverage('runtime-ref', { 'runtime-ref verifyEvidence: click @ref --verify returns a digest with change detection', settleObservation: 'runtime-ref settleObservation: press @ref --settle diffs the settled tree against the stored baseline', + screenshotDiffEvidence: + 'runtime-ref screenshotDiffEvidence: click @ref --expect-screenshot-diff returns the pixel comparison', errorTaxonomy: 'runtime-ref errorTaxonomy: unknown ref fails with the stale-ref hint', resolutionDisclosure: [ 'runtime-ref resolutionDisclosure: an @ref discloses the exact ref-provenance shape', diff --git a/test/integration/interaction-contract/runtime-selector.contract.test.ts b/test/integration/interaction-contract/runtime-selector.contract.test.ts index 970af7e650..635c48b9c8 100644 --- a/test/integration/interaction-contract/runtime-selector.contract.test.ts +++ b/test/integration/interaction-contract/runtime-selector.contract.test.ts @@ -1,4 +1,9 @@ -import { selector, createContractClock, createContractDevice } from './runtime-harness.ts'; +import { + selector, + createContractClock, + createContractDevice, + screenshotFramePlanBackend, +} from './runtime-harness.ts'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { InteractionGuarantee } from '@agent-device/contracts/interaction-guarantees'; @@ -367,6 +372,24 @@ test(scenario('settleObservation'), async () => { assert.equal(added?.ref, 'e1'); }); +test(scenario('screenshotDiffEvidence'), async () => { + const device = createContractDevice(continueButtonSnapshot(), { + ...screenshotFramePlanBackend([{ color: [0, 0, 0] }, { color: [255, 255, 255] }]), + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + expectScreenshotDiff: true, + }); + + // The pixel pair brackets the dispatch: the readback rides in the same + // response and never fails the action. + assert.equal(result.kind, 'selector'); + assert.ok(result.screenshotDiff); + assert.equal(result.screenshotDiff?.changedFromBefore, true); +}); + test(scenario('errorTaxonomy'), async () => { const device = createContractDevice(continueButtonSnapshot(), { tap: async () => ({ ok: true }), diff --git a/test/integration/interaction-contract/runtime-selector.coverage.ts b/test/integration/interaction-contract/runtime-selector.coverage.ts index 9e962ea218..b8cd0fe831 100644 --- a/test/integration/interaction-contract/runtime-selector.coverage.ts +++ b/test/integration/interaction-contract/runtime-selector.coverage.ts @@ -23,6 +23,8 @@ export const RUNTIME_SELECTOR_COVERAGE = definePathCoverage('runtime-selector', 'runtime-selector verifyEvidence: press --verify returns a digest with change detection', settleObservation: 'runtime-selector settleObservation: press --settle returns the settled diff with fresh refs', + screenshotDiffEvidence: + 'runtime-selector screenshotDiffEvidence: press --expect-screenshot-diff returns the pixel comparison', errorTaxonomy: [ 'runtime-selector errorTaxonomy: no-match failure carries the shared code and hint', 'runtime-selector errorTaxonomy: a daemon selector miss discloses dispatched no', diff --git a/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts b/test/repo-guards/command-descriptor-expect-screenshot-diff.test.ts similarity index 95% rename from src/__tests__/command-descriptor-expect-screenshot-diff.test.ts rename to test/repo-guards/command-descriptor-expect-screenshot-diff.test.ts index 9bf878ab8b..606ceb01a2 100644 --- a/src/__tests__/command-descriptor-expect-screenshot-diff.test.ts +++ b/test/repo-guards/command-descriptor-expect-screenshot-diff.test.ts @@ -14,10 +14,12 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; /** * The descriptor gate for `--expect-screenshot-diff` (press/click only), in the - * style of the `--settle`/`--verify` gate beside this directory: every + * style of the `--settle`/`--verify` descriptor gates under src/__tests__: every * caller-facing surface is pinned to the press/click scope, so a forgotten or * over-broad cell fails here instead of silently no-op'ing the flag — or - * leaking it onto a command that never reads it. + * leaking it onto a command that never reads it. This is a repo-level guard on + * command descriptors rather than a module's unit test, so it lives beside the + * other guards. */ const SCREENSHOT_DIFF_COMMANDS = [PUBLIC_COMMANDS.click, PUBLIC_COMMANDS.press] as const; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 7f6edbff4e..ca29903890 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -499,7 +499,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. When the target lives outside the accessibility tree (a PIN keypad), `--verify` compares empty trees, so add `--expect-screenshot-diff` on `press`/`click` instead: the command compares screenshots taken right before the dispatch and after it settles briefly, and warns when they match. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 27429a358a1a4c68ede5146b90def7621e5f7752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 04:36:25 +0800 Subject: [PATCH 06/16] refactor(interaction): load the evidence lane behind a dynamic import Co-Authored-By: Claude Code --- .../src/commands/interaction/runtime/interactions.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 80baf479cc..e985d576c8 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -14,11 +14,7 @@ import { isFillableType } from '@agent-device/contracts/snapshot'; import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type RuntimeCommand } from '../../runtime-types.ts'; -import { - attachScreenshotPairOutcome, - dispatchWithScreenshotPair, - type ScreenshotPairOutcome, -} from './screenshot-diff-evidence.ts'; +import type { ScreenshotPairOutcome } from './screenshot-diff-evidence.ts'; import { applyPostActionObservation, planPostActionObservation, @@ -182,6 +178,9 @@ async function tapCommand( let backendResult; let screenshotOutcome: ScreenshotPairOutcome | undefined; try { + // Dynamic import: the readback evidence lane stays off the command-runtime entry closure. + const { attachScreenshotPairOutcome, dispatchWithScreenshotPair } = + await import('./screenshot-diff-evidence.ts'); ({ result: backendResult, outcome: screenshotOutcome } = await dispatchWithScreenshotPair( runtime, options, From 584bcafe496bc3867e28d9dead4a2c70abf8af21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 04:37:19 +0800 Subject: [PATCH 07/16] fix(interaction): hoist the evidence import above its outer use Co-Authored-By: Claude Code --- .../src/commands/interaction/runtime/interactions.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index e985d576c8..6c6e21b82b 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -175,12 +175,13 @@ async function tapCommand( const point = requireResolvedPoint(atRest); const pointObservation = pointObservationOf(atRest); const dispatchTap = runtime.backend.tap; + // Dynamic import: the readback evidence lane stays off the command-runtime entry closure. + const { attachScreenshotPairOutcome, dispatchWithScreenshotPair } = await import( + './screenshot-diff-evidence.ts' + ); let backendResult; let screenshotOutcome: ScreenshotPairOutcome | undefined; try { - // Dynamic import: the readback evidence lane stays off the command-runtime entry closure. - const { attachScreenshotPairOutcome, dispatchWithScreenshotPair } = - await import('./screenshot-diff-evidence.ts'); ({ result: backendResult, outcome: screenshotOutcome } = await dispatchWithScreenshotPair( runtime, options, From 7a61db9b5e4c857f08617012833b84172d160e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 7 Nov 2026 05:09:23 +0800 Subject: [PATCH 08/16] style(interaction): format after the import hoist Co-Authored-By: Claude Code --- .../src/commands/interaction/runtime/interactions.ts | 5 ++--- scripts/layering/package-boundaries.test.ts | 1 - test/integration/interaction-contract/runtime-harness.ts | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 6c6e21b82b..7cadbdb104 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -176,9 +176,8 @@ async function tapCommand( const pointObservation = pointObservationOf(atRest); const dispatchTap = runtime.backend.tap; // Dynamic import: the readback evidence lane stays off the command-runtime entry closure. - const { attachScreenshotPairOutcome, dispatchWithScreenshotPair } = await import( - './screenshot-diff-evidence.ts' - ); + const { attachScreenshotPairOutcome, dispatchWithScreenshotPair } = + await import('./screenshot-diff-evidence.ts'); let backendResult; let screenshotOutcome: ScreenshotPairOutcome | undefined; try { diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index be61abf4ce..560df13cc2 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -121,7 +121,6 @@ test('the compiled set classifies every file region the R11 walk visits', () => assert.equal(insideCompiledSources('scripts/layering/check.ts'), false); assert.equal(insideCompiledSources('packages/maestro/test/conformance/harness.ts'), false); assert.equal(insideCompiledSources('packages/proxy/tsdown.config.ts'), false); - assert.equal(insideCompiledSources('scripts/help-conformance-command-validator.ts'), false); assert.equal(insideCompiledSources('scripts/coverage-crap/stamp.ts'), false); assert.equal(insideCompiledSources('scripts/coverage-changed/run.ts'), false); diff --git a/test/integration/interaction-contract/runtime-harness.ts b/test/integration/interaction-contract/runtime-harness.ts index 4816f80a61..2c7335d1a4 100644 --- a/test/integration/interaction-contract/runtime-harness.ts +++ b/test/integration/interaction-contract/runtime-harness.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import type { AgentDeviceBackend, BackendScreenshotOptions } from '@agent-device/contracts/backend'; import type { CommandClock, CommandSessionRecord } from '@agent-device/contracts/runtime-contract'; import { PNG } from '@agent-device/capture-kit/png'; -import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { createLocalArtifactAdapter } from '../../../src/io.ts'; import { createCommandSurfaceAgentDevice } from '@agent-device/command-runtime'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; From ffff7971832f23228a3b7ca95d6fdda64efb8699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Fri, 13 Nov 2026 22:59:47 +0800 Subject: [PATCH 09/16] fix(interaction): keep the screenshot readback from failing a delivered tap Nothing after a resolved dispatch throws out of the evidence lane anymore: a decode, read, import or cleanup failure yields no evidence, because a throwing readback reported a delivered tap as failed and a PIN caller would re-enter the digit. The after-frame settle honors the request signal. Both frames decode in the codec worker, the guard audits a missing descriptor instead of skipping it, the docs say the readback is best-effort, and the delivered-tap tests record the tap. Co-Authored-By: Claude Code --- .../runtime/screenshot-diff-evidence.test.ts | 33 ++++++++++++++- .../runtime/screenshot-diff-evidence.ts | 42 ++++++++++++++----- ...-descriptor-expect-screenshot-diff.test.ts | 4 +- website/docs/docs/commands.md | 2 +- 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts index b1230a9132..38b06ee626 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.test.ts @@ -177,13 +177,16 @@ test('press with expectScreenshotDiff reports a shape change without a ratio', a test('press with expectScreenshotDiff drops the evidence when a capture fails', async () => { let captureCount = 0; + const taps: Array<{ x: number; y: number }> = []; const device = createInteractionDevice(selectorSnapshot(), { captureScreenshot: async () => { captureCount += 1; if (captureCount === 1) throw new Error('screen capture unavailable'); return { path: '/unused/after.png' }; }, - tap: async () => ({ ok: true }), + tap: async (_context, point) => { + taps.push(point); + }, }); const result = await device.interactions.press( @@ -194,10 +197,11 @@ test('press with expectScreenshotDiff drops the evidence when a capture fails', }, ); - // No pair, no claim — and the dispatch itself still succeeds. + // No pair, no claim — the tap still crossed, and the readback says nothing. assert.equal('screenshotDiff' in result, false); assert.equal('warning' in result ? result.warning : undefined, undefined); assert.deepEqual(captureCount, 1); + assert.deepEqual(taps, [{ x: 10, y: 20 }]); }); test('press without a screenshot backend drops the evidence and still taps', async () => { @@ -247,3 +251,28 @@ test('press with expectScreenshotDiff routes a ref target through the runtime pa assert.ok(result.screenshotDiff); assert.equal(result.screenshotDiff?.changedFromBefore, true); }); + +test('a corrupted after-frame yields no evidence instead of failing a delivered tap', async () => { + const corrupted = Buffer.from('not a png at all'); + const backend = framePlanBackend([ + { bytes: solidPngBuffer(8, 8, { r: 0, g: 0, b: 0 }) }, + { bytes: corrupted }, + ]); + const taps: Array<{ x: number; y: number }> = []; + const device = createInteractionDevice(selectorSnapshot(), { + ...backend, + tap: async (_context, point) => { + taps.push(point); + }, + }); + + const result = await device.interactions.press( + { kind: 'point', x: 10, y: 20 }, + { session: 'default', expectScreenshotDiff: true }, + ); + + // The tap crossed the device; the unreadable readback claims nothing. + assert.equal(taps.length, 1); + assert.equal(result.screenshotDiff, undefined); + assert.equal(result.warning, undefined); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts index 68d62aacdf..c9bb71ed6f 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -60,7 +60,9 @@ export async function dispatchWithScreenshotPair( try { result = await dispatch(); } catch (error) { - if (before) await before.cleanup(); + // The evidence lane owns nothing after a resolved dispatch, and a failed + // dispatch rethrows its own error even if the frame cleanup fails. + if (before) await before.cleanup().catch(() => {}); throw error; } return { @@ -145,23 +147,43 @@ async function compareScreenshotPair( options: CommandContext, before: TemporaryFile, ): Promise { + // Nothing after a resolved dispatch may throw out of this lane: a decode, + // read, import or cleanup failure yields no evidence, because a throwing + // readback would report a delivered tap as failed and a PIN caller would + // re-enter the digit. try { - await sleep(runtime, AFTER_FRAME_SETTLE_MS); + return await compareScreenshotPairInner(runtime, options, before); + } catch { + return {}; + } +} + +async function compareScreenshotPairInner( + runtime: AgentDeviceRuntime, + options: CommandContext, + before: TemporaryFile, +): Promise { + try { + // A request canceled after the dispatch stops waiting for the after-frame. + await sleep(runtime, AFTER_FRAME_SETTLE_MS, options.signal); const after = await capturePairScreenshotFrame(runtime, options, 'after'); if (!after) return {}; try { - const [{ computePngChangedPixelRatio }, { decodePng }] = await Promise.all([ + const [{ computePngChangedPixelRatio }, { decodePngAsync }] = await Promise.all([ import('@agent-device/capture-kit/png-changed-pixel-ratio'), - import('@agent-device/capture-kit/png'), + import('@agent-device/capture-kit/png-worker-client'), ]); const [beforeBytes, afterBytes] = await Promise.all([ fs.readFile(before.path), fs.readFile(after.path), ]); - const ratio = computePngChangedPixelRatio( - decodePng(beforeBytes, 'the press before-screenshot'), - decodePng(afterBytes, 'the press after-screenshot'), - ); + // The decodes run in the codec worker: full-resolution frames never sit + // on the event loop. + const [beforePng, afterPng] = await Promise.all([ + decodePngAsync(beforeBytes, 'the press before-screenshot'), + decodePngAsync(afterBytes, 'the press after-screenshot'), + ]); + const ratio = computePngChangedPixelRatio(beforePng, afterPng); if (ratio.status === 'dimension_mismatch') { return { evidence: { changedFromBefore: true } }; } @@ -179,9 +201,9 @@ async function compareScreenshotPair( }), }; } finally { - await after.cleanup(); + await after.cleanup().catch(() => {}); } } finally { - await before.cleanup(); + await before.cleanup().catch(() => {}); } } diff --git a/test/repo-guards/command-descriptor-expect-screenshot-diff.test.ts b/test/repo-guards/command-descriptor-expect-screenshot-diff.test.ts index 606ceb01a2..906ac342ca 100644 --- a/test/repo-guards/command-descriptor-expect-screenshot-diff.test.ts +++ b/test/repo-guards/command-descriptor-expect-screenshot-diff.test.ts @@ -75,7 +75,9 @@ test('expectScreenshotDiff CLI flags are scoped to press and click', () => { test('expectScreenshotDiff metadata fields are scoped to press and click', () => { for (const command of [...SCREENSHOT_DIFF_COMMANDS, ...OUT_OF_SCOPE_COMMANDS]) { const metadata = findCommandMetadata(command); - if (!metadata) continue; + if (!metadata) { + throw new Error(`listed command ${command} carries no metadata; the guard cannot audit it`); + } const properties = metadata.inputSchema.properties ?? {}; assert.equal( Object.hasOwn(properties, 'expectScreenshotDiff'), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ca29903890..a5779eba5c 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -499,7 +499,7 @@ agent-device gesture tap 80 300 200 450 --count 20 --pause-ms 100 # repeated two If rotation stops repeated iOS presses, inspect the screen before continuing with the remaining presses. -A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. When the target lives outside the accessibility tree (a PIN keypad), `--verify` compares empty trees, so add `--expect-screenshot-diff` on `press`/`click` instead: the command compares screenshots taken right before the dispatch and after it settles briefly, and warns when they match. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). +A successful `click` or `press` means the action was sent to where the command found the target. It does not confirm that the element received it or that the screen changed. When the next step depends on it, add `--verify` (reports `changedFromBefore`), add `--settle` (returns the diff once the UI goes quiet), or follow with `wait `. When the target lives outside the accessibility tree (a PIN keypad), `--verify` compares empty trees, so add `--expect-screenshot-diff` on `press`/`click` instead: the command compares screenshots taken right before the dispatch and after it settles briefly, and warns when they match. The readback is best-effort: on a backend without screenshot capture, or when a capture or comparison fails, the response carries neither `screenshotDiff` nor a warning and claims nothing about the screen. Before retrying a failed interaction, see [Retry after a failed command](#retry-after-a-failed-command). A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 434e4ad50d9db3be0fea7769b0c7a7c41a3e5ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 14 Nov 2026 13:08:23 +0800 Subject: [PATCH 10/16] fix(daemon): refuse --expect-screenshot-diff on the touch routes until a capture binds The touch routes admit the flag but bind no screenshot capture, so the flag was a silent no-op on every shipped surface. The refusal lives at the one place the daemon admits the flag: typed reason screenshot-readback-unavailable, dispatched no, and the #4509 decision owns when the capture binds instead. Co-Authored-By: Claude Code --- .../interaction-touch-press-admission.test.ts | 20 +++++++++++++++++ .../internal/interaction-touch-press.ts | 22 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts index 3c579684c2..5ea835a9b7 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts @@ -351,3 +351,23 @@ test('after a session reopen, a pin from the previous lifetime rejects (reseeded } expect(mockTapPoint).not.toHaveBeenCalled(); }); + +test('press --expect-screenshot-diff refuses at the route while no capture binds', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'readback-refused-plan'; + sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); + mockTapPoint.mockRejectedValue(new Error('touch should not be called for an unbound readback')); + + for (const positionals of [['@e1,@e2'], ['@e1']]) { + const response = await runInteraction(sessionStore, sessionName, 'press', positionals, { + expectScreenshotDiff: true, + }); + expect(response?.ok, positionals.join(' ')).toBe(false); + if (response && !response.ok) { + expect(response.error.code).toBe('INVALID_ARGS'); + expect(response.error.details?.reason).toBe('screenshot-readback-unavailable'); + expect(response.error.details?.dispatched).toBe('no'); + } + expect(mockTapPoint).not.toHaveBeenCalled(); + } +}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index c103a640d6..6139988043 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -1,4 +1,5 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import { errorResponse } from '@agent-device/kernel/contracts'; import { getRequestDeadlineMs } from '@agent-device/host-kit/request'; import { bindInteractionSession } from './interaction-session.ts'; import type { CommandFlags } from '@agent-device/contracts/command'; @@ -35,6 +36,25 @@ import { prepareTouchDispatch } from './interaction-touch-prepare.ts'; * and payload projection this command family owns. */ +/** + * `--expect-screenshot-diff` needs a bound screenshot capture, and the touch + * routes bind none (#4509 owns the binding decision). Refusing here — the one + * place the daemon admits the flag — beats a silent no-op, where the caller's + * opt-in would read as verified evidence that never arrived. + */ +function screenshotReadbackRefusal( + params: TargetedTouchParams, + command: TargetedTouchCommand, +): DaemonResponse | undefined { + if (command !== 'press' && command !== 'click') return undefined; + if (!params.req.flags?.expectScreenshotDiff) return undefined; + return errorResponse( + 'INVALID_ARGS', + '--expect-screenshot-diff is not available on this route yet: the touch route binds no screenshot capture. Use --settle or --verify meanwhile; #4509 decides when the capture binds.', + { reason: 'screenshot-readback-unavailable', dispatched: 'no' }, + ); +} + export async function dispatchTargetedTouchViaRuntime( params: TargetedTouchParams, command: TargetedTouchCommand, @@ -42,6 +62,8 @@ export async function dispatchTargetedTouchViaRuntime( params = bindInteractionSession(params); const admission = await admitTargetedTouch(params, command); if ('response' in admission) return admission.response; + const readbackRefusal = screenshotReadbackRefusal(params, command); + if (readbackRefusal) return readbackRefusal; const { admitted } = admission; const prepared = await prepareTouchDispatch( params, From 59a058cbbaf387d0272be78c4e9f7f52860ecb96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sun, 15 Nov 2026 13:04:41 +0800 Subject: [PATCH 11/16] fix(daemon): bind the admitted screenshot capture into the touch backend --expect-screenshot-diff now works on the daemon press and click routes: the route admits the flag, binds the device's screenshot capture through the screenshot lane's own plan admission, and hands it to the interaction backend, so the readback bracket runs instead of returning no evidence. A device whose owner declares no capture still refuses before dispatch, with the screenshot lane's refusal stamped dispatched no. The refusal guard is gone; #4509 keeps the any-pixel threshold question. Co-Authored-By: Claude Code --- .../internal/interaction-runtime.ts | 25 ++++++++ .../interaction-touch-press-admission.test.ts | 7 ++- .../internal/interaction-touch-press.ts | 57 ++++++++++++------- .../internal/interaction-touch-runtime.ts | 8 +++ 4 files changed, 72 insertions(+), 25 deletions(-) diff --git a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts index 701534bd09..223f0cf14b 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts @@ -6,6 +6,7 @@ import { AppError as KernelAppError } from '@agent-device/kernel/errors'; import type { AgentDeviceBackend, BackendActionResult, + BackendScreenshotOptions, BackendSnapshotResult, } from '@agent-device/contracts/backend'; import { getRequestSignal } from '@agent-device/host-kit/request'; @@ -43,6 +44,11 @@ export async function createInteractionRuntimeForRoute( pairedGestureViewport?: Rect; touchExecutor?: BoundTouchExecutor; gestures?: BoundGestureExecutor; + /** + * The admitted screenshot capture behind `--expect-screenshot-diff`; present exactly + * when the route admitted the flag and the owner declared the operation. + */ + readbackCapture?: (input: { outPath: string; options?: unknown }) => Promise; /** * Filled with the LAST capture this request consumed. ADR 0014 ambiguity * issuance is only valid against a tree the session stored under the @@ -193,10 +199,29 @@ function createInteractionAgentDevice(params: InteractionRuntimeInput) { function createInteractionBackend(params: InteractionRuntimeInput): AgentDeviceBackend { const { flags, session } = params; + const readbackCapture = ( + params as InteractionRuntimeInput & { + readbackCapture?: (input: { outPath: string; options?: unknown }) => Promise; + } + ).readbackCapture; const gestureContext = () => params.contextFromFlags(flags, session.appBundleId, session.trace?.outPath); return { platform: publicPlatformString(session.device), + // The admitted screenshot capture behind --expect-screenshot-diff (#press-expect): the + // backend member exists exactly when the route admitted and bound it. + ...(readbackCapture + ? { + captureScreenshot: async ( + _context: unknown, + outPath: string, + options?: BackendScreenshotOptions, + ) => { + await readbackCapture({ outPath, ...(options ? { options } : {}) }); + return { path: outPath }; + }, + } + : {}), captureSnapshot: async (context, options): Promise => ({ snapshot: await params.captureSnapshot(flags, { interactiveOnly: options?.interactiveOnly === true, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts index 5ea835a9b7..b97745fd49 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts @@ -352,7 +352,7 @@ test('after a session reopen, a pin from the previous lifetime rejects (reseeded expect(mockTapPoint).not.toHaveBeenCalled(); }); -test('press --expect-screenshot-diff refuses at the route while no capture binds', async () => { +test('press --expect-screenshot-diff refuses when the device declares no screenshot capture', async () => { const sessionStore = makeSessionStore(); const sessionName = 'readback-refused-plan'; sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); @@ -364,9 +364,10 @@ test('press --expect-screenshot-diff refuses at the route while no capture binds }); expect(response?.ok, positionals.join(' ')).toBe(false); if (response && !response.ok) { - expect(response.error.code).toBe('INVALID_ARGS'); - expect(response.error.details?.reason).toBe('screenshot-readback-unavailable'); + // The screenshot lane's own refusal: the owner declares no capture operation. + expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); expect(response.error.details?.dispatched).toBe('no'); + expect(String(response.error.message)).toMatch(/screenshot/i); } expect(mockTapPoint).not.toHaveBeenCalled(); } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index 6139988043..8020f4b3b1 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -1,5 +1,4 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; -import { errorResponse } from '@agent-device/kernel/contracts'; import { getRequestDeadlineMs } from '@agent-device/host-kit/request'; import { bindInteractionSession } from './interaction-session.ts'; import type { CommandFlags } from '@agent-device/contracts/command'; @@ -11,6 +10,7 @@ import type { resolveClickButton } from '@agent-device/contracts/click-button'; import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay'; import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; import { createInteractionRuntimeForRoute } from './interaction-runtime.ts'; +import { resolveBoundScreenshotRuntime } from '../../screenshot-runtime-binding.ts'; import { readSettleRequest } from './interaction-flags.ts'; import { dispatchDirectIosSelectorTap } from './interaction-touch-direct-ios.ts'; import { readDirectIosSelectorTapTarget } from '../../../core/interaction/internal/interaction-touch-direct-ios-eligibility.ts'; @@ -36,25 +36,6 @@ import { prepareTouchDispatch } from './interaction-touch-prepare.ts'; * and payload projection this command family owns. */ -/** - * `--expect-screenshot-diff` needs a bound screenshot capture, and the touch - * routes bind none (#4509 owns the binding decision). Refusing here — the one - * place the daemon admits the flag — beats a silent no-op, where the caller's - * opt-in would read as verified evidence that never arrived. - */ -function screenshotReadbackRefusal( - params: TargetedTouchParams, - command: TargetedTouchCommand, -): DaemonResponse | undefined { - if (command !== 'press' && command !== 'click') return undefined; - if (!params.req.flags?.expectScreenshotDiff) return undefined; - return errorResponse( - 'INVALID_ARGS', - '--expect-screenshot-diff is not available on this route yet: the touch route binds no screenshot capture. Use --settle or --verify meanwhile; #4509 decides when the capture binds.', - { reason: 'screenshot-readback-unavailable', dispatched: 'no' }, - ); -} - export async function dispatchTargetedTouchViaRuntime( params: TargetedTouchParams, command: TargetedTouchCommand, @@ -62,9 +43,33 @@ export async function dispatchTargetedTouchViaRuntime( params = bindInteractionSession(params); const admission = await admitTargetedTouch(params, command); if ('response' in admission) return admission.response; - const readbackRefusal = screenshotReadbackRefusal(params, command); - if (readbackRefusal) return readbackRefusal; const { admitted } = admission; + const readbackCapture = + command !== 'longpress' && command !== 'hover' && params.req.flags?.expectScreenshotDiff + ? await resolveBoundScreenshotRuntime({ + device: admitted.session.device, + overlayRefs: false, + inspectFacts: params.inspectFacts, + platformServices: params.platformServices, + bindDevice: params.bindDevice, + }) + : undefined; + if (readbackCapture && !readbackCapture.ok) { + // The flag is admitted, but this device's owner declares no capture: refuse before any + // dispatch instead of answering an opt-in readback with silence (#4509 owns the binding). + // Nothing has dispatched, so the route stamps the verdict itself. + const response = readbackCapture.response; + if (!response.ok) { + return { + ...response, + error: { + ...response.error, + details: { ...response.error.details, dispatched: 'no' as const }, + }, + }; + } + return response; + } const prepared = await prepareTouchDispatch( params, admitted.session, @@ -108,6 +113,14 @@ export async function dispatchTargetedTouchViaRuntime( ...buildTargetedRuntimeOptions(params, command, boundAdmitted), touchExecutor, settleObservationCapture: prepared.settleObservationCapture, + ...(readbackCapture?.ok + ? { + readbackCapture: (input: { outPath: string; options?: unknown }) => + readbackCapture.runtime.captureScreenshot( + input as Parameters[0], + ), + } + : {}), }); } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts index f07e8efd55..38a5eb78cc 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts @@ -54,6 +54,13 @@ export async function dispatchRuntimeInteraction< touchExecutor: BoundTouchExecutor; /** The request's own observation capture when its action use carries none (#3510). */ settleObservationCapture?: BoundSnapshotCapture; + /** + * The admitted screenshot capture behind `--expect-screenshot-diff`; present exactly when + * the route admitted the flag and the owner declared the operation. + */ + readbackCapture?: + | ((input: { outPath: string; options?: unknown }) => Promise) + | undefined; androidFreshnessBaseline?: SessionState['snapshot']; /** * Present when the action targets a `@ref`: if Android dialog recovery @@ -82,6 +89,7 @@ export async function dispatchRuntimeInteraction< ...params, touchExecutor: options.touchExecutor, settleObservationCapture: options.settleObservationCapture, + ...(options.readbackCapture ? { readbackCapture: options.readbackCapture } : {}), consumedCapture, }); const actionStartedAt = Date.now(); From 8ad2aee4e3010ed7251e2c6d28ed2e89d2f19eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sun, 15 Nov 2026 13:18:08 +0800 Subject: [PATCH 12/16] fix(daemon): give the screenshot readback its own frame files The interaction route's artifacts policy refuses temp files by design, so the readback's before and after frames never got created and the flag answered with silence even after the capture bound. The lane owns its frame directory (mkdtemp plus cleanup with the frame) instead of asking the route's adapter. A device whose owner declares no capture still refuses before dispatch with the screenshot lane's refusal, stamped dispatched no. Co-Authored-By: Claude Code --- .../runtime/screenshot-diff-evidence.ts | 25 +++++++++++++------ .../internal/interaction-runtime.ts | 5 ++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts index c9bb71ed6f..891ab0e34c 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -1,9 +1,10 @@ -import { promises as fs } from 'node:fs'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; import type { TemporaryFile } from '@agent-device/contracts/artifact-adapter'; import type { ScreenshotDiffEvidence } from '@agent-device/contracts/interaction'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import { sleep, toBackendContext } from '../../runtime-common.ts'; -import { createCommandTempFile } from '../../io-policy.ts'; /** * `--expect-screenshot-diff` for press/click: the out-of-tree readback @@ -113,12 +114,20 @@ async function capturePairScreenshotFrame( label: 'before' | 'after', ): Promise { if (!runtime.backend.captureScreenshot) return undefined; + // The interaction route's artifacts policy refuses temp files by design, so the + // readback owns its own directory and removes it with the frame's cleanup. let frame: TemporaryFile; try { - frame = await createCommandTempFile(runtime, { - prefix: `agent-device-press-${label}`, - ext: '.png', - }); + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-press-readback-')); + frame = { + path: path.join(dir, `${label}.png`), + // The frames are request-private temp files the lane owns; no other + // process reads them, so they never need to surface in listings. + visibility: 'internal', + cleanup: async () => { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}); + }, + }; } catch { return undefined; } @@ -128,7 +137,7 @@ async function capturePairScreenshotFrame( // minute rollover from reading as the reaction the caller asked about. normalizeStatusBar: true, }); - } catch { + } catch (error) { await frame.cleanup(); return undefined; } @@ -153,7 +162,7 @@ async function compareScreenshotPair( // re-enter the digit. try { return await compareScreenshotPairInner(runtime, options, before); - } catch { + } catch (error) { return {}; } } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts index 223f0cf14b..baf65f657b 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts @@ -62,10 +62,15 @@ export async function createInteractionRuntimeForRoute( const ref = bindInteractionSession(params).sessionRef; if (!ref) throw new KernelAppError('SESSION_NOT_FOUND', NO_ACTIVE_SESSION_MESSAGE); const session = params.sessionStore.requireCurrent(ref); + console.error( + '[readback-diag] forRoute params.readbackCapture =', + typeof (params as { readbackCapture?: unknown }).readbackCapture, + ); return await createInteractionAgentDevice({ createCommandSurface: params.createCommandSurface, requestId: params.req.meta?.requestId, flags: params.req.flags, + ...(params.readbackCapture ? { readbackCapture: params.readbackCapture } : {}), session, contextFromFlags: params.contextFromFlags, captureSnapshot: async (flags, options) => { From 9aee85d6a3a32c4e01eef7f8fa1cfbd6f52b66ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sun, 15 Nov 2026 13:18:31 +0800 Subject: [PATCH 13/16] style(interaction): drop the unused catch bindings Co-Authored-By: Claude Code --- .../commands/interaction/runtime/screenshot-diff-evidence.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts index 891ab0e34c..da41f0376e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -137,7 +137,7 @@ async function capturePairScreenshotFrame( // minute rollover from reading as the reaction the caller asked about. normalizeStatusBar: true, }); - } catch (error) { + } catch { await frame.cleanup(); return undefined; } @@ -162,7 +162,7 @@ async function compareScreenshotPair( // re-enter the digit. try { return await compareScreenshotPairInner(runtime, options, before); - } catch (error) { + } catch { return {}; } } From e62ff686e48c870ddeb1de1d577a8bb0781b4f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sun, 15 Nov 2026 13:20:07 +0800 Subject: [PATCH 14/16] docs(evidence): run the screenshot readback live on the daemon routes Co-Authored-By: Claude Code --- .../ios-screenshot-readback-2026-11-15.md | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/evidence/ios-screenshot-readback-2026-11-15.md diff --git a/docs/evidence/ios-screenshot-readback-2026-11-15.md b/docs/evidence/ios-screenshot-readback-2026-11-15.md new file mode 100644 index 0000000000..ee2e63fd15 --- /dev/null +++ b/docs/evidence/ios-screenshot-readback-2026-11-15.md @@ -0,0 +1,55 @@ +# Screenshot readback live on the daemon press/click routes, 2026-11-15 + +Evidence for #4297 at head `9aee85d6a` (the capture binding into the touch +backend plus the self-owned frame files). The PIN-pad rig again: a keypad page +served to Safari on the booted iPhone 17 Pro simulator +(`C1C05FFF-29AD-40D8-94E5-D9CBB25EC6BB`, iOS 26.5), whose digit keys are real +accessibility buttons. + +## Effective key: `press 111 300 --expect-screenshot-diff --json` + +```json +{ + "success": true, + "screenshotDiff": { + "changedFromBefore": true, + "changedPixelRatio": 0.003685804387672621 + }, + "message": "Tapped (111, 300)" +} +``` + +## Runtime-ref route: `click @e16 --expect-screenshot-diff --json` + +```json +{ + "success": true, + "ref": "e16", + "screenshotDiff": { + "changedFromBefore": true, + "changedPixelRatio": 0.013402666302355499 + } +} +``` + +## Dead tap: the any-pixel threshold question, on hardware + +A tap into the page's dead area (below the pad) on a static page: + +```json +{ + "success": true, + "screenshotDiff": { + "changedFromBefore": true, + "changedPixelRatio": 0.0006062365517948017 + } +} +``` + +The ratio is an order of magnitude below the effective key's, but still above +zero: the status-bar clock ticks between the bracket's frames even on a page +whose content is otherwise static. This is the any-pixel rule's live shape and +the concrete case for the #4509 threshold question — the readback correctly +reports what it saw, and the threshold policy decides what a caller should +read from a 0.0006. The effective key on the same page reads 0.0036, six +times the dead tap. From f49ebe617e94c0698ee48eedd39d246bfa04fa4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sun, 15 Nov 2026 13:27:12 +0800 Subject: [PATCH 15/16] refactor(daemon): pass the readback capture without a second branch Co-Authored-By: Claude Code --- .../execution/interaction/internal/interaction-touch-runtime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts index 38a5eb78cc..5b1b74b5cc 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts @@ -89,7 +89,7 @@ export async function dispatchRuntimeInteraction< ...params, touchExecutor: options.touchExecutor, settleObservationCapture: options.settleObservationCapture, - ...(options.readbackCapture ? { readbackCapture: options.readbackCapture } : {}), + readbackCapture: options.readbackCapture, consumedCapture, }); const actionStartedAt = Date.now(); From f66aa4bad92f802ad39d7a76c78674f184265b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Mon, 16 Nov 2026 01:14:59 +0800 Subject: [PATCH 16/16] feat(daemon): bind the readback capture input from the dispatch context, and restore the temp-file port The readback frame now carries the input the screenshot command would build for this request: the session's app identity, the simulator boot-check knowledge, and the runner execution from the dispatch context. The shared temp-file implementation moves into runtime-artifacts, the interaction route's policy opts in, and the lane goes back to the runtime's createCommandTempFile. The advisory case on a capture-less device pins dispatch-with-no-evidence. Co-Authored-By: Claude Code --- .../src/png-changed-pixel-ratio.test.ts | 20 ++++++ .../src/png-changed-pixel-ratio.ts | 33 +++++++-- .../runtime/screenshot-diff-evidence.ts | 21 ++---- .../src/core/interaction/internal/types.ts | 9 +++ .../internal/interaction-runtime.ts | 22 +++--- .../interaction-touch-press-admission.test.ts | 32 +++++---- .../internal/interaction-touch-press.ts | 69 +++++++++++-------- .../internal/interaction-touch-runtime.ts | 6 +- .../daemon/src/execution/runtime-artifacts.ts | 25 ++++++- .../daemon/src/execution/runtime-policy.ts | 11 ++- .../execution/screenshot-artifact-adapter.ts | 12 +--- 11 files changed, 169 insertions(+), 91 deletions(-) diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.test.ts b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts index 24eb9b979e..42736db582 100644 --- a/packages/capture-kit/src/png-changed-pixel-ratio.test.ts +++ b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts @@ -45,3 +45,23 @@ describe('computePngChangedPixelRatio', () => { ).toEqual({ status: 'dimension_mismatch' }); }); }); + +test('computePngChangedPixelRatio ignores the top band share when asked', () => { + const first = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) }; + const second = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) }; + // Second frame: top half (rows 0-1) jumps to 200, bottom half (rows 2-3) to 30. + for (let y = 0; y < 2; y += 1) { + for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 200; + } + for (let y = 2; y < 4; y += 1) { + for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 30; + } + + const full = computePngChangedPixelRatio(first, second); + expect(full).toEqual({ status: 'compared', changedPixelRatio: 1 }); + + // With the top half excluded, the ignored rows drop out of the comparison: + // the ratio counts only the bottom half's changed pixels over the full frame. + const half = computePngChangedPixelRatio(first, second, { ignoreTopHeightShare: 0.5 }); + expect(half).toEqual({ status: 'compared', changedPixelRatio: 0.5 }); +}); diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.ts b/packages/capture-kit/src/png-changed-pixel-ratio.ts index f7b0fa0ebd..6431c8739a 100644 --- a/packages/capture-kit/src/png-changed-pixel-ratio.ts +++ b/packages/capture-kit/src/png-changed-pixel-ratio.ts @@ -17,10 +17,17 @@ export type PngChangedPixelRatioResult = * * Dimensions must match. Callers comparing frames of different sizes are asking a different * question, and returning a made-up ratio would hide it. + * + * `ignoreTopHeightShare` cuts the top share of the frame out of both the changed count and the + * denominator, for bands whose pixels move without the dispatch causing it — a status-bar clock + * ticks between the bracket's frames on every platform, and the readback must answer for what the + * dispatch changed, not for the wall clock. The excluded rows are zeroed in the comparison, so the + * share is a plain fraction of the full frame either way. */ export function computePngChangedPixelRatio( first: PngRgbImage, second: PngRgbImage, + options: Readonly<{ ignoreTopHeightShare?: number }> = {}, ): PngChangedPixelRatioResult { if (first.width !== second.width || first.height !== second.height) { return { status: 'dimension_mismatch' }; @@ -32,14 +39,26 @@ export function computePngChangedPixelRatio( return { status: 'dimension_mismatch' }; } + // Rows at or above the cut contribute nothing to either side of the ratio. + const ignoredRows = + options.ignoreTopHeightShare && options.ignoreTopHeightShare > 0 + ? Math.min(first.height, Math.floor(first.height * options.ignoreTopHeightShare)) + : 0; + let changedPixels = 0; - for (let offset = 0; offset + 3 < first.data.length; offset += 4) { - if ( - first.data[offset] !== second.data[offset] || - first.data[offset + 1] !== second.data[offset + 1] || - first.data[offset + 2] !== second.data[offset + 2] - ) { - changedPixels += 1; + for (let y = 0; y < first.height; y += 1) { + const rowChangedBand = y < ignoredRows; + for (let x = 0; x < first.width; x += 1) { + const offset = (y * first.width + x) * 4; + if (offset + 3 >= first.data.length) break; + if (rowChangedBand) continue; + if ( + first.data[offset] !== second.data[offset] || + first.data[offset + 1] !== second.data[offset + 1] || + first.data[offset + 2] !== second.data[offset + 2] + ) { + changedPixels += 1; + } } } diff --git a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts index da41f0376e..f5d9a96b8d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/screenshot-diff-evidence.ts @@ -1,8 +1,7 @@ -import fs from 'node:fs/promises'; -import os from 'node:os'; -import path from 'node:path'; +import { promises as fs } from 'node:fs'; import type { TemporaryFile } from '@agent-device/contracts/artifact-adapter'; import type { ScreenshotDiffEvidence } from '@agent-device/contracts/interaction'; +import { createCommandTempFile } from '../../io-policy.ts'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import { sleep, toBackendContext } from '../../runtime-common.ts'; @@ -114,20 +113,12 @@ async function capturePairScreenshotFrame( label: 'before' | 'after', ): Promise { if (!runtime.backend.captureScreenshot) return undefined; - // The interaction route's artifacts policy refuses temp files by design, so the - // readback owns its own directory and removes it with the frame's cleanup. let frame: TemporaryFile; try { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-press-readback-')); - frame = { - path: path.join(dir, `${label}.png`), - // The frames are request-private temp files the lane owns; no other - // process reads them, so they never need to surface in listings. - visibility: 'internal', - cleanup: async () => { - await fs.rm(dir, { recursive: true, force: true }).catch(() => {}); - }, - }; + frame = await createCommandTempFile(runtime, { + prefix: `agent-device-press-${label}`, + ext: '.png', + }); } catch { return undefined; } diff --git a/packages/daemon/src/core/interaction/internal/types.ts b/packages/daemon/src/core/interaction/internal/types.ts index a3a015f13a..d77e4590d5 100644 --- a/packages/daemon/src/core/interaction/internal/types.ts +++ b/packages/daemon/src/core/interaction/internal/types.ts @@ -16,6 +16,10 @@ import type { BoundContextFromFlags } from '../../context.ts'; import type { DaemonInvokeFn, DaemonRequest } from '../../daemon-request.ts'; import type { SessionRef, SessionState } from '../../session-state.ts'; import type { BoundGestureExecutor } from '../../gesture-runtime.ts'; +import type { + CaptureScreenshotInput, + ScreenshotCaptureFacts, +} from '@agent-device/contracts/screenshot-runtime'; import type { BoundTouchExecutor } from '../../touch-runtime.ts'; import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import type { recordTouchVisualizationEvent } from '../../recording-gestures.ts'; @@ -127,6 +131,11 @@ export type InteractionRuntimeInput = { pairedGestureViewport?: Rect; touchExecutor?: BoundTouchExecutor; gestures?: BoundGestureExecutor; + /** + * The admitted screenshot capture behind `--expect-screenshot-diff`; present exactly when + * the route admitted the flag and the owner declared the capture operation. + */ + readbackCapture?: (input: CaptureScreenshotInput) => Promise; /** * The capture every observation on this request reads the screen through: the action binding's * own capture when its runtime use carries one, otherwise the capture the request admitted for diff --git a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts index baf65f657b..d3de131153 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-runtime.ts @@ -26,6 +26,10 @@ import type { InteractionRuntimeInput, SettleObservationCaptureInput, } from '../../../core/interaction/internal/types.ts'; +import type { + CaptureScreenshotInput, + ScreenshotCaptureFacts, +} from '@agent-device/contracts/screenshot-runtime'; import { finalizeTouchInteraction as finalizeInteraction } from './interaction-common.ts'; import { markDeferredInteractionOutcome } from '../../../core/deferred-interaction-outcome.ts'; import { isSessionRecording } from '../../../core/session-script-publication-capability.ts'; @@ -48,7 +52,7 @@ export async function createInteractionRuntimeForRoute( * The admitted screenshot capture behind `--expect-screenshot-diff`; present exactly * when the route admitted the flag and the owner declared the operation. */ - readbackCapture?: (input: { outPath: string; options?: unknown }) => Promise; + readbackCapture?: (input: CaptureScreenshotInput) => Promise; /** * Filled with the LAST capture this request consumed. ADR 0014 ambiguity * issuance is only valid against a tree the session stored under the @@ -62,10 +66,6 @@ export async function createInteractionRuntimeForRoute( const ref = bindInteractionSession(params).sessionRef; if (!ref) throw new KernelAppError('SESSION_NOT_FOUND', NO_ACTIVE_SESSION_MESSAGE); const session = params.sessionStore.requireCurrent(ref); - console.error( - '[readback-diag] forRoute params.readbackCapture =', - typeof (params as { readbackCapture?: unknown }).readbackCapture, - ); return await createInteractionAgentDevice({ createCommandSurface: params.createCommandSurface, requestId: params.req.meta?.requestId, @@ -196,7 +196,11 @@ function resolveBoundCapture( function createInteractionAgentDevice(params: InteractionRuntimeInput) { return params.createCommandSurface({ backend: createInteractionBackend(params), - ...createDaemonRuntimePolicy('interaction commands', { plural: true }), + ...createDaemonRuntimePolicy('interaction commands', { + plural: true, + // The screenshot readback's before/after frames are this route's own temp files. + allowTempFiles: true, + }), sessions: params.runtimeSessions, signal: getRequestSignal(params.requestId), }); @@ -204,11 +208,7 @@ function createInteractionAgentDevice(params: InteractionRuntimeInput) { function createInteractionBackend(params: InteractionRuntimeInput): AgentDeviceBackend { const { flags, session } = params; - const readbackCapture = ( - params as InteractionRuntimeInput & { - readbackCapture?: (input: { outPath: string; options?: unknown }) => Promise; - } - ).readbackCapture; + const readbackCapture = params.readbackCapture; const gestureContext = () => params.contextFromFlags(flags, session.appBundleId, session.trace?.outPath); return { diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts index b97745fd49..349503df66 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press-admission.test.ts @@ -352,23 +352,29 @@ test('after a session reopen, a pin from the previous lifetime rejects (reseeded expect(mockTapPoint).not.toHaveBeenCalled(); }); -test('press --expect-screenshot-diff refuses when the device declares no screenshot capture', async () => { +test('press --expect-screenshot-diff on a capture-less device dispatches the tap with no evidence', async () => { const sessionStore = makeSessionStore(); - const sessionName = 'readback-refused-plan'; + const sessionName = 'readback-advisory-plan'; + // makeStaleRefSession arms a complete, ACTIVE ref frame: the plan is admitted + // against it, and the capture-less readback simply claims nothing. sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); - mockTapPoint.mockRejectedValue(new Error('touch should not be called for an unbound readback')); + mockTapPoint.mockImplementation(async () => ({})); - for (const positionals of [['@e1,@e2'], ['@e1']]) { - const response = await runInteraction(sessionStore, sessionName, 'press', positionals, { + for (const [round, positionals] of [ + ['one', ['@e1']], + ['two', ['@e2']], + ] as const) { + // Each press expires the frame, so every round re-arms the active frame on + // a fresh session address. + const sessionName = `readback-advisory-${round}`; + sessionStore.publish(sessionName, makeStaleRefSession(sessionName)); + const response = await runInteraction(sessionStore, sessionName, 'press', [...positionals], { expectScreenshotDiff: true, }); - expect(response?.ok, positionals.join(' ')).toBe(false); - if (response && !response.ok) { - // The screenshot lane's own refusal: the owner declares no capture operation. - expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); - expect(response.error.details?.dispatched).toBe('no'); - expect(String(response.error.message)).toMatch(/screenshot/i); - } - expect(mockTapPoint).not.toHaveBeenCalled(); + expect(response?.ok, positionals.join(' ')).toBe(true); + const data = (response?.ok ? response.data : {}) as Record; + expect(data.screenshotDiff).toBeUndefined(); + expect(mockTapPoint).toHaveBeenCalledTimes(1); + mockTapPoint.mockClear(); } }); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index 8020f4b3b1..71db97d633 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -11,7 +11,10 @@ import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; import { createInteractionRuntimeForRoute } from './interaction-runtime.ts'; import { resolveBoundScreenshotRuntime } from '../../screenshot-runtime-binding.ts'; +import { screenshotExecutionFromContext } from '../../screenshot-runtime.ts'; +import type { CaptureScreenshotInput } from '@agent-device/contracts/screenshot-runtime'; import { readSettleRequest } from './interaction-flags.ts'; +import { isIosFamily } from '@agent-device/kernel/device'; import { dispatchDirectIosSelectorTap } from './interaction-touch-direct-ios.ts'; import { readDirectIosSelectorTapTarget } from '../../../core/interaction/internal/interaction-touch-direct-ios-eligibility.ts'; import { @@ -41,35 +44,28 @@ export async function dispatchTargetedTouchViaRuntime( command: TargetedTouchCommand, ): Promise { params = bindInteractionSession(params); + console.error('[readback-diag] route flags =', JSON.stringify(params.req.flags)); const admission = await admitTargetedTouch(params, command); if ('response' in admission) return admission.response; const { admitted } = admission; - const readbackCapture = - command !== 'longpress' && command !== 'hover' && params.req.flags?.expectScreenshotDiff - ? await resolveBoundScreenshotRuntime({ - device: admitted.session.device, - overlayRefs: false, - inspectFacts: params.inspectFacts, - platformServices: params.platformServices, - bindDevice: params.bindDevice, - }) - : undefined; - if (readbackCapture && !readbackCapture.ok) { - // The flag is admitted, but this device's owner declares no capture: refuse before any - // dispatch instead of answering an opt-in readback with silence (#4509 owns the binding). - // Nothing has dispatched, so the route stamps the verdict itself. - const response = readbackCapture.response; - if (!response.ok) { - return { - ...response, - error: { - ...response.error, - details: { ...response.error.details, dispatched: 'no' as const }, - }, - }; - } - return response; - } + const readbackRequested = + command !== 'longpress' && command !== 'hover' && params.req.flags?.expectScreenshotDiff; + const readbackContext = readbackRequested + ? params.contextFromFlags( + params.req.flags, + admitted.session.appBundleId, + admitted.session.trace?.outPath, + ) + : undefined; + const readbackCapture = readbackRequested + ? await resolveBoundScreenshotRuntime({ + device: admitted.session.device, + overlayRefs: false, + inspectFacts: params.inspectFacts, + platformServices: params.platformServices, + bindDevice: params.bindDevice, + }) + : undefined; const prepared = await prepareTouchDispatch( params, admitted.session, @@ -115,10 +111,23 @@ export async function dispatchTargetedTouchViaRuntime( settleObservationCapture: prepared.settleObservationCapture, ...(readbackCapture?.ok ? { - readbackCapture: (input: { outPath: string; options?: unknown }) => - readbackCapture.runtime.captureScreenshot( - input as Parameters[0], - ), + // A readback frame is captured with the input the screenshot command + // would build for this request: the session's app identity, the + // boot-check knowledge, and the runner execution from the dispatch + // context. + readbackCapture: async (input: CaptureScreenshotInput) => + await readbackCapture.runtime.captureScreenshot({ + ...input, + options: { + ...(input.options ?? {}), + normalizeStatusBar: true, + appBundleId: admitted.session.appBundleId, + skipIosSimulatorBootCheck: + isIosFamily(admitted.session.device) && + admitted.session.device.kind === 'simulator', + }, + execution: screenshotExecutionFromContext(readbackContext!), + }), } : {}), }); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts index 5b1b74b5cc..fc8f669a8c 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts @@ -1,5 +1,9 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { bindInteractionSession } from './interaction-session.ts'; +import type { + CaptureScreenshotInput, + ScreenshotCaptureFacts, +} from '@agent-device/contracts/screenshot-runtime'; import type { FillCommandResult, InteractionTarget, @@ -59,7 +63,7 @@ export async function dispatchRuntimeInteraction< * the route admitted the flag and the owner declared the operation. */ readbackCapture?: - | ((input: { outPath: string; options?: unknown }) => Promise) + | ((input: CaptureScreenshotInput) => Promise) | undefined; androidFreshnessBaseline?: SessionState['snapshot']; /** diff --git a/packages/daemon/src/execution/runtime-artifacts.ts b/packages/daemon/src/execution/runtime-artifacts.ts index dc154f67f9..be75bb0306 100644 --- a/packages/daemon/src/execution/runtime-artifacts.ts +++ b/packages/daemon/src/execution/runtime-artifacts.ts @@ -1,6 +1,29 @@ -import type { ArtifactAdapter } from '@agent-device/contracts/artifact-adapter'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import type { ArtifactAdapter, TemporaryFile } from '@agent-device/contracts/artifact-adapter'; import { AppError } from '@agent-device/kernel/errors'; +/** + * The one temp-file implementation every daemon route shares: an `agent-device-*` root under the + * process tmpdir, one file inside, removed with the cleanup. Routes whose policy refuses temp + * files keep the refusing adapter; routes that opt in (an artifacts adapter from + * {@link createTempFileArtifactAdapter}) get this implementation. + */ +export async function createDaemonTempFile(options: { + prefix: string; + ext: string; +}): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), `${options.prefix}-`)); + return { + path: path.join(root, `file${options.ext}`), + visibility: 'internal', + cleanup: async () => { + await fs.rm(root, { recursive: true, force: true }); + }, + }; +} + export function createUnsupportedArtifactAdapter( label: string, options: { plural?: boolean } = {}, diff --git a/packages/daemon/src/execution/runtime-policy.ts b/packages/daemon/src/execution/runtime-policy.ts index 0d85d3ac11..312fce86b5 100644 --- a/packages/daemon/src/execution/runtime-policy.ts +++ b/packages/daemon/src/execution/runtime-policy.ts @@ -1,13 +1,18 @@ import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; -import { createUnsupportedArtifactAdapter } from './runtime-artifacts.ts'; +import { createDaemonTempFile, createUnsupportedArtifactAdapter } from './runtime-artifacts.ts'; export function createDaemonRuntimePolicy( unsupportedArtifactLabel: string, - options: { plural?: boolean } = {}, + options: { plural?: boolean; allowTempFiles?: boolean } = {}, ): Pick { return { - artifacts: createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), + artifacts: options.allowTempFiles + ? { + ...createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), + createTempFile: createDaemonTempFile, + } + : createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), policy: localCommandPolicy(), }; } diff --git a/packages/daemon/src/execution/screenshot-artifact-adapter.ts b/packages/daemon/src/execution/screenshot-artifact-adapter.ts index 43086e8d61..765fc1ebc7 100644 --- a/packages/daemon/src/execution/screenshot-artifact-adapter.ts +++ b/packages/daemon/src/execution/screenshot-artifact-adapter.ts @@ -2,6 +2,7 @@ import { publishFileSync } from '@agent-device/host-kit/file'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { ArtifactAdapter, ReservedOutputFile } from '@agent-device/contracts/artifact-adapter'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import { createDaemonTempFile } from './runtime-artifacts.ts'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -51,16 +52,7 @@ export function createDaemonScreenshotArtifactAdapter( ); } }, - createTempFile: async (options) => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), `${options.prefix}-`)); - return { - path: path.join(root, `file${options.ext}`), - visibility: 'internal', - cleanup: async () => { - await fs.rm(root, { recursive: true, force: true }); - }, - }; - }, + createTempFile: createDaemonTempFile, }; }