diff --git a/docs/evidence/ios-burst-capture-2026-11-13.md b/docs/evidence/ios-burst-capture-2026-11-13.md new file mode 100644 index 0000000000..1aff60dcbe --- /dev/null +++ b/docs/evidence/ios-burst-capture-2026-11-13.md @@ -0,0 +1,64 @@ +# Screenshot burst on an iOS simulator, 2026-11-13 + +Live evidence for #4291 at head `7f3741906` (the format-only commit after the +burst fixes; the behavior this documents is unchanged since): the burst path +(`--count`/`--interval-ms`) captured five frames of an animation on real +simulator dispatch. The gaps between the recorded `capturedAtMs` values run +724-909 ms — above the plan's 500 ms per-frame budget, because each frame also +pays the capture round trip the budget's doc comment excludes; the budget +bounds the admission wait per frame, not the full round trip. The screen was a page +served to Safari on the booted iPhone 17 Pro simulator +(`C1C05FFF-29AD-40D8-94E5-D9CBB25EC6BB`, iOS 26.5) whose content updates every +50 ms (a millisecond clock and a progress bar), so consecutive frames cannot +hash the same. + +## The run + +`agent-device screenshot /tmp/ad-burst/frames/page.png --count 5 --interval-ms +200 --platform ios --json` at the pushed head: + +```json +{ + "frames": [ + { "path": "/tmp/ad-burst/frames/page-0001.png", "capturedAtMs": 1794583109757 }, + { "path": "/tmp/ad-burst/frames/page-0002.png", "capturedAtMs": 1794583110561 }, + { "path": "/tmp/ad-burst/frames/page-0003.png", "capturedAtMs": 1794583111470 }, + { "path": "/tmp/ad-burst/frames/page-0004.png", "capturedAtMs": 1794583112194 }, + { "path": "/tmp/ad-burst/frames/page-0005.png", "capturedAtMs": 1794583112995 } + ] +} +``` + +`capturedAtMs` strictly increases. The five files on disk: + +``` +-rw-r--r-- 20251 page-0001.png +-rw-r--r-- 20457 page-0002.png +-rw-r--r-- 19326 page-0003.png +-rw-r--r-- 20858 page-0004.png +-rw-r--r-- 20986 page-0005.png +``` + +Five distinct md5 hashes — every frame captured a different render of the +animation, and all five landed on disk beside the `--out` path: + +``` +4a943cceb3b7468c40bc438b110980d4 page-0001.png +7b9451fa6875aa2a1147ddeec78aee46 page-0002.png +bda33d6c1e3ec2eb9b49c77dd690d265 page-0003.png +df7a320a07e242ff281abea1d0358ec1 page-0004.png +eb77c0c4882b8d2093c70a23b65e2cda page-0005.png +``` + +The command's `--json` envelope (abridged to the identifying fields): + +```json +{ + "success": true, + "data": { + "path": "/tmp/ad-burst/frames/page-0001.png", + "frames": [ "... five frames as shown above ..." ], + "message": "Saved 5 screenshots starting at /tmp/ad-burst/frames/page-0001.png" + } +} +``` diff --git a/packages/client/src/__tests__/screenshot-result.test.ts b/packages/client/src/__tests__/screenshot-result.test.ts index ad352cebc6..03b5fad9ec 100644 --- a/packages/client/src/__tests__/screenshot-result.test.ts +++ b/packages/client/src/__tests__/screenshot-result.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { SCREENSHOT_BURST_LIMITS } from '@agent-device/contracts/capture'; import { normalizeScreenshotCaptureResult, pickScreenshotResultData, @@ -57,3 +58,71 @@ test('normalizeScreenshotCaptureResult drops a display rotation outside the rota ); assert.ok(!('displayRotation' in result)); }); + +// The default daemon payload reaches this normalizer on the SDK's default path and, through +// `pickScreenshotResultData`, on the CLI's default `--json` path. A burst whose frames the +// normalizer dropped would break the result's "reports every frame" promise at both boundaries. +test('normalizeScreenshotCaptureResult carries the burst frames the daemon reported', () => { + const result = normalizeScreenshotCaptureResult( + { + path: '/tmp/burst/page-0001.png', + width: 100, + height: 50, + frames: [ + { path: '/tmp/burst/page-0001.png', capturedAtMs: 1_000, displayRotation: 'portrait' }, + { path: '/tmp/burst/page-0002.png', capturedAtMs: 1_400 }, + ], + }, + 'qa', + ); + assert.deepEqual(result.frames, [ + { path: '/tmp/burst/page-0001.png', capturedAtMs: 1_000, displayRotation: 'portrait' }, + { path: '/tmp/burst/page-0002.png', capturedAtMs: 1_400 }, + ]); + assert.equal(result.path, '/tmp/burst/page-0001.png'); + assert.deepEqual(result.identifiers, { session: 'qa' }); +}); + +test('normalizeScreenshotCaptureResult keeps only frames with a path and a wall clock, within the burst bound', () => { + const frames: unknown[] = [ + { path: '/tmp/burst/f-0.png', capturedAtMs: 0 }, + // Malformed INSIDE the bound: a missing wall clock filters out and the + // frames behind it slide forward to fill the slot. + { path: '/tmp/burst/f-1.png', capturedAtMs: Number.NaN }, + ...Array.from({ length: SCREENSHOT_BURST_LIMITS.maxCount }, (_, index) => ({ + path: `/tmp/burst/extra-${index}.png`, + capturedAtMs: 100 + index, + })), + { capturedAtMs: 9_999 }, + { path: '/tmp/burst/extra.png' }, + 'frame', + ]; + const result = normalizeScreenshotCaptureResult({ path: '/tmp/burst/f-0.png', frames }, 'qa'); + // The truncation cuts the raw list at the bound first; the malformed entry + // inside it then filters out, so the kept total is one below the bound. + assert.equal(result.frames?.length, SCREENSHOT_BURST_LIMITS.maxCount - 1); + assert.deepEqual(result.frames?.[0], { path: '/tmp/burst/f-0.png', capturedAtMs: 0 }); + // The malformed inside-the-bound frame is filtered, and the frame behind it + // slid forward into the slot with its own path and a finite wall clock. + assert.equal( + result.frames?.some((frame) => frame.path === '/tmp/burst/f-1.png'), + false, + ); + assert.equal(result.frames?.[1]?.path, '/tmp/burst/extra-0.png'); + assert.equal(Number.isFinite(result.frames?.[1]?.capturedAtMs), true); +}); + +test('normalizeScreenshotCaptureResult omits empty and absent frames', () => { + assert.ok(!('frames' in normalizeScreenshotCaptureResult({ path: '/tmp/a.png' }, 'qa'))); + assert.ok( + !('frames' in normalizeScreenshotCaptureResult({ path: '/tmp/a.png', frames: [] }, 'qa')), + ); +}); + +test('pickScreenshotResultData keeps the frames a normalized result carried', () => { + const frames = [{ path: '/tmp/a-0001.png', capturedAtMs: 1 }]; + assert.deepEqual(pickScreenshotResultData({ path: '/tmp/a-0001.png', frames }), { + path: '/tmp/a-0001.png', + frames, + }); +}); diff --git a/packages/client/src/screenshot-result.ts b/packages/client/src/screenshot-result.ts index 210c685066..db22d4c60a 100644 --- a/packages/client/src/screenshot-result.ts +++ b/packages/client/src/screenshot-result.ts @@ -1,4 +1,8 @@ -import type { ScreenshotResultData } from '@agent-device/contracts/capture'; +import { + SCREENSHOT_BURST_LIMITS, + type ScreenshotBurstFrame, + type ScreenshotResultData, +} from '@agent-device/contracts/capture'; import type { CaptureScreenshotResult } from '@agent-device/contracts/client'; import { isDeviceRotation } from '@agent-device/contracts/device'; import { isRecord, parsePoint, parseRect, readRequiredString } from '@agent-device/kernel/record'; @@ -6,6 +10,7 @@ import { readResponseWarnings } from '@agent-device/kernel/success-text'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; export function pickScreenshotResultData(value: ScreenshotResultData): ScreenshotResultData { + const frames = readScreenshotBurstFrames(value.frames); return { ...(typeof value.path === 'string' ? { path: value.path } : {}), ...(typeof value.width === 'number' ? { width: value.width } : {}), @@ -14,6 +19,7 @@ export function pickScreenshotResultData(value: ScreenshotResultData): Screensho ...(typeof value.logicalHeight === 'number' ? { logicalHeight: value.logicalHeight } : {}), ...(typeof value.pixelDensity === 'number' ? { pixelDensity: value.pixelDensity } : {}), ...(isDeviceRotation(value.displayRotation) ? { displayRotation: value.displayRotation } : {}), + ...(frames ? { frames } : {}), ...(value.overlayRefs ? { overlayRefs: value.overlayRefs } : {}), ...(value.warnings && value.warnings.length > 0 ? { warnings: value.warnings } : {}), }; @@ -33,6 +39,7 @@ export function normalizeScreenshotCaptureResult( logicalHeight: screenshot?.logicalHeight, pixelDensity: screenshot?.pixelDensity, ...(screenshot?.displayRotation ? { displayRotation: screenshot.displayRotation } : {}), + ...(screenshot?.frames ? { frames: screenshot.frames } : {}), overlayRefs: screenshot?.overlayRefs, ...(screenshot?.warnings ? { warnings: screenshot.warnings } : {}), identifiers: { session }, @@ -58,11 +65,37 @@ function readScreenshotResultData(value: unknown): ScreenshotResultData | undefi logicalHeight: readNumberField(value, 'logicalHeight'), pixelDensity: readNumberField(value, 'pixelDensity'), displayRotation: isDeviceRotation(value.displayRotation) ? value.displayRotation : undefined, + frames: readScreenshotBurstFrames(value.frames), overlayRefs: readScreenshotOverlayRefs(value.overlayRefs), ...(warnings !== undefined ? { warnings } : {}), }); } +/** + * The burst frames a payload may report: one plan admits at most `maxCount` frames, so a longer + * array is truncated, and a frame without a path and a wall clock is not evidence of a capture. + */ +function readScreenshotBurstFrames(value: unknown): ScreenshotBurstFrame[] | undefined { + if (!Array.isArray(value)) return undefined; + const frames = value + .slice(0, SCREENSHOT_BURST_LIMITS.maxCount) + .flatMap((frame) => readScreenshotBurstFrame(frame) ?? []); + return frames.length > 0 ? frames : undefined; +} + +function readScreenshotBurstFrame(frame: unknown): ScreenshotBurstFrame | undefined { + if (!isRecord(frame)) return undefined; + if (typeof frame.path !== 'string' || frame.path.length === 0) return undefined; + if (typeof frame.capturedAtMs !== 'number' || !Number.isFinite(frame.capturedAtMs)) { + return undefined; + } + return { + path: frame.path, + capturedAtMs: frame.capturedAtMs, + ...(isDeviceRotation(frame.displayRotation) ? { displayRotation: frame.displayRotation } : {}), + }; +} + function readNumberField(value: Record, key: string): number | undefined { const field = value[key]; return typeof field === 'number' ? field : undefined; diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index e03b62befb..85a731c613 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -21,7 +21,8 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 1, max: 200, usageLabel: '--count ', - usageDescription: 'Repeat count for press, swipe and gesture tap series', + usageDescription: + 'Repeat count for press, swipe and gesture tap series; screenshot: burst frame count (2-100) captured sequentially into a numbered file sequence', projectConfig: true, recorded: true, }, @@ -85,7 +86,8 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 0, max: 10_000, usageLabel: '--interval-ms ', - usageDescription: 'Delay between press iterations', + usageDescription: + 'Delay between press iterations; screenshot: wait between burst frames (100-10000, default 100)', projectConfig: true, recorded: true, }, diff --git a/packages/command-surface/src/capture/screenshot-options.test.ts b/packages/command-surface/src/capture/screenshot-options.test.ts index f07a6a60b6..38cbf0280a 100644 --- a/packages/command-surface/src/capture/screenshot-options.test.ts +++ b/packages/command-surface/src/capture/screenshot-options.test.ts @@ -6,10 +6,12 @@ import { SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS, readScreenshotScriptFlag, + screenshotBurstFramePath, screenshotFlagsFromOptions, screenshotFlagsFromPublicOptions, screenshotOptionsFromFlags, validateNoRetiredScreenshotMaxSize, + validateScreenshotBurst, validateScreenshotScale, type ScreenshotRequestFlags, } from '@agent-device/contracts/capture'; @@ -173,6 +175,8 @@ test('screenshot script flags use the shared recorded flag contract', () => { 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', 'screenshotCaptureBackend', + 'count', + 'intervalMs', ]); }); @@ -200,3 +204,98 @@ test('explicit screenshot stabilization survives SDK projection and script recor }); assert.deepEqual(screenshotOptionsFromFlags(recorded), { stabilize: true }); }); + +test('screenshot burst flags ride both projections', () => { + assert.deepEqual(screenshotOptionsFromFlags({ count: 5, intervalMs: 250 }), { + count: 5, + intervalMs: 250, + }); + assert.deepEqual(screenshotFlagsFromOptions({ count: 5, intervalMs: 250 }), { + count: 5, + intervalMs: 250, + }); + assert.deepEqual(screenshotFlagsFromPublicOptions({ count: 3 }), { count: 3 }); +}); + +test('screenshot burst admission bounds the count, the interval, and their pair', () => { + assert.doesNotThrow(() => validateScreenshotBurst({ count: 2 })); + assert.doesNotThrow(() => validateScreenshotBurst({ count: 5, intervalMs: 250 })); + assert.doesNotThrow(() => validateScreenshotBurst({})); + assert.throws(() => validateScreenshotBurst({ intervalMs: 250 }), { + message: /--interval-ms requires --count/, + }); + assert.throws(() => validateScreenshotBurst({ count: 1 }), { + message: /--count requires an integer from 2 to /, + }); + assert.throws(() => validateScreenshotBurst({ count: 101 }), { code: 'INVALID_ARGS' }); + assert.throws(() => validateScreenshotBurst({ count: 2.5 }), { code: 'INVALID_ARGS' }); + assert.throws(() => validateScreenshotBurst({ count: 2, intervalMs: 50 }), { + message: /--interval-ms requires an integer from 100 to /, + }); + assert.throws(() => validateScreenshotBurst({ count: 2, intervalMs: 20_000 }), { + code: 'INVALID_ARGS', + }); +}); + +test('screenshot burst admission refuses crop and overlay combinations and request-budget overruns', () => { + assert.throws(() => validateScreenshotBurst({ count: 3, cropOn: 'role=button label=Save' }), { + message: /--count cannot be combined with --crop-on or --overlay-refs/, + }); + assert.throws(() => validateScreenshotBurst({ count: 3, overlayRefs: true }), { + message: /--count cannot be combined with --crop-on or --overlay-refs/, + }); + assert.doesNotThrow(() => validateScreenshotBurst({ count: 3, cropOn: undefined })); + // The run budget covers the waits plus one capture allowance per frame, so the admitted burst + // stays inside the request envelope even when every frame costs its full allowance. + assert.throws( + () => validateScreenshotBurst({ count: 100, intervalMs: 200 }), + /keep the burst within 60000 ms/, + ); + assert.throws( + () => validateScreenshotBurst({ count: 100, intervalMs: 600 }), + /keep the burst within 60000 ms/, + ); + assert.doesNotThrow(() => validateScreenshotBurst({ count: 100 })); + assert.throws(() => validateScreenshotBurst({ count: 150 }), { + message: /--count requires an integer from 2 to 100/, + }); +}); + +test('screenshot burst frame paths number the user sequence in place', () => { + assert.equal(screenshotBurstFramePath('/tmp/page.png', 1), '/tmp/page-0001.png'); + assert.equal(screenshotBurstFramePath('/tmp/page.png', 12), '/tmp/page-0012.png'); + assert.equal(screenshotBurstFramePath('page.jpeg', 3), 'page-0003.jpeg'); + assert.equal(screenshotBurstFramePath('/tmp/page', 2), '/tmp/page-0002'); + assert.equal(screenshotBurstFramePath('/tmp/my.photos/page', 4), '/tmp/my.photos/page-0004'); + assert.equal(screenshotBurstFramePath('/tmp/.hidden', 5), '/tmp/.hidden-0005'); +}); + +test('screenshot burst flags round-trip through recorded scripts', () => { + const parts: string[] = []; + appendScreenshotScriptFlags(parts, { count: 5, intervalMs: 250 }); + assert.deepEqual(parts, ['--count', '5', '--interval-ms', '250']); + const recorded: Partial = {}; + assert.deepEqual(readScreenshotScriptFlag({ args: parts, index: 0, flags: recorded }), { + handled: true, + nextIndex: 1, + }); + assert.deepEqual(readScreenshotScriptFlag({ args: parts, index: 2, flags: recorded }), { + handled: true, + nextIndex: 3, + }); + assert.deepEqual(screenshotOptionsFromFlags(recorded), { count: 5, intervalMs: 250 }); + assert.throws(() => readScreenshotScriptFlag({ args: ['--count', '1'], index: 0, flags: {} }), { + code: 'INVALID_ARGS', + }); + assert.throws( + () => readScreenshotScriptFlag({ args: ['--interval-ms', '50'], index: 0, flags: {} }), + { code: 'INVALID_ARGS' }, + ); +}); + +test('the scripted pixel-density flag keeps its released error text', () => { + assert.throws( + () => readScreenshotScriptFlag({ args: ['--pixel-density', '0'], index: 0, flags: {} }), + { code: 'INVALID_ARGS', message: 'screenshot --pixel-density requires a positive integer' }, + ); +}); diff --git a/packages/command-surface/src/capture/screenshot.ts b/packages/command-surface/src/capture/screenshot.ts index 7ccd922b52..799ac54f1a 100644 --- a/packages/command-surface/src/capture/screenshot.ts +++ b/packages/command-surface/src/capture/screenshot.ts @@ -1,8 +1,10 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema'; import type { CaptureScreenshotOptions } from '@agent-device/contracts/client'; import { SESSION_SURFACES } from '@agent-device/contracts/session'; import { RETIRED_SCREENSHOT_MAX_SIZE, + SCREENSHOT_BURST_LIMITS, SCREENSHOT_CAPTURE_BACKENDS, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_SCALE_LIMITS, @@ -10,6 +12,7 @@ import { screenshotFlagsFromPublicOptions, screenshotOptionsFromFlags, validateNoRetiredScreenshotMaxSize, + validateScreenshotBurst, validateScreenshotScale, } from '@agent-device/contracts/capture'; import { @@ -55,6 +58,14 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( SCREENSHOT_CAPTURE_BACKENDS, 'Capture through the XCTest runner instead of the platform default, which leaves out the simulator-drawn Dynamic Island. Local Apple targets only; refused on provider sessions and on a macOS surface the helper captures, and with pixelDensity, screenMask or normalizeStatusBar.', ), + count: integerField( + 'Capture a burst of this many frames in one request; the command writes a -0001 style sequence and reports every frame. Cannot be combined with cropOn or overlayRefs.', + { min: SCREENSHOT_BURST_LIMITS.minCount, max: SCREENSHOT_BURST_LIMITS.maxCount }, + ), + intervalMs: integerField( + `Wait this many milliseconds between burst frames; requires count. Defaults to ${SCREENSHOT_BURST_LIMITS.defaultIntervalMs}.`, + { min: SCREENSHOT_BURST_LIMITS.minIntervalMs, max: SCREENSHOT_BURST_LIMITS.maxIntervalMs }, + ), surface: enumField(SESSION_SURFACES), }, ); @@ -62,7 +73,17 @@ const screenshotCommandMetadata = defineFieldCommandMetadata( const screenshotCliSchema = { positionalArgs: ['path?'], allowedFlags: SCREENSHOT_COMMAND_FLAG_KEYS, -} as const; + // A burst arms a multi-file write, so only the invocation that names the flags may start one. + // The count and wait serve press and swipe series too, and a repository `count` set for those + // must not turn a single capture into a burst, nor refuse --crop-on beside it. + explicitOnlyFlags: ['count', 'intervalMs'], + flagDescriptionOverrides: { + count: + 'Burst: capture this many frames of one animation in one request into a numbered file sequence (2-100). The CLI reads it only from this invocation; a config count for press and swipe series never arms a burst', + intervalMs: + 'Burst: wait this many milliseconds between frames (100-10000, default 100). Requires --count, and like --count it is read only from this invocation', + }, +} as const satisfies CommandSchemaOverride; export const screenshotCliReader: CliReader = (positionals, flags) => ({ ...commonInputFromFlags(flags), @@ -73,6 +94,7 @@ export const screenshotCliReader: CliReader = (positionals, flags) => ({ export const screenshotDaemonWriter: DaemonWriter = (input) => { validateNoRetiredScreenshotMaxSize('screenshot', input); validateScreenshotScale(input as CaptureScreenshotOptions); + validateScreenshotBurst(input as CaptureScreenshotOptions); return request(PUBLIC_COMMANDS.screenshot, optionalString(input.path), { ...input, ...screenshotFlagsFromPublicOptions(input as CaptureScreenshotOptions), @@ -84,7 +106,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, as is the small indicator window macOS adds in front of a window that a screen-sharing stream captures, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. Use --screen-mask black to fill the iOS simulator screen outline, including rounded corners, with black for this capture. It requires simctl: physical devices, tvOS and visionOS simulators, other platforms, provider captures and XCTest runner captures are refused. A masked capture fails if simctl fails instead of falling back to an unmasked runner capture. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators, Android emulators, and CoreDevice physical iPhones), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator or Android chrome. Physical iPhone crop verification covers portrait CoreDevice/devicectl captures only. XCTest, runner fallback, and physical iPad crops are refused; landscape iPhone crop is unverified. Android captures immediately by default; --normalize-status-bar also waits for transient UI to fade, and --no-stabilize keeps capture immediate. Use --count with --interval-ms to capture an animation burst in one request. The daemon captures frames sequentially and writes each frame as a numbered file beside the --out path, for example page-0001.png, and the result reports every frame path and capture time. A burst refuses --crop-on and --overlay-refs, because a crop binds one snapshot and overlay refs annotate one frame. Keep --count times --interval-ms plus 500 ms per frame within 60 seconds, because the frames must finish inside the request timeout. When --count is present without --interval-ms, the daemon waits 100ms between frames. A burst starts only when this invocation names --count; a count set for press and swipe in a config file never arms one, and a remote daemon refuses --count with REMOTE_SCREENSHOT_BURST_DENIED, so run a burst against a local daemon. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level (layer-0) window on the current Space: floating palettes and app-modal alerts at window level 8 are skipped, as is the small indicator window macOS adds in front of a window that a screen-sharing stream captures, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent. On iOS simulators a screenshot waits for that simulator's status bar to be free, and a request its caller cancelled while it waits reports the cancellation instead of capturing.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 3a1952b7fd..5783359a53 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -121,6 +121,14 @@ export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { /** `runner`: capture an Apple target through the XCTest runner instead of `simctl`. */ captureBackend?: ScreenshotCaptureBackend; surface?: SessionSurface; + /** + * Capture a burst of this many frames in one request; the daemon writes them as a + * `-0001` style sequence and reports each frame in `frames`. Cannot be combined + * with `cropOn` or `overlayRefs`. See `SCREENSHOT_BURST_LIMITS` for the bounds. + */ + count?: number; + /** Wait this many milliseconds between burst frames; requires `count`. */ + intervalMs?: number; }; export type CaptureScreenshotResult = ScreenshotResultData & { diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index e69e431077..3675fcc3a7 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -2,6 +2,7 @@ export type { DiffSnapshotCommandResult, SnapshotDiffLine, SnapshotDiffSummary } export { RETIRED_SCREENSHOT_MAX_SIZE, SCREENSHOT_ACTION_FLAG_KEYS, + SCREENSHOT_BURST_LIMITS, SCREENSHOT_CAPTURE_BACKENDS, SCREENSHOT_COMMAND_FLAG_KEYS, SCREENSHOT_CROP_REASONS, @@ -13,13 +14,16 @@ export { appendScreenshotScriptFlags, readScreenshotScriptFlag, retiredScreenshotMaxSizeFlagError, + screenshotBurstFramePath, screenshotFlagsFromOptions, screenshotFlagsFromPublicOptions, screenshotOptionsFromFlags, validateNoRetiredScreenshotMaxSize, + validateScreenshotBurst, validateScreenshotScale, } from '../screenshot.ts'; export type { + ScreenshotBurstFrame, ScreenshotCaptureBackend, ScreenshotCropReason, ScreenshotScreenMask, diff --git a/packages/contracts/src/screenshot.ts b/packages/contracts/src/screenshot.ts index 7b06ba3c6b..f2a9023e89 100644 --- a/packages/contracts/src/screenshot.ts +++ b/packages/contracts/src/screenshot.ts @@ -1,4 +1,5 @@ import { AppError } from '@agent-device/kernel/errors'; +import type { DeviceRotation } from './device-rotation.ts'; export const SCREENSHOT_SCREEN_MASKS = ['black'] as const; export type ScreenshotScreenMask = (typeof SCREENSHOT_SCREEN_MASKS)[number]; @@ -52,6 +53,13 @@ export function validateNoRetiredScreenshotMaxSize( } } +/** One frame of a `--count` burst: the written file and the wall clock the capture completed at. */ +export type ScreenshotBurstFrame = Readonly<{ + path: string; + capturedAtMs: number; + displayRotation?: DeviceRotation; +}>; + /** * Machine-readable `screenshot --crop-on` outcome taxonomy. Failures carry these * values in `error.details.reason`; `partialIntersection` doubles as the @@ -93,6 +101,25 @@ export const SCREENSHOT_CAPTURE_BACKENDS = ['runner'] as const; export type ScreenshotCaptureBackend = (typeof SCREENSHOT_CAPTURE_BACKENDS)[number]; +/** + * The burst (`--count`) bounds a screenshot request admits. The shared `--count` flag definition + * keeps the press/swipe ceiling, so this admission is the tighter screenshot bound. Frames run + * sequentially inside the request's standard daemon budget, so the admission models the whole run, + * not just the waits: `count * (intervalMs + perFrameBudgetMs)` stays within `maxRunMs`, which + * leaves the request envelope margin for binding, transport, and a capture that exceeds its + * allowance. A longer run belongs in a batch of single captures. + */ +export const SCREENSHOT_BURST_LIMITS = { + minCount: 2, + maxCount: 100, + minIntervalMs: 100, + maxIntervalMs: 10_000, + defaultIntervalMs: 100, + /** The capture cost one frame budgets for; a sequential capture on a healthy target stays below it. */ + perFrameBudgetMs: 500, + maxRunMs: 60_000, +} as const; + export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'out', 'overlayRefs', @@ -104,6 +131,8 @@ export const SCREENSHOT_COMMAND_FLAG_KEYS = [ 'screenshotNoStabilize', 'screenshotNormalizeStatusBar', 'screenshotCaptureBackend', + 'count', + 'intervalMs', ] as const; export const SCREENSHOT_ACTION_FLAG_KEYS = [ @@ -226,13 +255,33 @@ const SCREENSHOT_SCRIPT_BOOLEAN_FLAGS = [ { tokens: ['--normalize-status-bar'], key: 'screenshotNormalizeStatusBar' }, ] as const; -const SCREENSHOT_SCRIPT_INT_FLAGS = [ +const SCREENSHOT_SCRIPT_INT_FLAGS: readonly { + token: string; + key: 'screenshotPixelDensity' | 'count' | 'intervalMs'; + label: string; + min?: number; + max?: number; +}[] = [ { token: '--pixel-density', key: 'screenshotPixelDensity', label: 'screenshot --pixel-density', }, -] as const; + { + token: '--count', + key: 'count', + label: 'screenshot --count', + min: SCREENSHOT_BURST_LIMITS.minCount, + max: SCREENSHOT_BURST_LIMITS.maxCount, + }, + { + token: '--interval-ms', + key: 'intervalMs', + label: 'screenshot --interval-ms', + min: SCREENSHOT_BURST_LIMITS.minIntervalMs, + max: SCREENSHOT_BURST_LIMITS.maxIntervalMs, + }, +]; const SCREENSHOT_SCRIPT_NUMBER_FLAGS = [ { @@ -263,6 +312,8 @@ export type ScreenshotRequestFlags = { screenshotNoStabilize?: boolean; screenshotNormalizeStatusBar?: boolean; screenshotCaptureBackend?: ScreenshotCaptureBackend; + count?: number; + intervalMs?: number; }; export type ScreenshotDispatchFlags = Pick< @@ -297,6 +348,10 @@ export type ScreenshotPublicOptions = { stabilize?: boolean; normalizeStatusBar?: boolean; captureBackend?: ScreenshotCaptureBackend; + /** Capture a burst of this many frames in one request; see `SCREENSHOT_BURST_LIMITS`. */ + count?: number; + /** Wait this many milliseconds between burst frames; requires `count`. */ + intervalMs?: number; }; export type ScreenshotRuntimeOptions = { @@ -309,6 +364,8 @@ export type ScreenshotRuntimeOptions = { stabilize?: boolean; normalizeStatusBar?: boolean; captureBackend?: ScreenshotCaptureBackend; + count?: number; + intervalMs?: number; }; export function screenshotOptionsFromFlags( @@ -325,6 +382,8 @@ export function screenshotOptionsFromFlags( flags?.screenshotNoStabilize === undefined ? undefined : !flags.screenshotNoStabilize, normalizeStatusBar: flags?.screenshotNormalizeStatusBar, captureBackend: flags?.screenshotCaptureBackend, + count: flags?.count, + intervalMs: flags?.intervalMs, }); } @@ -346,6 +405,8 @@ export function screenshotFlagsFromOptions( screenshotNormalizeStatusBar: options.screenshotNormalizeStatusBar ?? options.normalizeStatusBar, screenshotCaptureBackend: options.screenshotCaptureBackend ?? options.captureBackend, + count: options.count, + intervalMs: options.intervalMs, }); } @@ -373,6 +434,77 @@ export function validateScreenshotScale(options: Pick, +): void { + const { count, intervalMs } = options; + if (count === undefined) { + if (intervalMs !== undefined) { + throw new AppError('INVALID_ARGS', 'screenshot --interval-ms requires --count'); + } + return; + } + if ( + !Number.isInteger(count) || + count < SCREENSHOT_BURST_LIMITS.minCount || + count > SCREENSHOT_BURST_LIMITS.maxCount + ) { + throw new AppError( + 'INVALID_ARGS', + `screenshot --count requires an integer from ${SCREENSHOT_BURST_LIMITS.minCount} to ${SCREENSHOT_BURST_LIMITS.maxCount}`, + ); + } + if ( + intervalMs !== undefined && + (!Number.isInteger(intervalMs) || + intervalMs < SCREENSHOT_BURST_LIMITS.minIntervalMs || + intervalMs > SCREENSHOT_BURST_LIMITS.maxIntervalMs) + ) { + throw new AppError( + 'INVALID_ARGS', + `screenshot --interval-ms requires an integer from ${SCREENSHOT_BURST_LIMITS.minIntervalMs} to ${SCREENSHOT_BURST_LIMITS.maxIntervalMs}`, + ); + } + const runMs = + count * + ((intervalMs ?? SCREENSHOT_BURST_LIMITS.defaultIntervalMs) + + SCREENSHOT_BURST_LIMITS.perFrameBudgetMs); + if (runMs > SCREENSHOT_BURST_LIMITS.maxRunMs) { + throw new AppError( + 'INVALID_ARGS', + `screenshot --count schedules ${runMs} ms of waits and captures; keep the burst within ${SCREENSHOT_BURST_LIMITS.maxRunMs} ms by lowering --count or --interval-ms, or capture in a batch`, + ); + } + if (options.cropOn !== undefined || options.overlayRefs === true) { + throw new AppError( + 'INVALID_ARGS', + 'screenshot --count cannot be combined with --crop-on or --overlay-refs', + ); + } +} + +/** + * The destination of one burst frame: `-0001.`, `-0002.`, ... so the user's + * `--out` names the whole sequence. A path without an extension keeps none. + */ +export function screenshotBurstFramePath(outPath: string, index: number): string { + const extension = screenshotPathExtension(outPath); + const stem = extension.length > 0 ? outPath.slice(0, outPath.length - extension.length) : outPath; + return `${stem}-${String(index).padStart(4, '0')}${extension}`; +} + +function screenshotPathExtension(outPath: string): string { + const separatorIndex = Math.max(outPath.lastIndexOf('/'), outPath.lastIndexOf('\\')); + const dotIndex = outPath.lastIndexOf('.'); + if (dotIndex <= separatorIndex + 1) return ''; + return outPath.slice(dotIndex); +} + export function appendScreenshotScriptFlags( parts: string[], flags: Partial | undefined, @@ -390,6 +522,12 @@ export function appendScreenshotScriptFlags( if (flags?.screenshotNoStabilize) parts.push('--no-stabilize'); if (flags?.screenshotNoStabilize === false) parts.push('--no-stabilize=false'); if (flags?.screenshotNormalizeStatusBar) parts.push('--normalize-status-bar'); + if (typeof flags?.count === 'number') { + parts.push('--count', String(flags.count)); + } + if (typeof flags?.intervalMs === 'number') { + parts.push('--interval-ms', String(flags.intervalMs)); + } } export function readScreenshotScriptFlag(params: { @@ -446,8 +584,15 @@ function readScreenshotIntScriptFlag(params: { const definition = SCREENSHOT_SCRIPT_INT_FLAGS.find((entry) => entry.token === params.token); if (!definition) return undefined; const parsed = Number(params.args[params.index + 1]); - if (!Number.isInteger(parsed) || parsed < 1) { - throw new AppError('INVALID_ARGS', `${definition.label} requires a positive integer`); + const min = definition.min ?? 1; + const max = definition.max ?? Number.MAX_SAFE_INTEGER; + if (!Number.isInteger(parsed) || parsed < min || parsed > max) { + throw new AppError( + 'INVALID_ARGS', + max === Number.MAX_SAFE_INTEGER + ? `${definition.label} requires a positive integer` + : `${definition.label} requires an integer from ${min} to ${max}`, + ); } params.flags[definition.key] = parsed; return { handled: true, nextIndex: params.index + 1 }; diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index 6df1bb1549..e1d6322550 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -10,6 +10,7 @@ import type { import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { DeviceRotation } from './device-rotation.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; +import type { ScreenshotBurstFrame } from './screenshot.ts'; export type ScreenshotResultData = { path?: string; @@ -20,6 +21,11 @@ export type ScreenshotResultData = { pixelDensity?: number; /** Display rotation the capture was rendered in; absent when the target reports none. */ displayRotation?: DeviceRotation; + /** + * The frames of a `--count` burst, in capture order; each carries its own path and capture + * wall clock. Absent for a single capture, where `path` is the whole result. + */ + frames?: ScreenshotBurstFrame[]; overlayRefs?: ScreenshotOverlayRef[]; warnings?: string[]; }; diff --git a/packages/daemon/src/execution/screenshot-runtime-burst.test.ts b/packages/daemon/src/execution/screenshot-runtime-burst.test.ts new file mode 100644 index 0000000000..aacfc120b0 --- /dev/null +++ b/packages/daemon/src/execution/screenshot-runtime-burst.test.ts @@ -0,0 +1,344 @@ +import { daemonPlatformServicesFixture } from '../__tests__/support/platform-services-fixture.ts'; +import fs from 'node:fs'; + +import path from 'node:path'; +import { expect, test, vi } from 'vitest'; +import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; +import { ANDROID_EMULATOR, mkdtempForTestSync } from '@agent-device/test-kit'; +import { makeSession } from '../__tests__/support/session-factories.ts'; +import type { GenericPlatformExecutionParams } from '../core/generic-leaf-execution.ts'; +import { resolveScreenshotGenericExecution } from './screenshot-runtime.ts'; +import { screenshotRuntimeFixture } from '../__tests__/support/screenshot-runtime-fixture.ts'; +import type { DaemonRequest } from '../index.ts'; +import type { SessionState } from '../core/session-state.ts'; +import { screenshotCommandSurface as testCreateCommandSurface } from '../__tests__/support/screenshot-command-surface-fake.ts'; + +function screenshotRequest(overrides: Partial = {}): DaemonRequest { + return { + command: 'screenshot', + token: 'test-token', + session: 'default', + positionals: [], + ...overrides, + }; +} + +function executionParams( + session: SessionState, + req: DaemonRequest, + dispatchContext: GenericPlatformExecutionParams['dispatchContext'] = {}, +): GenericPlatformExecutionParams { + return { + session, + sessionName: session.name, + logPath: path.join(mkdtempForTestSync('daemon'), 'daemon.log'), + command: 'screenshot', + request: req, + positionals: req.positionals ?? [], + out: req.flags?.out, + dispatchContext, + }; +} + +async function executeResult( + resolved: Awaited>, + session: SessionState, + req: DaemonRequest, + dispatchContext?: GenericPlatformExecutionParams['dispatchContext'], +): Promise> { + if (!resolved.ok) throw new Error('the screenshot plan must be admitted'); + const result = await resolved.execute(executionParams(session, req, dispatchContext)); + if (result === undefined) throw new Error('the screenshot leaf returns a result record'); + return result; +} + +test('a --count burst captures the numbered sequence in one request and reports every frame', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const outDir = mkdtempForTestSync('agent-device-burst'); + const outPath = path.join(outDir, 'page.png'); + const req = screenshotRequest({ positionals: [outPath], flags: { count: 3, intervalMs: 100 } }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + + const result = await executeResult(resolved, session, req, req.flags); + + expect(fixture.binds).toHaveLength(1); + expect(fixture.captureScreenshot).toHaveBeenCalledTimes(3); + const destinations = fixture.captureScreenshot.mock.calls.map((call) => call[0]?.outPath); + expect(destinations).toEqual([ + path.join(outDir, 'page-0001.png'), + path.join(outDir, 'page-0002.png'), + path.join(outDir, 'page-0003.png'), + ]); + for (const destination of destinations) { + expect(fs.existsSync(destination)).toBe(true); + } + const frames = result.frames as Array<{ path: string; capturedAtMs: number }>; + expect(frames.map((frame) => frame.path)).toEqual(destinations); + expect(frames[0]!.capturedAtMs).toBeLessThanOrEqual(frames[1]!.capturedAtMs); + expect(frames[1]!.capturedAtMs).toBeLessThanOrEqual(frames[2]!.capturedAtMs); + expect(result).toMatchObject({ + path: path.join(outDir, 'page-0001.png'), + width: 100, + height: 50, + message: `Saved 3 screenshots starting at ${path.join(outDir, 'page-0001.png')}`, + }); +}); + +test('a burst without an explicit path keeps every frame in one temp root', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const req = screenshotRequest({ flags: { count: 2 } }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!resolved.ok) throw new Error('screenshot admission must succeed'); + + const result = await executeResult(resolved, session, req, req.flags); + const frames = result.frames as Array<{ path: string }>; + expect(frames).toHaveLength(2); + expect(path.dirname(frames[0]!.path)).toBe(path.dirname(frames[1]!.path)); + expect(new Set(frames.map((frame) => frame.path)).size).toBe(2); + expect(fs.existsSync(frames[0]!.path)).toBe(true); + expect(fs.existsSync(frames[1]!.path)).toBe(true); +}); + +test('a burst that fails past frame 1 leaves no implicit temp root behind', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const req = screenshotRequest({ flags: { count: 3 } }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!resolved.ok) throw new Error('screenshot admission must succeed'); + fixture.captureScreenshot.mockImplementationOnce(async () => ({})); + fixture.captureScreenshot.mockRejectedValueOnce(new Error('second frame capture failed')); + + await expect(executeResult(resolved, session, req, req.flags)).rejects.toThrow( + 'second frame capture failed', + ); + expect(fixture.captureScreenshot).toHaveBeenCalledTimes(2); + + // The implicit root that held frame 1 is gone. Another concurrent run could + // own its own root, so this checks the one this burst created — the dirname + // of the frame the burst captured, not a global sweep. + const frameOneDir = path.dirname(fixture.captureScreenshot.mock.calls[0]![0]!.outPath); + expect(fs.existsSync(frameOneDir)).toBe(false); +}); + +test('burst admission refuses crop, overlay, and misbounded flags before any device work', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const refusals: Array<{ label: string; flags: Record }> = [ + { label: 'interval without count', flags: { intervalMs: 200 } }, + { label: 'count below two', flags: { count: 1 } }, + { label: 'overlong schedule', flags: { count: 150, intervalMs: 500 } }, + { label: 'crop beside burst', flags: { count: 3, screenshotCropOn: 'role=button label=Save' } }, + { label: 'overlay beside burst', flags: { count: 3, overlayRefs: true } }, + ]; + for (const refusal of refusals) { + const req = screenshotRequest({ flags: refusal.flags }); + await expect( + resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }), + refusal.label, + ).rejects.toMatchObject({ code: 'INVALID_ARGS' }); + } + expect(fixture.binds).toHaveLength(0); + expect(fixture.captureScreenshot).not.toHaveBeenCalled(); +}); + +test('a cancelled request stops the burst before the next frame', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const outDir = mkdtempForTestSync('agent-device-burst-cancel'); + const req = screenshotRequest({ + positionals: [path.join(outDir, 'page.png')], + flags: { count: 3, intervalMs: 100 }, + }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!resolved.ok) throw new Error('screenshot admission must succeed'); + + const controller = new AbortController(); + fixture.captureScreenshot.mockImplementation(async () => { + controller.abort(); + return {}; + }); + const execution = resolved.execute( + executionParams(session, req, { + count: req.flags?.count, + intervalMs: req.flags?.intervalMs, + signal: controller.signal, + }), + ); + await expect(execution).rejects.toThrow(); + // The abort lands in the first capture's callback, so exactly one frame may + // complete before the plan rejects; a later-frame cancellation would show 2. + expect(fixture.captureScreenshot.mock.calls.length).toBe(1); +}); + +test('an abort during the inter-frame wait stops the burst with the abort reason and one capture', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const outDir = mkdtempForTestSync('agent-device-burst'); + const req = screenshotRequest({ + positionals: [path.join(outDir, 'page.png')], + flags: { count: 3, intervalMs: 1000 }, + }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: testCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + if (!resolved.ok) throw new Error('screenshot admission must succeed'); + + const controller = new AbortController(); + const execution = resolved.execute( + executionParams(session, req, { + count: req.flags?.count, + intervalMs: req.flags?.intervalMs, + signal: controller.signal, + }), + ); + // Wait for frame 1 to complete; the abort then lands while the plan waits + // out intervalMs, so the wait must surface the abort instead of dispatching + // frame 2. + await vi.waitFor(() => expect(fixture.captureScreenshot).toHaveBeenCalledTimes(1)); + const abortTimer = setTimeout( + () => controller.abort(new Error('canceled during the interval')), + 20, + ); + await expect(execution).rejects.toThrow('canceled during the interval'); + clearTimeout(abortTimer); + expect(fixture.captureScreenshot.mock.calls.length).toBe(1); +}); + +/** A command surface whose every capture reports the given warnings, so a burst frames them. */ +function burstWarningSurface(warnings: string[]): CreateDaemonCommandSurface { + return async (config) => { + const forwarded = await testCreateCommandSurface(config); + return { + ...forwarded, + capture: { + ...forwarded.capture, + screenshot: async (options) => ({ + ...(await forwarded.capture.screenshot(options)), + warnings, + }), + }, + }; + }; +} + +test('a burst reports a warning the frames share once', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const outDir = mkdtempForTestSync('agent-device-burst-warnings'); + const req = screenshotRequest({ + positionals: [path.join(outDir, 'page.png')], + flags: { count: 3, intervalMs: 100 }, + }); + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: burstWarningSurface(['the status bar was busy; the capture waited']), + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + const result = await executeResult(resolved, session, req, req.flags); + expect(fixture.captureScreenshot).toHaveBeenCalledTimes(3); + expect(result.warnings).toEqual(['the status bar was busy; the capture waited']); +}); + +test('a burst keeps the warnings array within one note per frame', async () => { + const fixture = screenshotRuntimeFixture(); + const session = makeSession('default', { device: ANDROID_EMULATOR }); + const outDir = mkdtempForTestSync('agent-device-burst-warning-bound'); + const req = screenshotRequest({ + positionals: [path.join(outDir, 'page.png')], + flags: { count: 2, intervalMs: 100 }, + }); + let frame = 0; + const warningSurface: CreateDaemonCommandSurface = async (config) => { + const forwarded = await testCreateCommandSurface(config); + return { + ...forwarded, + capture: { + ...forwarded.capture, + screenshot: async (options) => { + frame += 1; + return { + ...(await forwarded.capture.screenshot(options)), + warnings: ['the frames share one note', `frame ${frame} contributes its own`], + }; + }, + }, + }; + }; + + const resolved = await resolveScreenshotGenericExecution({ + hostCwd: process.cwd(), + createCommandSurface: warningSurface, + platformServices: daemonPlatformServicesFixture(), + req, + session, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + const result = await executeResult(resolved, session, req, req.flags); + // The shared note collapses to one entry and frame 1 contributes its own; + // the bound (count 2) then stops frame 2's note — proving the aggregation + // consumed frame 2's list against the shared budget, not just frame 1's. + expect(result.warnings).toEqual(['the frames share one note', 'frame 1 contributes its own']); +}); diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index ab094426c2..14e955129f 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -1,16 +1,25 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import os from 'node:os'; import { createDaemonScreenshotArtifactAdapter } from './screenshot-artifact-adapter.ts'; import { expandSessionPath } from '@agent-device/host-kit/session-paths'; +import { sleep } from '@agent-device/host-kit/retry'; import type { CommandFlags } from '@agent-device/contracts/command'; import { retiredScreenshotMaxSizeFlagError, + SCREENSHOT_BURST_LIMITS, SCREENSHOT_CAPTURE_BACKENDS, + screenshotBurstFramePath, screenshotFlagsFromOptions, screenshotOptionsFromFlags, + validateScreenshotBurst, + type ScreenshotBurstFrame, } from '@agent-device/contracts/capture'; import type { DeviceRotation } from '@agent-device/contracts/device'; import type { ScreenshotRuntimeExecution } from '@agent-device/contracts/screenshot-runtime'; import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; +import { successText } from '@agent-device/kernel/success-text'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; @@ -63,6 +72,12 @@ export async function resolveScreenshotGenericExecution( if (retiredMaxSize) throw new AppError('INVALID_ARGS', retiredMaxSize); assertSupportedScreenshotPixelDensity(session.device, req.flags?.screenshotPixelDensity); assertSupportedScreenshotCaptureBackend(req.flags); + validateScreenshotBurst({ + count: req.flags?.count, + intervalMs: req.flags?.intervalMs, + cropOn: req.flags?.screenshotCropOn, + overlayRefs: req.flags?.overlayRefs === true ? true : undefined, + }); const request = readScreenshotRequest(req, params.hostCwd); const cropOn = @@ -199,6 +214,10 @@ async function executeScreenshot( }>, ): Promise> { const { session, runtime, flags } = params; + const burst = readScreenshotBurstPlan(params.dispatchContext); + if (burst) { + return await executeBurstScreenshot({ ...params, burst }); + } const crop = buildScreenshotCropBinding({ cropOn: params.cropOn, captureSnapshot: runtime.captureSnapshot, @@ -235,6 +254,112 @@ async function executeScreenshot( }; } +/** + * The admitted burst: the frame count and the wait between frames. The count rode the request's + * shared `count`/`intervalMs` flags through `DispatchContextFlags`; only the interval has a + * screenshot default, because those flags serve press and swipe series too. + */ +type ScreenshotBurstPlan = Readonly<{ count: number; intervalMs: number }>; + +function readScreenshotBurstPlan( + dispatchContext: DaemonCommandContext, +): ScreenshotBurstPlan | undefined { + const { count, intervalMs } = dispatchContext; + if (count === undefined) return undefined; + return { count, intervalMs: intervalMs ?? SCREENSHOT_BURST_LIMITS.defaultIntervalMs }; +} + +/** + * The `--count` burst: one admitted plan, captured sequentially frame by frame through the same + * single-capture path, each frame into its own numbered destination. The result reports every + * frame with its capture wall clock; the top-level fields describe the first frame, because every + * frame reads the same screen at the same size. Admission refuses a burst beside `--crop-on` and + * `--overlay-refs`, so this loop never carries the crop or overlay state. + */ +async function executeBurstScreenshot( + params: Readonly<{ + createCommandSurface: CreateDaemonCommandSurface; + session: SessionState; + sessionName: string; + logPath: string; + dispatchContext: DaemonCommandContext; + flags: CommandFlags | undefined; + outPath: string | undefined; + runtime: BoundScreenshotRuntime; + burst: ScreenshotBurstPlan; + }>, +): Promise> { + const { burst, dispatchContext } = params; + const signal = dispatchContext.signal; + const tempRoot = + params.outPath === undefined + ? await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-screenshot-burst-')) + : undefined; + const frameDestination = (index: number): string => + params.outPath !== undefined + ? screenshotBurstFramePath(params.outPath, index) + : path.join(tempRoot!, `screenshot-${String(index).padStart(4, '0')}.png`); + const frames: ScreenshotBurstFrame[] = []; + const warnings: string[] = []; + try { + for (let index = 1; index <= burst.count; index += 1) { + signal?.throwIfAborted(); + if (index > 1 && burst.intervalMs > 0) { + await sleep(burst.intervalMs, signal); + // The sleep resolves on an abort too, so this check is what stops a + // canceled request from dispatching the next frame. + signal?.throwIfAborted(); + } + const captured = await captureScreenshotArtifact({ + createCommandSurface: params.createCommandSurface, + session: params.session, + sessionName: params.sessionName, + outPath: frameDestination(index), + dispatchContext, + captureScreenshot: params.runtime.captureScreenshot, + }); + appendBurstWarnings(warnings, captured.warnings, burst.count); + frames.push({ + path: captured.path, + capturedAtMs: Date.now(), + ...(captured.displayRotation ? { displayRotation: captured.displayRotation } : {}), + }); + } + } catch (error) { + // A failed or canceled burst leaves no implicit root behind; an explicit + // --out path keeps whatever the caller named. + if (tempRoot !== undefined) + await fs.rm(tempRoot, { recursive: true, force: true }).catch(() => {}); + throw error; + } + const first = frames[0]!; + return { + path: first.path, + frames, + ...(first.displayRotation ? { displayRotation: first.displayRotation } : {}), + ...(warnings.length > 0 ? { warnings } : {}), + ...successText(`Saved ${burst.count} screenshots starting at ${first.path}`), + ...(await readScreenshotResultMetadata({ + device: params.session.device, + path: first.path, + requestedPixelDensity: params.flags?.screenshotPixelDensity, + scale: params.flags?.screenshotScale, + })), + }; +} + +/** + * The response's warning channel stays bounded: a note the frames share appears once, and the + * array holds at most one note per frame. A capture owner that degrades on every frame of the + * animation must not multiply its note by the frame count. + */ +function appendBurstWarnings(warnings: string[], additions: string[] | undefined, bound: number) { + for (const warning of additions ?? []) { + if (warnings.length >= bound) return; + if (!warnings.includes(warning)) warnings.push(warning); + } +} + /** Present exactly when the admitted plan carries a crop and the snapshot binding to serve it. */ function buildScreenshotCropBinding( params: Readonly<{ diff --git a/packages/daemon/src/server/__tests__/remote-host-input-policy.test.ts b/packages/daemon/src/server/__tests__/remote-host-input-policy.test.ts new file mode 100644 index 0000000000..c67f31a5d1 --- /dev/null +++ b/packages/daemon/src/server/__tests__/remote-host-input-policy.test.ts @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { restrictRemoteHostInputs } from '../remote-host-input-policy.ts'; +import type { DaemonRequest } from '../../core/daemon-request.ts'; + +function remoteRequest( + input: Record, + flags: Record = {}, + positionals: string[] = [], +) { + return { + command: 'screenshot', + positionals, + flags, + input, + internal: { publicNetworkOnly: true }, + } as unknown as DaemonRequest; +} + +test('a remote screenshot burst refuses with a typed reason before any device work', () => { + try { + restrictRemoteHostInputs(remoteRequest({}, { count: 5 }, ['/tmp/x.png'])); + assert.fail('a remote burst must refuse'); + } catch (error) { + assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); + assert.equal( + (error as { details?: { reason?: string } }).details?.reason, + 'REMOTE_SCREENSHOT_BURST_DENIED', + ); + } +}); + +test('a remote burst armed by a flags-level count refuses the same way', () => { + assert.throws( + () => restrictRemoteHostInputs(remoteRequest({}, { count: 2 }, ['/tmp/x.png'])), + /only the first frame would come back/, + ); +}); + +test('a remote single screenshot without count still passes the capture lane', () => { + const restricted = restrictRemoteHostInputs(remoteRequest({}, {}, ['/tmp/x.png'])); + assert.equal((restricted.input as Record).count, undefined); + assert.equal(restricted.command, 'screenshot'); +}); diff --git a/packages/daemon/src/server/remote-host-input-policy.ts b/packages/daemon/src/server/remote-host-input-policy.ts index 4aa9b62e3f..b0c0606ef5 100644 --- a/packages/daemon/src/server/remote-host-input-policy.ts +++ b/packages/daemon/src/server/remote-host-input-policy.ts @@ -158,7 +158,33 @@ function restrictRemoteBatchSteps(request: DaemonRequest): DaemonRequest { } function restrictRemoteCaptureOutput(request: DaemonRequest): DaemonRequest { - const { path: _path, out: _out, outPath: _outPath, ...input } = request.input ?? {}; + const { + path: _path, + out: _out, + outPath: _outPath, + count: _count, + intervalMs: _intervalMs, + ...input + } = request.input ?? {}; + // A burst writes every frame beside the --out path on the daemon host, and a + // remote client can only download the one artifact the response names — so a + // remote burst would return one usable frame and leak the rest. Refusing + // beats shipping N-1 unreadable paths. + if ( + request.command === 'screenshot' && + ((request.input as Record | undefined)?.count !== undefined || + request.flags?.count !== undefined) + ) { + throw new AppError( + 'INVALID_ARGS', + 'A remote screenshot request cannot run a burst: only the first frame would come back as a downloadable artifact.', + { + reason: 'REMOTE_SCREENSHOT_BURST_DENIED', + retriable: false, + hint: 'Run a local daemon against the device for a burst, or capture single screenshots and download each artifact by ID.', + }, + ); + } return { ...request, positionals: remoteCapturePositionals(request), diff --git a/src/cli/parser/__tests__/args-parse-interaction.test.ts b/src/cli/parser/__tests__/args-parse-interaction.test.ts index b1f7754205..ac6ccf2ab0 100644 --- a/src/cli/parser/__tests__/args-parse-interaction.test.ts +++ b/src/cli/parser/__tests__/args-parse-interaction.test.ts @@ -294,6 +294,10 @@ test('parseArgs recognizes screenshot flags', () => { '--normalize-status-bar', '--capture-backend', 'runner', + '--count', + '5', + '--interval-ms', + '250', ], { strictFlags: true, @@ -308,6 +312,8 @@ test('parseArgs recognizes screenshot flags', () => { assert.equal(parsed.flags.screenshotScale, 0.3); assert.equal(parsed.flags.screenshotNoStabilize, true); assert.equal(parsed.flags.screenshotNormalizeStatusBar, true); + assert.equal(parsed.flags.count, 5); + assert.equal(parsed.flags.intervalMs, 250); }); test('parseArgs recognizes viewport command', () => { diff --git a/src/cli/resolve-cli-options.test.ts b/src/cli/resolve-cli-options.test.ts index 71fe319333..49136baa0e 100644 --- a/src/cli/resolve-cli-options.test.ts +++ b/src/cli/resolve-cli-options.test.ts @@ -195,3 +195,66 @@ test('a selector-less command inherits the pinned device', () => { source: { kind: 'project', path: path.join(cwd, 'agent-device.json') }, }); }); + +// The screenshot burst flags are explicit-only: the same `count` key serves press and swipe series, +// and a repository `count` tuned for those must not turn a single capture into a burst, nor refuse +// `--crop-on` beside it. A typed flag still arms the burst, and press keeps the configured count. +function projectConfigDirWithCount(): string { + const cwd = mkdtempForTestSync('agent-device-cli-project-'); + fs.writeFileSync(path.join(cwd, 'agent-device.json'), JSON.stringify({ count: 3 })); + return cwd; +} + +test('a count from the project config never arms a screenshot burst', () => { + const parsed = resolveCliOptions(['screenshot'], { + cwd: projectConfigDirWithCount(), + env: isolatedEnv({}), + }); + + assert.equal(parsed.flags.count, undefined); + assert.equal(parsed.flags.intervalMs, undefined); +}); + +test('a typed --count still arms a screenshot burst', () => { + const parsed = resolveCliOptions(['screenshot', '--count', '5'], { + cwd: projectConfigDirWithCount(), + env: isolatedEnv({}), + }); + + assert.equal(parsed.flags.count, 5); +}); + +test('a configured intervalMs is removed from a screenshot request like the count', () => { + const cwd = projectConfigDirWithCount(); + fs.writeFileSync( + path.join(cwd, 'agent-device.json'), + JSON.stringify({ count: 3, intervalMs: 250 }), + ); + + const parsed = resolveCliOptions(['screenshot'], { cwd, env: isolatedEnv({}) }); + + assert.equal(parsed.flags.count, undefined); + assert.equal(parsed.flags.intervalMs, undefined); +}); + +test('a screenshot naming --crop-on beside a configured count is not refused as a burst', () => { + const cwd = projectConfigDirWithCount(); + fs.writeFileSync(path.join(cwd, 'agent-device.json'), JSON.stringify({ count: 3 })); + + const parsed = resolveCliOptions(['screenshot', '--crop-on', 'role=button label=Save'], { + cwd, + env: isolatedEnv({}), + }); + + assert.equal(parsed.flags.count, undefined); + assert.equal(parsed.flags.screenshotCropOn, 'role=button label=Save'); +}); + +test('the same project count keeps tuning press, which reads a configured repeat count', () => { + const parsed = resolveCliOptions(['press', '@e1'], { + cwd: projectConfigDirWithCount(), + env: isolatedEnv({}), + }); + + assert.equal(parsed.flags.count, 3); +}); diff --git a/src/mcp/__tests__/tool-input-config.test.ts b/src/mcp/__tests__/tool-input-config.test.ts new file mode 100644 index 0000000000..c30b165da8 --- /dev/null +++ b/src/mcp/__tests__/tool-input-config.test.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test } from 'vitest'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { resolveMcpConfigDefaults } from '../tool-input-config.ts'; + +/** + * Points `~` at an empty directory so a developer's own user config cannot join the resolution, + * and writes the project config the test asserts against into the isolated cwd. + */ +function projectHostWithConfig(config: Record): { + cwd: string; + host: { env: Record; cwd: string }; +} { + const home = mkdtempForTestSync('agent-device-mcp-home-'); + const cwd = mkdtempForTestSync('agent-device-mcp-project-'); + fs.writeFileSync(path.join(cwd, 'agent-device.json'), JSON.stringify(config)); + return { + cwd, + host: { env: { HOME: home, USERPROFILE: home, PATH: process.env.PATH ?? '' }, cwd }, + }; +} + +test('a project count for press series reaches a press tool call as a default', () => { + const { host } = projectHostWithConfig({ count: 3 }); + + const { input } = resolveMcpConfigDefaults('press', {}, { count: { type: 'integer' } }, host); + + assert.equal(input.count, 3); +}); + +// The screenshot burst flags are explicit-only (CommandSchema). The tool call is the typed +// invocation, so a repository `count` tuned for press and swipe series must not arm a burst. +test('a project count never reaches a screenshot tool call', () => { + const { host } = projectHostWithConfig({ count: 3 }); + + const { input } = resolveMcpConfigDefaults( + 'screenshot', + {}, + { count: { type: 'integer' } }, + host, + ); + + assert.equal(input.count, undefined); +}); + +test('a count the tool call typed reaches a screenshot, and wins over any default', () => { + const { host } = projectHostWithConfig({ count: 3 }); + + const { input } = resolveMcpConfigDefaults( + 'screenshot', + { count: 5 }, + { count: { type: 'integer' } }, + host, + ); + + assert.equal(input.count, 5); +}); diff --git a/src/mcp/tool-input-config.ts b/src/mcp/tool-input-config.ts index 9cb6ed5fee..99317d058a 100644 --- a/src/mcp/tool-input-config.ts +++ b/src/mcp/tool-input-config.ts @@ -2,6 +2,7 @@ import type { HostEnvironment } from '@agent-device/kernel/source-value'; import type { FlagKey } from '@agent-device/command-registry/flag-types'; import type { CliFlags } from '@agent-device/contracts/command'; import { + getCommandSchema, isFlagSupportedForCommand, mergeDefinedFlags, resolveBindingSettings, @@ -27,8 +28,12 @@ export function resolveMcpConfigDefaults( }); const defaults = configDefaults.flags; const devicePin = configDefaults.devicePin; + // The command's explicit-only flags mean what the CLI parser strips mean there: a config or env + // default never stands in for the caller naming the key, so the tool input must carry it. + const explicitOnly = getCommandSchema(name)?.explicitOnlyFlags; const applicableDefaults: Record = {}; for (const [key, value] of Object.entries(defaults)) { + if (explicitOnly?.includes(key as FlagKey)) continue; if ( Object.hasOwn(supportedProperties, key) && isFlagSupportedForCommand(key as FlagKey, name)