From 7ed1f6a9af2411face1bcf68a3a69bdc2b0401e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 17:49:35 +0100 Subject: [PATCH 1/6] perf(apple): read a single-panel simulator's rotation beside its screenshot A simulator whose panel table already resolved to one integrated panel never names a display, so its per-capture panel table read only supplies the display rotation. Start that read beside simctl io screenshot and attach its rotation within a 250 ms grace after the image is written; a later read is aborted and the rotation is absent, as the screenshot contract already states. Multi-panel and unknown devices keep the sequential read, so a folded device still names its lit panel. The resolved panel count moves from the foldable rotation module into the display inventory, which records it on every resolved read. Co-Authored-By: Claude Opus 5.5 --- packages/capture-kit/package.json | 4 + .../capture-kit/src/capture-fact-grace.ts | 21 ++++ packages/platform-android/src/screenshot.ts | 15 +-- .../core/__tests__/display-inventory.test.ts | 49 ++++++++ .../screenshot-rotation-read.test.ts | 105 ++++++++++++++++++ packages/platform-apple/src/core/config.ts | 5 + .../src/core/display-inventory.ts | 24 +++- .../platform-apple/src/core/screenshot.ts | 89 +++++++++++++-- .../src/foldable/orientation.ts | 21 ++-- 9 files changed, 294 insertions(+), 39 deletions(-) create mode 100644 packages/capture-kit/src/capture-fact-grace.ts create mode 100644 packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 581081453f..9386b67f87 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -54,6 +54,10 @@ "types": "./src/capture-admission/audio-probe-session-resource.ts", "default": "./src/capture-admission/audio-probe-session-resource.ts" }, + "./capture-fact-grace": { + "types": "./src/capture-fact-grace.ts", + "default": "./src/capture-fact-grace.ts" + }, "./durable-capture": { "types": "./src/durable-capture/index.ts", "default": "./src/durable-capture/index.ts" diff --git a/packages/capture-kit/src/capture-fact-grace.ts b/packages/capture-kit/src/capture-fact-grace.ts new file mode 100644 index 0000000000..93a671c41f --- /dev/null +++ b/packages/capture-kit/src/capture-fact-grace.ts @@ -0,0 +1,21 @@ +/** + * Waits at most `graceMs` for optional capture metadata that is read beside the image, such as + * the display rotation. A read still running at the deadline yields `undefined`; the caller then + * aborts it and reports the fact as absent. + */ +export async function settledWithin( + pending: Promise, + graceMs: number, +): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + pending, + new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), graceMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} diff --git a/packages/platform-android/src/screenshot.ts b/packages/platform-android/src/screenshot.ts index a7c9e4728e..82af57292a 100644 --- a/packages/platform-android/src/screenshot.ts +++ b/packages/platform-android/src/screenshot.ts @@ -6,6 +6,7 @@ import { runAndroidExecOut, runAndroidShell, sleep } from './adb.ts'; import { emitAndroidAdbDiagnostic, requireAndroidAdbHost } from './adb-host.ts'; import { androidAdbResultError } from './adb-executor.ts'; import { probeAndroidDisplayRotation } from './input-actions.ts'; +import { settledWithin } from '@agent-device/capture-kit/capture-fact-grace'; // PNG file signature: 0x89 P N G \r \n 0x1A \n const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); @@ -123,20 +124,6 @@ async function captureAndroidScreenshotWithRotation( } } -async function settledWithin(pending: Promise, graceMs: number): Promise { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - pending, - new Promise((resolve) => { - timer = setTimeout(() => resolve(undefined), graceMs); - }), - ]); - } finally { - clearTimeout(timer); - } -} - async function captureAndroidScreenshot(device: DeviceInfo, outPath: string): Promise { const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); const helper = await runAndroidAutomationHelper(device, { action: 'screenshot' }); diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 39ce493a6f..22ffefffbc 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -38,6 +38,7 @@ import { establishedLitPanel, buildInventory, parseCoreDeviceDisplays, + knownAppleMultiScreen, readPanelPower, resolveAppleCaptureDisplay, queryAppleDisplayInventory, @@ -567,6 +568,54 @@ describe('resolveAppleCaptureTarget', () => { }); }); +describe('knownAppleMultiScreen', () => { + beforeEach(() => { + mockRunXcrun.mockClear(); + mockEmitDiagnostic.mockClear(); + mockReadHostTextFile.mockReset(); + mockReadHostTextFile.mockResolvedValue(''); + }); + + test('keeps the panel count of the latest resolved table, not of an unresolved one', async () => { + const device = { ...IOS_TEST_SIMULATOR, id: `${IOS_TEST_SIMULATOR.id}-panel-count` }; + assert.equal(knownAppleMultiScreen(device), undefined); + + mockReadHostTextFile.mockResolvedValueOnce(JSON.stringify(IPHONE_17_SINGLE_PANEL)); + await queryAppleDisplayInventory(device, {}); + assert.equal(knownAppleMultiScreen(device), false); + + mockRunXcrun.mockResolvedValueOnce({ + exitCode: 1, + stdout: '', + stderr: 'The specified device was not found', + } as Awaited>); + assert.equal((await queryAppleDisplayInventory(device, {})).unresolved, true); + assert.equal(knownAppleMultiScreen(device), false); + + mockReadHostTextFile.mockResolvedValueOnce(JSON.stringify(IPHONE_DUO_CLOSED)); + await queryAppleDisplayInventory(device, {}); + assert.equal(knownAppleMultiScreen(device), true); + }); + + test('a read its caller aborted reports no unresolved-table diagnostic', async () => { + const device = { ...IOS_TEST_SIMULATOR, id: `${IOS_TEST_SIMULATOR.id}-aborted-read` }; + const controller = new AbortController(); + mockRunXcrun.mockImplementationOnce(async () => { + controller.abort(); + throw new Error('aborted'); + }); + const inventory = await queryAppleDisplayInventory(device, { signal: controller.signal }); + assert.equal(inventory.unresolved, true); + assert.equal( + mockEmitDiagnostic.mock.calls.some( + ([event]) => event.phase === 'apple_display_inventory_unresolved', + ), + false, + ); + assert.equal(knownAppleMultiScreen(device), undefined); + }); +}); + describe('appleSimulatorDisplayArgvFragment', () => { test('names the panel for capture commands that share the same implicit default', () => { assert.deepEqual(appleSimulatorDisplayArgvFragment(outerPanel), ['--display=LCD']); diff --git a/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts new file mode 100644 index 0000000000..1a5976cce0 --- /dev/null +++ b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts @@ -0,0 +1,105 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { bindTestCommandHost } from '@agent-device/test-kit'; +import { captureSimulatorScreenshotWithFallback } from '../screenshot.ts'; +import { IOS_TEST_SIMULATOR } from './apple-core-stub-helpers.ts'; + +bindTestCommandHost(); + +test('a known single-panel capture reads its rotation beside the image', async () => { + const events: string[] = []; + let finishProbe!: () => void; + const facts = await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { + env: process.env, + skipIosSimulatorBootCheck: true, + deps: { + ensureBooted: async () => {}, + normalizeDensity: async () => {}, + isKnownSinglePanel: () => true, + resolveCaptureTarget: async () => { + events.push('probe-start'); + await new Promise((resolve) => { + finishProbe = resolve; + }); + events.push('probe-end'); + return { displayRotation: 'landscape-left' }; + }, + captureWithRetry: async (_device, _outPath, _mask, display) => { + assert.equal(display, undefined); + events.push('capture'); + finishProbe(); + }, + }, + }); + assert.deepEqual(events, ['probe-start', 'capture', 'probe-end']); + assert.deepEqual(facts, { displayRotation: 'landscape-left' }); +}); + +test('a known single-panel capture reports no rotation when its read misses the grace', async () => { + let probeSignal: AbortSignal | undefined; + const facts = await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { + env: process.env, + skipIosSimulatorBootCheck: true, + deps: { + ensureBooted: async () => {}, + normalizeDensity: async () => {}, + isKnownSinglePanel: () => true, + resolveCaptureTarget: async (_device, options) => { + probeSignal = options?.signal; + return await new Promise(() => {}); + }, + captureWithRetry: async () => {}, + }, + }); + assert.deepEqual(facts, {}); + assert.equal(probeSignal?.aborted, true, 'the late read is aborted'); +}); + +test('a known single-panel capture reports no rotation when its read fails', async () => { + const facts = await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { + env: process.env, + skipIosSimulatorBootCheck: true, + deps: { + ensureBooted: async () => {}, + normalizeDensity: async () => {}, + isKnownSinglePanel: () => true, + resolveCaptureTarget: async () => { + throw new Error('devicectl failed'); + }, + captureWithRetry: async () => {}, + }, + }); + assert.deepEqual(facts, {}); +}); + +test('a device not known to be single-panel reads its panel table before the capture', async () => { + const events: string[] = []; + const panel = { + name: 'LCD-1', + displayId: 3, + power: 'lit', + primary: false, + widthPx: 2007, + heightPx: 2853, + pointScale: 3, + currentOrientation: 'rot90', + integrated: true, + } as const; + await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { + env: process.env, + skipIosSimulatorBootCheck: true, + deps: { + ensureBooted: async () => {}, + normalizeDensity: async () => {}, + isKnownSinglePanel: () => false, + resolveCaptureTarget: async () => { + events.push('probe'); + return { display: panel }; + }, + captureWithRetry: async (_device, _outPath, _mask, display) => { + events.push(`capture ${display?.name}`); + }, + }, + }); + assert.deepEqual(events, ['probe', 'capture LCD-1']); +}); diff --git a/packages/platform-apple/src/core/config.ts b/packages/platform-apple/src/core/config.ts index 728388eb24..ea6ad164c8 100644 --- a/packages/platform-apple/src/core/config.ts +++ b/packages/platform-apple/src/core/config.ts @@ -64,6 +64,11 @@ export const IOS_FOLDABLE_ORIENTATION_SETTLE_MS = 600; // CoreSimulator can briefly stall while it services the scale lookup immediately // after a keyboard transition. Keep this bounded below the full capture budget. export const IOS_SIMULATOR_SCREENSHOT_SCALE_TIMEOUT_MS = 15_000; +// A known single-panel capture reads its rotation beside the image, within this budget. +export const IOS_SIMULATOR_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS = 2_000; +// A rotation read still running this long after the image is written is aborted, and the capture +// reports no rotation: the rotation is optional metadata and must not delay the image. +export const IOS_SIMULATOR_SCREENSHOT_ROTATION_GRACE_MS = 250; /** * The boot-state read a skipped-boot-check capture makes after its own failure. It answers only diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index aeb192e121..1b477420e0 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -99,13 +99,28 @@ type AppleDisplayInventoryOptions = { const DISPLAYS_UNSUPPORTED_HINT = "This Xcode/CoreDevice toolchain does not report 'devicectl device info displays'. Update Xcode to a version that ships the display-information feature before targeting multi-display Apple devices."; +/** + * Each device's integrated panel count, from its latest resolved panel table. The count is a + * hardware fact, so it is kept for the daemon's lifetime; the table itself is not (see + * {@link queryAppleDisplayInventory}). + */ +const multiScreenByDevice = new Map(); + +/** + * Whether a resolved panel table for this device listed more than one integrated panel, or + * `undefined` when no table for it has resolved in this daemon. + */ +export function knownAppleMultiScreen(device: DeviceInfo): boolean | undefined { + return multiScreenByDevice.get(device.id); +} + /** * Reads the CoreDevice display table for one Apple device, or the guest's equivalent for a * simulator in a scoped device set or a caller that explicitly requests the guest table. * * Deliberately uncached: which panel is lit is exactly what changes when an * operator folds or opens the device, and a cached answer would resume capturing - * the dark panel. The probe costs ~0.2s against a ~5s simulator screenshot. + * the dark panel. Only the panel count it resolves is kept ({@link knownAppleMultiScreen}). * * A failure to ask is reported as an unresolved inventory instead of an error: * every caller has a correct pre-inventory path, and a missing host feature must @@ -120,10 +135,13 @@ export async function queryAppleDisplayInventory( ? await readGuestPanelTable(device, options) : await readCoreDevicePanelTable(device, options); if (!table.ok) { - emitUnresolvedDiagnostic(device, table.reason, table.detail); + // A caller that aborted the read already knows it did not answer. + if (!options.signal?.aborted) emitUnresolvedDiagnostic(device, table.reason, table.detail); return unresolvedInventory(); } - return inventoryFromPanelTable(device, table.payload); + const inventory = inventoryFromPanelTable(device, table.payload); + if (!inventory.unresolved) multiScreenByDevice.set(device.id, inventory.multiScreen); + return inventory; } type PanelTableRead = diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index bfb3809faf..36affe249d 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -17,12 +17,15 @@ import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor- import { resizePngFile } from '@agent-device/capture-kit/png-resize'; import { readPngSize } from '@agent-device/capture-kit/png-size'; +import { settledWithin } from '@agent-device/capture-kit/capture-fact-grace'; import { computeDensityScaledScreenshotSize } from '@agent-device/capture-kit/screenshot-density'; import { IOS_RUNNER_SCREENSHOT_COPY_TIMEOUT_MS, IOS_SIMULATOR_SCREENSHOT_RETRY_BASE_DELAY_MS, IOS_SIMULATOR_SCREENSHOT_RETRY_MAX_ATTEMPTS, IOS_SIMULATOR_SCREENSHOT_RETRY_MAX_DELAY_MS, + IOS_SIMULATOR_SCREENSHOT_ROTATION_GRACE_MS, + IOS_SIMULATOR_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS, IOS_SIMULATOR_SCREENSHOT_SCALE_TIMEOUT_MS, IOS_SIMULATOR_SCREENSHOT_STATE_PROBE_TIMEOUT_MS, IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS, @@ -38,6 +41,7 @@ import { } from './screenshot-status-bar.ts'; import { appleSimulatorDisplayArgvFragment, + knownAppleMultiScreen, resolveAppleCaptureTarget, type AppleCaptureTarget, type AppleDeviceDisplay, @@ -70,7 +74,15 @@ type SimulatorScreenshotFlowDeps = { capture: () => Promise, signal?: AbortSignal, ) => Promise; - resolveCaptureTarget: (device: DeviceInfo) => Promise; + /** + * Whether a resolved panel table for this device listed one integrated panel. Such a capture + * never names a display, so its panel table read only supplies the rotation and runs beside it. + */ + isKnownSinglePanel: (device: DeviceInfo) => boolean; + resolveCaptureTarget: ( + device: DeviceInfo, + options?: { timeoutMs?: number; signal?: AbortSignal }, + ) => Promise; captureWithRetry: ( device: DeviceInfo, outPath: string, @@ -118,6 +130,7 @@ const defaultSimulatorScreenshotFlowDeps: SimulatorScreenshotFlowDeps = { ensureBooted: ensureBootedSimulator, withNormalizedStatusBar: withNormalizedSimulatorStatusBar, withStatusBarHeld: withSimulatorStatusBarHeld, + isKnownSinglePanel: (device) => knownAppleMultiScreen(device) === false, resolveCaptureTarget: resolveAppleCaptureTarget, captureWithRetry: captureSimulatorScreenshotWithRetry, normalizeDensity: normalizeIosSimulatorScreenshotDensity, @@ -176,10 +189,11 @@ export async function captureSimulatorScreenshotWithFallback( if (!options.skipIosSimulatorBootCheck) { await deps.ensureBooted(device); } - // A foldable lights one panel at a time, so the capture must name the panel the - // system is currently showing rather than accept simctl's implicit default. - const { display, displayRotation } = await deps.resolveCaptureTarget(device); - const captureFacts: ScreenshotCaptureFacts = displayRotation ? { displayRotation } : {}; + const { display, readCaptureFacts, cancelRotationRead } = await planCaptureTarget( + device, + deps, + options.runnerOptions?.signal, + ); const captureAndNormalize = async () => { await deps.captureWithRetry(device, outPath, options.screenMask, display); await deps.normalizeDensity(device, outPath, options.pixelDensity, display?.pointScale); @@ -198,14 +212,14 @@ export async function captureSimulatorScreenshotWithFallback( if (resolveAppleToolProvider().simulatorScreenshotBackend !== 'runner') { try { await captureAndNormalize(); - return captureFacts; + return await readCaptureFacts(); } catch (error) { let screenshotError = error; if (await mayRecoverSkippedBootCheck(error)) { await deps.ensureBooted(device); try { await captureAndNormalize(); - return captureFacts; + return await readCaptureFacts(); } catch (retryError) { screenshotError = retryError; } @@ -229,12 +243,69 @@ export async function captureSimulatorScreenshotWithFallback( // A runner that reports nothing measured nothing: the probe stays the pre-panel answer rather // than borrowing a scale from a panel nobody captured (#2728). await deps.normalizeDensity(device, outPath, options.pixelDensity, captured?.pixelsPerPoint); - return captureFacts; + return await readCaptureFacts(); }; const holdStatusBar = options.normalizeStatusBar === true ? deps.withNormalizedStatusBar : deps.withStatusBarHeld; - return await holdStatusBar(device, captureWithFallback, options.runnerOptions?.signal); + try { + return await holdStatusBar(device, captureWithFallback, options.runnerOptions?.signal); + } finally { + cancelRotationRead(); + } +} + +type CaptureTargetPlan = { + display: AppleDeviceDisplay | undefined; + /** The facts to report once the image is written. */ + readCaptureFacts: () => Promise; + /** Stops a rotation read that runs beside the capture. */ + cancelRotationRead: () => void; +}; + +/** + * Decides which display the capture names and how its rotation is read. + * + * A foldable lights one panel at a time, so a multi-panel capture must name the panel the system + * is currently showing rather than accept simctl's implicit default. That panel changes when the + * device folds, so a device not known to be single-panel reads its panel table before every + * capture. A known single-panel capture never names a display: its read only supplies the + * rotation, so it runs beside the capture, and a read still running + * `IOS_SIMULATOR_SCREENSHOT_ROTATION_GRACE_MS` after the image is written is aborted and the + * capture reports no rotation. + */ +async function planCaptureTarget( + device: DeviceInfo, + deps: SimulatorScreenshotFlowDeps, + signal: AbortSignal | undefined, +): Promise { + if (!deps.isKnownSinglePanel(device)) { + const { display, displayRotation } = await deps.resolveCaptureTarget(device); + const facts: ScreenshotCaptureFacts = displayRotation ? { displayRotation } : {}; + return { display, readCaptureFacts: async () => facts, cancelRotationRead: () => {} }; + } + const probeController = new AbortController(); + const rotation = deps + .resolveCaptureTarget(device, { + timeoutMs: IOS_SIMULATOR_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS, + signal: signal ? AbortSignal.any([signal, probeController.signal]) : probeController.signal, + }) + .then( + (target) => target.displayRotation, + () => undefined, + ); + return { + display: undefined, + readCaptureFacts: async () => { + const displayRotation = await settledWithin( + rotation, + IOS_SIMULATOR_SCREENSHOT_ROTATION_GRACE_MS, + ); + probeController.abort(); + return displayRotation ? { displayRotation } : {}; + }, + cancelRotationRead: () => probeController.abort(), + }; } export async function captureSimulatorScreenshotWithRetry( diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index a6ec1d45b5..0c27a5688a 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -11,6 +11,7 @@ import { } from '../core/config.ts'; import { establishedLitPanel, + knownAppleMultiScreen, panelContentRotation, queryAppleDisplayInventory, type AppleDeviceDisplay, @@ -26,9 +27,6 @@ const UPSIDE_DOWN_UNSUPPORTED_HINT = const ORIENTATION_UNVERIFIED_HINT = 'The lit panel did not show the requested orientation, or the simulator never reported a single lit panel; the app in front may not support that orientation. Take a screenshot to see the screen, then retry.'; -/** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ -const multiPanelByDevice = new Map(); - // A device whose panel count was never read routes as single-panel. A full-budget read that failed // for it is not repeated for a few seconds, because every surface command would pay it again. const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); @@ -37,12 +35,9 @@ const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); let panelTableReads = 0; const latestResolvedRead = new Map(); -function recordPanelTable( - device: DeviceInfo, - inventory: AppleDisplayInventory, - read: number, -): void { - multiPanelByDevice.set(device.id, inventory.multiScreen); +// Resolved panel counts are hardware facts that the panel table read keeps per daemon +// (knownAppleMultiScreen), so each simulator is probed once. +function recordResolvedRead(device: DeviceInfo, read: number): void { unansweredPanelTables.delete(device.id); latestResolvedRead.set(device.id, Math.max(read, latestResolvedRead.get(device.id) ?? 0)); } @@ -57,14 +52,14 @@ export async function isAppleFoldableSimulator( signal: AbortSignal | undefined, ): Promise { if (!appleFoldableFacts(device).setFoldPose.available) return false; - const known = multiPanelByDevice.get(device.id); + const known = knownAppleMultiScreen(device); if (known !== undefined) return known; const read = ++panelTableReads; const inventory = await queryAppleDisplayInventory(device, { signal, }); if (inventory.unresolved) return false; - recordPanelTable(device, inventory, read); + recordResolvedRead(device, read); return inventory.multiScreen; } @@ -84,7 +79,7 @@ export async function readFoldablePanelTable( options: { signal?: AbortSignal; timeoutMs: number }, ): Promise { if (!appleFoldableFacts(device).setFoldPose.available) return { kind: 'single-panel' }; - const known = multiPanelByDevice.get(device.id); + const known = knownAppleMultiScreen(device); if (known === false) return { kind: 'single-panel' }; if (known === undefined && unansweredPanelTables.get(device.id)) return { kind: 'single-panel' }; const read = ++panelTableReads; @@ -98,7 +93,7 @@ export async function readFoldablePanelTable( } return { kind: 'single-panel' }; } - recordPanelTable(device, inventory, read); + recordResolvedRead(device, read); return inventory.multiScreen ? { kind: 'multi-panel', inventory } : { kind: 'single-panel' }; } From 38ae500cde5713ed45f264c06dc4b15a0df6a707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 18:01:40 +0100 Subject: [PATCH 2/6] fix(apple): report a stale single-panel count and amend ADR 0025 A read beside a single-panel capture that resolves a named panel now emits apple_display_single_panel_count_stale. ADR 0025 records that only the panel count is kept and that a single-panel table read runs beside the capture. The runner-fallback inventory tests pin the sequential path explicitly. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0025-foldable-apple-panels.md | 19 +++++++- .../core/__tests__/display-inventory.test.ts | 3 ++ .../screenshot-rotation-read.test.ts | 40 +++++++++++++++- .../platform-apple/src/core/screenshot.ts | 46 ++++++++++++------- 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index e240779db2..7093c7b89f 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -27,7 +27,7 @@ Only inference answers the second question. | More than one integrated panel | Pass `--display=` on every `simctl io screenshot`. Never use the implicit default. | | Zero or multiple lit panels with a `primary` panel | Name the `primary` panel. Emit `apple_display_capture_ambiguous`. Report pose as `unknown`. | | No sole lit panel and no `primary` panel | Refuse with `COMMAND_FAILED` and `details.reason: capture-panel-unidentified`. Never choose a panel by array order. | -| One integrated panel | Preserve pre-panel behavior exactly: no display flag, no pose, unchanged scale probe. | +| One integrated panel | Preserve pre-panel behavior exactly: no display flag, no pose, unchanged scale probe. Once a resolved table shows one panel, read the table beside the capture for `displayRotation` only. | | Density normalization | Use the captured panel's `pointScale`. For runner fallback, use that capture's reported pixels-per-point. | | Pose report | From panel power alone, report only `closed`, `fully-open`, or `unknown`, never a narrower pose. `fold` reports an exact pose only after hinge readings verify it. | | Pose request | `agent-device fold ` sends private HID inside the simulator. Then it reads the hinge angle back inside the simulator until the angle agrees. For `half-open`, two consecutive readings must agree. If the readings never agree, refuse the pose. | @@ -101,7 +101,7 @@ It is the only reader of panel tables, and it selects the source. It leaves exact pose to the app under test through `UIHinge.status`. If a caller needs pose, it asks the operator and takes another snapshot. - **No cache.** - The probe executes per capture. + The probe executes per capture. Only the resolved panel count is kept (see the 2026-11-16 amendment). An operator changes the lit panel through a fold or open action. Cached inventory could resume capture of the dark panel and recreate the failure. `IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS` limits the probe to 5s, below the subsequent 20s screenshot deadline. @@ -755,3 +755,18 @@ Consequences: - `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` must equal that frame. Snapshot points are translated by that frame's origin, as for a placed iPad window, and a synthesized turn is about the lit panel's screen in the app's interface orientation, not the window's shape: an app beside the inner panel's 80 pt status strip still maps into the whole screen. `SYNTHESIZED_RECORD displayID=` must equal `lit=`. - A recording keeps the panel that its `recordStart` named. The runner cannot read panel power, so it does not notice a fold during the recording: SpringBoard keeps windows on the panel that went dark, and later frames show that dark panel. Start a new recording after a fold. - Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. + +## Amendment: a single-panel table read runs beside the capture (2026-11-16) + +This amendment narrows the capture rule "first resolve the panel table" to devices not known to be single-panel. +The display inventory keeps each device's panel count from its latest resolved table for the daemon's lifetime; an unresolved read does not change it. +When that count is one, the capture names no display, so the table read only supplies `displayRotation`. +That read then starts beside `simctl io screenshot` with a 2s budget, in both simulator sets. +A read still running 250ms after the image is written is aborted, and the capture reports no rotation, as for an unresolved table. +A multi-panel device or a device with no resolved count keeps the read before the capture, so a fold still names the lit panel. +If a read beside a capture resolves to a named panel, the capture emits `apple_display_single_panel_count_stale`, and the new count routes the next capture. + +Rejected alternatives: + +- **A cached panel table: rejected.** The lit panel changes on a fold, and rotation changes without an agent-device command. +- **Waiting for the read without a bound: rejected.** Rotation is optional metadata and must not delay the image. diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 22ffefffbc..33bb6791f8 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -670,6 +670,7 @@ describe('captureSimulatorScreenshotWithFallback runner fallback', () => { pixelDensity: 2, deps: { ensureBooted: async () => {}, + isKnownSinglePanel: () => false, resolveCaptureTarget: async () => ({ display: litPanel }), captureWithRetry: async () => { throw new Error('simctl screenshot failed'); @@ -702,6 +703,7 @@ describe('captureSimulatorScreenshotWithFallback runner fallback', () => { pixelDensity: 2, deps: { ensureBooted: async () => {}, + isKnownSinglePanel: () => false, resolveCaptureTarget: async () => ({ display: outerPanel }), captureWithRetry: async () => { throw new Error('simctl screenshot failed'); @@ -729,6 +731,7 @@ describe('captureSimulatorScreenshotWithFallback runner fallback', () => { pixelDensity: 2, deps: { ensureBooted: async () => {}, + isKnownSinglePanel: () => false, resolveCaptureTarget: async () => ({ display: outerPanel }), captureWithRetry: async () => {}, normalizeDensity: async (_device, _path, _density, sourcePixelDensity) => { diff --git a/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts index 1a5976cce0..f295faf476 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts @@ -1,6 +1,13 @@ -import { test } from 'vitest'; +import { test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { bindTestCommandHost } from '@agent-device/test-kit'; + +vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, emitDiagnostic: vi.fn() }; +}); + +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { captureSimulatorScreenshotWithFallback } from '../screenshot.ts'; import { IOS_TEST_SIMULATOR } from './apple-core-stub-helpers.ts'; @@ -103,3 +110,34 @@ test('a device not known to be single-panel reads its panel table before the cap }); assert.deepEqual(events, ['probe', 'capture LCD-1']); }); + +test('a read beside the capture that names a panel reports the stale single-panel count', async () => { + const panel = { + name: 'LCD', + displayId: 1, + power: 'lit', + primary: true, + widthPx: 1398, + heightPx: 2034, + pointScale: 3, + currentOrientation: 'rot0', + integrated: true, + } as const; + const facts = await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { + env: process.env, + skipIosSimulatorBootCheck: true, + deps: { + ensureBooted: async () => {}, + normalizeDensity: async () => {}, + isKnownSinglePanel: () => true, + resolveCaptureTarget: async () => ({ display: panel }), + captureWithRetry: async () => {}, + }, + }); + assert.deepEqual(facts, {}); + assert.ok( + vi + .mocked(emitDiagnostic) + .mock.calls.some(([event]) => event.phase === 'apple_display_single_panel_count_stale'), + ); +}); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index 36affe249d..24af8ff548 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -264,15 +264,10 @@ type CaptureTargetPlan = { }; /** - * Decides which display the capture names and how its rotation is read. - * - * A foldable lights one panel at a time, so a multi-panel capture must name the panel the system - * is currently showing rather than accept simctl's implicit default. That panel changes when the - * device folds, so a device not known to be single-panel reads its panel table before every - * capture. A known single-panel capture never names a display: its read only supplies the - * rotation, so it runs beside the capture, and a read still running - * `IOS_SIMULATOR_SCREENSHOT_ROTATION_GRACE_MS` after the image is written is aborted and the - * capture reports no rotation. + * Decides which display the capture names and how its rotation is read. A foldable's lit panel + * changes when it folds, so a device not known to be single-panel reads its panel table before + * every capture. A known single-panel capture never names a display, so its read only supplies + * the rotation and runs beside the capture. */ async function planCaptureTarget( device: DeviceInfo, @@ -291,23 +286,42 @@ async function planCaptureTarget( signal: signal ? AbortSignal.any([signal, probeController.signal]) : probeController.signal, }) .then( - (target) => target.displayRotation, + (target) => { + if (target.display) emitSinglePanelCountStaleDiagnostic(device, target.display); + return target.displayRotation; + }, () => undefined, ); return { display: undefined, readCaptureFacts: async () => { - const displayRotation = await settledWithin( - rotation, - IOS_SIMULATOR_SCREENSHOT_ROTATION_GRACE_MS, - ); - probeController.abort(); - return displayRotation ? { displayRotation } : {}; + try { + const displayRotation = await settledWithin( + rotation, + IOS_SIMULATOR_SCREENSHOT_ROTATION_GRACE_MS, + ); + return displayRotation ? { displayRotation } : {}; + } finally { + probeController.abort(); + } }, cancelRotationRead: () => probeController.abort(), }; } +/** The read resolved a named panel, so this capture went out without the display it needed. */ +function emitSinglePanelCountStaleDiagnostic(device: DeviceInfo, display: AppleDeviceDisplay) { + emitDiagnostic({ + level: 'warn', + phase: 'apple_display_single_panel_count_stale', + data: { + deviceId: device.id, + display: display.name, + hint: 'The device was known to have one panel, but its panel table now names a panel to capture. Retake the screenshot; the next capture names that panel.', + }, + }); +} + export async function captureSimulatorScreenshotWithRetry( device: DeviceInfo, outPath: string, From bcb1edb32c250a9c35286e9faa17921120014436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 18:18:38 +0100 Subject: [PATCH 3/6] test(apple): pin the single-panel rotation read budgets; clarify the verdict cache The rotation read test now asserts the 2 s budget and drives the 250 ms grace with fake timers. ADR 0025 states that a stale single-panel verdict sends one capture without --display, and that the resolved read replaces the verdict so the next capture names the lit panel. The cache is described as a multi-panel verdict, not a count. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0025-foldable-apple-panels.md | 15 ++--- .../screenshot-rotation-read.test.ts | 56 +++++++++++++------ .../src/core/display-inventory.ts | 8 +-- .../src/foldable/orientation.ts | 2 +- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 7093c7b89f..60ef0770d2 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -100,8 +100,8 @@ It is the only reader of panel tables, and it selects the source. The inventory reports measured `primary`, panel power, geometry, and orientation. It leaves exact pose to the app under test through `UIHinge.status`. If a caller needs pose, it asks the operator and takes another snapshot. -- **No cache.** - The probe executes per capture. Only the resolved panel count is kept (see the 2026-11-16 amendment). +- **No cached lit-panel inventory.** + The probe executes per capture. The daemon caches only each device's multi-panel verdict, a hardware fact (see the 2026-11-16 amendment). An operator changes the lit panel through a fold or open action. Cached inventory could resume capture of the dark panel and recreate the failure. `IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS` limits the probe to 5s, below the subsequent 20s screenshot deadline. @@ -758,13 +758,14 @@ Consequences: ## Amendment: a single-panel table read runs beside the capture (2026-11-16) -This amendment narrows the capture rule "first resolve the panel table" to devices not known to be single-panel. -The display inventory keeps each device's panel count from its latest resolved table for the daemon's lifetime; an unresolved read does not change it. -When that count is one, the capture names no display, so the table read only supplies `displayRotation`. +This amendment narrows the capture rule "first resolve the panel table" to devices known to be single-panel. +The display inventory caches each device's multi-panel verdict from its latest resolved table for the daemon's lifetime; an unresolved read does not change it. +When the verdict is single-panel, the capture names no display, so the table read only supplies `displayRotation`. That read then starts beside `simctl io screenshot` with a 2s budget, in both simulator sets. A read still running 250ms after the image is written is aborted, and the capture reports no rotation, as for an unresolved table. -A multi-panel device or a device with no resolved count keeps the read before the capture, so a fold still names the lit panel. -If a read beside a capture resolves to a named panel, the capture emits `apple_display_single_panel_count_stale`, and the new count routes the next capture. +A multi-panel device or a device with no resolved verdict keeps the read before the capture, so a fold still names the lit panel. +If a read beside a capture resolves to a multi-panel table, that capture has already gone out without `--display`. It takes simctl's implicit default and can be the dark-panel black image. +The capture then emits `apple_display_single_panel_count_stale` with a retake hint, and the resolved read replaces the verdict, so the next capture reads the table before it captures and names the lit panel. Rejected alternatives: diff --git a/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts index f295faf476..0b1a28e536 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts @@ -42,24 +42,46 @@ test('a known single-panel capture reads its rotation beside the image', async ( assert.deepEqual(facts, { displayRotation: 'landscape-left' }); }); -test('a known single-panel capture reports no rotation when its read misses the grace', async () => { - let probeSignal: AbortSignal | undefined; - const facts = await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { - env: process.env, - skipIosSimulatorBootCheck: true, - deps: { - ensureBooted: async () => {}, - normalizeDensity: async () => {}, - isKnownSinglePanel: () => true, - resolveCaptureTarget: async (_device, options) => { - probeSignal = options?.signal; - return await new Promise(() => {}); +test('a known single-panel read gets a 2 s budget and 250 ms after the image, then is aborted', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + let probeOptions: { timeoutMs?: number; signal?: AbortSignal } | undefined; + let captured = false; + let facts: object | undefined; + const capture = captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { + env: process.env, + skipIosSimulatorBootCheck: true, + deps: { + ensureBooted: async () => {}, + normalizeDensity: async () => {}, + isKnownSinglePanel: () => true, + resolveCaptureTarget: async (_device, options) => { + probeOptions = options; + return await new Promise(() => {}); + }, + captureWithRetry: async () => { + captured = true; + }, }, - captureWithRetry: async () => {}, - }, - }); - assert.deepEqual(facts, {}); - assert.equal(probeSignal?.aborted, true, 'the late read is aborted'); + }); + void capture.then((result) => { + facts = result; + }); + + await vi.advanceTimersByTimeAsync(249); + assert.equal(captured, true); + assert.equal(probeOptions?.timeoutMs, 2_000); + assert.equal(facts, undefined, 'the capture waits out the grace'); + assert.equal(probeOptions?.signal?.aborted, false); + + await vi.advanceTimersByTimeAsync(1); + assert.deepEqual(facts, {}, 'the capture reports no rotation once the grace ends'); + assert.equal(probeOptions?.signal?.aborted, true, 'the late read is aborted'); + } finally { + // A failed assertion must not leave the capture holding the simulator's status bar lock. + await vi.runAllTimersAsync(); + vi.useRealTimers(); + } }); test('a known single-panel capture reports no rotation when its read fails', async () => { diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 1b477420e0..a5c49b43ad 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -100,9 +100,9 @@ const DISPLAYS_UNSUPPORTED_HINT = "This Xcode/CoreDevice toolchain does not report 'devicectl device info displays'. Update Xcode to a version that ships the display-information feature before targeting multi-display Apple devices."; /** - * Each device's integrated panel count, from its latest resolved panel table. The count is a - * hardware fact, so it is kept for the daemon's lifetime; the table itself is not (see - * {@link queryAppleDisplayInventory}). + * Each device's multi-panel verdict: whether its latest resolved panel table listed more than one + * integrated panel. The verdict is a hardware fact, so it is kept for the daemon's lifetime; the + * table itself is not (see {@link queryAppleDisplayInventory}). */ const multiScreenByDevice = new Map(); @@ -120,7 +120,7 @@ export function knownAppleMultiScreen(device: DeviceInfo): boolean | undefined { * * Deliberately uncached: which panel is lit is exactly what changes when an * operator folds or opens the device, and a cached answer would resume capturing - * the dark panel. Only the panel count it resolves is kept ({@link knownAppleMultiScreen}). + * the dark panel. Only the multi-panel verdict it resolves is kept ({@link knownAppleMultiScreen}). * * A failure to ask is reported as an unresolved inventory instead of an error: * every caller has a correct pre-inventory path, and a missing host feature must diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 0c27a5688a..0fc70e3d4a 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -35,7 +35,7 @@ const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); let panelTableReads = 0; const latestResolvedRead = new Map(); -// Resolved panel counts are hardware facts that the panel table read keeps per daemon +// A resolved multi-panel verdict is a hardware fact that the panel table read keeps per daemon // (knownAppleMultiScreen), so each simulator is probed once. function recordResolvedRead(device: DeviceInfo, read: number): void { unansweredPanelTables.delete(device.id); From 78f9971064c498bc17bb4286e0fef6db07a43a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 18:20:27 +0100 Subject: [PATCH 4/6] fix(apple): name a canceled panel table read instead of suppressing its diagnostic Fold and orientation hints point at apple_display_inventory_unresolved, so a caller-canceled read must still emit it. The diagnostic fires again for every unresolved read; a read whose caller aborted it reports reason probe-canceled instead of probe-failed, without the toolchain hint. Co-Authored-By: Claude Opus 5.5 --- .../src/core/__tests__/display-inventory.test.ts | 15 ++++++++------- .../platform-apple/src/core/display-inventory.ts | 6 +++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 33bb6791f8..1ad9e25e15 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -597,7 +597,8 @@ describe('knownAppleMultiScreen', () => { assert.equal(knownAppleMultiScreen(device), true); }); - test('a read its caller aborted reports no unresolved-table diagnostic', async () => { + test('a read its caller aborted reports the cancellation, not a toolchain failure', async () => { + // Callers' hints (fold, orientation) point at this diagnostic, so it must fire and say why. const device = { ...IOS_TEST_SIMULATOR, id: `${IOS_TEST_SIMULATOR.id}-aborted-read` }; const controller = new AbortController(); mockRunXcrun.mockImplementationOnce(async () => { @@ -606,12 +607,12 @@ describe('knownAppleMultiScreen', () => { }); const inventory = await queryAppleDisplayInventory(device, { signal: controller.signal }); assert.equal(inventory.unresolved, true); - assert.equal( - mockEmitDiagnostic.mock.calls.some( - ([event]) => event.phase === 'apple_display_inventory_unresolved', - ), - false, - ); + const unresolved = mockEmitDiagnostic.mock.calls + .map(([event]) => event) + .filter((event) => event.phase === 'apple_display_inventory_unresolved'); + assert.equal(unresolved.length, 1); + assert.equal(unresolved[0]?.data?.reason, 'probe-canceled'); + assert.equal(unresolved[0]?.data?.hint, undefined); assert.equal(knownAppleMultiScreen(device), undefined); }); }); diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index a5c49b43ad..571d4d17f9 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -135,8 +135,7 @@ export async function queryAppleDisplayInventory( ? await readGuestPanelTable(device, options) : await readCoreDevicePanelTable(device, options); if (!table.ok) { - // A caller that aborted the read already knows it did not answer. - if (!options.signal?.aborted) emitUnresolvedDiagnostic(device, table.reason, table.detail); + emitUnresolvedDiagnostic(device, table.reason, table.detail); return unresolvedInventory(); } const inventory = inventoryFromPanelTable(device, table.payload); @@ -163,7 +162,8 @@ async function readCoreDevicePanelTable( return { ok: false, reason: outcome.reason, detail: outcome.result.stderr.trim() }; } catch (error) { // exec classifies its own kills, so a probe that never answered is told apart - // from a toolchain that refused the subcommand. + // from a toolchain that refused the subcommand, and a caller that stopped the read is named. + if (options.signal?.aborted) return probeFailure(error, 'probe-canceled'); return probeFailure(error, isCommandTimeoutError(error) ? 'probe-timed-out' : 'probe-failed'); } } From 5cd98288d91d9f463ec14a04b8cfbbb211c479a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 18:23:20 +0100 Subject: [PATCH 5/6] chore(gates): declare the capture-kit capture-fact-grace export R11 pins capture-kit's export list; the shared settledWithin helper that Android and Apple screenshots use is a new deliberate subpath. Co-Authored-By: Claude Opus 5.5 --- scripts/layering/package-boundaries.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index cee6d6dc56..409252455a 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -483,6 +483,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/audio-probe-recovery', '@agent-device/capture-kit/audio-probe-resource-store', '@agent-device/capture-kit/audio-probe-session-resource', + '@agent-device/capture-kit/capture-fact-grace', '@agent-device/capture-kit/durable-capture', '@agent-device/capture-kit/durable-capture-admission-ledger', '@agent-device/capture-kit/durable-capture-resource', From 8ee29b532ff065f079d771699b1a1d49b30b3968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 18:32:05 +0100 Subject: [PATCH 6/6] fix(apple): report a stale single-panel verdict when the panel table names no panel A read beside a single-panel capture that refuses with capture-panel-unidentified resolved a multi-panel table too, so that capture also went out without --display. It now emits apple_display_single_panel_count_stale. The refused read already records the multi-panel verdict, so the next capture reads the table first; a test pins that. Other read failures stay silent. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0025-foldable-apple-panels.md | 2 +- .../core/__tests__/display-inventory.test.ts | 27 +++++++++++++ .../screenshot-rotation-read.test.ts | 38 +++++++++++++++++++ .../platform-apple/src/core/screenshot.ts | 21 +++++++--- 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 60ef0770d2..f7f9612caa 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -764,7 +764,7 @@ When the verdict is single-panel, the capture names no display, so the table rea That read then starts beside `simctl io screenshot` with a 2s budget, in both simulator sets. A read still running 250ms after the image is written is aborted, and the capture reports no rotation, as for an unresolved table. A multi-panel device or a device with no resolved verdict keeps the read before the capture, so a fold still names the lit panel. -If a read beside a capture resolves to a multi-panel table, that capture has already gone out without `--display`. It takes simctl's implicit default and can be the dark-panel black image. +If a read beside a capture resolves to a multi-panel table, including one that refuses with `capture-panel-unidentified`, that capture has already gone out without `--display`. It takes simctl's implicit default and can be the dark-panel black image. The capture then emits `apple_display_single_panel_count_stale` with a retake hint, and the resolved read replaces the verdict, so the next capture reads the table before it captures and names the lit panel. Rejected alternatives: diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 1ad9e25e15..f0aae79352 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -597,6 +597,33 @@ describe('knownAppleMultiScreen', () => { assert.equal(knownAppleMultiScreen(device), true); }); + test('a refused multi-panel table still records the multi-panel verdict', async () => { + // A single-panel capture's read beside it relies on this to send the next capture through the + // table read, even when the table names no panel to capture. + const device = { ...IOS_TEST_SIMULATOR, id: `${IOS_TEST_SIMULATOR.id}-unidentified` }; + mockReadHostTextFile.mockResolvedValueOnce(JSON.stringify(IPHONE_17_SINGLE_PANEL)); + await queryAppleDisplayInventory(device, {}); + assert.equal(knownAppleMultiScreen(device), false); + + mockReadHostTextFile.mockResolvedValueOnce( + JSON.stringify({ + result: { + displays: IPHONE_DUO_CLOSED.result.displays.map((entry) => ({ + ...entry, + primary: false, + backlightState: 'activeOn', + })), + }, + }), + ); + await assert.rejects( + resolveAppleCaptureTarget(device), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'capture-panel-unidentified', + ); + assert.equal(knownAppleMultiScreen(device), true); + }); + test('a read its caller aborted reports the cancellation, not a toolchain failure', async () => { // Callers' hints (fold, orientation) point at this diagnostic, so it must fire and say why. const device = { ...IOS_TEST_SIMULATOR, id: `${IOS_TEST_SIMULATOR.id}-aborted-read` }; diff --git a/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts index 0b1a28e536..359127b623 100644 --- a/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts +++ b/packages/platform-apple/src/core/__tests__/screenshot-rotation-read.test.ts @@ -8,6 +8,7 @@ vi.mock('@agent-device/host-kit/diagnostics', async (importOriginal) => { }); import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { AppError } from '@agent-device/kernel/errors'; import { captureSimulatorScreenshotWithFallback } from '../screenshot.ts'; import { IOS_TEST_SIMULATOR } from './apple-core-stub-helpers.ts'; @@ -163,3 +164,40 @@ test('a read beside the capture that names a panel reports the stale single-pane .mock.calls.some(([event]) => event.phase === 'apple_display_single_panel_count_stale'), ); }); + +test('a read beside the capture that refuses an unidentified panel reports the stale count', async () => { + vi.mocked(emitDiagnostic).mockClear(); + const captureStale = async (error: Error) => { + vi.mocked(emitDiagnostic).mockClear(); + const facts = await captureSimulatorScreenshotWithFallback(IOS_TEST_SIMULATOR, '/tmp/s.png', { + env: process.env, + skipIosSimulatorBootCheck: true, + deps: { + ensureBooted: async () => {}, + normalizeDensity: async () => {}, + isKnownSinglePanel: () => true, + resolveCaptureTarget: async () => { + throw error; + }, + captureWithRetry: async () => {}, + }, + }); + assert.deepEqual(facts, {}); + return vi + .mocked(emitDiagnostic) + .mock.calls.some(([event]) => event.phase === 'apple_display_single_panel_count_stale'); + }; + + assert.equal( + await captureStale( + new AppError('COMMAND_FAILED', 'which panel?', { reason: 'capture-panel-unidentified' }), + ), + true, + ); + // Only the typed refusal says the table listed several panels; other failures say nothing. + assert.equal( + await captureStale(new AppError('COMMAND_FAILED', 'capture-panel-unidentified')), + false, + ); + assert.equal(await captureStale(new Error('devicectl failed')), false); +}); diff --git a/packages/platform-apple/src/core/screenshot.ts b/packages/platform-apple/src/core/screenshot.ts index 24af8ff548..270c6e2b09 100644 --- a/packages/platform-apple/src/core/screenshot.ts +++ b/packages/platform-apple/src/core/screenshot.ts @@ -287,10 +287,16 @@ async function planCaptureTarget( }) .then( (target) => { - if (target.display) emitSinglePanelCountStaleDiagnostic(device, target.display); + if (target.display) emitSinglePanelCountStaleDiagnostic(device, target.display.name); return target.displayRotation; }, - () => undefined, + (error: unknown) => { + // A multi-panel table that names no panel refuses; this capture still went out unnamed. + if (error instanceof AppError && error.details?.reason === 'capture-panel-unidentified') { + emitSinglePanelCountStaleDiagnostic(device, undefined); + } + return undefined; + }, ); return { display: undefined, @@ -309,15 +315,18 @@ async function planCaptureTarget( }; } -/** The read resolved a named panel, so this capture went out without the display it needed. */ -function emitSinglePanelCountStaleDiagnostic(device: DeviceInfo, display: AppleDeviceDisplay) { +/** + * The read resolved a multi-panel table, so this capture went out without the display it needed. + * That read already recorded the multi-panel verdict, so the next capture reads the table first. + */ +function emitSinglePanelCountStaleDiagnostic(device: DeviceInfo, display: string | undefined) { emitDiagnostic({ level: 'warn', phase: 'apple_display_single_panel_count_stale', data: { deviceId: device.id, - display: display.name, - hint: 'The device was known to have one panel, but its panel table now names a panel to capture. Retake the screenshot; the next capture names that panel.', + ...(display ? { display } : {}), + hint: 'The device was known to have one panel, but its panel table now lists several, so this capture named no panel and may show a dark one. Retake the screenshot; the next capture reads the panel table first.', }, }); }