From 9fe6ad3c8b6e107b8aa7e077879f3206fe3862cd Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 14:35:29 +0000 Subject: [PATCH 01/16] fix(ios): rotate a foldable in a scoped simulator set Dispatch through the fold helper and confirm from the lit panel's currentOrientation, choosing each picker value for the turn still needed because the guest applies picker events relatively. Map portrait-upside-down to the picker's pud wire spelling in the fold helper. --- .changeset/foldable-scoped-rotation.md | 5 + apple/fold-helper/Fold.m | 12 +- packages/command-surface/src/system/index.ts | 2 +- packages/platform-apple/src/core/config.ts | 8 + .../src/foldable/fold-helper.test.ts | 15 ++ .../src/foldable/fold-helper.ts | 14 +- .../src/foldable/orientation.test.ts | 184 +++++++++++++++++- .../src/foldable/orientation.ts | 148 ++++++++++++-- .../src/foldable/picker-delta.test.ts | 138 +++++++++++++ .../src/foldable/picker-delta.ts | 110 +++++++++++ website/docs/docs/commands.md | 2 +- 11 files changed, 609 insertions(+), 29 deletions(-) create mode 100644 .changeset/foldable-scoped-rotation.md create mode 100644 packages/platform-apple/src/foldable/picker-delta.test.ts create mode 100644 packages/platform-apple/src/foldable/picker-delta.ts diff --git a/.changeset/foldable-scoped-rotation.md b/.changeset/foldable-scoped-rotation.md new file mode 100644 index 0000000000..fb3f301149 --- /dev/null +++ b/.changeset/foldable-scoped-rotation.md @@ -0,0 +1,5 @@ +--- +'agent-device': patch +--- + +Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and confirmed from the panel table's lit-panel `currentOrientation`. Because the guest applies each orientation-picker event relative to the previous picker value (the lit panel reads picker + a fixed per-panel offset, +90 degrees on the inner panel), each dispatch is chosen for the turn still needed and re-read until the panel matches, so the first command lands. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the delta dispatch needs to reach every orientation; a user target of `portrait-upside-down` stays refused. diff --git a/apple/fold-helper/Fold.m b/apple/fold-helper/Fold.m index cc2c21ca47..60ba17772c 100644 --- a/apple/fold-helper/Fold.m +++ b/apple/fold-helper/Fold.m @@ -106,14 +106,18 @@ static int runSet(NSString *argument) { return dispatchEvents(^BOOL(CFTypeRef client) { return runFrames(client, frames); }); } -// The orientation control has no upside-down value. +// The upside-down picker value is spelled "pud" on the wire; "portrait-upside-down" is +// ignored by the guest. It matters because each picker event rotates the display by +// (new - previous) picker value: without a 180 value, one of the four display orientations +// can never be landed (measured on iPhone Duo). static int runRotate(NSString *orientation) { - if (![@[@"portrait", @"landscape-left", @"landscape-right"] containsObject:orientation]) { - fprintf(stderr, "Expected orientation portrait, landscape-left or landscape-right\n"); + NSString *value = [orientation isEqualToString:@"portrait-upside-down"] ? @"pud" : orientation; + if (![@[@"portrait", @"landscape-left", @"landscape-right", @"pud"] containsObject:value]) { + fprintf(stderr, "Expected orientation portrait, landscape-left, landscape-right or portrait-upside-down\n"); return 2; } return dispatchEvents(^BOOL(CFTypeRef client) { - return sendControl(client, @"orientation-picker-control", @"enum", orientation); + return sendControl(client, @"orientation-picker-control", @"enum", value); }); } diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 8c9ff9ecb4..fd7a9eb540 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -290,7 +290,7 @@ const backCommandDescription = const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = - 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down, and refuses every rotation in a scoped simulator set.'; + 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down; in a scoped simulator set it rotates through the fold helper and confirms the rotation from the panel table.'; const foldGuidance = { hingeEvent: 'simulator HID hinge event', permissions: 'Device Hub and host Accessibility permission are not required.', diff --git a/packages/platform-apple/src/core/config.ts b/packages/platform-apple/src/core/config.ts index 898dfe8fd2..f8cd79174a 100644 --- a/packages/platform-apple/src/core/config.ts +++ b/packages/platform-apple/src/core/config.ts @@ -63,6 +63,14 @@ export const IOS_FOLD_POSE_STABLE_DEGREES = 0.5; */ export const IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS = 8; export const IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS = 2_000; +/** + * How many read/dispatch rounds a scoped-set foldable rotation gets. The picker delta lands + * in at most two dispatches once the first one calibrates the guest's picker state; the rest + * of the budget covers panel reads that do not answer yet. Verified on iPhone Duo with the + * settle delay below between a dispatch and the panel read that confirms it. + */ +export const IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS = 6; +export const IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS = 600; // CoreSimulator can briefly stall while it services the scale lookup immediately // after a keyboard transition. Keep this bounded below the full capture budget. diff --git a/packages/platform-apple/src/foldable/fold-helper.test.ts b/packages/platform-apple/src/foldable/fold-helper.test.ts index d2afc0a755..6049158393 100644 --- a/packages/platform-apple/src/foldable/fold-helper.test.ts +++ b/packages/platform-apple/src/foldable/fold-helper.test.ts @@ -114,6 +114,21 @@ test('every mode addresses the UDID inside its scoped simulator set', async () = ]); }); +test('rotate dispatches every picker value, upside-down included', async () => { + const { calls } = await withHelper(scopedDuo, { stdout: '', exitCode: 0 }, async (helper) => { + await helper.rotate('portrait'); + await helper.rotate('landscape-left'); + await helper.rotate('landscape-right'); + await helper.rotate('portrait-upside-down'); + }); + expect(calls.map((call) => call.args.slice(-2))).toEqual([ + ['rotate', 'portrait'], + ['rotate', 'landscape-left'], + ['rotate', 'landscape-right'], + ['rotate', 'portrait-upside-down'], + ]); +}); + test('reads one guest hinge sample without sending HID', async () => { const { value, calls } = await withHelper( scopedDuo, diff --git a/packages/platform-apple/src/foldable/fold-helper.ts b/packages/platform-apple/src/foldable/fold-helper.ts index 1891ed319f..2f554a4f58 100644 --- a/packages/platform-apple/src/foldable/fold-helper.ts +++ b/packages/platform-apple/src/foldable/fold-helper.ts @@ -19,9 +19,17 @@ const HINGE_ANGLE_NO_SAMPLE_HINT = const OUTPUT_VERSION = 1; -/** The orientations the guest's device-state control offers; it has no upside-down value. */ +/** The orientations a foldable accepts as a user target; upside-down stays refused there. */ export type FoldableRotation = Exclude; +/** + * Every value the guest's orientation picker accepts, upside-down included: its wire spelling + * is `pud`, which the fold helper maps from `portrait-upside-down`. The picker needs all four + * values because each event turns the lit panel by the value's delta, so reaching one target + * can require dispatching another value. Callers keep user targets to {@link FoldableRotation}. + */ +export type FoldPickerValue = DeviceRotation; + /** The guest panel table in the `devicectl device info displays` payload shape. */ export type SimulatorPanelPayload = { result: { displays: unknown[] } }; @@ -33,8 +41,8 @@ export type SimulatorPanelPayload = { result: { displays: unknown[] } }; export type FoldHelper = Readonly<{ /** Dispatches a preset pose or a keyframe trajectory through simulator HID. */ setPose(pose: FoldPose | readonly FoldKeyframe[], signal?: AbortSignal): Promise; - /** Dispatches a device orientation through the same simulator HID control channel. */ - rotate(orientation: FoldableRotation, signal?: AbortSignal): Promise; + /** Dispatches one picker value through the same simulator HID control channel. */ + rotate(orientation: FoldPickerValue, signal?: AbortSignal): Promise; /** * One fresh CoreMotion `CMAngleManager` subscription, the source `devicectl device motion * hinge-angle` relays. It reports the latest sample once `windowMs` has passed since the first, diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 85420a7236..ecf9e977ab 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -152,18 +152,190 @@ test('a physical device keeps the runner rotation without a CoreDevice probe', a expect(mockRunner).toHaveBeenCalledTimes(1); }); -test('a foldable simulator in a scoped set refuses rotation before any HID dispatch', async () => { +/** + * A scoped-set Duo as measured on the device: the guest does not treat the + * `orientation-picker-control` value as absolute. Each picker event rotates the lit panel by + * (new value - previous picker value), so the lit panel reads `picker + K` for a fixed + * per-panel offset K. Angles follow CADisplay `rotN`: portrait 0, landscape-right 90, + * portrait-upside-down 180, landscape-left 270. The dark panel reports a misleading rotation: + * only the lit panel's `currentOrientation` may confirm a rotation. + */ +const PANEL_ANGLE: Record = { + portrait: 0, + 'landscape-right': 90, + 'portrait-upside-down': 180, + 'landscape-left': 270, +}; + +function fakeScopedDuo( + device: DeviceInfo, + options: { + lit: 'inner' | 'outer'; + picker: DeviceRotation; + offsets: { inner: number; outer: number }; + followsHid?: boolean; + }, +) { + const helpers: string[][] = []; + const devicectl: string[] = []; + let pickerAngle = PANEL_ANGLE[options.picker]; + let litScreenAngle = (pickerAngle + options.offsets[options.lit]) % 360; + const panel = (kind: 'inner' | 'outer') => { + const lit = kind === options.lit; + return { + backlightState: lit ? 'activeOn' : 'off', + currentOrientation: lit ? `rot${litScreenAngle}` : 'rot180', + displayId: kind === 'outer' ? 1 : 3, + name: kind === 'outer' ? 'LCD' : 'LCD-1', + nativeSize: kind === 'outer' ? [1398, 2034] : [2007, 2853], + pointScale: 3, + primary: kind === 'outer', + type: { integrated: {} }, + }; + }; + const provider = createLocalAppleToolProvider({ + runCommand: async (_command, args) => { + const [tool, ...rest] = args; + if (tool !== 'simctl') { + devicectl.push(args.join(' ')); + throw new Error(`unexpected tool call in a scoped set: ${args.join(' ')}`); + } + const spawn = rest.slice(rest.indexOf('spawn')); + expect(spawn.slice(0, 3)).toEqual(['spawn', device.id, '/cache/fold-helper']); + const mode = spawn.slice(3); + helpers.push(mode); + if (mode[0] === 'panels') { + const displays = [panel('outer'), panel('inner')]; + return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; + } + expect(mode[0]).toBe('rotate'); + if (options.followsHid !== false) { + const nextPickerAngle = PANEL_ANGLE[mode[1]! as DeviceRotation]; + litScreenAngle = (litScreenAngle + nextPickerAngle - pickerAngle + 360) % 360; + pickerAngle = nextPickerAngle; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + }); + return { + helpers, + devicectl, + rotations: () => helpers.filter((mode) => mode[0] === 'rotate').map((mode) => mode[1]!), + litScreen: () => `rot${litScreenAngle}`, + rotate: (rotation: DeviceRotation) => + withAppleToolProvider(provider, () => + createAppleInteractor(device, {}, createLiveTestHostEnvironment()).setOrientation(rotation), + ), + }; +} + +/** Runs one scoped rotation under fake timers: the settle delay must not cost real time. */ +async function rotateScoped( + sim: ReturnType, + rotation: DeviceRotation, +): Promise { + vi.useFakeTimers(); + try { + const outcome = sim.rotate(rotation).then( + (value) => value, + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(30_000); + return await outcome; + } finally { + vi.useRealTimers(); + } +} + +test('a foldable simulator in a scoped set rotates on the first command, confirmed by the panel table', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeSimulator(scoped, DUO_DISPLAYS); - await expect(sim.rotate('landscape-left')).rejects.toMatchObject({ - code: 'UNSUPPORTED_OPERATION', - details: { reason: 'unsupported-device-scope' }, + // The inner panel lights at rot90 while the picker sits at portrait (K = +90, measured). + const sim = fakeScopedDuo(scoped, { + lit: 'inner', + picker: 'portrait', + offsets: { inner: 90, outer: 0 }, }); - expect(sim.helpers).toEqual([['panels']]); + expect(sim.litScreen()).toBe('rot90'); + await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + expect(sim.litScreen()).toBe('rot0'); + expect(sim.rotations().length).toBeLessThanOrEqual(2); expect(sim.devicectl).toEqual([]); expect(mockRunner).not.toHaveBeenCalled(); }); +test('a scoped rotation to landscape-left dispatches the upside-down picker value when the delta needs it', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { + lit: 'inner', + picker: 'landscape-left', + offsets: { inner: 90, outer: 0 }, + }); + expect(sim.litScreen()).toBe('rot0'); + await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + expect(sim.litScreen()).toBe('rot270'); + expect(sim.rotations()).toContain('portrait-upside-down'); + expect(sim.devicectl).toEqual([]); +}); + +test('a scoped rotation on the outer panel converges with that panel offset', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { + lit: 'outer', + picker: 'portrait', + offsets: { inner: 90, outer: 270 }, + }); + await expect(rotateScoped(sim, 'landscape-right')).resolves.toEqual({ + orientation: 'landscape-right', + }); + expect(sim.litScreen()).toBe('rot90'); + expect(sim.rotations().length).toBeLessThanOrEqual(2); + expect(sim.devicectl).toEqual([]); +}); + +test('a scoped rotation to the orientation already showing dispatches nothing', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { + lit: 'inner', + picker: 'landscape-left', + offsets: { inner: 90, outer: 0 }, + }); + expect(sim.litScreen()).toBe('rot0'); + await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + expect(sim.rotations()).toEqual([]); +}); + +test('a scoped rotation the panel table never reports is refused as unverified', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { + lit: 'inner', + picker: 'portrait', + offsets: { inner: 90, outer: 0 }, + followsHid: false, + }); + await expect(rotateScoped(sim, 'portrait')).resolves.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-unverified-scoped-set', requestedOrientation: 'portrait' }, + }); + expect(sim.devicectl).toEqual([]); +}); + +test('a foldable simulator in a scoped set still refuses portrait-upside-down as a target', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { + lit: 'inner', + picker: 'portrait', + offsets: { inner: 90, outer: 0 }, + }); + await expect(sim.rotate('portrait-upside-down')).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'unsupported-foldable-orientation' }, + }); + expect(sim.rotations()).toEqual([]); + expect(mockRunner).not.toHaveBeenCalled(); +}); + test('a single-panel simulator in a scoped set keeps the runner rotation', async () => { const sim = fakeSimulator( simulator({ simulatorSetPath: '/tmp/scoped-set' }), diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index a67802bab1..1cd7973ecd 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -8,25 +8,35 @@ import { IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS, IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS, + IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS, + IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS, } from '../core/config.ts'; import { readAppleDeviceOrientation } from '../core/device-orientation.ts'; import { queryAppleDisplayInventory, + type AppleDeviceDisplay, type AppleDisplayInventory, } from '../core/display-inventory.ts'; import { simulatorAddressFor } from '../core/simctl.ts'; -import { createFoldHelper } from './fold-helper.ts'; +import { createFoldHelper, type FoldHelper } from './fold-helper.ts'; +import { + measuredPanelPickerOffset, + nextPickerValue, + panelPickerOffset, + panelRotationForOrientation, + type PanelQuarterTurn, +} from './picker-delta.ts'; import { appleFoldableFacts } from './runtime.ts'; const UPSIDE_DOWN_UNSUPPORTED_HINT = 'A foldable simulator accepts portrait, landscape-left, and landscape-right. Its device-state control has no upside-down orientation.'; -const SCOPED_SET_UNSUPPORTED_HINT = - 'CoreDevice resolves only simulators in the default device set, so the rotation of a foldable simulator in a scoped set cannot be verified. Rotate it in the default device set.'; - const ORIENTATION_UNVERIFIED_HINT = 'The simulator HID command completed, but CoreDevice did not report the new orientation. Read it with xcrun devicectl device orientation get --device , then retry.'; +const SCOPED_ORIENTATION_UNVERIFIED_HINT = + 'Rotation was dispatched through the fold helper, which needs relative-picker compensation in a scoped simulator set where CoreDevice cannot read orientation, and the lit panel did not settle on the requested orientation. Take a screenshot to see the screen, then retry.'; + /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); @@ -101,7 +111,11 @@ export async function readFoldablePanelTable( return inventory.multiScreen ? { kind: 'multi-panel', inventory } : { kind: 'single-panel' }; } -/** Rotates a foldable simulator through guest HID and verifies the rotation through CoreDevice. */ +/** + * Rotates a foldable simulator through guest HID. The default device set verifies the rotation + * through CoreDevice; a scoped set, which CoreDevice cannot resolve, verifies it from the + * fold helper's panel table instead (see {@link setScopedFoldableOrientation}). + */ export async function setAppleFoldableOrientation( device: DeviceInfo, rotation: DeviceRotation, @@ -117,15 +131,7 @@ export async function setAppleFoldableOrientation( }); } if (simulatorAddressFor(device).simulatorSetPath) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - `${device.name} cannot rotate in a scoped simulator set`, - { - deviceId: device.id, - reason: 'unsupported-device-scope', - hint: SCOPED_SET_UNSUPPORTED_HINT, - }, - ); + return await setScopedFoldableOrientation(device, rotation, signal); } await createFoldHelper(device).rotate(rotation, signal); emitDiagnostic({ @@ -154,3 +160,117 @@ export async function setAppleFoldableOrientation( }, ); } + +/** + * Rotates a foldable simulator in a scoped set, where only the fold helper can dispatch and + * only its panel table can confirm. Measured on iPhone Duo, the guest applies each + * `orientation-picker-control` event relatively — the lit panel turns by (new value - + * previous picker value), i.e. it reads picker + a fixed per-panel offset — so one absolute + * dispatch lands a quarter-turn away and looks one command late. This loop reads the lit + * panel, dispatches the picker value {@link nextPickerValue} chooses for the turn still + * needed, and re-reads until the panel table reports the target: at most two dispatches once + * the first one calibrates the guest's picker state. A repeat of the showing orientation is + * a read-only success. The picker state is deliberately not kept between commands: a fold in + * between can light the other panel, and recalibrating costs one dispatch. + */ +async function setScopedFoldableOrientation( + device: DeviceInfo, + rotation: DeviceRotation, + signal: AbortSignal | undefined, +): Promise<{ orientation: DeviceRotation }> { + const helper = createFoldHelper(device); + let pickerValue: DeviceRotation | undefined; + let panelOffset: PanelQuarterTurn | undefined; + let litDisplayId: number | undefined; + let observed: DeviceRotation | undefined; + let rawPanelOrientation: string | undefined; + for (let attempt = 1; attempt <= IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS; attempt += 1) { + signal?.throwIfAborted(); + const lit = await readLitPanel(device, helper, signal); + if (lit) { + // A different panel lit up mid-command: the picker state and offset were calibrated + // against the previous panel, so start that panel's calibration over. + if (litDisplayId !== undefined && lit.panel.displayId !== litDisplayId) { + pickerValue = undefined; + panelOffset = undefined; + } + litDisplayId = lit.panel.displayId; + observed = lit.rotation; + rawPanelOrientation = lit.panel.currentOrientation; + if (observed === rotation) return { orientation: rotation }; + panelOffset = + pickerValue === undefined + ? (panelOffset ?? measuredPanelPickerOffset(lit.panel)) + : panelPickerOffset(observed, pickerValue); + } + const next = nextPickerValue({ + currentRotation: lit?.rotation, + targetRotation: rotation, + panelOffset, + currentPickerValue: pickerValue, + }); + if (next === undefined) { + await settleScopedRotation(); + continue; + } + await helper.rotate(next, signal); + pickerValue = next; + emitDiagnostic({ + level: 'info', + phase: 'apple_foldable_orientation_dispatched', + data: { + deviceId: device.id, + orientation: next, + requestedOrientation: rotation, + scopedSet: true, + attempt, + ...(observed === undefined ? {} : { observedOrientation: observed }), + }, + }); + await settleScopedRotation(); + } + throw new AppError( + 'COMMAND_FAILED', + `${device.name} did not confirm rotation to ${rotation}: the panel table reports ${observed ?? 'no orientation'}`, + { + deviceId: device.id, + requestedOrientation: rotation, + ...(observed === undefined ? {} : { observedOrientation: observed }), + ...(rawPanelOrientation === undefined ? {} : { rawPanelOrientation }), + reason: 'orientation-unverified-scoped-set', + hint: SCOPED_ORIENTATION_UNVERIFIED_HINT, + }, + ); +} + +/** + * The one lit panel and the rotation it shows, from one fresh panel-table read. Anything + * less decided — an unreadable table, no lit panel, an unrecognized `rotN` — is no reading: + * the rotation loop must not confirm or compensate from a panel that is not showing. + */ +async function readLitPanel( + device: DeviceInfo, + helper: FoldHelper, + signal: AbortSignal | undefined, +): Promise<{ panel: AppleDeviceDisplay; rotation: DeviceRotation } | undefined> { + let inventory: AppleDisplayInventory; + try { + inventory = await queryAppleDisplayInventory(device, { signal, foldHelper: helper }); + } catch (error) { + if (signal?.aborted) throw error; + return undefined; + } + if (inventory.unresolved || !inventory.multiScreen) return undefined; + const litPanels = inventory.displays.filter((display) => display.power === 'lit'); + const panel = litPanels.length === 1 ? litPanels[0]! : undefined; + if (!panel) return undefined; + const rotation = panelRotationForOrientation(panel.currentOrientation); + return rotation === undefined ? undefined : { panel, rotation }; +} + +/** The gap a dispatched picker event needs before the panel table shows its effect. */ +async function settleScopedRotation(): Promise { + await new Promise((resolve) => { + setTimeout(resolve, IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS); + }); +} diff --git a/packages/platform-apple/src/foldable/picker-delta.test.ts b/packages/platform-apple/src/foldable/picker-delta.test.ts new file mode 100644 index 0000000000..59473e2029 --- /dev/null +++ b/packages/platform-apple/src/foldable/picker-delta.test.ts @@ -0,0 +1,138 @@ +import { expect, test } from 'vitest'; +import type { DeviceRotation } from '@agent-device/contracts/device'; +import { + measuredPanelPickerOffset, + nextPickerValue, + panelPickerOffset, + panelRotationAngle, + panelRotationForOrientation, + rotationAtPanelAngle, + type PanelQuarterTurn, +} from './picker-delta.ts'; + +const ROTATIONS: readonly DeviceRotation[] = [ + 'portrait', + 'landscape-right', + 'portrait-upside-down', + 'landscape-left', +]; +const OFFSETS: readonly PanelQuarterTurn[] = [0, 90, 180, 270]; + +/** Applies one picker event the way the guest does: the panel turns by (new - previous). */ +function applyPickerEvent( + screen: DeviceRotation, + previousPicker: DeviceRotation, + nextPicker: DeviceRotation, +): DeviceRotation { + return rotationAtPanelAngle( + panelRotationAngle(screen) + + panelRotationAngle(nextPicker) - + panelRotationAngle(previousPicker), + ); +} + +test('panel angles follow CADisplay rotN in both directions', () => { + expect(ROTATIONS.map(panelRotationAngle)).toEqual([0, 90, 180, 270]); + for (const rotation of ROTATIONS) { + expect(rotationAtPanelAngle(panelRotationAngle(rotation))).toBe(rotation); + expect(rotationAtPanelAngle(panelRotationAngle(rotation) + 360)).toBe(rotation); + expect(rotationAtPanelAngle(panelRotationAngle(rotation) - 360)).toBe(rotation); + } + expect(panelRotationForOrientation('rot0')).toBe('portrait'); + expect(panelRotationForOrientation('rot90')).toBe('landscape-right'); + expect(panelRotationForOrientation('rot180')).toBe('portrait-upside-down'); + expect(panelRotationForOrientation('rot270')).toBe('landscape-left'); + expect(panelRotationForOrientation('unknown')).toBeUndefined(); +}); + +test('with the picker known, one dispatch lands every current/target/offset/picker state', () => { + for (const current of ROTATIONS) { + for (const target of ROTATIONS) { + for (const offset of OFFSETS) { + for (const picker of ROTATIONS) { + const next = nextPickerValue({ + currentRotation: current, + targetRotation: target, + panelOffset: offset, + currentPickerValue: picker, + }); + expect(next, `${current} -> ${target}, K=${offset}, picker=${picker}`).toBeDefined(); + expect(applyPickerEvent(current, picker, next!)).toBe(target); + // The offset observed after the dispatch is still the guest's own relation: + // re-deriving it from the landed state must give a consistent quarter turn. + expect(panelPickerOffset(target, next!)).toBe( + ((panelRotationAngle(target) - panelRotationAngle(next!) + 360) % + 360) as PanelQuarterTurn, + ); + } + } + } + } +}); + +test('with only the offset known, one dispatch lands every consistent state', () => { + for (const target of ROTATIONS) { + for (const offset of OFFSETS) { + for (const picker of ROTATIONS) { + // A consistent guest state: the panel reads picker + offset. + const current = rotationAtPanelAngle(panelRotationAngle(picker) + offset); + const next = nextPickerValue({ + currentRotation: current, + targetRotation: target, + panelOffset: offset, + currentPickerValue: undefined, + }); + expect(next).toBe(rotationAtPanelAngle(panelRotationAngle(target) - offset)); + expect(applyPickerEvent(current, picker, next!)).toBe(target); + } + } + } +}); + +test('with neither known, the calibration dispatch plus one delta dispatch lands every state', () => { + for (const target of ROTATIONS) { + for (const offset of OFFSETS) { + for (const picker of ROTATIONS) { + const start = rotationAtPanelAngle(panelRotationAngle(picker) + offset); + const calibration = nextPickerValue({ + currentRotation: start, + targetRotation: target, + panelOffset: undefined, + currentPickerValue: undefined, + }); + expect(calibration).toBe(target); + const afterCalibration = applyPickerEvent(start, picker, calibration!); + if (afterCalibration === target) continue; + const learned = panelPickerOffset(afterCalibration, calibration!); + expect(learned).toBe(offset); + const correction = nextPickerValue({ + currentRotation: afterCalibration, + targetRotation: target, + panelOffset: learned, + currentPickerValue: calibration, + }); + expect(applyPickerEvent(afterCalibration, calibration!, correction!)).toBe(target); + } + } + } +}); + +test('without a current rotation there is no dispatch once the picker is known', () => { + for (const target of ROTATIONS) { + for (const picker of ROTATIONS) { + expect( + nextPickerValue({ + currentRotation: undefined, + targetRotation: target, + panelOffset: 90, + currentPickerValue: picker, + }), + ).toBeUndefined(); + } + } +}); + +test('the measured inner-panel offset is +90 and the outer panel is never guessed', () => { + expect(measuredPanelPickerOffset({ primary: false })).toBe(90); + expect(measuredPanelPickerOffset({ primary: true })).toBeUndefined(); +}); diff --git a/packages/platform-apple/src/foldable/picker-delta.ts b/packages/platform-apple/src/foldable/picker-delta.ts new file mode 100644 index 0000000000..315e21bcf5 --- /dev/null +++ b/packages/platform-apple/src/foldable/picker-delta.ts @@ -0,0 +1,110 @@ +import type { DeviceRotation } from '@agent-device/contracts/device'; + +/** + * Panel-frame quarter turns, following CADisplay `rotN`: portrait 0, landscape-right 90, + * portrait-upside-down 180, landscape-left 270. This frame is a panel-table fact, not the + * Android `Surface.ROTATION_*` frame in `DEVICE_ROTATION_SURFACE_INDEX` (which turns the + * other way), so the two tables must not be shared. + */ +export type PanelQuarterTurn = 0 | 90 | 180 | 270; + +const PANEL_ROTATION_ANGLE: Readonly> = { + portrait: 0, + 'landscape-right': 90, + 'portrait-upside-down': 180, + 'landscape-left': 270, +}; + +const ROTATION_BY_QUARTER: readonly DeviceRotation[] = [ + 'portrait', + 'landscape-right', + 'portrait-upside-down', + 'landscape-left', +]; + +const PANEL_ORIENTATIONS: ReadonlyMap = new Map([ + ['rot0', 'portrait'], + ['rot90', 'landscape-right'], + ['rot180', 'portrait-upside-down'], + ['rot270', 'landscape-left'], +]); + +/** The panel-frame angle of one rotation. */ +export function panelRotationAngle(rotation: DeviceRotation): PanelQuarterTurn { + return PANEL_ROTATION_ANGLE[rotation]; +} + +/** The rotation at one panel-frame angle. The angle is normalized into one full turn. */ +export function rotationAtPanelAngle(angle: number): DeviceRotation { + const quarter = (((Math.round(angle / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3; + return ROTATION_BY_QUARTER[quarter]!; +} + +/** The lit-panel rotation a panel table `currentOrientation` names, or nothing when unread. */ +export function panelRotationForOrientation( + currentOrientation: string, +): DeviceRotation | undefined { + return PANEL_ORIENTATIONS.get(currentOrientation); +} + +/** + * The offset between the orientation picker and the lit panel it drives: the panel reads + * `picker + offset`. Learning it from one dispatched picker value and the panel rotation + * that followed it is what calibrates a panel whose offset was never measured. + */ +export function panelPickerOffset( + currentRotation: DeviceRotation, + pickerValue: DeviceRotation, +): PanelQuarterTurn { + const angle = (panelRotationAngle(currentRotation) - panelRotationAngle(pickerValue) + 360) % 360; + return angle as PanelQuarterTurn; +} + +/** + * Measured picker offsets by panel. The `primary` panel is the one that stays lit while the + * device is closed (the outer panel); the other integrated panel is the inner one. A measured + * offset only saves the calibration dispatch: the caller still confirms every rotation from + * the panel table and re-derives the offset from what it observes, so a panel that disagrees + * with this table still converges. An unmeasured panel returns nothing rather than a guess. + */ +export function measuredPanelPickerOffset(panel: { + primary: boolean; +}): PanelQuarterTurn | undefined { + // Measured on iPhone Duo: the inner panel lights at rot90 while the picker sits at portrait. + return panel.primary ? undefined : 90; +} + +/** + * The next picker value to dispatch, in a guest that applies picker events relatively: each + * event turns the lit panel by (new value - previous picker value), so an absolute dispatch + * lands a panel offset away from its target. The choice is, in order: + * + * - Picker known: the value whose delta from the current picker value is exactly the turn + * still needed. Needs the current lit-panel rotation; without it there is no safe dispatch, + * so the caller re-reads instead (nothing is returned). + * - Picker unknown, offset known: the value that leaves the panel at the target once the + * offset is added back. + * - Both unknown: the target itself, as a calibration dispatch. Wherever it lands, one panel + * read afterwards fixes the offset through {@link panelPickerOffset}, and the next dispatch + * is exact. + */ +export function nextPickerValue(input: { + currentRotation: DeviceRotation | undefined; + targetRotation: DeviceRotation; + panelOffset: PanelQuarterTurn | undefined; + currentPickerValue: DeviceRotation | undefined; +}): DeviceRotation | undefined { + const { currentRotation, targetRotation, panelOffset, currentPickerValue } = input; + if (currentPickerValue !== undefined) { + if (currentRotation === undefined) return undefined; + return rotationAtPanelAngle( + panelRotationAngle(currentPickerValue) + + panelRotationAngle(targetRotation) - + panelRotationAngle(currentRotation), + ); + } + if (panelOffset !== undefined) { + return rotationAtPanelAngle(panelRotationAngle(targetRotation) - panelOffset); + } + return targetRotation; +} diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 7f6edbff4e..d8a7732bd5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -132,7 +132,7 @@ Back, orientation, and hardware buttons: - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. - An app with a left-edge drawer may open the drawer instead of going back. When a back or close control is visible, press it by its ref. - `back --system` sends system back input: the Android back keyevent, the iOS back swipe, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. -- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. A foldable iPhone simulator (iPhone Duo) refuses `portrait-upside-down`, and refuses every rotation in a scoped simulator set (`--ios-simulator-device-set`). +- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. A foldable iPhone simulator (iPhone Duo) refuses `portrait-upside-down`; in a scoped simulator set (`--ios-simulator-device-set`) it rotates through the fold helper and confirms the rotation from the panel table. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`. - `action-button` presses the iPhone or iPad Action Button once. It takes no arguments, no hold duration, and no `--settle`, and it doesn't activate the session app or drive Camera Control. Other platforms refuse it, and a model without the button (an iPhone SE, most iPad simulators) fails with `UNSUPPORTED_OPERATION`. - `action-button` reports that the press was dispatched, not what it triggered. Simulators run no Shortcuts or App Intents, so verify the outcome on a physical iPhone. Expect a press to take about five seconds on a Simulator. From b35461bf214c416d4bc2c7bf26993f133305297e Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 14:39:41 +0000 Subject: [PATCH 02/16] fix(ios): confirm the final scoped dispatch and keep picker angles private --- .../src/foldable/orientation.ts | 9 ++++- .../src/foldable/picker-delta.test.ts | 40 +++++++++---------- .../src/foldable/picker-delta.ts | 4 +- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 1cd7973ecd..2f38342d08 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -29,7 +29,7 @@ import { import { appleFoldableFacts } from './runtime.ts'; const UPSIDE_DOWN_UNSUPPORTED_HINT = - 'A foldable simulator accepts portrait, landscape-left, and landscape-right. Its device-state control has no upside-down orientation.'; + 'A foldable simulator accepts portrait, landscape-left, and landscape-right as rotation targets.'; const ORIENTATION_UNVERIFIED_HINT = 'The simulator HID command completed, but CoreDevice did not report the new orientation. Read it with xcrun devicectl device orientation get --device , then retry.'; @@ -229,6 +229,13 @@ async function setScopedFoldableOrientation( }); await settleScopedRotation(); } + // The last dispatch gets its confirming read too: the loop only reads before dispatching. + const settled = await readLitPanel(device, helper, signal); + if (settled) { + observed = settled.rotation; + rawPanelOrientation = settled.panel.currentOrientation; + if (observed === rotation) return { orientation: rotation }; + } throw new AppError( 'COMMAND_FAILED', `${device.name} did not confirm rotation to ${rotation}: the panel table reports ${observed ?? 'no orientation'}`, diff --git a/packages/platform-apple/src/foldable/picker-delta.test.ts b/packages/platform-apple/src/foldable/picker-delta.test.ts index 59473e2029..d2ef7134c1 100644 --- a/packages/platform-apple/src/foldable/picker-delta.test.ts +++ b/packages/platform-apple/src/foldable/picker-delta.test.ts @@ -4,9 +4,7 @@ import { measuredPanelPickerOffset, nextPickerValue, panelPickerOffset, - panelRotationAngle, panelRotationForOrientation, - rotationAtPanelAngle, type PanelQuarterTurn, } from './picker-delta.ts'; @@ -18,26 +16,29 @@ const ROTATIONS: readonly DeviceRotation[] = [ ]; const OFFSETS: readonly PanelQuarterTurn[] = [0, 90, 180, 270]; +/** The measured guest model in CADisplay `rotN` angles, stated here independently of the code. */ +const ANGLE: Record = { + portrait: 0, + 'landscape-right': 90, + 'portrait-upside-down': 180, + 'landscape-left': 270, +}; +const AT_ANGLE: readonly DeviceRotation[] = ROTATIONS; + +function atAngle(angle: number): DeviceRotation { + return AT_ANGLE[(((angle / 90) % 4) + 4) % 4]!; +} + /** Applies one picker event the way the guest does: the panel turns by (new - previous). */ function applyPickerEvent( screen: DeviceRotation, previousPicker: DeviceRotation, nextPicker: DeviceRotation, ): DeviceRotation { - return rotationAtPanelAngle( - panelRotationAngle(screen) + - panelRotationAngle(nextPicker) - - panelRotationAngle(previousPicker), - ); + return atAngle(ANGLE[screen] + ANGLE[nextPicker] - ANGLE[previousPicker]); } -test('panel angles follow CADisplay rotN in both directions', () => { - expect(ROTATIONS.map(panelRotationAngle)).toEqual([0, 90, 180, 270]); - for (const rotation of ROTATIONS) { - expect(rotationAtPanelAngle(panelRotationAngle(rotation))).toBe(rotation); - expect(rotationAtPanelAngle(panelRotationAngle(rotation) + 360)).toBe(rotation); - expect(rotationAtPanelAngle(panelRotationAngle(rotation) - 360)).toBe(rotation); - } +test('panel table orientations read as panel-frame rotations', () => { expect(panelRotationForOrientation('rot0')).toBe('portrait'); expect(panelRotationForOrientation('rot90')).toBe('landscape-right'); expect(panelRotationForOrientation('rot180')).toBe('portrait-upside-down'); @@ -58,11 +59,8 @@ test('with the picker known, one dispatch lands every current/target/offset/pick }); expect(next, `${current} -> ${target}, K=${offset}, picker=${picker}`).toBeDefined(); expect(applyPickerEvent(current, picker, next!)).toBe(target); - // The offset observed after the dispatch is still the guest's own relation: - // re-deriving it from the landed state must give a consistent quarter turn. expect(panelPickerOffset(target, next!)).toBe( - ((panelRotationAngle(target) - panelRotationAngle(next!) + 360) % - 360) as PanelQuarterTurn, + ((ANGLE[target] - ANGLE[next!] + 360) % 360) as PanelQuarterTurn, ); } } @@ -75,14 +73,14 @@ test('with only the offset known, one dispatch lands every consistent state', () for (const offset of OFFSETS) { for (const picker of ROTATIONS) { // A consistent guest state: the panel reads picker + offset. - const current = rotationAtPanelAngle(panelRotationAngle(picker) + offset); + const current = atAngle(ANGLE[picker] + offset); const next = nextPickerValue({ currentRotation: current, targetRotation: target, panelOffset: offset, currentPickerValue: undefined, }); - expect(next).toBe(rotationAtPanelAngle(panelRotationAngle(target) - offset)); + expect(next).toBe(atAngle(ANGLE[target] - offset)); expect(applyPickerEvent(current, picker, next!)).toBe(target); } } @@ -93,7 +91,7 @@ test('with neither known, the calibration dispatch plus one delta dispatch lands for (const target of ROTATIONS) { for (const offset of OFFSETS) { for (const picker of ROTATIONS) { - const start = rotationAtPanelAngle(panelRotationAngle(picker) + offset); + const start = atAngle(ANGLE[picker] + offset); const calibration = nextPickerValue({ currentRotation: start, targetRotation: target, diff --git a/packages/platform-apple/src/foldable/picker-delta.ts b/packages/platform-apple/src/foldable/picker-delta.ts index 315e21bcf5..b0dee10f64 100644 --- a/packages/platform-apple/src/foldable/picker-delta.ts +++ b/packages/platform-apple/src/foldable/picker-delta.ts @@ -30,12 +30,12 @@ const PANEL_ORIENTATIONS: ReadonlyMap = new Map([ ]); /** The panel-frame angle of one rotation. */ -export function panelRotationAngle(rotation: DeviceRotation): PanelQuarterTurn { +function panelRotationAngle(rotation: DeviceRotation): PanelQuarterTurn { return PANEL_ROTATION_ANGLE[rotation]; } /** The rotation at one panel-frame angle. The angle is normalized into one full turn. */ -export function rotationAtPanelAngle(angle: number): DeviceRotation { +function rotationAtPanelAngle(angle: number): DeviceRotation { const quarter = (((Math.round(angle / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3; return ROTATION_BY_QUARTER[quarter]!; } From 48a90b65571b455f016f131a8de0d66bf3d253ee Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 14:44:33 +0000 Subject: [PATCH 03/16] refactor(ios): split scoped rotation state out of the dispatch loop --- .../src/foldable/orientation.ts | 136 +++++++++++------- 1 file changed, 87 insertions(+), 49 deletions(-) diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 2f38342d08..3fae9b9c35 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -18,7 +18,7 @@ import { type AppleDisplayInventory, } from '../core/display-inventory.ts'; import { simulatorAddressFor } from '../core/simctl.ts'; -import { createFoldHelper, type FoldHelper } from './fold-helper.ts'; +import { createFoldHelper, type FoldHelper, type FoldableRotation } from './fold-helper.ts'; import { measuredPanelPickerOffset, nextPickerValue, @@ -175,75 +175,113 @@ export async function setAppleFoldableOrientation( */ async function setScopedFoldableOrientation( device: DeviceInfo, - rotation: DeviceRotation, + rotation: FoldableRotation, signal: AbortSignal | undefined, ): Promise<{ orientation: DeviceRotation }> { const helper = createFoldHelper(device); - let pickerValue: DeviceRotation | undefined; - let panelOffset: PanelQuarterTurn | undefined; - let litDisplayId: number | undefined; - let observed: DeviceRotation | undefined; - let rawPanelOrientation: string | undefined; + const state: ScopedRotationState = { + pickerValue: undefined, + panelOffset: undefined, + litDisplayId: undefined, + observed: undefined, + rawPanelOrientation: undefined, + }; for (let attempt = 1; attempt <= IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS; attempt += 1) { signal?.throwIfAborted(); const lit = await readLitPanel(device, helper, signal); if (lit) { - // A different panel lit up mid-command: the picker state and offset were calibrated - // against the previous panel, so start that panel's calibration over. - if (litDisplayId !== undefined && lit.panel.displayId !== litDisplayId) { - pickerValue = undefined; - panelOffset = undefined; - } - litDisplayId = lit.panel.displayId; - observed = lit.rotation; - rawPanelOrientation = lit.panel.currentOrientation; - if (observed === rotation) return { orientation: rotation }; - panelOffset = - pickerValue === undefined - ? (panelOffset ?? measuredPanelPickerOffset(lit.panel)) - : panelPickerOffset(observed, pickerValue); + noteLitPanel(state, lit); + if (state.observed === rotation) return { orientation: rotation }; } const next = nextPickerValue({ currentRotation: lit?.rotation, targetRotation: rotation, - panelOffset, - currentPickerValue: pickerValue, + panelOffset: state.panelOffset, + currentPickerValue: state.pickerValue, }); - if (next === undefined) { - await settleScopedRotation(); - continue; + if (next !== undefined) { + await dispatchScopedPicker(device, helper, state, next, rotation, attempt, signal); } - await helper.rotate(next, signal); - pickerValue = next; - emitDiagnostic({ - level: 'info', - phase: 'apple_foldable_orientation_dispatched', - data: { - deviceId: device.id, - orientation: next, - requestedOrientation: rotation, - scopedSet: true, - attempt, - ...(observed === undefined ? {} : { observedOrientation: observed }), - }, - }); await settleScopedRotation(); } // The last dispatch gets its confirming read too: the loop only reads before dispatching. const settled = await readLitPanel(device, helper, signal); - if (settled) { - observed = settled.rotation; - rawPanelOrientation = settled.panel.currentOrientation; - if (observed === rotation) return { orientation: rotation }; + if (settled) noteLitPanel(state, settled); + if (state.observed === rotation) return { orientation: rotation }; + throw scopedUnverifiedError(device, rotation, state); +} + +type ScopedRotationState = { + pickerValue: DeviceRotation | undefined; + panelOffset: PanelQuarterTurn | undefined; + litDisplayId: number | undefined; + observed: DeviceRotation | undefined; + rawPanelOrientation: string | undefined; +}; + +type LitPanel = { panel: AppleDeviceDisplay; rotation: DeviceRotation }; + +/** + * Folds one lit-panel reading into the rotation state. A different panel lit up mid-command: + * the picker state and offset were calibrated against the previous panel, so start that + * panel's calibration over. Otherwise the reading fixes the panel's offset — from the picker + * value this command dispatched when there is one, else from the measured table. + */ +function noteLitPanel(state: ScopedRotationState, lit: LitPanel): void { + if (state.litDisplayId !== undefined && lit.panel.displayId !== state.litDisplayId) { + state.pickerValue = undefined; + state.panelOffset = undefined; } - throw new AppError( + state.litDisplayId = lit.panel.displayId; + state.observed = lit.rotation; + state.rawPanelOrientation = lit.panel.currentOrientation; + state.panelOffset = + state.pickerValue === undefined + ? (state.panelOffset ?? measuredPanelPickerOffset(lit.panel)) + : panelPickerOffset(lit.rotation, state.pickerValue); +} + +/** Dispatches one picker value and records it as the guest's picker state for this command. */ +async function dispatchScopedPicker( + device: DeviceInfo, + helper: FoldHelper, + state: ScopedRotationState, + next: DeviceRotation, + rotation: FoldableRotation, + attempt: number, + signal: AbortSignal | undefined, +): Promise { + await helper.rotate(next, signal); + state.pickerValue = next; + emitDiagnostic({ + level: 'info', + phase: 'apple_foldable_orientation_dispatched', + data: { + deviceId: device.id, + orientation: next, + requestedOrientation: rotation, + scopedSet: true, + attempt, + ...(state.observed === undefined ? {} : { observedOrientation: state.observed }), + }, + }); +} + +function scopedUnverifiedError( + device: DeviceInfo, + rotation: FoldableRotation, + state: ScopedRotationState, +): AppError { + return new AppError( 'COMMAND_FAILED', - `${device.name} did not confirm rotation to ${rotation}: the panel table reports ${observed ?? 'no orientation'}`, + `${device.name} did not confirm rotation to ${rotation}: the panel table reports ${state.observed ?? 'no orientation'}`, { deviceId: device.id, requestedOrientation: rotation, - ...(observed === undefined ? {} : { observedOrientation: observed }), - ...(rawPanelOrientation === undefined ? {} : { rawPanelOrientation }), + ...(state.observed === undefined ? {} : { observedOrientation: state.observed }), + ...(state.rawPanelOrientation === undefined + ? {} + : { rawPanelOrientation: state.rawPanelOrientation }), reason: 'orientation-unverified-scoped-set', hint: SCOPED_ORIENTATION_UNVERIFIED_HINT, }, @@ -259,7 +297,7 @@ async function readLitPanel( device: DeviceInfo, helper: FoldHelper, signal: AbortSignal | undefined, -): Promise<{ panel: AppleDeviceDisplay; rotation: DeviceRotation } | undefined> { +): Promise { let inventory: AppleDisplayInventory; try { inventory = await queryAppleDisplayInventory(device, { signal, foldHelper: helper }); From 4245f65d34bba0841f9efe4767ecb1424d941745 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 17:56:32 +0000 Subject: [PATCH 04/16] fix(ios): use the measured outer-panel picker offset on iPhone Duo The outer panel reads the orientation-picker value itself (offset 0) and the inner panel reads it plus 90 degrees. Measured on iPhone Duo (iOS 27.3) against the panel table over 27 dispatches and five pose changes in both directions: the picker value survives a fold, and only the lit panel's offset changes. After a fold that left the picker at the upside-down value, which the outer panel's UI could not show, one dispatch of the target with the measured offset still landed it. The guest model in the tests now follows that measurement: one picker value, a per-panel offset, folds that keep the picker, and the few seconds after a fold to closed when the table names both panels lit. Co-Authored-By: Claude Opus 5.5 --- .changeset/foldable-scoped-rotation.md | 2 +- .../src/foldable/orientation.test.ts | 155 ++++++++++++++---- .../src/foldable/orientation.ts | 22 +-- .../src/foldable/picker-delta.test.ts | 76 ++++++++- .../src/foldable/picker-delta.ts | 36 ++-- 5 files changed, 236 insertions(+), 55 deletions(-) diff --git a/.changeset/foldable-scoped-rotation.md b/.changeset/foldable-scoped-rotation.md index fb3f301149..89fefac592 100644 --- a/.changeset/foldable-scoped-rotation.md +++ b/.changeset/foldable-scoped-rotation.md @@ -2,4 +2,4 @@ 'agent-device': patch --- -Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and confirmed from the panel table's lit-panel `currentOrientation`. Because the guest applies each orientation-picker event relative to the previous picker value (the lit panel reads picker + a fixed per-panel offset, +90 degrees on the inner panel), each dispatch is chosen for the turn still needed and re-read until the panel matches, so the first command lands. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the delta dispatch needs to reach every orientation; a user target of `portrait-upside-down` stays refused. +Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and confirmed from the panel table's lit-panel `currentOrientation`. The lit panel reads the orientation-picker value plus a fixed per-panel offset, measured on iPhone Duo as 0 degrees on the outer panel and +90 degrees on the inner panel, and the picker value survives a fold. Each dispatch therefore subtracts the lit panel's offset and is re-read until the panel matches, so the first command lands. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the inner panel needs to reach landscape-left; a user target of `portrait-upside-down` stays refused. diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index ecf9e977ab..3e818bbe28 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -153,12 +153,16 @@ test('a physical device keeps the runner rotation without a CoreDevice probe', a }); /** - * A scoped-set Duo as measured on the device: the guest does not treat the - * `orientation-picker-control` value as absolute. Each picker event rotates the lit panel by - * (new value - previous picker value), so the lit panel reads `picker + K` for a fixed - * per-panel offset K. Angles follow CADisplay `rotN`: portrait 0, landscape-right 90, - * portrait-upside-down 180, landscape-left 270. The dark panel reports a misleading rotation: - * only the lit panel's `currentOrientation` may confirm a rotation. + * A scoped-set Duo as measured on the device (iOS 27.3): the guest keeps one + * `orientation-picker-control` value, and the lit panel reads that value plus a fixed + * per-panel offset — outer 0, inner +90 — so successive events look relative on the inner + * panel. The picker survives a fold: the panel that lights afterwards reads it plus its own + * offset. A panel whose foreground UI cannot show `picker + offset` keeps the rotation it last + * showed (measured: after a fold left the picker at the upside-down value, the outer panel kept + * landscape-left). For `bothLitReads` panel reads after a fold to closed, both panels report + * lit, as the Duo's table does for a few seconds. Angles follow CADisplay `rotN`: portrait 0, + * landscape-right 90, portrait-upside-down 180, landscape-left 270. A dark panel reports a + * misleading rotation: only the lit panel's `currentOrientation` may confirm a rotation. */ const PANEL_ANGLE: Record = { portrait: 0, @@ -166,25 +170,52 @@ const PANEL_ANGLE: Record = { 'portrait-upside-down': 180, 'landscape-left': 270, }; +const MEASURED_OFFSETS = { inner: 90, outer: 0 } as const; + +type DuoPanel = 'inner' | 'outer'; + +function rotationAtAngle(angle: number): DeviceRotation { + const rotation = (Object.keys(PANEL_ANGLE) as DeviceRotation[]).find( + (candidate) => PANEL_ANGLE[candidate] === ((angle % 360) + 360) % 360, + ); + return rotation!; +} function fakeScopedDuo( device: DeviceInfo, options: { - lit: 'inner' | 'outer'; + lit: DuoPanel; picker: DeviceRotation; - offsets: { inner: number; outer: number }; + offsets?: { inner: number; outer: number }; followsHid?: boolean; + /** Rotations a panel's foreground UI cannot show; the panel keeps its last rotation. */ + cannotShow?: Partial>; + /** The rotation a dark panel last showed, for the fold that lights it. */ + lastShown?: Partial>; }, ) { const helpers: string[][] = []; const devicectl: string[] = []; + const offsets = options.offsets ?? MEASURED_OFFSETS; + let lit = options.lit; let pickerAngle = PANEL_ANGLE[options.picker]; - let litScreenAngle = (pickerAngle + options.offsets[options.lit]) % 360; - const panel = (kind: 'inner' | 'outer') => { - const lit = kind === options.lit; + let bothLitReads = 0; + let panelReads = 0; + const shown: Record = { + inner: PANEL_ANGLE[options.lastShown?.inner ?? 'portrait'], + outer: PANEL_ANGLE[options.lastShown?.outer ?? 'portrait'], + }; + /** The panel shows `picker + offset` unless its foreground UI cannot. */ + const show = (kind: DuoPanel) => { + const angle = (pickerAngle + offsets[kind]) % 360; + if (!options.cannotShow?.[kind]?.includes(rotationAtAngle(angle))) shown[kind] = angle; + }; + show(lit); + const panel = (kind: DuoPanel, bothLit: boolean) => { + const isLit = bothLit || kind === lit; return { - backlightState: lit ? 'activeOn' : 'off', - currentOrientation: lit ? `rot${litScreenAngle}` : 'rot180', + backlightState: isLit ? 'activeOn' : 'off', + currentOrientation: isLit ? `rot${shown[kind]}` : 'rot180', displayId: kind === 'outer' ? 1 : 3, name: kind === 'outer' ? 'LCD' : 'LCD-1', nativeSize: kind === 'outer' ? [1398, 2034] : [2007, 2853], @@ -205,14 +236,16 @@ function fakeScopedDuo( const mode = spawn.slice(3); helpers.push(mode); if (mode[0] === 'panels') { - const displays = [panel('outer'), panel('inner')]; + panelReads += 1; + const bothLit = bothLitReads > 0; + if (bothLit) bothLitReads -= 1; + const displays = [panel('outer', bothLit), panel('inner', bothLit)]; return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; } expect(mode[0]).toBe('rotate'); if (options.followsHid !== false) { - const nextPickerAngle = PANEL_ANGLE[mode[1]! as DeviceRotation]; - litScreenAngle = (litScreenAngle + nextPickerAngle - pickerAngle + 360) % 360; - pickerAngle = nextPickerAngle; + pickerAngle = PANEL_ANGLE[mode[1]! as DeviceRotation]; + show(lit); } return { stdout: '', stderr: '', exitCode: 0 }; }, @@ -220,8 +253,16 @@ function fakeScopedDuo( return { helpers, devicectl, + panelReads: () => panelReads, rotations: () => helpers.filter((mode) => mode[0] === 'rotate').map((mode) => mode[1]!), - litScreen: () => `rot${litScreenAngle}`, + litScreen: () => `rot${shown[lit]}`, + /** A pose change made outside the rotation: the picker stays, the other panel lights. */ + fold: (kind: DuoPanel, foldOptions: { bothLitReads?: number } = {}) => { + lit = kind; + bothLitReads = foldOptions.bothLitReads ?? 0; + show(kind); + helpers.length = 0; + }, rotate: (rotation: DeviceRotation) => withAppleToolProvider(provider, () => createAppleInteractor(device, {}, createLiveTestHostEnvironment()).setOrientation(rotation), @@ -250,15 +291,12 @@ async function rotateScoped( test('a foldable simulator in a scoped set rotates on the first command, confirmed by the panel table', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); // The inner panel lights at rot90 while the picker sits at portrait (K = +90, measured). - const sim = fakeScopedDuo(scoped, { - lit: 'inner', - picker: 'portrait', - offsets: { inner: 90, outer: 0 }, - }); + const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'portrait' }); expect(sim.litScreen()).toBe('rot90'); await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); expect(sim.litScreen()).toBe('rot0'); - expect(sim.rotations().length).toBeLessThanOrEqual(2); + // The measured inner offset makes the first dispatch exact: portrait - 90 is landscape-left. + expect(sim.rotations()).toEqual(['landscape-left']); expect(sim.devicectl).toEqual([]); expect(mockRunner).not.toHaveBeenCalled(); }); @@ -268,7 +306,6 @@ test('a scoped rotation to landscape-left dispatches the upside-down picker valu const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left', - offsets: { inner: 90, outer: 0 }, }); expect(sim.litScreen()).toBe('rot0'); await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ @@ -279,7 +316,19 @@ test('a scoped rotation to landscape-left dispatches the upside-down picker valu expect(sim.devicectl).toEqual([]); }); -test('a scoped rotation on the outer panel converges with that panel offset', async () => { +test('the outer panel reads the picker value itself, so its first dispatch is the target', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { lit: 'outer', picker: 'portrait' }); + expect(sim.litScreen()).toBe('rot0'); + for (const rotation of ['landscape-left', 'landscape-right', 'portrait'] as const) { + sim.helpers.length = 0; + await expect(rotateScoped(sim, rotation)).resolves.toEqual({ orientation: rotation }); + expect(sim.rotations()).toEqual([rotation]); + } + expect(sim.devicectl).toEqual([]); +}); + +test('a panel whose offset differs from the measured one still converges', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); const sim = fakeScopedDuo(scoped, { lit: 'outer', @@ -294,12 +343,62 @@ test('a scoped rotation on the outer panel converges with that panel offset', as expect(sim.devicectl).toEqual([]); }); +test('the picker survives a fold in both directions and each panel adds its own offset', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + // Measured sequence: the inner panel at rot0 with the picker at landscape-left. + const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); + expect(sim.litScreen()).toBe('rot0'); + + sim.fold('outer'); + expect(sim.litScreen()).toBe('rot270'); + await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + expect(sim.rotations()).toEqual(['portrait']); + + sim.fold('inner'); + expect(sim.litScreen()).toBe('rot90'); + await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + expect(sim.rotations()).toEqual(['portrait-upside-down']); +}); + +test('a fold that leaves the picker where the outer panel cannot follow still lands in one dispatch', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + // Measured: the inner panel at landscape-left needs the picker at the upside-down value; the + // outer panel's foreground UI has no upside-down, so after the fold it kept landscape-left. + const sim = fakeScopedDuo(scoped, { + lit: 'inner', + picker: 'portrait-upside-down', + cannotShow: { outer: ['portrait-upside-down'] }, + lastShown: { outer: 'landscape-left' }, + }); + expect(sim.litScreen()).toBe('rot270'); + sim.fold('outer'); + expect(sim.litScreen()).toBe('rot270'); + await expect(rotateScoped(sim, 'landscape-right')).resolves.toEqual({ + orientation: 'landscape-right', + }); + expect(sim.rotations()).toEqual(['landscape-right']); +}); + +test('reads with both panels lit after a fold to closed confirm nothing', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); + // The outer panel already shows the target, but the table names two lit panels for two reads. + sim.fold('outer', { bothLitReads: 2 }); + expect(sim.litScreen()).toBe('rot270'); + await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + expect(sim.panelReads()).toBeGreaterThanOrEqual(3); + expect(sim.rotations()).toEqual(['landscape-left']); +}); + test('a scoped rotation to the orientation already showing dispatches nothing', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left', - offsets: { inner: 90, outer: 0 }, }); expect(sim.litScreen()).toBe('rot0'); await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); @@ -311,7 +410,6 @@ test('a scoped rotation the panel table never reports is refused as unverified', const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'portrait', - offsets: { inner: 90, outer: 0 }, followsHid: false, }); await expect(rotateScoped(sim, 'portrait')).resolves.toMatchObject({ @@ -326,7 +424,6 @@ test('a foldable simulator in a scoped set still refuses portrait-upside-down as const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'portrait', - offsets: { inner: 90, outer: 0 }, }); await expect(sim.rotate('portrait-upside-down')).rejects.toMatchObject({ code: 'UNSUPPORTED_OPERATION', diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 3fae9b9c35..cb8f50f28c 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -163,15 +163,15 @@ export async function setAppleFoldableOrientation( /** * Rotates a foldable simulator in a scoped set, where only the fold helper can dispatch and - * only its panel table can confirm. Measured on iPhone Duo, the guest applies each - * `orientation-picker-control` event relatively — the lit panel turns by (new value - - * previous picker value), i.e. it reads picker + a fixed per-panel offset — so one absolute - * dispatch lands a quarter-turn away and looks one command late. This loop reads the lit - * panel, dispatches the picker value {@link nextPickerValue} chooses for the turn still - * needed, and re-reads until the panel table reports the target: at most two dispatches once - * the first one calibrates the guest's picker state. A repeat of the showing orientation is - * a read-only success. The picker state is deliberately not kept between commands: a fold in - * between can light the other panel, and recalibrating costs one dispatch. + * only its panel table can confirm. Measured on iPhone Duo, the lit panel reads the + * `orientation-picker-control` value plus a fixed per-panel offset (outer panel 0, inner + * panel +90), so one absolute dispatch of the target lands a quarter-turn away on the inner + * panel and looks one command late. This loop reads the lit panel, dispatches the picker + * value {@link nextPickerValue} chooses, and re-reads until the panel table reports the + * target: one dispatch when the lit panel's offset is the measured one, at most two when the + * first dispatch has to calibrate it. A repeat of the showing orientation is a read-only + * success. The picker state is deliberately not kept between commands: a fold in between can + * light the other panel, whose offset differs, and the measured offset covers that case. */ async function setScopedFoldableOrientation( device: DeviceInfo, @@ -291,7 +291,9 @@ function scopedUnverifiedError( /** * The one lit panel and the rotation it shows, from one fresh panel-table read. Anything * less decided — an unreadable table, no lit panel, an unrecognized `rotN` — is no reading: - * the rotation loop must not confirm or compensate from a panel that is not showing. + * the rotation loop must not confirm or compensate from a panel that is not showing. On + * iPhone Duo the table reports both panels lit for a few seconds after a fold to closed, so + * the first reads after that fold are no reading by design. */ async function readLitPanel( device: DeviceInfo, diff --git a/packages/platform-apple/src/foldable/picker-delta.test.ts b/packages/platform-apple/src/foldable/picker-delta.test.ts index d2ef7134c1..dba2f51675 100644 --- a/packages/platform-apple/src/foldable/picker-delta.test.ts +++ b/packages/platform-apple/src/foldable/picker-delta.test.ts @@ -130,7 +130,79 @@ test('without a current rotation there is no dispatch once the picker is known', } }); -test('the measured inner-panel offset is +90 and the outer panel is never guessed', () => { +test('the measured offsets are 0 on the outer (primary) panel and +90 on the inner panel', () => { + expect(measuredPanelPickerOffset({ primary: true })).toBe(0); expect(measuredPanelPickerOffset({ primary: false })).toBe(90); - expect(measuredPanelPickerOffset({ primary: true })).toBeUndefined(); +}); + +/** + * Raw iPhone Duo readings (iOS 27.3, scoped simulator set, Safari in front), in the order they + * were taken: the picker value the guest held and the lit panel's `currentOrientation` that the + * panel table reported next. Rows marked `fold` are the first read after a pose change, with + * the picker value from before it. Every row gives its panel's measured offset, so the picker + * value survives a fold in both directions and only the panel's offset changes. + */ +const DUO_READINGS: readonly { + panel: 'inner' | 'outer'; + picker: DeviceRotation; + reading: string; + fold?: true; +}[] = [ + { panel: 'inner', picker: 'portrait', reading: 'rot90' }, + { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, + { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, + { panel: 'inner', picker: 'portrait-upside-down', reading: 'rot270' }, + { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, + { panel: 'inner', picker: 'portrait', reading: 'rot90' }, + { panel: 'inner', picker: 'portrait', reading: 'rot90' }, + { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, + { panel: 'outer', picker: 'landscape-left', reading: 'rot270', fold: true }, + { panel: 'outer', picker: 'portrait', reading: 'rot0' }, + { panel: 'outer', picker: 'landscape-left', reading: 'rot270' }, + { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, + { panel: 'outer', picker: 'portrait', reading: 'rot0' }, + { panel: 'outer', picker: 'landscape-left', reading: 'rot270' }, + { panel: 'inner', picker: 'landscape-left', reading: 'rot0', fold: true }, + { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, + { panel: 'inner', picker: 'portrait', reading: 'rot90' }, + { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, + { panel: 'inner', picker: 'portrait-upside-down', reading: 'rot270' }, + // (fold to closed with the picker at portrait-upside-down: see the next test) + { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, + { panel: 'outer', picker: 'portrait', reading: 'rot0' }, + { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, + { panel: 'inner', picker: 'landscape-right', reading: 'rot180', fold: true }, + { panel: 'inner', picker: 'portrait', reading: 'rot90' }, + { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, + { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, + { panel: 'inner', picker: 'portrait', reading: 'rot90' }, + { panel: 'outer', picker: 'portrait', reading: 'rot0', fold: true }, + { panel: 'outer', picker: 'landscape-left', reading: 'rot270' }, + { panel: 'inner', picker: 'landscape-left', reading: 'rot0', fold: true }, + { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, + { panel: 'inner', picker: 'portrait', reading: 'rot90' }, +]; + +test('every measured Duo reading gives its panel the measured offset, across folds', () => { + for (const [index, row] of DUO_READINGS.entries()) { + const reading = panelRotationForOrientation(row.reading); + expect(reading, `row ${index}`).toBeDefined(); + expect(panelPickerOffset(reading!, row.picker), `row ${index}`).toBe( + measuredPanelPickerOffset({ primary: row.panel === 'outer' }), + ); + } + expect(DUO_READINGS.filter((row) => row.fold)).toHaveLength(5); +}); + +test('after a fold that left the picker where the outer panel could not follow, the measured offset still lands the target', () => { + // Measured: picker at portrait-upside-down, outer panel lit showing landscape-left (its UI has + // no upside-down). One dispatch of landscape-right landed landscape-right. A purely relative + // guest would have turned the panel by (90 - 180) to portrait-upside-down instead. + const next = nextPickerValue({ + currentRotation: 'landscape-left', + targetRotation: 'landscape-right', + panelOffset: measuredPanelPickerOffset({ primary: true }), + currentPickerValue: undefined, + }); + expect(next).toBe('landscape-right'); }); diff --git a/packages/platform-apple/src/foldable/picker-delta.ts b/packages/platform-apple/src/foldable/picker-delta.ts index b0dee10f64..0f85d7b3ee 100644 --- a/packages/platform-apple/src/foldable/picker-delta.ts +++ b/packages/platform-apple/src/foldable/picker-delta.ts @@ -60,30 +60,40 @@ export function panelPickerOffset( return angle as PanelQuarterTurn; } +/** Outer-panel offset, measured on iPhone Duo: the outer panel reads the picker value itself. */ +const OUTER_PANEL_PICKER_OFFSET: PanelQuarterTurn = 0; +/** Inner-panel offset, measured on iPhone Duo: it lights at rot90 while the picker sits at portrait. */ +const INNER_PANEL_PICKER_OFFSET: PanelQuarterTurn = 90; + /** * Measured picker offsets by panel. The `primary` panel is the one that stays lit while the - * device is closed (the outer panel); the other integrated panel is the inner one. A measured - * offset only saves the calibration dispatch: the caller still confirms every rotation from - * the panel table and re-derives the offset from what it observes, so a panel that disagrees - * with this table still converges. An unmeasured panel returns nothing rather than a guess. + * device is closed (the outer panel); the other integrated panel is the inner one (lit open + * and half-open). Measured on iPhone Duo (iOS 27.3) against the panel table's lit-panel + * `currentOrientation`: the outer panel reads `picker + 0` and the inner panel `picker + 90`, + * across fold/unfold in both directions. The guest keeps one picker value through a fold, so + * the panel that lights afterwards reads that same value plus its own offset. + * + * A measured offset only saves the calibration dispatch: the caller still confirms every + * rotation from the panel table and re-derives the offset from what it observes after each + * dispatch, so a panel or runtime that disagrees with this table still converges. */ -export function measuredPanelPickerOffset(panel: { - primary: boolean; -}): PanelQuarterTurn | undefined { - // Measured on iPhone Duo: the inner panel lights at rot90 while the picker sits at portrait. - return panel.primary ? undefined : 90; +export function measuredPanelPickerOffset(panel: { primary: boolean }): PanelQuarterTurn { + return panel.primary ? OUTER_PANEL_PICKER_OFFSET : INNER_PANEL_PICKER_OFFSET; } /** - * The next picker value to dispatch, in a guest that applies picker events relatively: each - * event turns the lit panel by (new value - previous picker value), so an absolute dispatch - * lands a panel offset away from its target. The choice is, in order: + * The next picker value to dispatch. The guest shows `picker + offset` on the lit panel, with + * a fixed offset per panel, so an absolute dispatch of the target lands an offset away from + * it, and successive events look as if each one turned the panel by (new value - previous + * picker value). The choice is, in order: * * - Picker known: the value whose delta from the current picker value is exactly the turn * still needed. Needs the current lit-panel rotation; without it there is no safe dispatch, * so the caller re-reads instead (nothing is returned). * - Picker unknown, offset known: the value that leaves the panel at the target once the - * offset is added back. + * offset is added back. This holds even when the lit panel is not showing `picker + offset` + * yet, for example after a fold left the picker at a value the newly lit panel's + * foreground UI does not show. * - Both unknown: the target itself, as a calibration dispatch. Wherever it lands, one panel * read afterwards fixes the offset through {@link panelPickerOffset}, and the next dispatch * is exact. From 47874e07d3cf3af3f6b6bf433b5ecd319e39ac79 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 19:23:32 +0000 Subject: [PATCH 05/16] refactor(ios): one panel-content parser and one offset step for scoped foldable rotation Review of #4440: - display-inventory owns the only currentOrientation parser (panelContentRotation); appleDisplayRotation and the scoped foldable rotation both use it, and picker-delta drops its copied map. - The scoped loop has one target-minus-offset step. The offset comes from this command's last dispatch and a fresh reading of the same panel, and from the measured table only before that. The persistent panelOffset and the separate delta branch are gone. - A read that names no single lit panel neither confirms nor dispatches; there is no blind calibration dispatch any more. - The unverified error names only what the final read saw. - Tests assert exact dispatch sequences; the measurement fixture keeps the one contradicting reading, marked as the exception. - Docs, help text and changeset name the failure and qualify the model. Co-Authored-By: Claude Opus 5.5 --- .changeset/foldable-scoped-rotation.md | 2 +- packages/command-surface/src/system/index.ts | 2 +- .../core/__tests__/display-inventory.test.ts | 13 ++ packages/platform-apple/src/core/config.ts | 9 +- .../src/core/display-inventory.ts | 17 +- .../src/foldable/orientation.test.ts | 89 +++++++++- .../src/foldable/orientation.ts | 139 +++++++-------- .../src/foldable/picker-delta.test.ts | 164 ++++++------------ .../src/foldable/picker-delta.ts | 82 +++------ website/docs/docs/commands.md | 2 +- 10 files changed, 250 insertions(+), 269 deletions(-) diff --git a/.changeset/foldable-scoped-rotation.md b/.changeset/foldable-scoped-rotation.md index 89fefac592..65f47d2f3b 100644 --- a/.changeset/foldable-scoped-rotation.md +++ b/.changeset/foldable-scoped-rotation.md @@ -2,4 +2,4 @@ 'agent-device': patch --- -Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and confirmed from the panel table's lit-panel `currentOrientation`. The lit panel reads the orientation-picker value plus a fixed per-panel offset, measured on iPhone Duo as 0 degrees on the outer panel and +90 degrees on the inner panel, and the picker value survives a fold. Each dispatch therefore subtracts the lit panel's offset and is re-read until the panel matches, so the first command lands. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the inner panel needs to reach landscape-left; a user target of `portrait-upside-down` stays refused. +Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and succeeds only when the lit panel shows the requested orientation, otherwise failing with `orientation-unverified-scoped-set`. Measured on iPhone Duo, the lit panel usually reads the orientation-picker value plus a fixed per-panel offset (0 degrees on the outer panel, +90 degrees on the inner panel), and the picker value survives a fold; a panel whose foreground app cannot show that orientation keeps its previous one instead. Each dispatch is therefore the target minus the lit panel's offset, taken from the measured value at first and then recalibrated from the last dispatch and a fresh reading of the same panel, so the first command lands. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the inner panel needs to reach landscape-left; a user target of `portrait-upside-down` stays refused. diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index fd7a9eb540..e593315548 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -290,7 +290,7 @@ const backCommandDescription = const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = - 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down; in a scoped simulator set it rotates through the fold helper and confirms the rotation from the panel table.'; + 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In a scoped simulator set it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified-scoped-set.'; const foldGuidance = { hingeEvent: 'simulator HID hinge event', permissions: 'Device Hub and host Accessibility permission are not required.', diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 69a1576ece..305cd0c1b1 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -39,6 +39,7 @@ import { readPanelPower, resolveAppleCaptureDisplay, queryAppleDisplayInventory, + panelContentRotation, resolveAppleCaptureTarget, type AppleDeviceDisplay, } from '../display-inventory.ts'; @@ -362,6 +363,18 @@ describe('resolveAppleCaptureDisplay', () => { }); }); +describe('panelContentRotation', () => { + test('parses each panel content rotation and nothing else', () => { + assert.equal(panelContentRotation('rot0'), 'portrait'); + assert.equal(panelContentRotation('rot90'), 'landscape-right'); + assert.equal(panelContentRotation('rot180'), 'portrait-upside-down'); + assert.equal(panelContentRotation('rot270'), 'landscape-left'); + for (const value of ['unknown', 'constructor', 'rot45', '']) { + assert.equal(panelContentRotation(value), undefined); + } + }); +}); + describe('resolveAppleCaptureTarget', () => { beforeEach(() => { mockRunXcrun.mockClear(); diff --git a/packages/platform-apple/src/core/config.ts b/packages/platform-apple/src/core/config.ts index f8cd79174a..798dc8b0ba 100644 --- a/packages/platform-apple/src/core/config.ts +++ b/packages/platform-apple/src/core/config.ts @@ -64,10 +64,11 @@ export const IOS_FOLD_POSE_STABLE_DEGREES = 0.5; export const IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS = 8; export const IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS = 2_000; /** - * How many read/dispatch rounds a scoped-set foldable rotation gets. The picker delta lands - * in at most two dispatches once the first one calibrates the guest's picker state; the rest - * of the budget covers panel reads that do not answer yet. Verified on iPhone Duo with the - * settle delay below between a dispatch and the panel read that confirms it. + * How many read/dispatch rounds a scoped-set foldable rotation gets. A panel at its measured + * picker offset lands in one dispatch, and a panel that disagrees with the measured offset in + * two, the second calibrated by the first. The rest of the budget covers reads that name no + * single lit panel, such as the few seconds after an iPhone Duo folds closed. Verified on + * iPhone Duo with the settle delay below between a dispatch and the read that confirms it. */ export const IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS = 6; export const IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS = 600; diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 476cd6b774..c9afca8fb5 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -487,15 +487,28 @@ export type AppleCaptureTarget = * `rot270` is what the display shows after a rotation to landscape-left. An unrecognized or * missing value maps to no rotation. */ -const CORE_DEVICE_ROTATIONS: ReadonlyMap = new Map([ +const PANEL_CONTENT_ROTATIONS: ReadonlyMap = new Map< + string, + DeviceRotation +>([ ['rot0', 'portrait'], ['rot90', 'landscape-right'], ['rot180', 'portrait-upside-down'], ['rot270', 'landscape-left'], ]); +/** + * The content rotation a panel table `currentOrientation` names, in that panel's own frame, or + * nothing for an unrecognized value. This is the one parser of the field: a single-panel capture + * reads it as the device rotation, and the foldable rotation controller reads it as the lit + * panel's rotation before compensating that panel's picker offset. + */ +export function panelContentRotation(currentOrientation: string): DeviceRotation | undefined { + return PANEL_CONTENT_ROTATIONS.get(currentOrientation); +} + function appleDisplayRotation(display: AppleDeviceDisplay): DeviceRotation | undefined { - return CORE_DEVICE_ROTATIONS.get(display.currentOrientation); + return panelContentRotation(display.currentOrientation); } /** diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 3e818bbe28..86ec879445 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -192,6 +192,8 @@ function fakeScopedDuo( cannotShow?: Partial>; /** The rotation a dark panel last showed, for the fold that lights it. */ lastShown?: Partial>; + /** From this panel read on (1-based, the routing probe included), both panels report lit. */ + bothLitFromRead?: number; }, ) { const helpers: string[][] = []; @@ -237,8 +239,10 @@ function fakeScopedDuo( helpers.push(mode); if (mode[0] === 'panels') { panelReads += 1; - const bothLit = bothLitReads > 0; - if (bothLit) bothLitReads -= 1; + const bothLit = + bothLitReads > 0 || + (options.bothLitFromRead !== undefined && panelReads >= options.bothLitFromRead); + if (bothLitReads > 0) bothLitReads -= 1; const displays = [panel('outer', bothLit), panel('inner', bothLit)]; return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; } @@ -262,6 +266,7 @@ function fakeScopedDuo( bothLitReads = foldOptions.bothLitReads ?? 0; show(kind); helpers.length = 0; + panelReads = 0; }, rotate: (rotation: DeviceRotation) => withAppleToolProvider(provider, () => @@ -312,7 +317,7 @@ test('a scoped rotation to landscape-left dispatches the upside-down picker valu orientation: 'landscape-left', }); expect(sim.litScreen()).toBe('rot270'); - expect(sim.rotations()).toContain('portrait-upside-down'); + expect(sim.rotations()).toEqual(['portrait-upside-down']); expect(sim.devicectl).toEqual([]); }); @@ -339,7 +344,8 @@ test('a panel whose offset differs from the measured one still converges', async orientation: 'landscape-right', }); expect(sim.litScreen()).toBe('rot90'); - expect(sim.rotations().length).toBeLessThanOrEqual(2); + // The measured outer offset (0) lands a quarter-turn away; that reading calibrates the second. + expect(sim.rotations()).toEqual(['landscape-right', 'portrait-upside-down']); expect(sim.devicectl).toEqual([]); }); @@ -381,17 +387,67 @@ test('a fold that leaves the picker where the outer panel cannot follow still la expect(sim.rotations()).toEqual(['landscape-right']); }); -test('reads with both panels lit after a fold to closed confirm nothing', async () => { +test('reads with both panels lit after a fold to closed neither confirm nor dispatch', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); + // A first command routes the device as a foldable (one panel probe) and finds portrait showing. + await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); // The outer panel already shows the target, but the table names two lit panels for two reads. sim.fold('outer', { bothLitReads: 2 }); expect(sim.litScreen()).toBe('rot270'); await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ orientation: 'landscape-left', }); - expect(sim.panelReads()).toBeGreaterThanOrEqual(3); - expect(sim.rotations()).toEqual(['landscape-left']); + // Confirmed by the third read, the first to name one lit panel; nothing was dispatched blind. + expect(sim.panelReads()).toBe(3); + expect(sim.rotations()).toEqual([]); +}); + +test('after the both-lit reads, the first single-lit reading drives one exact dispatch', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); + await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + sim.fold('outer', { bothLitReads: 2 }); + await expect(rotateScoped(sim, 'landscape-right')).resolves.toEqual({ + orientation: 'landscape-right', + }); + expect(sim.panelReads()).toBe(4); + expect(sim.rotations()).toEqual(['landscape-right']); +}); + +test('a table that never names a single lit panel dispatches nothing and is refused', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + const sim = fakeScopedDuo(scoped, { lit: 'outer', picker: 'portrait', bothLitFromRead: 1 }); + const outcome = await rotateScoped(sim, 'landscape-left'); + expect(outcome).toMatchObject({ + code: 'COMMAND_FAILED', + details: { + reason: 'orientation-unverified-scoped-set', + requestedOrientation: 'landscape-left', + }, + }); + expect((outcome as AppError).message).toContain('named no single lit panel'); + expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); + expect(sim.rotations()).toEqual([]); +}); + +test('an undecided final read does not report an earlier reading as the current one', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + // The routing probe and two decided rotation reads, then both panels report lit to the end. + const sim = fakeScopedDuo(scoped, { + lit: 'inner', + picker: 'portrait', + followsHid: false, + bothLitFromRead: 4, + }); + const outcome = await rotateScoped(sim, 'portrait'); + expect(outcome).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-unverified-scoped-set' }, + }); + expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); + expect((outcome as AppError).details).not.toHaveProperty('rawPanelOrientation'); + expect(sim.rotations()).toEqual(['landscape-left', 'portrait-upside-down']); }); test('a scoped rotation to the orientation already showing dispatches nothing', async () => { @@ -405,7 +461,7 @@ test('a scoped rotation to the orientation already showing dispatches nothing', expect(sim.rotations()).toEqual([]); }); -test('a scoped rotation the panel table never reports is refused as unverified', async () => { +test('a scoped rotation the lit panel never shows is refused as unverified', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); const sim = fakeScopedDuo(scoped, { lit: 'inner', @@ -414,8 +470,23 @@ test('a scoped rotation the panel table never reports is refused as unverified', }); await expect(rotateScoped(sim, 'portrait')).resolves.toMatchObject({ code: 'COMMAND_FAILED', - details: { reason: 'orientation-unverified-scoped-set', requestedOrientation: 'portrait' }, + message: expect.stringContaining('the lit panel shows landscape-right'), + details: { + reason: 'orientation-unverified-scoped-set', + requestedOrientation: 'portrait', + observedOrientation: 'landscape-right', + rawPanelOrientation: 'rot90', + }, }); + // Each dispatch re-derives the offset from the unchanged reading: six rounds, then refusal. + expect(sim.rotations()).toEqual([ + 'landscape-left', + 'portrait-upside-down', + 'landscape-right', + 'portrait', + 'landscape-left', + 'portrait-upside-down', + ]); expect(sim.devicectl).toEqual([]); }); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index cb8f50f28c..e98e9fab00 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -13,6 +13,7 @@ import { } from '../core/config.ts'; import { readAppleDeviceOrientation } from '../core/device-orientation.ts'; import { + panelContentRotation, queryAppleDisplayInventory, type AppleDeviceDisplay, type AppleDisplayInventory, @@ -21,9 +22,8 @@ import { simulatorAddressFor } from '../core/simctl.ts'; import { createFoldHelper, type FoldHelper, type FoldableRotation } from './fold-helper.ts'; import { measuredPanelPickerOffset, - nextPickerValue, panelPickerOffset, - panelRotationForOrientation, + pickerValueForTarget, type PanelQuarterTurn, } from './picker-delta.ts'; import { appleFoldableFacts } from './runtime.ts'; @@ -35,7 +35,7 @@ const ORIENTATION_UNVERIFIED_HINT = 'The simulator HID command completed, but CoreDevice did not report the new orientation. Read it with xcrun devicectl device orientation get --device , then retry.'; const SCOPED_ORIENTATION_UNVERIFIED_HINT = - 'Rotation was dispatched through the fold helper, which needs relative-picker compensation in a scoped simulator set where CoreDevice cannot read orientation, and the lit panel did not settle on the requested orientation. Take a screenshot to see the screen, then retry.'; + 'In a scoped simulator set the rotation goes through the fold helper and only the lit panel can confirm it. The lit panel did not show the requested orientation, or the simulator never reported a single lit panel; the app in front may not support that orientation. Take a screenshot to see the screen, then retry.'; /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); @@ -164,14 +164,14 @@ export async function setAppleFoldableOrientation( /** * Rotates a foldable simulator in a scoped set, where only the fold helper can dispatch and * only its panel table can confirm. Measured on iPhone Duo, the lit panel reads the - * `orientation-picker-control` value plus a fixed per-panel offset (outer panel 0, inner - * panel +90), so one absolute dispatch of the target lands a quarter-turn away on the inner - * panel and looks one command late. This loop reads the lit panel, dispatches the picker - * value {@link nextPickerValue} chooses, and re-reads until the panel table reports the - * target: one dispatch when the lit panel's offset is the measured one, at most two when the - * first dispatch has to calibrate it. A repeat of the showing orientation is a read-only - * success. The picker state is deliberately not kept between commands: a fold in between can - * light the other panel, whose offset differs, and the measured offset covers that case. + * `orientation-picker-control` value plus a per-panel offset (outer panel 0, inner panel +90), + * so one absolute dispatch of the target lands a quarter-turn away on the inner panel and looks + * one command late. One bounded loop reads the lit panel and, until it shows the target, + * dispatches `target - offset`: the offset comes from this command's last dispatch and a fresh + * reading of the same panel once there is one, and from the measured table before that. A read + * that names no single lit panel neither confirms nor dispatches; the loop waits for the next. + * A repeat of the showing orientation is a read-only success. Nothing is kept between commands: + * a fold in between can light the other panel, and the measured table covers that panel. */ async function setScopedFoldableOrientation( device: DeviceInfo, @@ -179,109 +179,92 @@ async function setScopedFoldableOrientation( signal: AbortSignal | undefined, ): Promise<{ orientation: DeviceRotation }> { const helper = createFoldHelper(device); - const state: ScopedRotationState = { - pickerValue: undefined, - panelOffset: undefined, - litDisplayId: undefined, - observed: undefined, - rawPanelOrientation: undefined, - }; + let dispatched: ScopedDispatch | undefined; + let reading: LitPanel | undefined; for (let attempt = 1; attempt <= IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS; attempt += 1) { signal?.throwIfAborted(); - const lit = await readLitPanel(device, helper, signal); - if (lit) { - noteLitPanel(state, lit); - if (state.observed === rotation) return { orientation: rotation }; - } - const next = nextPickerValue({ - currentRotation: lit?.rotation, - targetRotation: rotation, - panelOffset: state.panelOffset, - currentPickerValue: state.pickerValue, - }); - if (next !== undefined) { - await dispatchScopedPicker(device, helper, state, next, rotation, attempt, signal); + reading = await readLitPanel(device, helper, signal); + if (reading?.rotation === rotation) return { orientation: rotation }; + if (reading) { + const panelOffset = scopedPanelOffset(dispatched, reading); + const value = pickerValueForTarget(rotation, panelOffset); + await helper.rotate(value, signal); + dispatched = { value, displayId: reading.panel.displayId }; + emitScopedDispatch(device, rotation, attempt, value, panelOffset, reading); } await settleScopedRotation(); } // The last dispatch gets its confirming read too: the loop only reads before dispatching. - const settled = await readLitPanel(device, helper, signal); - if (settled) noteLitPanel(state, settled); - if (state.observed === rotation) return { orientation: rotation }; - throw scopedUnverifiedError(device, rotation, state); + reading = await readLitPanel(device, helper, signal); + if (reading?.rotation === rotation) return { orientation: rotation }; + throw scopedUnverifiedError(device, rotation, reading); } -type ScopedRotationState = { - pickerValue: DeviceRotation | undefined; - panelOffset: PanelQuarterTurn | undefined; - litDisplayId: number | undefined; - observed: DeviceRotation | undefined; - rawPanelOrientation: string | undefined; -}; +/** The picker value this command last dispatched, and the panel that was lit for it. */ +type ScopedDispatch = { value: DeviceRotation; displayId: number }; type LitPanel = { panel: AppleDeviceDisplay; rotation: DeviceRotation }; /** - * Folds one lit-panel reading into the rotation state. A different panel lit up mid-command: - * the picker state and offset were calibrated against the previous panel, so start that - * panel's calibration over. Otherwise the reading fixes the panel's offset — from the picker - * value this command dispatched when there is one, else from the measured table. + * The lit panel's picker offset. After this command dispatched to the same panel, the fresh + * reading and that dispatch give the offset this panel really has. Before any dispatch, or when + * a fold lit another panel mid-command, the measured offset of the panel now lit applies. */ -function noteLitPanel(state: ScopedRotationState, lit: LitPanel): void { - if (state.litDisplayId !== undefined && lit.panel.displayId !== state.litDisplayId) { - state.pickerValue = undefined; - state.panelOffset = undefined; - } - state.litDisplayId = lit.panel.displayId; - state.observed = lit.rotation; - state.rawPanelOrientation = lit.panel.currentOrientation; - state.panelOffset = - state.pickerValue === undefined - ? (state.panelOffset ?? measuredPanelPickerOffset(lit.panel)) - : panelPickerOffset(lit.rotation, state.pickerValue); +function scopedPanelOffset( + dispatched: ScopedDispatch | undefined, + reading: LitPanel, +): PanelQuarterTurn { + return dispatched?.displayId === reading.panel.displayId + ? panelPickerOffset(reading.rotation, dispatched.value) + : measuredPanelPickerOffset(reading.panel); } -/** Dispatches one picker value and records it as the guest's picker state for this command. */ -async function dispatchScopedPicker( +function emitScopedDispatch( device: DeviceInfo, - helper: FoldHelper, - state: ScopedRotationState, - next: DeviceRotation, rotation: FoldableRotation, attempt: number, - signal: AbortSignal | undefined, -): Promise { - await helper.rotate(next, signal); - state.pickerValue = next; + value: DeviceRotation, + panelOffset: PanelQuarterTurn, + reading: LitPanel, +): void { emitDiagnostic({ level: 'info', phase: 'apple_foldable_orientation_dispatched', data: { deviceId: device.id, - orientation: next, + orientation: value, requestedOrientation: rotation, scopedSet: true, attempt, - ...(state.observed === undefined ? {} : { observedOrientation: state.observed }), + observedOrientation: reading.rotation, + litDisplay: reading.panel.name, + panelOffset, }, }); } +/** The error names only what the final read saw: an undecided final read reports no rotation. */ function scopedUnverifiedError( device: DeviceInfo, rotation: FoldableRotation, - state: ScopedRotationState, + reading: LitPanel | undefined, ): AppError { return new AppError( 'COMMAND_FAILED', - `${device.name} did not confirm rotation to ${rotation}: the panel table reports ${state.observed ?? 'no orientation'}`, + `${device.name} did not confirm rotation to ${rotation}: ${ + reading + ? `the lit panel shows ${reading.rotation}` + : 'the last panel table read named no single lit panel' + }`, { deviceId: device.id, requestedOrientation: rotation, - ...(state.observed === undefined ? {} : { observedOrientation: state.observed }), - ...(state.rawPanelOrientation === undefined - ? {} - : { rawPanelOrientation: state.rawPanelOrientation }), + ...(reading + ? { + observedOrientation: reading.rotation, + rawPanelOrientation: reading.panel.currentOrientation, + } + : {}), reason: 'orientation-unverified-scoped-set', hint: SCOPED_ORIENTATION_UNVERIFIED_HINT, }, @@ -291,8 +274,8 @@ function scopedUnverifiedError( /** * The one lit panel and the rotation it shows, from one fresh panel-table read. Anything * less decided — an unreadable table, no lit panel, an unrecognized `rotN` — is no reading: - * the rotation loop must not confirm or compensate from a panel that is not showing. On - * iPhone Duo the table reports both panels lit for a few seconds after a fold to closed, so + * the rotation loop must not confirm, compensate or dispatch from a panel that is not showing. + * On iPhone Duo the table reports both panels lit for a few seconds after a fold to closed, so * the first reads after that fold are no reading by design. */ async function readLitPanel( @@ -311,7 +294,7 @@ async function readLitPanel( const litPanels = inventory.displays.filter((display) => display.power === 'lit'); const panel = litPanels.length === 1 ? litPanels[0]! : undefined; if (!panel) return undefined; - const rotation = panelRotationForOrientation(panel.currentOrientation); + const rotation = panelContentRotation(panel.currentOrientation); return rotation === undefined ? undefined : { panel, rotation }; } diff --git a/packages/platform-apple/src/foldable/picker-delta.test.ts b/packages/platform-apple/src/foldable/picker-delta.test.ts index dba2f51675..48ad007254 100644 --- a/packages/platform-apple/src/foldable/picker-delta.test.ts +++ b/packages/platform-apple/src/foldable/picker-delta.test.ts @@ -1,10 +1,10 @@ import { expect, test } from 'vitest'; import type { DeviceRotation } from '@agent-device/contracts/device'; +import { panelContentRotation } from '../core/display-inventory.ts'; import { measuredPanelPickerOffset, - nextPickerValue, panelPickerOffset, - panelRotationForOrientation, + pickerValueForTarget, type PanelQuarterTurn, } from './picker-delta.ts'; @@ -23,111 +23,44 @@ const ANGLE: Record = { 'portrait-upside-down': 180, 'landscape-left': 270, }; -const AT_ANGLE: readonly DeviceRotation[] = ROTATIONS; function atAngle(angle: number): DeviceRotation { - return AT_ANGLE[(((angle / 90) % 4) + 4) % 4]!; + return ROTATIONS[(((angle / 90) % 4) + 4) % 4]!; } -/** Applies one picker event the way the guest does: the panel turns by (new - previous). */ -function applyPickerEvent( - screen: DeviceRotation, - previousPicker: DeviceRotation, - nextPicker: DeviceRotation, -): DeviceRotation { - return atAngle(ANGLE[screen] + ANGLE[nextPicker] - ANGLE[previousPicker]); +/** The lit panel after one picker event, as measured: the picker value plus the panel offset. */ +function litPanelFor(picker: DeviceRotation, offset: number): DeviceRotation { + return atAngle(ANGLE[picker] + offset); } -test('panel table orientations read as panel-frame rotations', () => { - expect(panelRotationForOrientation('rot0')).toBe('portrait'); - expect(panelRotationForOrientation('rot90')).toBe('landscape-right'); - expect(panelRotationForOrientation('rot180')).toBe('portrait-upside-down'); - expect(panelRotationForOrientation('rot270')).toBe('landscape-left'); - expect(panelRotationForOrientation('unknown')).toBeUndefined(); -}); - -test('with the picker known, one dispatch lands every current/target/offset/picker state', () => { - for (const current of ROTATIONS) { - for (const target of ROTATIONS) { - for (const offset of OFFSETS) { - for (const picker of ROTATIONS) { - const next = nextPickerValue({ - currentRotation: current, - targetRotation: target, - panelOffset: offset, - currentPickerValue: picker, - }); - expect(next, `${current} -> ${target}, K=${offset}, picker=${picker}`).toBeDefined(); - expect(applyPickerEvent(current, picker, next!)).toBe(target); - expect(panelPickerOffset(target, next!)).toBe( - ((ANGLE[target] - ANGLE[next!] + 360) % 360) as PanelQuarterTurn, - ); - } - } - } - } -}); - -test('with only the offset known, one dispatch lands every consistent state', () => { +test('the target minus the offset lands every target at every offset in one dispatch', () => { for (const target of ROTATIONS) { for (const offset of OFFSETS) { - for (const picker of ROTATIONS) { - // A consistent guest state: the panel reads picker + offset. - const current = atAngle(ANGLE[picker] + offset); - const next = nextPickerValue({ - currentRotation: current, - targetRotation: target, - panelOffset: offset, - currentPickerValue: undefined, - }); - expect(next).toBe(atAngle(ANGLE[target] - offset)); - expect(applyPickerEvent(current, picker, next!)).toBe(target); - } + const value = pickerValueForTarget(target, offset); + expect(litPanelFor(value, offset), `${target}, K=${offset}`).toBe(target); } } }); -test('with neither known, the calibration dispatch plus one delta dispatch lands every state', () => { +test('a panel whose offset differs from the measured one lands on the dispatch its reading calibrates', () => { for (const target of ROTATIONS) { - for (const offset of OFFSETS) { - for (const picker of ROTATIONS) { - const start = atAngle(ANGLE[picker] + offset); - const calibration = nextPickerValue({ - currentRotation: start, - targetRotation: target, - panelOffset: undefined, - currentPickerValue: undefined, - }); - expect(calibration).toBe(target); - const afterCalibration = applyPickerEvent(start, picker, calibration!); - if (afterCalibration === target) continue; - const learned = panelPickerOffset(afterCalibration, calibration!); - expect(learned).toBe(offset); - const correction = nextPickerValue({ - currentRotation: afterCalibration, - targetRotation: target, - panelOffset: learned, - currentPickerValue: calibration, - }); - expect(applyPickerEvent(afterCalibration, calibration!, correction!)).toBe(target); + for (const actual of OFFSETS) { + for (const measured of OFFSETS) { + const first = pickerValueForTarget(target, measured); + const reading = litPanelFor(first, actual); + if (reading === target) continue; + const learned = panelPickerOffset(reading, first); + expect(learned).toBe(actual); + expect(litPanelFor(pickerValueForTarget(target, learned), actual)).toBe(target); } } } }); -test('without a current rotation there is no dispatch once the picker is known', () => { - for (const target of ROTATIONS) { - for (const picker of ROTATIONS) { - expect( - nextPickerValue({ - currentRotation: undefined, - targetRotation: target, - panelOffset: 90, - currentPickerValue: picker, - }), - ).toBeUndefined(); - } - } +test('the inner panel reaches landscape-left only through the upside-down picker value', () => { + expect(pickerValueForTarget('landscape-left', 90)).toBe('portrait-upside-down'); + expect(pickerValueForTarget('portrait', 90)).toBe('landscape-left'); + expect(pickerValueForTarget('landscape-right', 90)).toBe('portrait'); }); test('the measured offsets are 0 on the outer (primary) panel and +90 on the inner panel', () => { @@ -136,17 +69,19 @@ test('the measured offsets are 0 on the outer (primary) panel and +90 on the inn }); /** - * Raw iPhone Duo readings (iOS 27.3, scoped simulator set, Safari in front), in the order they - * were taken: the picker value the guest held and the lit panel's `currentOrientation` that the - * panel table reported next. Rows marked `fold` are the first read after a pose change, with - * the picker value from before it. Every row gives its panel's measured offset, so the picker - * value survives a fold in both directions and only the panel's offset changes. + * Every iPhone Duo reading taken for the measurement (iOS 27.3, scoped simulator set, Safari in + * front), in order: the picker value the guest held and the lit panel's `currentOrientation` + * that the panel table reported next. Rows marked `fold` are the first read after a pose change, + * with the picker value from before it. One row contradicts the outer panel's offset and is kept + * with its `exception`: a fold left the picker at the upside-down value, which the outer panel's + * foreground app cannot show, so that panel kept the rotation it last showed. */ const DUO_READINGS: readonly { panel: 'inner' | 'outer'; picker: DeviceRotation; reading: string; fold?: true; + exception?: string; }[] = [ { panel: 'inner', picker: 'portrait', reading: 'rot90' }, { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, @@ -167,7 +102,13 @@ const DUO_READINGS: readonly { { panel: 'inner', picker: 'portrait', reading: 'rot90' }, { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, { panel: 'inner', picker: 'portrait-upside-down', reading: 'rot270' }, - // (fold to closed with the picker at portrait-upside-down: see the next test) + { + panel: 'outer', + picker: 'portrait-upside-down', + reading: 'rot270', + fold: true, + exception: 'the outer panel app has no upside-down; the panel kept landscape-left', + }, { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, { panel: 'outer', picker: 'portrait', reading: 'rot0' }, { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, @@ -183,26 +124,25 @@ const DUO_READINGS: readonly { { panel: 'inner', picker: 'portrait', reading: 'rot90' }, ]; -test('every measured Duo reading gives its panel the measured offset, across folds', () => { +test('every measured Duo reading but the marked exception gives its panel the measured offset', () => { for (const [index, row] of DUO_READINGS.entries()) { - const reading = panelRotationForOrientation(row.reading); + const reading = panelContentRotation(row.reading); expect(reading, `row ${index}`).toBeDefined(); - expect(panelPickerOffset(reading!, row.picker), `row ${index}`).toBe( - measuredPanelPickerOffset({ primary: row.panel === 'outer' }), - ); + const offset = panelPickerOffset(reading!, row.picker); + const measured = measuredPanelPickerOffset({ primary: row.panel === 'outer' }); + if (row.exception) expect(offset, `row ${index}`).not.toBe(measured); + else expect(offset, `row ${index}`).toBe(measured); } - expect(DUO_READINGS.filter((row) => row.fold)).toHaveLength(5); + expect(DUO_READINGS).toHaveLength(33); + expect(DUO_READINGS.filter((row) => row.fold)).toHaveLength(6); + expect(DUO_READINGS.filter((row) => row.exception)).toHaveLength(1); }); -test('after a fold that left the picker where the outer panel could not follow, the measured offset still lands the target', () => { - // Measured: picker at portrait-upside-down, outer panel lit showing landscape-left (its UI has - // no upside-down). One dispatch of landscape-right landed landscape-right. A purely relative - // guest would have turned the panel by (90 - 180) to portrait-upside-down instead. - const next = nextPickerValue({ - currentRotation: 'landscape-left', - targetRotation: 'landscape-right', - panelOffset: measuredPanelPickerOffset({ primary: true }), - currentPickerValue: undefined, - }); - expect(next).toBe('landscape-right'); +test('after the exception, the measured outer offset still lands the next target in one dispatch', () => { + // Measured: picker at portrait-upside-down, outer panel lit showing landscape-left. One + // dispatch of landscape-right landed landscape-right (the row after the exception). A purely + // relative guest would have turned the panel by (90 - 180) to portrait-upside-down instead. + expect( + pickerValueForTarget('landscape-right', measuredPanelPickerOffset({ primary: true })), + ).toBe('landscape-right'); }); diff --git a/packages/platform-apple/src/foldable/picker-delta.ts b/packages/platform-apple/src/foldable/picker-delta.ts index 0f85d7b3ee..72131d2b07 100644 --- a/packages/platform-apple/src/foldable/picker-delta.ts +++ b/packages/platform-apple/src/foldable/picker-delta.ts @@ -4,7 +4,9 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; * Panel-frame quarter turns, following CADisplay `rotN`: portrait 0, landscape-right 90, * portrait-upside-down 180, landscape-left 270. This frame is a panel-table fact, not the * Android `Surface.ROTATION_*` frame in `DEVICE_ROTATION_SURFACE_INDEX` (which turns the - * other way), so the two tables must not be shared. + * other way), so the two tables must not be shared. Parsing a panel's `currentOrientation` + * into a rotation is `panelContentRotation` in the display inventory; this module only does + * the picker arithmetic on rotations already parsed. */ export type PanelQuarterTurn = 0 | 90 | 180 | 270; @@ -22,13 +24,6 @@ const ROTATION_BY_QUARTER: readonly DeviceRotation[] = [ 'landscape-left', ]; -const PANEL_ORIENTATIONS: ReadonlyMap = new Map([ - ['rot0', 'portrait'], - ['rot90', 'landscape-right'], - ['rot180', 'portrait-upside-down'], - ['rot270', 'landscape-left'], -]); - /** The panel-frame angle of one rotation. */ function panelRotationAngle(rotation: DeviceRotation): PanelQuarterTurn { return PANEL_ROTATION_ANGLE[rotation]; @@ -40,17 +35,10 @@ function rotationAtPanelAngle(angle: number): DeviceRotation { return ROTATION_BY_QUARTER[quarter]!; } -/** The lit-panel rotation a panel table `currentOrientation` names, or nothing when unread. */ -export function panelRotationForOrientation( - currentOrientation: string, -): DeviceRotation | undefined { - return PANEL_ORIENTATIONS.get(currentOrientation); -} - /** * The offset between the orientation picker and the lit panel it drives: the panel reads - * `picker + offset`. Learning it from one dispatched picker value and the panel rotation - * that followed it is what calibrates a panel whose offset was never measured. + * `picker + offset`. The picker value is the one this command last dispatched, and the + * rotation is a fresh reading of the same panel taken after it. */ export function panelPickerOffset( currentRotation: DeviceRotation, @@ -66,55 +54,27 @@ const OUTER_PANEL_PICKER_OFFSET: PanelQuarterTurn = 0; const INNER_PANEL_PICKER_OFFSET: PanelQuarterTurn = 90; /** - * Measured picker offsets by panel. The `primary` panel is the one that stays lit while the - * device is closed (the outer panel); the other integrated panel is the inner one (lit open - * and half-open). Measured on iPhone Duo (iOS 27.3) against the panel table's lit-panel - * `currentOrientation`: the outer panel reads `picker + 0` and the inner panel `picker + 90`, - * across fold/unfold in both directions. The guest keeps one picker value through a fold, so - * the panel that lights afterwards reads that same value plus its own offset. - * - * A measured offset only saves the calibration dispatch: the caller still confirms every - * rotation from the panel table and re-derives the offset from what it observes after each - * dispatch, so a panel or runtime that disagrees with this table still converges. + * Measured picker offsets by panel, used only before a command has calibrated the lit panel + * with a dispatch of its own. The `primary` panel is the one that stays lit while the device + * is closed (the outer panel); the other integrated panel is the inner one (lit open and + * half-open). Measured on iPhone Duo (iOS 27.3) against the lit panel's `currentOrientation`: + * the outer panel reads `picker + 0` and the inner panel `picker + 90`, across fold and unfold + * in both directions. The guest keeps one picker value through a fold, so the panel that + * lights afterwards reads that same value plus its own offset — unless its foreground app + * cannot show that orientation, in which case it keeps the rotation it last showed. */ export function measuredPanelPickerOffset(panel: { primary: boolean }): PanelQuarterTurn { return panel.primary ? OUTER_PANEL_PICKER_OFFSET : INNER_PANEL_PICKER_OFFSET; } /** - * The next picker value to dispatch. The guest shows `picker + offset` on the lit panel, with - * a fixed offset per panel, so an absolute dispatch of the target lands an offset away from - * it, and successive events look as if each one turned the panel by (new value - previous - * picker value). The choice is, in order: - * - * - Picker known: the value whose delta from the current picker value is exactly the turn - * still needed. Needs the current lit-panel rotation; without it there is no safe dispatch, - * so the caller re-reads instead (nothing is returned). - * - Picker unknown, offset known: the value that leaves the panel at the target once the - * offset is added back. This holds even when the lit panel is not showing `picker + offset` - * yet, for example after a fold left the picker at a value the newly lit panel's - * foreground UI does not show. - * - Both unknown: the target itself, as a calibration dispatch. Wherever it lands, one panel - * read afterwards fixes the offset through {@link panelPickerOffset}, and the next dispatch - * is exact. + * The picker value that leaves the lit panel at `targetRotation` once the panel's offset is + * added back: `target - offset`. The guest shows `picker + offset`, so dispatching the target + * itself lands an offset away from it, which is what made a rotation look one command late. */ -export function nextPickerValue(input: { - currentRotation: DeviceRotation | undefined; - targetRotation: DeviceRotation; - panelOffset: PanelQuarterTurn | undefined; - currentPickerValue: DeviceRotation | undefined; -}): DeviceRotation | undefined { - const { currentRotation, targetRotation, panelOffset, currentPickerValue } = input; - if (currentPickerValue !== undefined) { - if (currentRotation === undefined) return undefined; - return rotationAtPanelAngle( - panelRotationAngle(currentPickerValue) + - panelRotationAngle(targetRotation) - - panelRotationAngle(currentRotation), - ); - } - if (panelOffset !== undefined) { - return rotationAtPanelAngle(panelRotationAngle(targetRotation) - panelOffset); - } - return targetRotation; +export function pickerValueForTarget( + targetRotation: DeviceRotation, + panelOffset: PanelQuarterTurn, +): DeviceRotation { + return rotationAtPanelAngle(panelRotationAngle(targetRotation) - panelOffset); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d8a7732bd5..39187b1a1f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -132,7 +132,7 @@ Back, orientation, and hardware buttons: - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. - An app with a left-edge drawer may open the drawer instead of going back. When a back or close control is visible, press it by its ref. - `back --system` sends system back input: the Android back keyevent, the iOS back swipe, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. -- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. A foldable iPhone simulator (iPhone Duo) refuses `portrait-upside-down`; in a scoped simulator set (`--ios-simulator-device-set`) it rotates through the fold helper and confirms the rotation from the panel table. +- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. A foldable iPhone simulator (iPhone Duo) refuses `portrait-upside-down`. In a scoped simulator set (`--ios-simulator-device-set`) it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified-scoped-set`. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`. - `action-button` presses the iPhone or iPad Action Button once. It takes no arguments, no hold duration, and no `--settle`, and it doesn't activate the session app or drive Camera Control. Other platforms refuse it, and a model without the button (an iPhone SE, most iPad simulators) fails with `UNSUPPORTED_OPERATION`. - `action-button` reports that the press was dispatched, not what it triggered. Simulators run no Shortcuts or App Intents, so verify the outcome on a physical iPhone. Expect a press to take about five seconds on a Simulator. From a92989be1868dc21b968116a81a407ae0aed79ce Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 20:11:00 +0000 Subject: [PATCH 06/16] fix(ios): a scoped rotation learns nothing from a panel that did not move Review of #4440 (cubic): the offset re-derived from the last dispatch and a same-panel reading assumed that dispatch had landed. A reading taken before it landed, or a panel whose app cannot show the target, taught a wrong offset; on the device a portrait-only app made the loop cycle through six picker values, upside-down included. A reading that still shows what the panel showed before the last dispatch now teaches nothing and gets no further dispatch: the loop re-reads, and a panel that never follows fails unverified after one dispatch. Also qualify the attempt-budget comment and note that both panels report lit for a few seconds after an unfold as well as a fold. Co-Authored-By: Claude Opus 5.5 --- .changeset/foldable-scoped-rotation.md | 2 +- packages/platform-apple/src/core/config.ts | 5 ++- .../src/foldable/orientation.test.ts | 42 ++++++++++++++----- .../src/foldable/orientation.ts | 39 ++++++++++++----- 4 files changed, 64 insertions(+), 24 deletions(-) diff --git a/.changeset/foldable-scoped-rotation.md b/.changeset/foldable-scoped-rotation.md index 65f47d2f3b..ce8b451108 100644 --- a/.changeset/foldable-scoped-rotation.md +++ b/.changeset/foldable-scoped-rotation.md @@ -2,4 +2,4 @@ 'agent-device': patch --- -Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and succeeds only when the lit panel shows the requested orientation, otherwise failing with `orientation-unverified-scoped-set`. Measured on iPhone Duo, the lit panel usually reads the orientation-picker value plus a fixed per-panel offset (0 degrees on the outer panel, +90 degrees on the inner panel), and the picker value survives a fold; a panel whose foreground app cannot show that orientation keeps its previous one instead. Each dispatch is therefore the target minus the lit panel's offset, taken from the measured value at first and then recalibrated from the last dispatch and a fresh reading of the same panel, so the first command lands. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the inner panel needs to reach landscape-left; a user target of `portrait-upside-down` stays refused. +Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and succeeds only when the lit panel shows the requested orientation, otherwise failing with `orientation-unverified-scoped-set`. Measured on iPhone Duo, the lit panel usually reads the orientation-picker value plus a fixed per-panel offset (0 degrees on the outer panel, +90 degrees on the inner panel), and the picker value survives a fold; a panel whose foreground app cannot show that orientation keeps its previous one instead. Each dispatch is therefore the target minus the lit panel's offset, taken from the measured value at first and then recalibrated from the last dispatch and a fresh reading of the same panel, so the first command lands. A reading that has not moved since the last dispatch teaches nothing and is not followed by another dispatch, so a panel that cannot show the target fails after one dispatch. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the inner panel needs to reach landscape-left; a user target of `portrait-upside-down` stays refused. diff --git a/packages/platform-apple/src/core/config.ts b/packages/platform-apple/src/core/config.ts index 798dc8b0ba..cb22e27c12 100644 --- a/packages/platform-apple/src/core/config.ts +++ b/packages/platform-apple/src/core/config.ts @@ -67,8 +67,9 @@ export const IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS = 2_000; * How many read/dispatch rounds a scoped-set foldable rotation gets. A panel at its measured * picker offset lands in one dispatch, and a panel that disagrees with the measured offset in * two, the second calibrated by the first. The rest of the budget covers reads that name no - * single lit panel, such as the few seconds after an iPhone Duo folds closed. Verified on - * iPhone Duo with the settle delay below between a dispatch and the read that confirms it. + * single lit panel, such as the few seconds after an iPhone Duo folds or unfolds, and reads taken + * before a dispatch has landed. Exercised on an iPhone Duo simulator (iOS 27.3) with this loop and + * the settle delay below between a dispatch and the read that confirms it. */ export const IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS = 6; export const IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS = 600; diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 86ec879445..c99493e82b 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -194,6 +194,8 @@ function fakeScopedDuo( lastShown?: Partial>; /** From this panel read on (1-based, the routing probe included), both panels report lit. */ bothLitFromRead?: number; + /** How many panel reads after a picker event still show the panel as it was before it. */ + hidLagReads?: number; }, ) { const helpers: string[][] = []; @@ -201,6 +203,8 @@ function fakeScopedDuo( const offsets = options.offsets ?? MEASURED_OFFSETS; let lit = options.lit; let pickerAngle = PANEL_ANGLE[options.picker]; + let pendingPicker: number | undefined; + let lagReadsLeft = 0; let bothLitReads = 0; let panelReads = 0; const shown: Record = { @@ -239,6 +243,12 @@ function fakeScopedDuo( helpers.push(mode); if (mode[0] === 'panels') { panelReads += 1; + if (lagReadsLeft > 0) lagReadsLeft -= 1; + else if (pendingPicker !== undefined) { + pickerAngle = pendingPicker; + pendingPicker = undefined; + show(lit); + } const bothLit = bothLitReads > 0 || (options.bothLitFromRead !== undefined && panelReads >= options.bothLitFromRead); @@ -247,7 +257,10 @@ function fakeScopedDuo( return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; } expect(mode[0]).toBe('rotate'); - if (options.followsHid !== false) { + if (options.followsHid !== false && options.hidLagReads) { + pendingPicker = PANEL_ANGLE[mode[1]! as DeviceRotation]; + lagReadsLeft = options.hidLagReads; + } else if (options.followsHid !== false) { pickerAngle = PANEL_ANGLE[mode[1]! as DeviceRotation]; show(lit); } @@ -447,7 +460,7 @@ test('an undecided final read does not report an earlier reading as the current }); expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); expect((outcome as AppError).details).not.toHaveProperty('rawPanelOrientation'); - expect(sim.rotations()).toEqual(['landscape-left', 'portrait-upside-down']); + expect(sim.rotations()).toEqual(['landscape-left']); }); test('a scoped rotation to the orientation already showing dispatches nothing', async () => { @@ -461,6 +474,19 @@ test('a scoped rotation to the orientation already showing dispatches nothing', expect(sim.rotations()).toEqual([]); }); +test('a reading taken before the dispatch lands teaches nothing and is not followed by another dispatch', async () => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + // The first two panel reads after each picker event still show the panel as it was before it. + const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'portrait', hidLagReads: 2 }); + expect(sim.litScreen()).toBe('rot90'); + await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + // One exact dispatch (landscape-left - 90 = portrait-upside-down); the stale reads in between + // neither re-learned an offset nor dispatched again. + expect(sim.rotations()).toEqual(['portrait-upside-down']); +}); + test('a scoped rotation the lit panel never shows is refused as unverified', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); const sim = fakeScopedDuo(scoped, { @@ -478,15 +504,9 @@ test('a scoped rotation the lit panel never shows is refused as unverified', asy rawPanelOrientation: 'rot90', }, }); - // Each dispatch re-derives the offset from the unchanged reading: six rounds, then refusal. - expect(sim.rotations()).toEqual([ - 'landscape-left', - 'portrait-upside-down', - 'landscape-right', - 'portrait', - 'landscape-left', - 'portrait-upside-down', - ]); + // The panel never moved after the one dispatch, so it taught no offset and nothing more was + // dispatched: the loop re-read until its budget ended, then refused. + expect(sim.rotations()).toEqual(['landscape-left']); expect(sim.devicectl).toEqual([]); }); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index e98e9fab00..e211257e56 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -168,8 +168,9 @@ export async function setAppleFoldableOrientation( * so one absolute dispatch of the target lands a quarter-turn away on the inner panel and looks * one command late. One bounded loop reads the lit panel and, until it shows the target, * dispatches `target - offset`: the offset comes from this command's last dispatch and a fresh - * reading of the same panel once there is one, and from the measured table before that. A read - * that names no single lit panel neither confirms nor dispatches; the loop waits for the next. + * reading of the same panel that moved since it, and from the measured table before that. A read + * that names no single lit panel, or a panel that has not moved since the last dispatch, neither + * confirms nor dispatches; the loop waits for the next read. * A repeat of the showing orientation is a read-only success. Nothing is kept between commands: * a fold in between can light the other panel, and the measured table covers that panel. */ @@ -185,11 +186,11 @@ async function setScopedFoldableOrientation( signal?.throwIfAborted(); reading = await readLitPanel(device, helper, signal); if (reading?.rotation === rotation) return { orientation: rotation }; - if (reading) { + if (reading && !awaitingDispatchEffect(dispatched, reading)) { const panelOffset = scopedPanelOffset(dispatched, reading); const value = pickerValueForTarget(rotation, panelOffset); await helper.rotate(value, signal); - dispatched = { value, displayId: reading.panel.displayId }; + dispatched = { value, displayId: reading.panel.displayId, before: reading.rotation }; emitScopedDispatch(device, rotation, attempt, value, panelOffset, reading); } await settleScopedRotation(); @@ -200,14 +201,32 @@ async function setScopedFoldableOrientation( throw scopedUnverifiedError(device, rotation, reading); } -/** The picker value this command last dispatched, and the panel that was lit for it. */ -type ScopedDispatch = { value: DeviceRotation; displayId: number }; +/** + * The picker value this command last dispatched, the panel that was lit for it, and the + * rotation that panel showed before the dispatch. + */ +type ScopedDispatch = { value: DeviceRotation; displayId: number; before: DeviceRotation }; type LitPanel = { panel: AppleDeviceDisplay; rotation: DeviceRotation }; /** - * The lit panel's picker offset. After this command dispatched to the same panel, the fresh - * reading and that dispatch give the offset this panel really has. Before any dispatch, or when + * The same panel still shows what it showed before the last dispatch: the dispatch has not taken + * effect yet, or the app in front cannot show that orientation. Such a reading teaches no offset, + * so nothing is dispatched on it; the loop re-reads, and a panel that never follows ends the + * command unverified after one dispatch instead of cycling through picker values. + */ +function awaitingDispatchEffect( + dispatched: ScopedDispatch | undefined, + reading: LitPanel, +): boolean { + return ( + dispatched?.displayId === reading.panel.displayId && reading.rotation === dispatched.before + ); +} + +/** + * The lit panel's picker offset. After this command dispatched to the same panel and the panel + * moved, the fresh reading and that dispatch give the offset this panel really has. Before any dispatch, or when * a fold lit another panel mid-command, the measured offset of the panel now lit applies. */ function scopedPanelOffset( @@ -275,8 +294,8 @@ function scopedUnverifiedError( * The one lit panel and the rotation it shows, from one fresh panel-table read. Anything * less decided — an unreadable table, no lit panel, an unrecognized `rotN` — is no reading: * the rotation loop must not confirm, compensate or dispatch from a panel that is not showing. - * On iPhone Duo the table reports both panels lit for a few seconds after a fold to closed, so - * the first reads after that fold are no reading by design. + * On iPhone Duo the table reports both panels lit for a few seconds after a pose change in either + * direction, so the first reads after a fold or an unfold are no reading by design. */ async function readLitPanel( device: DeviceInfo, From 674f6705e3e4da5b2359b08529ca98698def30d5 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 22:06:48 +0000 Subject: [PATCH 07/16] fix(ios): a scoped rotation needs one established lit panel Review of #4440: readLitPanel filtered the parsed panels for a single lit one, so a table with a second lit integrated entry that lacked identity or geometry (which the parser drops) still produced a reading. The rotation then confirmed, or dispatched compensation, without one established lit panel. display-inventory now owns the strict selection (establishedLitPanel): a resolved multi-panel table that parsed every integrated entry, and so named an active display, with exactly one lit panel, which is that display. appleLitPanel and the scoped rotation both select through it; the incomplete-table capture refusal is unchanged and picker calibration stays in the foldable module. Regression: an interactor test with two activeOn integrated entries, one without geometry, for a target the parsed panel already shows and for another target: no dispatch, no confirmation, orientation-unverified- scoped-set. On c75affa it returned success in both cases. Co-Authored-By: Claude Opus 5.5 --- .../core/__tests__/display-inventory.test.ts | 52 +++++++++++++++++++ .../src/core/display-inventory.ts | 26 ++++++++-- .../src/foldable/orientation.test.ts | 33 +++++++++++- .../src/foldable/orientation.ts | 10 ++-- 4 files changed, 109 insertions(+), 12 deletions(-) diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 305cd0c1b1..660b3a9ff7 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -33,7 +33,9 @@ vi.mock('@agent-device/host-kit/host-file', async (importOriginal) => { }); import { + appleLitPanel, appleSimulatorDisplayArgvFragment, + establishedLitPanel, buildInventory, parseCoreDeviceDisplays, readPanelPower, @@ -375,6 +377,56 @@ describe('panelContentRotation', () => { }); }); +describe('establishedLitPanel', () => { + beforeEach(() => { + mockReadHostTextFile.mockReset(); + mockReadHostTextFile.mockResolvedValue(''); + }); + + function duoTable(mapEntry: (entry: Record) => Record) { + return JSON.stringify({ + result: { displays: IPHONE_DUO_CLOSED.result.displays.map(mapEntry) }, + }); + } + + test('a complete table with one lit panel establishes that panel for both selectors', () => { + const inventory = buildInventory([outerPanel, innerPanel]); + assert.equal(establishedLitPanel(inventory), outerPanel); + assert.deepEqual(appleLitPanel(inventory), { displayId: 1, widthPt: 466, heightPt: 678 }); + }); + + test('two lit panels establish none', () => { + const inventory = buildInventory([outerPanel, { ...innerPanel, power: 'lit' }]); + assert.equal(establishedLitPanel(inventory), undefined); + assert.equal(appleLitPanel(inventory), undefined); + }); + + test('an incomplete table establishes no panel even when one parsed panel is the only lit one', async () => { + // Two activeOn integrated entries; the inner one has no geometry and does not parse. + mockReadHostTextFile.mockResolvedValue( + duoTable((entry) => { + const lit: Record = { ...entry, backlightState: 'activeOn' }; + if (entry.name !== 'LCD-1') return lit; + const { nativeSize: _nativeSize, ...withoutGeometry } = lit; + return withoutGeometry; + }), + ); + const inventory = await queryAppleDisplayInventory(IOS_TEST_SIMULATOR, {}); + assert.equal(inventory.multiScreen, true); + assert.equal(inventory.displays.filter((display) => display.power === 'lit').length, 1); + assert.equal(establishedLitPanel(inventory), undefined); + assert.equal(appleLitPanel(inventory), undefined); + }); + + test('a single-panel or unresolved table establishes no foldable panel', () => { + assert.equal(establishedLitPanel(buildInventory([outerPanel])), undefined); + assert.equal( + establishedLitPanel({ displays: [], multiScreen: false, ambiguous: false, unresolved: true }), + undefined, + ); + }); +}); + describe('resolveAppleCaptureTarget', () => { beforeEach(() => { mockRunXcrun.mockClear(); diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index c9afca8fb5..70fd471e3a 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -411,14 +411,30 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi } /** - * The panel a runner request names, only when panel power decided exactly one panel. Unlike a - * capture, a surface request does not fall back to the `primary` panel: it can keep the runner's - * historic route instead of guessing. Display 0 is the runtime's "no display" identity. + * The one panel the panel table establishes as lit, or nothing. It takes a resolved multi-panel + * table that parsed every integrated entry, and so named an active display, with exactly one lit + * panel, which is that active display. A table with an integrated entry missing its identity or + * geometry establishes no panel, even when the panels that did parse show a single lit one: the + * entry that did not parse may be lit too. Runner routing (`appleLitPanel`) and the foldable + * rotation controller select through this one rule. */ -export function appleLitPanel(inventory: AppleDisplayInventory): RunnerCommand['litPanel'] { +export function establishedLitPanel( + inventory: AppleDisplayInventory, +): AppleDeviceDisplay | undefined { if (!inventory.multiScreen || inventory.unresolved || !inventory.activeDisplay) return undefined; const litPanels = inventory.displays.filter((display) => display.power === 'lit'); - const panel = litPanels.length === 1 ? litPanels[0]! : undefined; + if (litPanels.length !== 1 || litPanels[0] !== inventory.activeDisplay) return undefined; + return litPanels[0]; +} + +/** + * The panel a runner request names, only when the panel table establishes one lit panel (see + * {@link establishedLitPanel}). Unlike a capture, a surface request does not fall back to the + * `primary` panel: it can keep the runner's historic route instead of guessing. Display 0 is the + * runtime's "no display" identity. + */ +export function appleLitPanel(inventory: AppleDisplayInventory): RunnerCommand['litPanel'] { + const panel = establishedLitPanel(inventory); if (!panel || panel.displayId <= 0) return undefined; return { displayId: panel.displayId, diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index c99493e82b..cf9c5cd4c5 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -196,6 +196,8 @@ function fakeScopedDuo( bothLitFromRead?: number; /** How many panel reads after a picker event still show the panel as it was before it. */ hidLagReads?: number; + /** A panel the table reports lit (activeOn) but without geometry: an incomplete table. */ + incompleteLitPanel?: DuoPanel; }, ) { const helpers: string[][] = []; @@ -218,13 +220,14 @@ function fakeScopedDuo( }; show(lit); const panel = (kind: DuoPanel, bothLit: boolean) => { - const isLit = bothLit || kind === lit; + const incomplete = options.incompleteLitPanel === kind; + const isLit = bothLit || kind === lit || incomplete; return { backlightState: isLit ? 'activeOn' : 'off', currentOrientation: isLit ? `rot${shown[kind]}` : 'rot180', displayId: kind === 'outer' ? 1 : 3, name: kind === 'outer' ? 'LCD' : 'LCD-1', - nativeSize: kind === 'outer' ? [1398, 2034] : [2007, 2853], + ...(incomplete ? {} : { nativeSize: kind === 'outer' ? [1398, 2034] : [2007, 2853] }), pointScale: 3, primary: kind === 'outer', type: { integrated: {} }, @@ -474,6 +477,32 @@ test('a scoped rotation to the orientation already showing dispatches nothing', expect(sim.rotations()).toEqual([]); }); +test.each([ + ['the parsed panel already shows the target', 'landscape-left'], + ['the parsed panel shows another rotation', 'landscape-right'], +] as const)( + 'an incomplete panel table never confirms or dispatches (%s)', + async (_case, target) => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + // Two activeOn integrated entries; the inner one has no geometry, so it does not parse. The + // outer panel that did parse is the only lit one left, but no single lit panel is established. + const sim = fakeScopedDuo(scoped, { + lit: 'outer', + picker: 'landscape-left', + incompleteLitPanel: 'inner', + }); + expect(sim.litScreen()).toBe('rot270'); + const outcome = await rotateScoped(sim, target); + expect(outcome).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-unverified-scoped-set', requestedOrientation: target }, + }); + expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); + expect(sim.rotations()).toEqual([]); + expect(sim.devicectl).toEqual([]); + }, +); + test('a reading taken before the dispatch lands teaches nothing and is not followed by another dispatch', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); // The first two panel reads after each picker event still show the panel as it was before it. diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index e211257e56..a03d13b1fb 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -13,6 +13,7 @@ import { } from '../core/config.ts'; import { readAppleDeviceOrientation } from '../core/device-orientation.ts'; import { + establishedLitPanel, panelContentRotation, queryAppleDisplayInventory, type AppleDeviceDisplay, @@ -291,8 +292,9 @@ function scopedUnverifiedError( } /** - * The one lit panel and the rotation it shows, from one fresh panel-table read. Anything - * less decided — an unreadable table, no lit panel, an unrecognized `rotN` — is no reading: + * The one lit panel and the rotation it shows, from one fresh panel-table read, selected by the + * display inventory's {@link establishedLitPanel}. Anything less decided — an unreadable table, + * an incomplete one, no lit panel or several, an unrecognized `rotN` — is no reading: * the rotation loop must not confirm, compensate or dispatch from a panel that is not showing. * On iPhone Duo the table reports both panels lit for a few seconds after a pose change in either * direction, so the first reads after a fold or an unfold are no reading by design. @@ -309,9 +311,7 @@ async function readLitPanel( if (signal?.aborted) throw error; return undefined; } - if (inventory.unresolved || !inventory.multiScreen) return undefined; - const litPanels = inventory.displays.filter((display) => display.power === 'lit'); - const panel = litPanels.length === 1 ? litPanels[0]! : undefined; + const panel = establishedLitPanel(inventory); if (!panel) return undefined; const rotation = panelContentRotation(panel.currentOrientation); return rotation === undefined ? undefined : { panel, rotation }; From 64b4ea65b30a7db82f51c40328c6cac8f8bb5891 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Wed, 11 Nov 2026 22:20:10 +0000 Subject: [PATCH 08/16] fix(ios): a panel of unknown power blocks the established lit panel Review of #4440 (cubic): establishedLitPanel counted only power 'lit', so a lit panel beside a sibling of unknown power (an unrecognized backlight state, which readPanelPower deliberately does not call dark) was still established, and the scoped rotation could confirm or dispatch from it. Any panel of unknown power now establishes none, for runner routing and the scoped rotation alike; the capture fallback is unchanged. Fail-first: on bfdbd9b the new selector test and both interactor cases (target already showing, another target) failed - the rotation returned success in both. Co-Authored-By: Claude Opus 5.5 --- .../core/__tests__/display-inventory.test.ts | 7 +++++ .../src/core/display-inventory.ts | 9 ++++-- .../src/foldable/orientation.test.ts | 31 ++++++++++++++++++- .../src/foldable/orientation.ts | 3 +- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 660b3a9ff7..7d3a7e0b0a 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -401,6 +401,13 @@ describe('establishedLitPanel', () => { assert.equal(appleLitPanel(inventory), undefined); }); + test('a lit panel beside a panel of unknown power establishes none', () => { + // Unknown power may be showing content, so the lit panel is not the only one that may be lit. + const inventory = buildInventory([outerPanel, { ...innerPanel, power: 'unknown' }]); + assert.equal(establishedLitPanel(inventory), undefined); + assert.equal(appleLitPanel(inventory), undefined); + }); + test('an incomplete table establishes no panel even when one parsed panel is the only lit one', async () => { // Two activeOn integrated entries; the inner one has no geometry and does not parse. mockReadHostTextFile.mockResolvedValue( diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 70fd471e3a..47a10ccb7d 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -413,15 +413,18 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi /** * The one panel the panel table establishes as lit, or nothing. It takes a resolved multi-panel * table that parsed every integrated entry, and so named an active display, with exactly one lit - * panel, which is that active display. A table with an integrated entry missing its identity or - * geometry establishes no panel, even when the panels that did parse show a single lit one: the - * entry that did not parse may be lit too. Runner routing (`appleLitPanel`) and the foldable + * panel, which is that active display, and no panel of unknown power. A table with an integrated + * entry missing its identity or geometry, or with a panel whose power is unknown, establishes no + * panel, even when the others show a single lit one: that entry may be lit too. Runner routing (`appleLitPanel`) and the foldable * rotation controller select through this one rule. */ export function establishedLitPanel( inventory: AppleDisplayInventory, ): AppleDeviceDisplay | undefined { if (!inventory.multiScreen || inventory.unresolved || !inventory.activeDisplay) return undefined; + // Unknown power may be showing content (see readPanelPower), so beside it no lit panel is the + // only one that may be lit. + if (inventory.displays.some((display) => display.power === 'unknown')) return undefined; const litPanels = inventory.displays.filter((display) => display.power === 'lit'); if (litPanels.length !== 1 || litPanels[0] !== inventory.activeDisplay) return undefined; return litPanels[0]; diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index cf9c5cd4c5..815e1c5fdf 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -198,6 +198,8 @@ function fakeScopedDuo( hidLagReads?: number; /** A panel the table reports lit (activeOn) but without geometry: an incomplete table. */ incompleteLitPanel?: DuoPanel; + /** A panel whose backlight state the table reports in an unrecognized form: power unknown. */ + unknownPowerPanel?: DuoPanel; }, ) { const helpers: string[][] = []; @@ -221,9 +223,10 @@ function fakeScopedDuo( show(lit); const panel = (kind: DuoPanel, bothLit: boolean) => { const incomplete = options.incompleteLitPanel === kind; + const unknownPower = options.unknownPowerPanel === kind; const isLit = bothLit || kind === lit || incomplete; return { - backlightState: isLit ? 'activeOn' : 'off', + backlightState: unknownPower ? 'transitioning' : isLit ? 'activeOn' : 'off', currentOrientation: isLit ? `rot${shown[kind]}` : 'rot180', displayId: kind === 'outer' ? 1 : 3, name: kind === 'outer' ? 'LCD' : 'LCD-1', @@ -503,6 +506,32 @@ test.each([ }, ); +test.each([ + ['the lit panel already shows the target', 'landscape-left'], + ['the lit panel shows another rotation', 'landscape-right'], +] as const)( + 'a panel of unknown power beside the lit one never confirms or dispatches (%s)', + async (_case, target) => { + const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); + // The outer panel reads lit; the inner one reports a backlight state the parser does not + // recognize, so its power is unknown and it may be lit too. + const sim = fakeScopedDuo(scoped, { + lit: 'outer', + picker: 'landscape-left', + unknownPowerPanel: 'inner', + }); + expect(sim.litScreen()).toBe('rot270'); + const outcome = await rotateScoped(sim, target); + expect(outcome).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-unverified-scoped-set', requestedOrientation: target }, + }); + expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); + expect(sim.rotations()).toEqual([]); + expect(sim.devicectl).toEqual([]); + }, +); + test('a reading taken before the dispatch lands teaches nothing and is not followed by another dispatch', async () => { const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); // The first two panel reads after each picker event still show the panel as it was before it. diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index a03d13b1fb..2c876dd5ac 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -294,7 +294,8 @@ function scopedUnverifiedError( /** * The one lit panel and the rotation it shows, from one fresh panel-table read, selected by the * display inventory's {@link establishedLitPanel}. Anything less decided — an unreadable table, - * an incomplete one, no lit panel or several, an unrecognized `rotN` — is no reading: + * an incomplete one, no lit panel or several, a panel of unknown power, an unrecognized `rotN` — + * is no reading: * the rotation loop must not confirm, compensate or dispatch from a panel that is not showing. * On iPhone Duo the table reports both panels lit for a few seconds after a pose change in either * direction, so the first reads after a fold or an unfold are no reading by design. From 84861b57958f5530d49d5b0a88e205aad97da790 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 13:47:43 +0000 Subject: [PATCH 09/16] refactor(ios): confirm every foldable rotation from its lit panel --- .changeset/foldable-scoped-rotation.md | 4 +- docs/adr/0025-foldable-apple-panels.md | 2 + docs/agents/device-foldable.md | 3 +- packages/command-surface/src/system/index.ts | 2 +- .../core/__tests__/display-inventory.test.ts | 8 +- packages/platform-apple/src/core/config.ts | 25 +- .../src/core/device-orientation.ts | 46 -- .../src/core/display-inventory.ts | 21 +- .../src/foldable/fold-helper.ts | 2 +- .../src/foldable/orientation.test.ts | 659 ++++++++---------- .../src/foldable/orientation.ts | 205 ++---- .../src/foldable/picker-delta.test.ts | 148 ---- .../src/foldable/picker-delta.ts | 80 --- packages/platform-apple/src/foldable/pose.ts | 5 +- .../src/foldable/runner-panel.test.ts | 16 +- src/commands/schema/cli-help-foldable.ts | 2 +- .../provider-scenarios/ios-fold.test.ts | 29 +- website/docs/docs/commands.md | 2 +- 18 files changed, 419 insertions(+), 840 deletions(-) delete mode 100644 packages/platform-apple/src/core/device-orientation.ts delete mode 100644 packages/platform-apple/src/foldable/picker-delta.test.ts delete mode 100644 packages/platform-apple/src/foldable/picker-delta.ts diff --git a/.changeset/foldable-scoped-rotation.md b/.changeset/foldable-scoped-rotation.md index ce8b451108..9219daaeeb 100644 --- a/.changeset/foldable-scoped-rotation.md +++ b/.changeset/foldable-scoped-rotation.md @@ -2,4 +2,6 @@ 'agent-device': patch --- -Fix `orientation` on a foldable iPhone simulator (iPhone Duo) in a scoped simulator set. It no longer refuses with `unsupported-device-scope`: the fold helper already reaches a scoped set, so the rotation is dispatched through it and succeeds only when the lit panel shows the requested orientation, otherwise failing with `orientation-unverified-scoped-set`. Measured on iPhone Duo, the lit panel usually reads the orientation-picker value plus a fixed per-panel offset (0 degrees on the outer panel, +90 degrees on the inner panel), and the picker value survives a fold; a panel whose foreground app cannot show that orientation keeps its previous one instead. Each dispatch is therefore the target minus the lit panel's offset, taken from the measured value at first and then recalibrated from the last dispatch and a fresh reading of the same panel, so the first command lands. A reading that has not moved since the last dispatch teaches nothing and is not followed by another dispatch, so a panel that cannot show the target fails after one dispatch. The fold helper also maps `portrait-upside-down` to the picker's `pud` wire spelling, which the inner panel needs to reach landscape-left; a user target of `portrait-upside-down` stays refused. +Fix foldable iPhone simulator rotation in scoped sets and confirm the lit panel in both default and scoped sets. Both routes use the fold helper, with no CoreDevice orientation polling. An already visible target dispatches nothing. Incomplete, unknown-power and multi-lit panel tables neither confirm nor dispatch. A target the panel never shows fails with `orientation-unverified`. + +Use the fixed iPhone Duo picker offsets measured in both sets (outer 0 degrees, inner +90 degrees), replacing runtime calibration. Inner landscape-left sends `portrait-upside-down`, mapped to the HID wire value `pud`; it remains unsupported as a user target. Delayed reads wait without resending the same picker value to the same panel. diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 19f55cf345..e32cea8253 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -58,6 +58,8 @@ It is the only reader of panel tables, and it selects the source. For a simulator in a scoped set, the fold helper's `panels` mode is the screen authority. It emits the same entry shape from the guest sources that `dtdeviceinfod` relays to CoreDevice: `CADisplay` and `BLSBacklight`. The guest lists every integrated panel and omits only the facts it cannot read. + Foldable `fold` and `orientation` operations use the guest table in either simulator set. + Rotation confirms the lit panel because CoreDevice orientation can match the picker while the visible panel shows another orientation. - **Completeness.** Both sources list one entry per integrated panel. A panel that the parser drops is not single-panel evidence. If two or more integrated entries exist and not all parse, the device is multi-panel without a capture panel. Capture refuses with `capture-panel-unidentified`. diff --git a/docs/agents/device-foldable.md b/docs/agents/device-foldable.md index 0719bb2e99..70b0167201 100644 --- a/docs/agents/device-foldable.md +++ b/docs/agents/device-foldable.md @@ -2,7 +2,8 @@ Before you change capture behavior on a multi-panel device, read ADR 0025. To list the panels, run `xcrun devicectl device info displays --device `. -In a scoped set, run `xcrun simctl --set spawn ~/.agent-device/fold-helper//fold-helper panels`. +For foldable orientation in either set, read the guest table with `xcrun simctl spawn ~/.agent-device/fold-helper//fold-helper panels`. +For a scoped set, add `--set ` after `simctl`. - Panels: the list shows each integrated panel with `backlightState`. Only the lit panel permits capture. A capture of the dark panel exits 0 and writes an all-black PNG. diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index e593315548..5a5ccef8d2 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -290,7 +290,7 @@ const backCommandDescription = const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = - 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In a scoped simulator set it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified-scoped-set.'; + 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified.'; const foldGuidance = { hingeEvent: 'simulator HID hinge event', permissions: 'Device Hub and host Accessibility permission are not required.', diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 7d3a7e0b0a..86d8b15aff 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -750,7 +750,10 @@ describe('scoped simulator panel table', () => { assert.equal(mockRunXcrun.mock.calls.length, 0); }); - test('reads with the fold helper the caller already opened', async () => { + test.each([ + ['scoped', scopedDuo], + ['default', { ...scopedDuo, simulatorSetPath: undefined }], + ] as const)('reads with the supplied fold helper in the %s set', async (_set, device) => { vi.mocked(ensureFoldHelperBinary).mockClear(); const readPanelPayload = vi.fn(async () => ({ result: { displays: (JSON.parse(GUEST_PANELS_OPEN) as { displays: unknown[] }).displays }, @@ -762,12 +765,13 @@ describe('scoped simulator panel table', () => { readPanelPayload, }; - const inventory = await queryAppleDisplayInventory(scopedDuo, { foldHelper }); + const inventory = await queryAppleDisplayInventory(device, { foldHelper }); assert.equal(inventory.activeDisplay?.name, 'LCD-1'); assert.equal(readPanelPayload.mock.calls.length, 1); assert.equal(vi.mocked(ensureFoldHelperBinary).mock.calls.length, 0); assert.equal(mockRunSimctlForDevice.mock.calls.length, 0); + assert.equal(mockRunXcrun.mock.calls.length, 0); }); test('applies the same rule to a CoreDevice table whose second panel does not parse', async () => { diff --git a/packages/platform-apple/src/core/config.ts b/packages/platform-apple/src/core/config.ts index cb22e27c12..b81ecc0233 100644 --- a/packages/platform-apple/src/core/config.ts +++ b/packages/platform-apple/src/core/config.ts @@ -31,10 +31,10 @@ export const IOS_SIMULATOR_SCREENSHOT_TIMEOUT_MS = 20_000; // request-level daemon reset. Measured probe cost is ~0.2s. export const IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS = 5_000; /** - * A scoped simulator's panel table comes from a guest helper that the first read on a host also + * A guest panel table comes from a helper that the first read on a host also * builds. This ceiling on the whole read, build included, stays below the capture's own budget. */ -export const IOS_SCOPED_DISPLAY_PROBE_BUDGET_MS = 15_000; +export const IOS_GUEST_DISPLAY_PROBE_BUDGET_MS = 15_000; /** * One guest hinge read, before its sampling window: the fold helper waits at most 3s for its first * CoreMotion sample, so this exec deadline only guards a wedged `simctl spawn`. @@ -55,24 +55,11 @@ export const IOS_FOLD_POSE_READ_INTERVAL_MS = 1_000; /** Two consecutive reads this close together mean the hinge has stopped moving. */ export const IOS_FOLD_POSE_STABLE_DEGREES = 0.5; /** - * How many fresh CoreDevice orientation reads a dispatched foldable rotation gets before it is - * refused. One read costs ~0.2s, and the iPhone Duo reported the new orientation on the first - * fresh read; the margin covers a loaded host. A read that times out counts as one that saw no - * orientation. With the routing probe, a cold fold-helper build, and the HID dispatch, the worst - * case stays inside the default 90s request envelope. + * Bounded lit-panel reads cover delayed HID effects and the two-lit interval after a fold. + * A fixed picker value is sent once per lit panel; the remaining reads only observe. */ -export const IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS = 8; -export const IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS = 2_000; -/** - * How many read/dispatch rounds a scoped-set foldable rotation gets. A panel at its measured - * picker offset lands in one dispatch, and a panel that disagrees with the measured offset in - * two, the second calibrated by the first. The rest of the budget covers reads that name no - * single lit panel, such as the few seconds after an iPhone Duo folds or unfolds, and reads taken - * before a dispatch has landed. Exercised on an iPhone Duo simulator (iOS 27.3) with this loop and - * the settle delay below between a dispatch and the read that confirms it. - */ -export const IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS = 6; -export const IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS = 600; +export const IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS = 6; +export const IOS_FOLDABLE_ORIENTATION_SETTLE_MS = 600; // CoreSimulator can briefly stall while it services the scale lookup immediately // after a keyboard transition. Keep this bounded below the full capture budget. diff --git a/packages/platform-apple/src/core/device-orientation.ts b/packages/platform-apple/src/core/device-orientation.ts deleted file mode 100644 index d4ad18301c..0000000000 --- a/packages/platform-apple/src/core/device-orientation.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { DeviceRotation } from '@agent-device/contracts/device'; -import type { DeviceInfo } from '@agent-device/kernel/device'; -import { isCommandTimeoutError } from '@agent-device/host-kit/command'; - -import { runIosDevicectlJsonRequest, type IosDevicectlJsonOutcome } from './devicectl.ts'; - -/** CoreDevice `deviceOrientation` names, which are UIDeviceOrientation case names. */ -const CORE_DEVICE_ORIENTATIONS: ReadonlyMap = new Map< - unknown, - DeviceRotation ->([ - ['portrait', 'portrait'], - ['portraitUpsideDown', 'portrait-upside-down'], - ['landscapeLeft', 'landscape-left'], - ['landscapeRight', 'landscape-right'], -]); - -/** - * Reads the orientation of the device itself through `devicectl device orientation get`. - * The display table's `currentDeviceOrientation` is not a replacement: on the iPhone Duo inner - * panel, it is a quarter-turn from the device orientation. - * - * CoreDevice answers each query with the orientation it sampled at the previous query, so the - * first read after a change can be stale. A failed or timed-out read, face-up, and face-down - * return nothing. - */ -export async function readAppleDeviceOrientation( - device: DeviceInfo, - options: { timeoutMs: number; signal?: AbortSignal }, -): Promise { - let outcome: IosDevicectlJsonOutcome; - try { - outcome = await runIosDevicectlJsonRequest({ - jsonPrefix: 'agent-device-apple-orientation', - args: ['devicectl', 'device', 'orientation', 'get', '--device', device.id], - timeoutMs: options.timeoutMs, - signal: options.signal, - }); - } catch (error) { - if (options.signal?.aborted || !isCommandTimeoutError(error)) throw error; - return undefined; - } - if (!outcome.ok) return undefined; - const payload = outcome.payload as { result?: { deviceOrientation?: unknown } } | null; - return CORE_DEVICE_ORIENTATIONS.get(payload?.result?.deviceOrientation); -} diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 47a10ccb7d..3b835437e5 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -5,10 +5,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { - IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, - IOS_SCOPED_DISPLAY_PROBE_BUDGET_MS, -} from './config.ts'; +import { IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, IOS_GUEST_DISPLAY_PROBE_BUDGET_MS } from './config.ts'; import type { FoldHelper } from '../foldable/fold-helper.ts'; import { runIosDevicectlJsonRequest } from './devicectl.ts'; import { simulatorAddressFor } from './simctl.ts'; @@ -93,7 +90,7 @@ type CoreDeviceDisplayPayload = { type AppleDisplayInventoryOptions = { timeoutMs?: number; signal?: AbortSignal; - /** A fold helper the caller already holds, so a scoped read does not probe its build again. */ + /** Selects the guest table in either simulator set and reuses this helper. */ foldHelper?: FoldHelper; }; @@ -102,7 +99,7 @@ const DISPLAYS_UNSUPPORTED_HINT = /** * Reads the CoreDevice display table for one Apple device, or the guest's equivalent for a - * simulator in a scoped device set. + * simulator in a scoped device set or a caller that supplies its fold helper. * * Deliberately uncached: which panel is lit is exactly what changes when an * operator folds or opens the device, and a cached answer would resume capturing @@ -116,9 +113,10 @@ export async function queryAppleDisplayInventory( device: DeviceInfo, options: AppleDisplayInventoryOptions, ): Promise { - const table = simulatorAddressFor(device).simulatorSetPath - ? await readGuestPanelTable(device, options) - : await readCoreDevicePanelTable(device, options); + const table = + options.foldHelper || simulatorAddressFor(device).simulatorSetPath + ? await readGuestPanelTable(device, options) + : await readCoreDevicePanelTable(device, options); if (!table.ok) { emitUnresolvedDiagnostic(device, table.reason, table.detail); return unresolvedInventory(); @@ -153,14 +151,15 @@ async function readCoreDevicePanelTable( /** * CoreDevice resolves a simulator only in the default device set, so a scoped simulator's panel * table comes from the fold helper inside the guest. It reads the same CADisplay and BLSBacklight - * sources `dtdeviceinfod` reports to CoreDevice, in CoreDevice's payload shape. + * sources `dtdeviceinfod` reports to CoreDevice, in CoreDevice's payload shape. Foldable + * operations use this guest source in either set. */ async function readGuestPanelTable( device: DeviceInfo, options: AppleDisplayInventoryOptions, ): Promise { // The first read on a host also builds the helper; the budget covers that build too. - const budget = AbortSignal.timeout(IOS_SCOPED_DISPLAY_PROBE_BUDGET_MS); + const budget = AbortSignal.timeout(IOS_GUEST_DISPLAY_PROBE_BUDGET_MS); const signal = options.signal ? AbortSignal.any([options.signal, budget]) : budget; try { const { createFoldHelper } = await import('../foldable/fold-helper.ts'); diff --git a/packages/platform-apple/src/foldable/fold-helper.ts b/packages/platform-apple/src/foldable/fold-helper.ts index 2f554a4f58..ef1908af54 100644 --- a/packages/platform-apple/src/foldable/fold-helper.ts +++ b/packages/platform-apple/src/foldable/fold-helper.ts @@ -25,7 +25,7 @@ export type FoldableRotation = Exclude; /** * Every value the guest's orientation picker accepts, upside-down included: its wire spelling * is `pud`, which the fold helper maps from `portrait-upside-down`. The picker needs all four - * values because each event turns the lit panel by the value's delta, so reaching one target + * values because the lit panel adds a fixed offset to the picker, so reaching one target * can require dispatching another value. Callers keep user targets to {@link FoldableRotation}. */ export type FoldPickerValue = DeviceRotation; diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 815e1c5fdf..371d7da78c 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -1,5 +1,5 @@ import fs from 'node:fs'; -import { beforeEach, expect, test, vi } from 'vitest'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; import { bindTestCommandHost, createLiveTestHostEnvironment } from '@agent-device/test-kit'; vi.mock('./fold-helper-cache.ts', () => ({ @@ -17,7 +17,7 @@ import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/too import { createAppleInteractor } from '../interactor.ts'; import { readFoldablePanelTable } from './orientation.ts'; import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; -import { IPHONE_17_SINGLE_PANEL, IPHONE_DUO_CLOSED } from '../__tests__/display-fixtures.ts'; +import { IPHONE_17_SINGLE_PANEL } from '../__tests__/display-fixtures.ts'; bindTestCommandHost(); @@ -34,63 +34,24 @@ function simulator(overrides: Partial = {}): DeviceInfo { return { ...IOS_SIMULATOR, id: `${IOS_SIMULATOR.id}-${simulatorCount}`, ...overrides }; } -const DUO_DISPLAYS = IPHONE_DUO_CLOSED.result.displays; const SINGLE_PANEL_DISPLAYS = IPHONE_17_SINGLE_PANEL.result.displays; -const CORE_DEVICE_NAMES: Record = { - 'landscape-left': 'landscapeLeft', - 'landscape-right': 'landscapeRight', - portrait: 'portrait', -}; - -/** - * A simulator as CoreDevice and simctl see it. A guest HID orientation event takes effect at once - * unless `followsHid` is false. As measured on iPhone Duo, `devicectl device orientation get` - * answers with the orientation it sampled at the previous query, starting from `sampled`. The - * first `wedgedReads` orientation reads time out. The fold helper's `panels` mode serves the - * displays of a simulator in a scoped set. - */ -function fakeSimulator( - device: DeviceInfo, - displays: readonly object[], - options: { followsHid?: boolean; sampled?: string; wedgedReads?: number } = {}, -) { +/** Single-panel and physical targets keep their runner rotation. */ +function fakeSimulator(device: DeviceInfo, displays: readonly object[]) { const helpers: string[][] = []; const devicectl: string[] = []; - let orientation = 'portrait'; - let sampled = options.sampled ?? orientation; - let wedgedReads = options.wedgedReads ?? 0; const provider = createLocalAppleToolProvider({ runCommand: async (_command, args) => { - const [tool, ...rest] = args; - if (tool === 'simctl') { - const spawn = rest.slice(rest.indexOf('spawn')); - expect(spawn.slice(0, 3)).toEqual(['spawn', device.id, '/cache/fold-helper']); - const mode = spawn.slice(3); - helpers.push(mode); - if (mode[0] === 'panels') { - return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; - } - expect(mode[0]).toBe('rotate'); - if (options.followsHid !== false) orientation = CORE_DEVICE_NAMES[mode[1]!]!; - return { stdout: '', stderr: '', exitCode: 0 }; - } - const jsonPath = args.at(-1)!; - const query = args.slice(0, 4).join(' '); - expect(args.slice(4)).toEqual(['--device', device.id, '--json-output', jsonPath]); - devicectl.push(query); - if (query === 'devicectl device info displays') { - fs.writeFileSync(jsonPath, JSON.stringify({ result: { displays } })); - return { stdout: '', stderr: '', exitCode: 0 }; - } - expect(query).toBe('devicectl device orientation get'); - if (wedgedReads > 0) { - wedgedReads -= 1; - throw new AppError('COMMAND_FAILED', 'xcrun timed out', { timeoutMs: 2_000 }); + if (args[0] !== 'simctl') { + devicectl.push(args.join(' ')); + throw new Error(`unexpected tool call: ${args.join(' ')}`); } - fs.writeFileSync(jsonPath, JSON.stringify({ result: { deviceOrientation: sampled } })); - sampled = orientation; - return { stdout: '', stderr: '', exitCode: 0 }; + const spawn = args.slice(args.indexOf('spawn')); + expect(spawn.slice(0, 3)).toEqual(['spawn', device.id, '/cache/fold-helper']); + const mode = spawn.slice(3); + helpers.push(mode); + expect(mode).toEqual(['panels']); + return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; }, }); return { @@ -103,41 +64,20 @@ function fakeSimulator( }; } -test.each(['landscape-left', 'portrait', 'landscape-right'] as const)( - 'a foldable simulator rotates to %s through guest HID once CoreDevice reports it', - async (rotation) => { - const device = simulator(); - const sim = fakeSimulator(device, DUO_DISPLAYS, { - sampled: rotation === 'portrait' ? 'landscapeLeft' : 'portrait', - }); - await expect(sim.rotate(rotation)).resolves.toEqual({ orientation: rotation }); - expect(sim.helpers).toEqual([['rotate', rotation]]); - // The routing probe, the read that can be stale, and the read that reports the rotation. - expect(sim.devicectl).toEqual([ - 'devicectl device info displays', - 'devicectl device orientation get', - 'devicectl device orientation get', - ]); - expect(mockRunner).not.toHaveBeenCalled(); - }, -); - -test('a simulator is probed for its panels once', async () => { - const sim = fakeSimulator(simulator(), DUO_DISPLAYS); - await sim.rotate('landscape-left'); - await sim.rotate('portrait'); - expect(sim.devicectl.filter((query) => query.endsWith('displays'))).toHaveLength(1); - expect(sim.helpers).toEqual([ - ['rotate', 'landscape-left'], - ['rotate', 'portrait'], - ]); +test('a simulator is probed for its panel count once', async () => { + const sim = fakeDuo(simulator(), { lit: 'outer', picker: 'portrait' }); + await rotateDuo(sim, 'landscape-left'); + await rotateDuo(sim, 'portrait'); + expect(sim.panelReads()).toBe(5); + expect(sim.rotations()).toEqual(['landscape-left', 'portrait']); + expect(sim.devicectl).toEqual([]); }); test('a single-panel simulator keeps the runner rotation', async () => { const device = simulator(); const sim = fakeSimulator(device, SINGLE_PANEL_DISPLAYS); await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); - expect(sim.helpers).toEqual([]); + expect(sim.helpers).toEqual([['panels']]); expect(mockRunner).toHaveBeenCalledWith( device, expect.objectContaining({ command: 'rotate', orientation: 'landscape-left' }), @@ -146,14 +86,14 @@ test('a single-panel simulator keeps the runner rotation', async () => { }); test('a physical device keeps the runner rotation without a CoreDevice probe', async () => { - const sim = fakeSimulator(IOS_DEVICE, DUO_DISPLAYS); + const sim = fakeSimulator(IOS_DEVICE, []); await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); expect(sim.devicectl).toEqual([]); expect(mockRunner).toHaveBeenCalledTimes(1); }); /** - * A scoped-set Duo as measured on the device (iOS 27.3): the guest keeps one + * A Duo as measured on the device (iOS 27.3): the guest keeps one * `orientation-picker-control` value, and the lit panel reads that value plus a fixed * per-panel offset — outer 0, inner +90 — so successive events look relative on the inner * panel. The picker survives a fold: the panel that lights afterwards reads it plus its own @@ -181,7 +121,7 @@ function rotationAtAngle(angle: number): DeviceRotation { return rotation!; } -function fakeScopedDuo( +function fakeDuo( device: DeviceInfo, options: { lit: DuoPanel; @@ -241,6 +181,18 @@ function fakeScopedDuo( const [tool, ...rest] = args; if (tool !== 'simctl') { devicectl.push(args.join(' ')); + if (!device.simulatorSetPath && tool === 'devicectl') { + const displays = [panel('outer', false), panel('inner', false)]; + const rotation = rotationAtAngle(shown[lit]); + const deviceOrientation = rotation.replaceAll(/-([a-z])/g, (_, letter: string) => + letter.toUpperCase(), + ); + fs.writeFileSync( + args.at(-1)!, + JSON.stringify({ result: { displays, deviceOrientation } }), + ); + return { stdout: '', stderr: '', exitCode: 0 }; + } throw new Error(`unexpected tool call in a scoped set: ${args.join(' ')}`); } const spawn = rest.slice(rest.indexOf('spawn')); @@ -294,9 +246,9 @@ function fakeScopedDuo( }; } -/** Runs one scoped rotation under fake timers: the settle delay must not cost real time. */ -async function rotateScoped( - sim: ReturnType, +/** Runs one rotation under fake timers: the settle delay must not cost real time. */ +async function rotateDuo( + sim: ReturnType, rotation: DeviceRotation, ): Promise { vi.useFakeTimers(); @@ -312,274 +264,297 @@ async function rotateScoped( } } -test('a foldable simulator in a scoped set rotates on the first command, confirmed by the panel table', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - // The inner panel lights at rot90 while the picker sits at portrait (K = +90, measured). - const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'portrait' }); - expect(sim.litScreen()).toBe('rot90'); - await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); - expect(sim.litScreen()).toBe('rot0'); - // The measured inner offset makes the first dispatch exact: portrait - 90 is landscape-left. - expect(sim.rotations()).toEqual(['landscape-left']); - expect(sim.devicectl).toEqual([]); - expect(mockRunner).not.toHaveBeenCalled(); -}); +describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simulatorSetPath) => { + test.each([ + ['outer', 'portrait', []], + ['outer', 'landscape-left', ['landscape-left']], + ['outer', 'landscape-right', ['landscape-right']], + ['inner', 'portrait', ['landscape-left']], + ['inner', 'landscape-left', ['portrait-upside-down']], + ['inner', 'landscape-right', []], + ] as const)( + 'confirms %s %s from the lit panel without CoreDevice', + async (lit, target, dispatched) => { + const sim = fakeDuo(simulator({ simulatorSetPath }), { lit, picker: 'portrait' }); + await expect(rotateDuo(sim, target)).resolves.toEqual({ orientation: target }); + expect(sim.litScreen()).toBe(`rot${PANEL_ANGLE[target]}`); + expect(sim.rotations()).toEqual(dispatched); + expect(sim.devicectl).toEqual([]); + expect(mockRunner).not.toHaveBeenCalled(); + }, + ); -test('a scoped rotation to landscape-left dispatches the upside-down picker value when the delta needs it', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { - lit: 'inner', - picker: 'landscape-left', - }); - expect(sim.litScreen()).toBe('rot0'); - await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ - orientation: 'landscape-left', + test('a foldable rotates on the first command, confirmed by the panel table', async () => { + const scoped = simulator({ simulatorSetPath }); + // The inner panel lights at rot90 while the picker sits at portrait (K = +90, measured). + const sim = fakeDuo(scoped, { lit: 'inner', picker: 'portrait' }); + expect(sim.litScreen()).toBe('rot90'); + await expect(rotateDuo(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + expect(sim.litScreen()).toBe('rot0'); + // The measured inner offset makes the first dispatch exact: portrait - 90 is landscape-left. + expect(sim.rotations()).toEqual(['landscape-left']); + expect(sim.devicectl).toEqual([]); + expect(mockRunner).not.toHaveBeenCalled(); }); - expect(sim.litScreen()).toBe('rot270'); - expect(sim.rotations()).toEqual(['portrait-upside-down']); - expect(sim.devicectl).toEqual([]); -}); - -test('the outer panel reads the picker value itself, so its first dispatch is the target', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { lit: 'outer', picker: 'portrait' }); - expect(sim.litScreen()).toBe('rot0'); - for (const rotation of ['landscape-left', 'landscape-right', 'portrait'] as const) { - sim.helpers.length = 0; - await expect(rotateScoped(sim, rotation)).resolves.toEqual({ orientation: rotation }); - expect(sim.rotations()).toEqual([rotation]); - } - expect(sim.devicectl).toEqual([]); -}); -test('a panel whose offset differs from the measured one still converges', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { - lit: 'outer', - picker: 'portrait', - offsets: { inner: 90, outer: 270 }, - }); - await expect(rotateScoped(sim, 'landscape-right')).resolves.toEqual({ - orientation: 'landscape-right', + test('inner landscape-left dispatches the upside-down picker value', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { + lit: 'inner', + picker: 'landscape-left', + }); + expect(sim.litScreen()).toBe('rot0'); + await expect(rotateDuo(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + expect(sim.litScreen()).toBe('rot270'); + expect(sim.rotations()).toEqual(['portrait-upside-down']); + expect(sim.devicectl).toEqual([]); }); - expect(sim.litScreen()).toBe('rot90'); - // The measured outer offset (0) lands a quarter-turn away; that reading calibrates the second. - expect(sim.rotations()).toEqual(['landscape-right', 'portrait-upside-down']); - expect(sim.devicectl).toEqual([]); -}); - -test('the picker survives a fold in both directions and each panel adds its own offset', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - // Measured sequence: the inner panel at rot0 with the picker at landscape-left. - const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); - expect(sim.litScreen()).toBe('rot0'); - sim.fold('outer'); - expect(sim.litScreen()).toBe('rot270'); - await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); - expect(sim.rotations()).toEqual(['portrait']); - - sim.fold('inner'); - expect(sim.litScreen()).toBe('rot90'); - await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ - orientation: 'landscape-left', + test('the outer panel reads the picker value itself, so its first dispatch is the target', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { lit: 'outer', picker: 'portrait' }); + expect(sim.litScreen()).toBe('rot0'); + for (const rotation of ['landscape-left', 'landscape-right', 'portrait'] as const) { + sim.helpers.length = 0; + await expect(rotateDuo(sim, rotation)).resolves.toEqual({ orientation: rotation }); + expect(sim.rotations()).toEqual([rotation]); + } + expect(sim.devicectl).toEqual([]); }); - expect(sim.rotations()).toEqual(['portrait-upside-down']); -}); -test('a fold that leaves the picker where the outer panel cannot follow still lands in one dispatch', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - // Measured: the inner panel at landscape-left needs the picker at the upside-down value; the - // outer panel's foreground UI has no upside-down, so after the fold it kept landscape-left. - const sim = fakeScopedDuo(scoped, { - lit: 'inner', - picker: 'portrait-upside-down', - cannotShow: { outer: ['portrait-upside-down'] }, - lastShown: { outer: 'landscape-left' }, - }); - expect(sim.litScreen()).toBe('rot270'); - sim.fold('outer'); - expect(sim.litScreen()).toBe('rot270'); - await expect(rotateScoped(sim, 'landscape-right')).resolves.toEqual({ - orientation: 'landscape-right', + test('an unexpected panel response cannot calibrate another picker value', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { + lit: 'outer', + picker: 'portrait', + offsets: { inner: 90, outer: 270 }, + }); + const outcome = await rotateDuo(sim, 'landscape-right'); + expect(outcome).toMatchObject({ + code: 'COMMAND_FAILED', + details: { requestedOrientation: 'landscape-right', observedOrientation: 'portrait' }, + }); + expect(sim.litScreen()).toBe('rot0'); + // An unexpected reading is not evidence that an unsupported picker value is safe to send. + expect(sim.rotations()).toEqual(['landscape-right']); + expect(sim.devicectl).toEqual([]); }); - expect(sim.rotations()).toEqual(['landscape-right']); -}); -test('reads with both panels lit after a fold to closed neither confirm nor dispatch', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); - // A first command routes the device as a foldable (one panel probe) and finds portrait showing. - await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); - // The outer panel already shows the target, but the table names two lit panels for two reads. - sim.fold('outer', { bothLitReads: 2 }); - expect(sim.litScreen()).toBe('rot270'); - await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ - orientation: 'landscape-left', - }); - // Confirmed by the third read, the first to name one lit panel; nothing was dispatched blind. - expect(sim.panelReads()).toBe(3); - expect(sim.rotations()).toEqual([]); -}); + test('the picker survives a fold in both directions and each panel adds its own offset', async () => { + const scoped = simulator({ simulatorSetPath }); + // Measured sequence: the inner panel at rot0 with the picker at landscape-left. + const sim = fakeDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); + expect(sim.litScreen()).toBe('rot0'); -test('after the both-lit reads, the first single-lit reading drives one exact dispatch', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); - await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); - sim.fold('outer', { bothLitReads: 2 }); - await expect(rotateScoped(sim, 'landscape-right')).resolves.toEqual({ - orientation: 'landscape-right', - }); - expect(sim.panelReads()).toBe(4); - expect(sim.rotations()).toEqual(['landscape-right']); -}); + sim.fold('outer'); + expect(sim.litScreen()).toBe('rot270'); + await expect(rotateDuo(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + expect(sim.rotations()).toEqual(['portrait']); -test('a table that never names a single lit panel dispatches nothing and is refused', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { lit: 'outer', picker: 'portrait', bothLitFromRead: 1 }); - const outcome = await rotateScoped(sim, 'landscape-left'); - expect(outcome).toMatchObject({ - code: 'COMMAND_FAILED', - details: { - reason: 'orientation-unverified-scoped-set', - requestedOrientation: 'landscape-left', - }, + sim.fold('inner'); + expect(sim.litScreen()).toBe('rot90'); + await expect(rotateDuo(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + expect(sim.rotations()).toEqual(['portrait-upside-down']); }); - expect((outcome as AppError).message).toContain('named no single lit panel'); - expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); - expect(sim.rotations()).toEqual([]); -}); -test('an undecided final read does not report an earlier reading as the current one', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - // The routing probe and two decided rotation reads, then both panels report lit to the end. - const sim = fakeScopedDuo(scoped, { - lit: 'inner', - picker: 'portrait', - followsHid: false, - bothLitFromRead: 4, - }); - const outcome = await rotateScoped(sim, 'portrait'); - expect(outcome).toMatchObject({ - code: 'COMMAND_FAILED', - details: { reason: 'orientation-unverified-scoped-set' }, + test('a fold that leaves the picker where the outer panel cannot follow still lands in one dispatch', async () => { + const scoped = simulator({ simulatorSetPath }); + // Measured: the inner panel at landscape-left needs the picker at the upside-down value; the + // outer panel's foreground UI has no upside-down, so after the fold it kept landscape-left. + const sim = fakeDuo(scoped, { + lit: 'inner', + picker: 'portrait-upside-down', + cannotShow: { outer: ['portrait-upside-down'] }, + lastShown: { outer: 'landscape-left' }, + }); + expect(sim.litScreen()).toBe('rot270'); + sim.fold('outer'); + expect(sim.litScreen()).toBe('rot270'); + await expect(rotateDuo(sim, 'landscape-right')).resolves.toEqual({ + orientation: 'landscape-right', + }); + expect(sim.rotations()).toEqual(['landscape-right']); }); - expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); - expect((outcome as AppError).details).not.toHaveProperty('rawPanelOrientation'); - expect(sim.rotations()).toEqual(['landscape-left']); -}); -test('a scoped rotation to the orientation already showing dispatches nothing', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { - lit: 'inner', - picker: 'landscape-left', + test('reads with both panels lit after a fold to closed neither confirm nor dispatch', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); + // A first command routes the device as a foldable (one panel probe) and finds portrait showing. + await expect(rotateDuo(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + // The outer panel already shows the target, but the table names two lit panels for two reads. + sim.fold('outer', { bothLitReads: 2 }); + expect(sim.litScreen()).toBe('rot270'); + await expect(rotateDuo(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + // Confirmed by the third read, the first to name one lit panel; nothing was dispatched blind. + expect(sim.panelReads()).toBe(3); + expect(sim.rotations()).toEqual([]); }); - expect(sim.litScreen()).toBe('rot0'); - await expect(rotateScoped(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); - expect(sim.rotations()).toEqual([]); -}); -test.each([ - ['the parsed panel already shows the target', 'landscape-left'], - ['the parsed panel shows another rotation', 'landscape-right'], -] as const)( - 'an incomplete panel table never confirms or dispatches (%s)', - async (_case, target) => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - // Two activeOn integrated entries; the inner one has no geometry, so it does not parse. The - // outer panel that did parse is the only lit one left, but no single lit panel is established. - const sim = fakeScopedDuo(scoped, { - lit: 'outer', - picker: 'landscape-left', - incompleteLitPanel: 'inner', + test('after the both-lit reads, the first single-lit reading drives one exact dispatch', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { lit: 'inner', picker: 'landscape-left' }); + await expect(rotateDuo(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + sim.fold('outer', { bothLitReads: 2 }); + await expect(rotateDuo(sim, 'landscape-right')).resolves.toEqual({ + orientation: 'landscape-right', }); - expect(sim.litScreen()).toBe('rot270'); - const outcome = await rotateScoped(sim, target); + expect(sim.panelReads()).toBe(4); + expect(sim.rotations()).toEqual(['landscape-right']); + }); + + test('a table that never names a single lit panel dispatches nothing and is refused', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { lit: 'outer', picker: 'portrait', bothLitFromRead: 1 }); + const outcome = await rotateDuo(sim, 'landscape-left'); expect(outcome).toMatchObject({ code: 'COMMAND_FAILED', - details: { reason: 'orientation-unverified-scoped-set', requestedOrientation: target }, + details: { + reason: 'orientation-unverified', + requestedOrientation: 'landscape-left', + }, }); + expect((outcome as AppError).message).toContain('named no single lit panel'); expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); expect(sim.rotations()).toEqual([]); - expect(sim.devicectl).toEqual([]); - }, -); - -test.each([ - ['the lit panel already shows the target', 'landscape-left'], - ['the lit panel shows another rotation', 'landscape-right'], -] as const)( - 'a panel of unknown power beside the lit one never confirms or dispatches (%s)', - async (_case, target) => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - // The outer panel reads lit; the inner one reports a backlight state the parser does not - // recognize, so its power is unknown and it may be lit too. - const sim = fakeScopedDuo(scoped, { - lit: 'outer', - picker: 'landscape-left', - unknownPowerPanel: 'inner', + }); + + test('an undecided final read does not report an earlier reading as the current one', async () => { + const scoped = simulator({ simulatorSetPath }); + // The routing probe and two decided rotation reads, then both panels report lit to the end. + const sim = fakeDuo(scoped, { + lit: 'inner', + picker: 'portrait', + followsHid: false, + bothLitFromRead: 4, }); - expect(sim.litScreen()).toBe('rot270'); - const outcome = await rotateScoped(sim, target); + const outcome = await rotateDuo(sim, 'portrait'); expect(outcome).toMatchObject({ code: 'COMMAND_FAILED', - details: { reason: 'orientation-unverified-scoped-set', requestedOrientation: target }, + details: { reason: 'orientation-unverified' }, }); expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); - expect(sim.rotations()).toEqual([]); - expect(sim.devicectl).toEqual([]); - }, -); - -test('a reading taken before the dispatch lands teaches nothing and is not followed by another dispatch', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - // The first two panel reads after each picker event still show the panel as it was before it. - const sim = fakeScopedDuo(scoped, { lit: 'inner', picker: 'portrait', hidLagReads: 2 }); - expect(sim.litScreen()).toBe('rot90'); - await expect(rotateScoped(sim, 'landscape-left')).resolves.toEqual({ - orientation: 'landscape-left', + expect((outcome as AppError).details).not.toHaveProperty('rawPanelOrientation'); + expect(sim.rotations()).toEqual(['landscape-left']); }); - // One exact dispatch (landscape-left - 90 = portrait-upside-down); the stale reads in between - // neither re-learned an offset nor dispatched again. - expect(sim.rotations()).toEqual(['portrait-upside-down']); -}); -test('a scoped rotation the lit panel never shows is refused as unverified', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { - lit: 'inner', - picker: 'portrait', - followsHid: false, + test('a rotation to the orientation already showing dispatches nothing', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { + lit: 'inner', + picker: 'landscape-left', + }); + expect(sim.litScreen()).toBe('rot0'); + await expect(rotateDuo(sim, 'portrait')).resolves.toEqual({ orientation: 'portrait' }); + expect(sim.rotations()).toEqual([]); }); - await expect(rotateScoped(sim, 'portrait')).resolves.toMatchObject({ - code: 'COMMAND_FAILED', - message: expect.stringContaining('the lit panel shows landscape-right'), - details: { - reason: 'orientation-unverified-scoped-set', - requestedOrientation: 'portrait', - observedOrientation: 'landscape-right', - rawPanelOrientation: 'rot90', + + test.each([ + ['the parsed panel already shows the target', 'landscape-left'], + ['the parsed panel shows another rotation', 'landscape-right'], + ] as const)( + 'an incomplete panel table never confirms or dispatches (%s)', + async (_case, target) => { + const scoped = simulator({ simulatorSetPath }); + // Two activeOn integrated entries; the inner one has no geometry, so it does not parse. The + // outer panel that did parse is the only lit one left, but no single lit panel is established. + const sim = fakeDuo(scoped, { + lit: 'outer', + picker: 'landscape-left', + incompleteLitPanel: 'inner', + }); + expect(sim.litScreen()).toBe('rot270'); + const outcome = await rotateDuo(sim, target); + expect(outcome).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-unverified', requestedOrientation: target }, + }); + expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); + expect(sim.rotations()).toEqual([]); + expect(sim.devicectl).toEqual([]); }, + ); + + test.each([ + ['the lit panel already shows the target', 'landscape-left'], + ['the lit panel shows another rotation', 'landscape-right'], + ] as const)( + 'a panel of unknown power beside the lit one never confirms or dispatches (%s)', + async (_case, target) => { + const scoped = simulator({ simulatorSetPath }); + // The outer panel reads lit; the inner one reports a backlight state the parser does not + // recognize, so its power is unknown and it may be lit too. + const sim = fakeDuo(scoped, { + lit: 'outer', + picker: 'landscape-left', + unknownPowerPanel: 'inner', + }); + expect(sim.litScreen()).toBe('rot270'); + const outcome = await rotateDuo(sim, target); + expect(outcome).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-unverified', requestedOrientation: target }, + }); + expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); + expect(sim.rotations()).toEqual([]); + expect(sim.devicectl).toEqual([]); + }, + ); + + test('delayed panel reads wait without resending the picker value', async () => { + const scoped = simulator({ simulatorSetPath }); + // The first two panel reads after each picker event still show the panel as it was before it. + const sim = fakeDuo(scoped, { lit: 'inner', picker: 'portrait', hidLagReads: 2 }); + expect(sim.litScreen()).toBe('rot90'); + await expect(rotateDuo(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + // One exact dispatch (landscape-left - 90 = portrait-upside-down); the stale reads in between + // did not cause another dispatch. + expect(sim.rotations()).toEqual(['portrait-upside-down']); }); - // The panel never moved after the one dispatch, so it taught no offset and nothing more was - // dispatched: the loop re-read until its budget ended, then refused. - expect(sim.rotations()).toEqual(['landscape-left']); - expect(sim.devicectl).toEqual([]); -}); -test('a foldable simulator in a scoped set still refuses portrait-upside-down as a target', async () => { - const scoped = simulator({ simulatorSetPath: '/tmp/scoped-set' }); - const sim = fakeScopedDuo(scoped, { - lit: 'inner', - picker: 'portrait', + test('a rotation the lit panel never shows is refused as unverified', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { + lit: 'inner', + picker: 'portrait', + followsHid: false, + }); + await expect(rotateDuo(sim, 'portrait')).resolves.toMatchObject({ + code: 'COMMAND_FAILED', + message: expect.stringContaining('the lit panel shows landscape-right'), + details: { + reason: 'orientation-unverified', + requestedOrientation: 'portrait', + observedOrientation: 'landscape-right', + rawPanelOrientation: 'rot90', + }, + }); + // The panel never moved after the one dispatch, so nothing more was + // dispatched: the loop re-read until its budget ended, then refused. + expect(sim.rotations()).toEqual(['landscape-left']); + expect(sim.devicectl).toEqual([]); }); - await expect(sim.rotate('portrait-upside-down')).rejects.toMatchObject({ - code: 'UNSUPPORTED_OPERATION', - details: { reason: 'unsupported-foldable-orientation' }, + + test('a foldable still refuses portrait-upside-down as a target', async () => { + const scoped = simulator({ simulatorSetPath }); + const sim = fakeDuo(scoped, { + lit: 'inner', + picker: 'portrait', + }); + await expect(sim.rotate('portrait-upside-down')).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'unsupported-foldable-orientation' }, + }); + expect(sim.rotations()).toEqual([]); + expect(mockRunner).not.toHaveBeenCalled(); }); - expect(sim.rotations()).toEqual([]); - expect(mockRunner).not.toHaveBeenCalled(); }); test('a single-panel simulator in a scoped set keeps the runner rotation', async () => { @@ -591,39 +566,11 @@ test('a single-panel simulator in a scoped set keeps the runner rotation', async expect(mockRunner).toHaveBeenCalledTimes(1); }); -test('a foldable simulator refuses portrait-upside-down before any HID dispatch', async () => { - const sim = fakeSimulator(simulator(), DUO_DISPLAYS); - await expect(sim.rotate('portrait-upside-down')).rejects.toMatchObject({ - code: 'UNSUPPORTED_OPERATION', - details: { reason: 'unsupported-foldable-orientation' }, - }); - expect(sim.helpers).toEqual([]); - expect(mockRunner).not.toHaveBeenCalled(); -}); - -test('a timed-out CoreDevice read does not refuse a rotation that a later read reports', async () => { - const sim = fakeSimulator(simulator(), DUO_DISPLAYS, { wedgedReads: 2 }); - await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); -}); - -test('a foldable rotation that CoreDevice never reports is refused as unverified', async () => { - const sim = fakeSimulator(simulator(), DUO_DISPLAYS, { followsHid: false }); - await expect(sim.rotate('landscape-right')).rejects.toMatchObject({ - code: 'COMMAND_FAILED', - details: { - reason: 'orientation-unverified', - requestedOrientation: 'landscape-right', - observedOrientation: 'portrait', - }, - }); - expect(sim.helpers).toHaveLength(1); -}); - /** A rotation reads the panel table itself: the surface probe's retry window never holds it. */ test('a rotation is not routed by a pending panel-table retry', async () => { const device = simulator(); let tableFailures = 0; - const sim = fakeSimulator(device, DUO_DISPLAYS); + const sim = fakeDuo(device, { lit: 'outer', picker: 'portrait' }); const failing = createLocalAppleToolProvider({ runCommand: async (_command, args) => { if (args.slice(0, 4).join(' ') === 'devicectl device info displays') { @@ -638,17 +585,9 @@ test('a rotation is not routed by a pending panel-table retry', async () => { await withAppleToolProvider(failing, () => readFoldablePanelTable(device, { timeoutMs: 5_000 })); expect(tableFailures).toBe(1); - await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); - expect(sim.helpers).toEqual([['rotate', 'landscape-left']]); - expect(mockRunner).not.toHaveBeenCalled(); -}); - -test('a stale CoreDevice sample cannot verify a rotation that did not happen', async () => { - const sim = fakeSimulator(simulator(), DUO_DISPLAYS, { - followsHid: false, - sampled: 'landscapeLeft', - }); - await expect(sim.rotate('landscape-left')).rejects.toMatchObject({ - details: { reason: 'orientation-unverified', observedOrientation: 'portrait' }, + await expect(rotateDuo(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', }); + expect(sim.rotations()).toEqual(['landscape-left']); + expect(mockRunner).not.toHaveBeenCalled(); }); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 2c876dd5ac..5d3349b6d6 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -6,12 +6,9 @@ import { createTtlMemo } from '@agent-device/kernel/ttl-memo'; import { IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, - IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS, IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS, - IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS, - IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS, + IOS_FOLDABLE_ORIENTATION_SETTLE_MS, } from '../core/config.ts'; -import { readAppleDeviceOrientation } from '../core/device-orientation.ts'; import { establishedLitPanel, panelContentRotation, @@ -21,22 +18,13 @@ import { } from '../core/display-inventory.ts'; import { simulatorAddressFor } from '../core/simctl.ts'; import { createFoldHelper, type FoldHelper, type FoldableRotation } from './fold-helper.ts'; -import { - measuredPanelPickerOffset, - panelPickerOffset, - pickerValueForTarget, - type PanelQuarterTurn, -} from './picker-delta.ts'; import { appleFoldableFacts } from './runtime.ts'; const UPSIDE_DOWN_UNSUPPORTED_HINT = 'A foldable simulator accepts portrait, landscape-left, and landscape-right as rotation targets.'; const ORIENTATION_UNVERIFIED_HINT = - 'The simulator HID command completed, but CoreDevice did not report the new orientation. Read it with xcrun devicectl device orientation get --device , then retry.'; - -const SCOPED_ORIENTATION_UNVERIFIED_HINT = - 'In a scoped simulator set the rotation goes through the fold helper and only the lit panel can confirm it. The lit panel did not show the requested orientation, or the simulator never reported a single lit panel; the app in front may not support that orientation. Take a screenshot to see the screen, then retry.'; + 'The lit panel did not show the requested orientation, or the simulator never reported a single lit panel; the app in front may not support that orientation. Take a screenshot to see the screen, then retry.'; /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); @@ -72,7 +60,10 @@ export async function isAppleFoldableSimulator( const known = multiPanelByDevice.get(device.id); if (known !== undefined) return known; const read = ++panelTableReads; - const inventory = await queryAppleDisplayInventory(device, { signal }); + const inventory = await queryAppleDisplayInventory(device, { + signal, + foldHelper: createFoldHelper(device), + }); if (inventory.unresolved) return false; recordPanelTable(device, inventory, read); return inventory.multiScreen; @@ -112,11 +103,29 @@ export async function readFoldablePanelTable( return inventory.multiScreen ? { kind: 'multi-panel', inventory } : { kind: 'single-panel' }; } -/** - * Rotates a foldable simulator through guest HID. The default device set verifies the rotation - * through CoreDevice; a scoped set, which CoreDevice cannot resolve, verifies it from the - * fold helper's panel table instead (see {@link setScopedFoldableOrientation}). - */ +/** iPhone Duo (iOS 27.3) renders picker + offset: outer 0 degrees, inner +90. */ +const PANEL_PICKERS = { + outer: { + offset: 0, + values: { + portrait: 'portrait', + 'landscape-left': 'landscape-left', + 'landscape-right': 'landscape-right', + }, + }, + inner: { + offset: 90, + values: { + portrait: 'landscape-left', + 'landscape-left': 'portrait-upside-down', + 'landscape-right': 'portrait', + }, + }, +} as const; + +type LitPanel = { panel: AppleDeviceDisplay; rotation: DeviceRotation }; + +/** Dispatches through guest HID and confirms from the lit panel in either simulator set. */ export async function setAppleFoldableOrientation( device: DeviceInfo, rotation: DeviceRotation, @@ -131,140 +140,42 @@ export async function setAppleFoldableOrientation( hint: UPSIDE_DOWN_UNSUPPORTED_HINT, }); } - if (simulatorAddressFor(device).simulatorSetPath) { - return await setScopedFoldableOrientation(device, rotation, signal); - } - await createFoldHelper(device).rotate(rotation, signal); - emitDiagnostic({ - level: 'info', - phase: 'apple_foldable_orientation_dispatched', - data: { deviceId: device.id, orientation: rotation }, - }); - const read = { signal, timeoutMs: IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS }; - // This read can return the orientation CoreDevice sampled before the dispatch. - await readAppleDeviceOrientation(device, read); - let observed: DeviceRotation | undefined; - for (let attempt = 1; attempt <= IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS; attempt += 1) { - signal?.throwIfAborted(); - observed = await readAppleDeviceOrientation(device, read); - if (observed === rotation) return { orientation: rotation }; - } - throw new AppError( - 'COMMAND_FAILED', - `${device.name} did not rotate to ${rotation}: CoreDevice reports ${observed ?? 'no orientation'}`, - { - deviceId: device.id, - requestedOrientation: rotation, - ...(observed === undefined ? {} : { observedOrientation: observed }), - reason: 'orientation-unverified', - hint: ORIENTATION_UNVERIFIED_HINT, - }, - ); -} - -/** - * Rotates a foldable simulator in a scoped set, where only the fold helper can dispatch and - * only its panel table can confirm. Measured on iPhone Duo, the lit panel reads the - * `orientation-picker-control` value plus a per-panel offset (outer panel 0, inner panel +90), - * so one absolute dispatch of the target lands a quarter-turn away on the inner panel and looks - * one command late. One bounded loop reads the lit panel and, until it shows the target, - * dispatches `target - offset`: the offset comes from this command's last dispatch and a fresh - * reading of the same panel that moved since it, and from the measured table before that. A read - * that names no single lit panel, or a panel that has not moved since the last dispatch, neither - * confirms nor dispatches; the loop waits for the next read. - * A repeat of the showing orientation is a read-only success. Nothing is kept between commands: - * a fold in between can light the other panel, and the measured table covers that panel. - */ -async function setScopedFoldableOrientation( - device: DeviceInfo, - rotation: FoldableRotation, - signal: AbortSignal | undefined, -): Promise<{ orientation: DeviceRotation }> { const helper = createFoldHelper(device); - let dispatched: ScopedDispatch | undefined; + let dispatchedDisplayId: number | undefined; let reading: LitPanel | undefined; - for (let attempt = 1; attempt <= IOS_FOLDABLE_SCOPED_ORIENTATION_ATTEMPTS; attempt += 1) { + for (let attempt = 1; attempt <= IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS; attempt += 1) { signal?.throwIfAborted(); reading = await readLitPanel(device, helper, signal); if (reading?.rotation === rotation) return { orientation: rotation }; - if (reading && !awaitingDispatchEffect(dispatched, reading)) { - const panelOffset = scopedPanelOffset(dispatched, reading); - const value = pickerValueForTarget(rotation, panelOffset); + if (reading && reading.panel.displayId !== dispatchedDisplayId) { + const mapping = reading.panel.primary ? PANEL_PICKERS.outer : PANEL_PICKERS.inner; + const value = mapping.values[rotation]; await helper.rotate(value, signal); - dispatched = { value, displayId: reading.panel.displayId, before: reading.rotation }; - emitScopedDispatch(device, rotation, attempt, value, panelOffset, reading); + dispatchedDisplayId = reading.panel.displayId; + emitDiagnostic({ + level: 'info', + phase: 'apple_foldable_orientation_dispatched', + data: { + deviceId: device.id, + orientation: value, + requestedOrientation: rotation, + scopedSet: Boolean(simulatorAddressFor(device).simulatorSetPath), + attempt, + observedOrientation: reading.rotation, + litDisplay: reading.panel.name, + panelOffset: mapping.offset, + }, + }); } - await settleScopedRotation(); + await settleRotation(); } - // The last dispatch gets its confirming read too: the loop only reads before dispatching. reading = await readLitPanel(device, helper, signal); if (reading?.rotation === rotation) return { orientation: rotation }; - throw scopedUnverifiedError(device, rotation, reading); -} - -/** - * The picker value this command last dispatched, the panel that was lit for it, and the - * rotation that panel showed before the dispatch. - */ -type ScopedDispatch = { value: DeviceRotation; displayId: number; before: DeviceRotation }; - -type LitPanel = { panel: AppleDeviceDisplay; rotation: DeviceRotation }; - -/** - * The same panel still shows what it showed before the last dispatch: the dispatch has not taken - * effect yet, or the app in front cannot show that orientation. Such a reading teaches no offset, - * so nothing is dispatched on it; the loop re-reads, and a panel that never follows ends the - * command unverified after one dispatch instead of cycling through picker values. - */ -function awaitingDispatchEffect( - dispatched: ScopedDispatch | undefined, - reading: LitPanel, -): boolean { - return ( - dispatched?.displayId === reading.panel.displayId && reading.rotation === dispatched.before - ); -} - -/** - * The lit panel's picker offset. After this command dispatched to the same panel and the panel - * moved, the fresh reading and that dispatch give the offset this panel really has. Before any dispatch, or when - * a fold lit another panel mid-command, the measured offset of the panel now lit applies. - */ -function scopedPanelOffset( - dispatched: ScopedDispatch | undefined, - reading: LitPanel, -): PanelQuarterTurn { - return dispatched?.displayId === reading.panel.displayId - ? panelPickerOffset(reading.rotation, dispatched.value) - : measuredPanelPickerOffset(reading.panel); -} - -function emitScopedDispatch( - device: DeviceInfo, - rotation: FoldableRotation, - attempt: number, - value: DeviceRotation, - panelOffset: PanelQuarterTurn, - reading: LitPanel, -): void { - emitDiagnostic({ - level: 'info', - phase: 'apple_foldable_orientation_dispatched', - data: { - deviceId: device.id, - orientation: value, - requestedOrientation: rotation, - scopedSet: true, - attempt, - observedOrientation: reading.rotation, - litDisplay: reading.panel.name, - panelOffset, - }, - }); + throw unverifiedError(device, rotation, reading); } -/** The error names only what the final read saw: an undecided final read reports no rotation. */ -function scopedUnverifiedError( +/** An undecided final read reports no rotation, even if an earlier read named one. */ +function unverifiedError( device: DeviceInfo, rotation: FoldableRotation, reading: LitPanel | undefined, @@ -285,8 +196,8 @@ function scopedUnverifiedError( rawPanelOrientation: reading.panel.currentOrientation, } : {}), - reason: 'orientation-unverified-scoped-set', - hint: SCOPED_ORIENTATION_UNVERIFIED_HINT, + reason: 'orientation-unverified', + hint: ORIENTATION_UNVERIFIED_HINT, }, ); } @@ -296,7 +207,7 @@ function scopedUnverifiedError( * display inventory's {@link establishedLitPanel}. Anything less decided — an unreadable table, * an incomplete one, no lit panel or several, a panel of unknown power, an unrecognized `rotN` — * is no reading: - * the rotation loop must not confirm, compensate or dispatch from a panel that is not showing. + * the rotation loop must not confirm or dispatch from a panel that is not showing. * On iPhone Duo the table reports both panels lit for a few seconds after a pose change in either * direction, so the first reads after a fold or an unfold are no reading by design. */ @@ -319,8 +230,8 @@ async function readLitPanel( } /** The gap a dispatched picker event needs before the panel table shows its effect. */ -async function settleScopedRotation(): Promise { +async function settleRotation(): Promise { await new Promise((resolve) => { - setTimeout(resolve, IOS_FOLDABLE_SCOPED_ORIENTATION_SETTLE_MS); + setTimeout(resolve, IOS_FOLDABLE_ORIENTATION_SETTLE_MS); }); } diff --git a/packages/platform-apple/src/foldable/picker-delta.test.ts b/packages/platform-apple/src/foldable/picker-delta.test.ts deleted file mode 100644 index 48ad007254..0000000000 --- a/packages/platform-apple/src/foldable/picker-delta.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { expect, test } from 'vitest'; -import type { DeviceRotation } from '@agent-device/contracts/device'; -import { panelContentRotation } from '../core/display-inventory.ts'; -import { - measuredPanelPickerOffset, - panelPickerOffset, - pickerValueForTarget, - type PanelQuarterTurn, -} from './picker-delta.ts'; - -const ROTATIONS: readonly DeviceRotation[] = [ - 'portrait', - 'landscape-right', - 'portrait-upside-down', - 'landscape-left', -]; -const OFFSETS: readonly PanelQuarterTurn[] = [0, 90, 180, 270]; - -/** The measured guest model in CADisplay `rotN` angles, stated here independently of the code. */ -const ANGLE: Record = { - portrait: 0, - 'landscape-right': 90, - 'portrait-upside-down': 180, - 'landscape-left': 270, -}; - -function atAngle(angle: number): DeviceRotation { - return ROTATIONS[(((angle / 90) % 4) + 4) % 4]!; -} - -/** The lit panel after one picker event, as measured: the picker value plus the panel offset. */ -function litPanelFor(picker: DeviceRotation, offset: number): DeviceRotation { - return atAngle(ANGLE[picker] + offset); -} - -test('the target minus the offset lands every target at every offset in one dispatch', () => { - for (const target of ROTATIONS) { - for (const offset of OFFSETS) { - const value = pickerValueForTarget(target, offset); - expect(litPanelFor(value, offset), `${target}, K=${offset}`).toBe(target); - } - } -}); - -test('a panel whose offset differs from the measured one lands on the dispatch its reading calibrates', () => { - for (const target of ROTATIONS) { - for (const actual of OFFSETS) { - for (const measured of OFFSETS) { - const first = pickerValueForTarget(target, measured); - const reading = litPanelFor(first, actual); - if (reading === target) continue; - const learned = panelPickerOffset(reading, first); - expect(learned).toBe(actual); - expect(litPanelFor(pickerValueForTarget(target, learned), actual)).toBe(target); - } - } - } -}); - -test('the inner panel reaches landscape-left only through the upside-down picker value', () => { - expect(pickerValueForTarget('landscape-left', 90)).toBe('portrait-upside-down'); - expect(pickerValueForTarget('portrait', 90)).toBe('landscape-left'); - expect(pickerValueForTarget('landscape-right', 90)).toBe('portrait'); -}); - -test('the measured offsets are 0 on the outer (primary) panel and +90 on the inner panel', () => { - expect(measuredPanelPickerOffset({ primary: true })).toBe(0); - expect(measuredPanelPickerOffset({ primary: false })).toBe(90); -}); - -/** - * Every iPhone Duo reading taken for the measurement (iOS 27.3, scoped simulator set, Safari in - * front), in order: the picker value the guest held and the lit panel's `currentOrientation` - * that the panel table reported next. Rows marked `fold` are the first read after a pose change, - * with the picker value from before it. One row contradicts the outer panel's offset and is kept - * with its `exception`: a fold left the picker at the upside-down value, which the outer panel's - * foreground app cannot show, so that panel kept the rotation it last showed. - */ -const DUO_READINGS: readonly { - panel: 'inner' | 'outer'; - picker: DeviceRotation; - reading: string; - fold?: true; - exception?: string; -}[] = [ - { panel: 'inner', picker: 'portrait', reading: 'rot90' }, - { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, - { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, - { panel: 'inner', picker: 'portrait-upside-down', reading: 'rot270' }, - { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, - { panel: 'inner', picker: 'portrait', reading: 'rot90' }, - { panel: 'inner', picker: 'portrait', reading: 'rot90' }, - { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, - { panel: 'outer', picker: 'landscape-left', reading: 'rot270', fold: true }, - { panel: 'outer', picker: 'portrait', reading: 'rot0' }, - { panel: 'outer', picker: 'landscape-left', reading: 'rot270' }, - { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, - { panel: 'outer', picker: 'portrait', reading: 'rot0' }, - { panel: 'outer', picker: 'landscape-left', reading: 'rot270' }, - { panel: 'inner', picker: 'landscape-left', reading: 'rot0', fold: true }, - { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, - { panel: 'inner', picker: 'portrait', reading: 'rot90' }, - { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, - { panel: 'inner', picker: 'portrait-upside-down', reading: 'rot270' }, - { - panel: 'outer', - picker: 'portrait-upside-down', - reading: 'rot270', - fold: true, - exception: 'the outer panel app has no upside-down; the panel kept landscape-left', - }, - { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, - { panel: 'outer', picker: 'portrait', reading: 'rot0' }, - { panel: 'outer', picker: 'landscape-right', reading: 'rot90' }, - { panel: 'inner', picker: 'landscape-right', reading: 'rot180', fold: true }, - { panel: 'inner', picker: 'portrait', reading: 'rot90' }, - { panel: 'inner', picker: 'landscape-left', reading: 'rot0' }, - { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, - { panel: 'inner', picker: 'portrait', reading: 'rot90' }, - { panel: 'outer', picker: 'portrait', reading: 'rot0', fold: true }, - { panel: 'outer', picker: 'landscape-left', reading: 'rot270' }, - { panel: 'inner', picker: 'landscape-left', reading: 'rot0', fold: true }, - { panel: 'inner', picker: 'landscape-right', reading: 'rot180' }, - { panel: 'inner', picker: 'portrait', reading: 'rot90' }, -]; - -test('every measured Duo reading but the marked exception gives its panel the measured offset', () => { - for (const [index, row] of DUO_READINGS.entries()) { - const reading = panelContentRotation(row.reading); - expect(reading, `row ${index}`).toBeDefined(); - const offset = panelPickerOffset(reading!, row.picker); - const measured = measuredPanelPickerOffset({ primary: row.panel === 'outer' }); - if (row.exception) expect(offset, `row ${index}`).not.toBe(measured); - else expect(offset, `row ${index}`).toBe(measured); - } - expect(DUO_READINGS).toHaveLength(33); - expect(DUO_READINGS.filter((row) => row.fold)).toHaveLength(6); - expect(DUO_READINGS.filter((row) => row.exception)).toHaveLength(1); -}); - -test('after the exception, the measured outer offset still lands the next target in one dispatch', () => { - // Measured: picker at portrait-upside-down, outer panel lit showing landscape-left. One - // dispatch of landscape-right landed landscape-right (the row after the exception). A purely - // relative guest would have turned the panel by (90 - 180) to portrait-upside-down instead. - expect( - pickerValueForTarget('landscape-right', measuredPanelPickerOffset({ primary: true })), - ).toBe('landscape-right'); -}); diff --git a/packages/platform-apple/src/foldable/picker-delta.ts b/packages/platform-apple/src/foldable/picker-delta.ts deleted file mode 100644 index 72131d2b07..0000000000 --- a/packages/platform-apple/src/foldable/picker-delta.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { DeviceRotation } from '@agent-device/contracts/device'; - -/** - * Panel-frame quarter turns, following CADisplay `rotN`: portrait 0, landscape-right 90, - * portrait-upside-down 180, landscape-left 270. This frame is a panel-table fact, not the - * Android `Surface.ROTATION_*` frame in `DEVICE_ROTATION_SURFACE_INDEX` (which turns the - * other way), so the two tables must not be shared. Parsing a panel's `currentOrientation` - * into a rotation is `panelContentRotation` in the display inventory; this module only does - * the picker arithmetic on rotations already parsed. - */ -export type PanelQuarterTurn = 0 | 90 | 180 | 270; - -const PANEL_ROTATION_ANGLE: Readonly> = { - portrait: 0, - 'landscape-right': 90, - 'portrait-upside-down': 180, - 'landscape-left': 270, -}; - -const ROTATION_BY_QUARTER: readonly DeviceRotation[] = [ - 'portrait', - 'landscape-right', - 'portrait-upside-down', - 'landscape-left', -]; - -/** The panel-frame angle of one rotation. */ -function panelRotationAngle(rotation: DeviceRotation): PanelQuarterTurn { - return PANEL_ROTATION_ANGLE[rotation]; -} - -/** The rotation at one panel-frame angle. The angle is normalized into one full turn. */ -function rotationAtPanelAngle(angle: number): DeviceRotation { - const quarter = (((Math.round(angle / 90) % 4) + 4) % 4) as 0 | 1 | 2 | 3; - return ROTATION_BY_QUARTER[quarter]!; -} - -/** - * The offset between the orientation picker and the lit panel it drives: the panel reads - * `picker + offset`. The picker value is the one this command last dispatched, and the - * rotation is a fresh reading of the same panel taken after it. - */ -export function panelPickerOffset( - currentRotation: DeviceRotation, - pickerValue: DeviceRotation, -): PanelQuarterTurn { - const angle = (panelRotationAngle(currentRotation) - panelRotationAngle(pickerValue) + 360) % 360; - return angle as PanelQuarterTurn; -} - -/** Outer-panel offset, measured on iPhone Duo: the outer panel reads the picker value itself. */ -const OUTER_PANEL_PICKER_OFFSET: PanelQuarterTurn = 0; -/** Inner-panel offset, measured on iPhone Duo: it lights at rot90 while the picker sits at portrait. */ -const INNER_PANEL_PICKER_OFFSET: PanelQuarterTurn = 90; - -/** - * Measured picker offsets by panel, used only before a command has calibrated the lit panel - * with a dispatch of its own. The `primary` panel is the one that stays lit while the device - * is closed (the outer panel); the other integrated panel is the inner one (lit open and - * half-open). Measured on iPhone Duo (iOS 27.3) against the lit panel's `currentOrientation`: - * the outer panel reads `picker + 0` and the inner panel `picker + 90`, across fold and unfold - * in both directions. The guest keeps one picker value through a fold, so the panel that - * lights afterwards reads that same value plus its own offset — unless its foreground app - * cannot show that orientation, in which case it keeps the rotation it last showed. - */ -export function measuredPanelPickerOffset(panel: { primary: boolean }): PanelQuarterTurn { - return panel.primary ? OUTER_PANEL_PICKER_OFFSET : INNER_PANEL_PICKER_OFFSET; -} - -/** - * The picker value that leaves the lit panel at `targetRotation` once the panel's offset is - * added back: `target - offset`. The guest shows `picker + offset`, so dispatching the target - * itself lands an offset away from it, which is what made a rotation look one command late. - */ -export function pickerValueForTarget( - targetRotation: DeviceRotation, - panelOffset: PanelQuarterTurn, -): DeviceRotation { - return rotationAtPanelAngle(panelRotationAngle(targetRotation) - panelOffset); -} diff --git a/packages/platform-apple/src/foldable/pose.ts b/packages/platform-apple/src/foldable/pose.ts index 5985e520ca..f80e781532 100644 --- a/packages/platform-apple/src/foldable/pose.ts +++ b/packages/platform-apple/src/foldable/pose.ts @@ -26,7 +26,7 @@ const FOLDABLE_REQUIRED_HINT = 'fold sets the hinge angle of a foldable simulator such as iPhone Duo; this simulator reports one integrated panel, so it has no hinge to pose.'; const INVENTORY_REQUIRED_HINT = - "fold needs the simulator's panel table to tell a foldable from a single-panel simulator: 'devicectl device info displays' in the default device set, or the fold helper's guest panel read in a scoped set. The apple_display_inventory_unresolved diagnostic names why the read failed."; + "fold needs the simulator's integrated displays to identify a foldable: the fold helper reads their panel table in either simulator set. The apple_display_inventory_unresolved diagnostic names why the read failed."; const DISPATCHED_UNVERIFIED_HINT = 'The pose request was sent and the device may have followed it, but the first hinge-angle read returned no sample, so fold cannot confirm it. Take a screenshot or snapshot to see which panel is lit; hingeReadFailure names why the read failed.'; @@ -45,8 +45,7 @@ export async function setAppleFoldPose( const intent = input; const targetAngle = intent.keyframes?.at(-1)?.angle; const pose = intent.pose ?? foldPoseForHingeAngle(targetAngle!)!; - // One helper serves the scoped panel reads, the dispatch, and every hinge read. It builds on its - // first use, so a default-set device that cannot fold is refused before any build. + // One helper serves the panel reads, the dispatch, and every hinge read. const helper = createFoldHelper(device); const inventory = await queryAppleDisplayInventory(device, { signal: options.signal, diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index b1c76fc015..e6d3236dee 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -16,6 +16,7 @@ vi.mock('../core/devicectl.ts', async (importOriginal) => ({ })), })); +import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/tool-provider.ts'; import { runIosDevicectlJsonRequest } from '../core/devicectl.ts'; import { runAppleRunnerCommand } from '../core/runner-client.ts'; import { iosRunnerOverrides } from '../interactions.ts'; @@ -166,7 +167,20 @@ test('a resolved read lifts the back-off a failed one opened', async () => { assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); answering(IPHONE_DUO_CLOSED); // A rotation reads the table on its own route; its answer clears the back-off for surface reads. - assert.equal(await isAppleFoldableSimulator(device, undefined), true); + const guest = createLocalAppleToolProvider({ + runCommand: async (_command, args) => { + assert.equal(args.at(-1), 'panels'); + return { + exitCode: 0, + stdout: JSON.stringify({ version: 1, displays: IPHONE_DUO_CLOSED.result.displays }), + stderr: '', + }; + }, + }); + assert.equal( + await withAppleToolProvider(guest, () => isAppleFoldableSimulator(device, undefined)), + true, + ); assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); }); diff --git a/src/commands/schema/cli-help-foldable.ts b/src/commands/schema/cli-help-foldable.ts index decced21ea..2e754b9b0e 100644 --- a/src/commands/schema/cli-help-foldable.ts +++ b/src/commands/schema/cli-help-foldable.ts @@ -15,7 +15,7 @@ Changing the pose: Expect a fold to take 2-4 seconds: half-open waits for two agreeing hinge reads one second apart, and the first fold after a boot can take longer while the hinge settles. Re-snapshot after every fold; refs and coordinates from before it are stale, and the command's message says so. Timed motion: fold --keyframes '[{"atMs":0,"angle":0},{"atMs":5000,"angle":180}]'. Use 2–64 frames starting at 0ms, increasing integer timestamps up to 60000ms, and angles from 0 to 180. The last timestamp sets motion duration, excluding setup and verification. Equal angles hold; cancellation stops motion. See the fold examples in the command and Node API documentation for trajectories. - Requirements: an iOS simulator session on a foldable device and an Xcode toolchain with the iOS simulator SDK and foldable HID support (verified on Xcode 27.1). Device Hub and host Accessibility permission are not required. The command runs a small helper through simctl spawn for the session UDID; the helper is built once per Fold.m source hash and Xcode toolchain, cached under ~/.agent-device/fold-helper, and rebuilt only when the source or the toolchain changes. The same helper reads the hinge and, in a scoped set, the panels. Build or dispatch failures are reported without a UI fallback. The app under test reads the resulting pose as UIHinge.status. + Requirements: an iOS simulator session on a foldable device and an Xcode toolchain with the iOS simulator SDK and foldable HID support (verified on Xcode 27.1). Device Hub and host Accessibility permission are not required. The command runs a small helper through simctl spawn for the session UDID; the helper is built once per Fold.m source hash and Xcode toolchain, cached under ~/.agent-device/fold-helper, and rebuilt only when the source or the toolchain changes. The same helper reads the hinge and the panels in either simulator set. Build or dispatch failures are reported without a UI fallback. The app under test reads the resulting pose as UIHinge.status. Android emulators: fold sets the posture through the emulator console (adb emu posture 1|2|3), then reads adb shell cmd device_state print-state until the guest commits CLOSED, HALF_OPENED, or OPENED, and reports the angle read from the hinge sensor. It can differ from the AVD profile's per-posture angles: 0, 90 and 180 degrees on the Pixel folds, 15, 90 and 165 on the generic 7.6in Foldable. Folding to the cover display can raise Android's "Swipe up to continue" lock screen; fold dismisses it so the app stays reachable. An emulator profile without those device states (a phone) fails with UNSUPPORTED_OPERATION and reason single-panel-device, and a physical device is refused. Keyframes are not supported on Android, and the response carries no panel geometry: re-snapshot after every fold. If a task asserts behavior for more than one pose, fold to each pose and re-snapshot, and report which poses the run covered.`, } as const; diff --git a/test/integration/provider-scenarios/ios-fold.test.ts b/test/integration/provider-scenarios/ios-fold.test.ts index 4dc0f01229..87a6d1ed3c 100644 --- a/test/integration/provider-scenarios/ios-fold.test.ts +++ b/test/integration/provider-scenarios/ios-fold.test.ts @@ -43,17 +43,19 @@ function guestHingeReading(angleDegrees: number): string { return `${JSON.stringify({ version: 1, angleDegrees, angleValid: true })}\n`; } -/** Writes the `devicectl … displays --json-output ` result both fold fakes below answer. */ -function writeDisplayInventoryFixture(jsonOutputPath: string): void { +/** The fold helper's `panels` mode output for a Duo with its inner panel lit. */ +function guestPanelReading(): string { const displays = [0, 1].map((displayId) => ({ name: `LCD-${displayId}`, displayId, nativeSize: [2007, 2853], pointScale: 3, type: { integrated: {} }, - active: displayId === 1, + primary: displayId === 0, + backlightState: displayId === 1 ? 'activeOn' : 'off', + currentOrientation: 'rot0', })); - fs.writeFileSync(jsonOutputPath, JSON.stringify({ result: { displays } })); + return `${JSON.stringify({ version: 1, displays })}\n`; } /** @@ -97,6 +99,7 @@ test('timed fold keyframes reach simulator HID through the public client and dae if (args[0] === 'launch') return ok; assert.equal(args[0], 'spawn'); assert.equal(args[1], PROVIDER_SCENARIO_IOS_SIMULATOR.id); + if (args[3] === 'panels') return { ...ok, stdout: guestPanelReading() }; if (args[3] === 'hinge') return { ...ok, stdout: guestHingeReading(angle) }; assert.equal(args[3], 'set'); assert.deepEqual(JSON.parse(args[4]!), trajectory); @@ -104,11 +107,6 @@ test('timed fold keyframes reach simulator HID through the public client and dae angle = 100; return ok; }, - devicectl: async (args) => { - assert.ok(args.includes('displays')); - writeDisplayInventoryFixture(args[args.indexOf('--json-output') + 1]!); - return ok; - }, }); let builds = 0; const daemon = await createProviderScenarioHarness({ @@ -226,6 +224,9 @@ function createFoldLedgerAppleToolProvider(params: { } if (args[0] === 'launch') return ok; await record('simctl', args, options); + if (args[0] === 'spawn' && args[3] === 'panels') { + return { ...ok, stdout: guestPanelReading() }; + } if (args[0] === 'spawn' && args[3] === 'hinge') { const angle = params.hingeAngleAt(hingeReads); hingeReads += 1; @@ -233,12 +234,6 @@ function createFoldLedgerAppleToolProvider(params: { } return ok; }, - devicectl: async (args, options) => { - await record('devicectl', args, options); - assert.ok(args.includes('displays'), `unexpected devicectl call: ${args.join(' ')}`); - writeDisplayInventoryFixture(args[args.indexOf('--json-output') + 1]!); - return ok; - }, }); const provider: AppleToolProvider = { ...recording.provider, @@ -347,7 +342,7 @@ function assertFoldLedgerPhaseCoverage(ledger: readonly FoldLedgerCall[]): void const isHingeRead = (call: FoldLedgerCall) => call.tool === 'simctl' && call.args[0] === 'spawn' && call.args[3] === 'hinge'; const isDisplayInventory = (call: FoldLedgerCall) => - call.tool === 'devicectl' && call.args.includes('displays'); + call.tool === 'simctl' && call.args[0] === 'spawn' && call.args[3] === 'panels'; assert.ok(ledger.some(isBuild), 'ledger is missing the helper-build phase'); assert.ok(ledger.some(isSpawn), 'ledger is missing the HID-dispatch (simctl spawn) phase'); @@ -355,7 +350,7 @@ function assertFoldLedgerPhaseCoverage(ledger: readonly FoldLedgerCall[]): void assert.equal( ledger.filter(isDisplayInventory).length, 2, - 'ledger must record both the foldable-check and the lit-panel display inventory reads', + 'ledger must record both the foldable-check and the lit-panel guest inventory reads', ); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 39187b1a1f..65dabe8bbf 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -132,7 +132,7 @@ Back, orientation, and hardware buttons: - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. - An app with a left-edge drawer may open the drawer instead of going back. When a back or close control is visible, press it by its ref. - `back --system` sends system back input: the Android back keyevent, the iOS back swipe, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. -- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. A foldable iPhone simulator (iPhone Duo) refuses `portrait-upside-down`. In a scoped simulator set (`--ios-simulator-device-set`) it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified-scoped-set`. +- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. A foldable iPhone simulator (iPhone Duo) refuses `portrait-upside-down`. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`. - `action-button` presses the iPhone or iPad Action Button once. It takes no arguments, no hold duration, and no `--settle`, and it doesn't activate the session app or drive Camera Control. Other platforms refuse it, and a model without the button (an iPhone SE, most iPad simulators) fails with `UNSUPPORTED_OPERATION`. - `action-button` reports that the press was dispatched, not what it triggered. Simulators run no Shortcuts or App Intents, so verify the outcome on a physical iPhone. Expect a press to take about five seconds on a Simulator. From 5f3f6f22df8bb65a126ad5190c1ddb308f6d3018 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 15:12:15 +0000 Subject: [PATCH 10/16] docs(agents): keep foldable guidance within its context budget --- docs/agents/device-foldable.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/agents/device-foldable.md b/docs/agents/device-foldable.md index 70b0167201..777cdb8e22 100644 --- a/docs/agents/device-foldable.md +++ b/docs/agents/device-foldable.md @@ -2,8 +2,7 @@ Before you change capture behavior on a multi-panel device, read ADR 0025. To list the panels, run `xcrun devicectl device info displays --device `. -For foldable orientation in either set, read the guest table with `xcrun simctl spawn ~/.agent-device/fold-helper//fold-helper panels`. -For a scoped set, add `--set ` after `simctl`. +For rotation, run `xcrun simctl [--set ] spawn ~/.agent-device/fold-helper//fold-helper panels`. - Panels: the list shows each integrated panel with `backlightState`. Only the lit panel permits capture. A capture of the dark panel exits 0 and writes an all-black PNG. From 0b3e642738afa4e8907735a7c224e59a8e619108 Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 15:50:36 +0000 Subject: [PATCH 11/16] docs(ios): reconcile foldable rotation contract and fixtures --- docs/adr/0025-foldable-apple-panels.md | 27 +++++++++---------- docs/agents/device-foldable.md | 13 ++++----- .../src/foldable/orientation.test.ts | 15 +---------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index e32cea8253..88d40ccbfe 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -106,7 +106,7 @@ It is the only reader of panel tables, and it selects the source. `IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS` limits the probe to 5s, below the subsequent 20s screenshot deadline. Thus, an unresponsive CoreDevice cannot consume the capture's own budget. Measured cost is 0.16–0.26s, compared with 4.93s for the screenshot. - The guest probe costs about 0.3s. `IOS_SCOPED_DISPLAY_PROBE_BUDGET_MS` limits it to 15s, including the first helper build. + The guest probe costs about 0.3s. `IOS_GUEST_DISPLAY_PROBE_BUDGET_MS` limits it to 15s, including the first helper build. ## Why the default capture had to stop being implicit @@ -372,19 +372,23 @@ Rejected alternatives: `XCUIDevice.orientation` has no effect on an iPhone Duo simulator, and the runner readback repeats the requested value. `devicectl device orientation set` reports success and has no effect on a simulator that `simctl boot` started. `orientation` therefore rotates a multi-panel simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). -The control has no upside-down value, so `portrait-upside-down` is refused with `unsupported-foldable-orientation`. +The picker accepts `pud` for upside-down dispatch, which the inner panel needs to show landscape-left. +The user target `portrait-upside-down` remains refused with `unsupported-foldable-orientation`. Routing uses the `fold` admission and the panel table. A resolved panel count is a hardware fact, so the daemon probes each simulator once. -The verdict comes from `devicectl device orientation get`, a process that never sent the HID event. -CoreDevice answers each query with the value that it sampled at the previous query, so the first read after a dispatch is discarded. -A rotation that no fresh read reports is refused with `orientation-unverified`. -A simulator in a scoped set is refused with `unsupported-device-scope` before dispatch, because no reader can verify its rotation. +Verification was updated on 2026-11-12 under [#4440](https://github.com/callstack/agent-device/pull/4440). +In both simulator sets, the helper reads the guest panel table and dispatches the fixed picker value for the established lit panel. +The table maps the outer panel at zero offset and the inner panel at +90 degrees. +Only a fresh `currentOrientation` from an established lit panel confirms the requested visible orientation. +An incomplete table, unknown-power sibling, or multiple lit panels cannot confirm or dispatch. +A repeated visible target dispatches nothing; delayed reads wait without resending to the same panel. +A target that never appears is refused with `orientation-unverified` in either set. Rejected alternatives: -- **The panel table's `orientation.currentDeviceOrientation`: rejected.** It is screen-relative. On the Duo inner panel, it is a quarter-turn from the device orientation. -- **A guest orientation read: rejected.** CoreMotion `CMDeviceOrientationManager` reports `Ambiguous`, and `BKSHIDServicesIsOrientationLockedWithOrientation` reports 0, after a rotation that CoreDevice and the screenshots confirm. -- **Dispatch without a verdict in a scoped set: rejected.** The runner path already reported a false success there. +- **CoreDevice orientation readback: rejected.** Its picker sample can lag and can match the request while the visible inner panel is a quarter-turn away. +- **Other guest device-orientation APIs: rejected.** CoreMotion `CMDeviceOrientationManager` reports `Ambiguous`, and `BKSHIDServicesIsOrientationLockedWithOrientation` reports 0 after a rotation. +- **Dispatch without a lit-panel verdict: rejected.** An acknowledgement alone can report false success. ## Pose is derived from panel power, and official control does not exist @@ -680,11 +684,6 @@ The silent misses investigated here came from default outer-display routes and i `AgentDeviceMacOSHelper` also reaches it through Screen Capture Kit. Only iOS branches changed target. Desktop behavior remains unmeasured here. -- **Quarter-turn detection.** - Both Duo panels report `currentOrientation: rot90`. - Inventory carries orientation without a test against changed orientation. - `orientation` now rotates a Duo simulator in the default set through guest HID (2026-10-26). - The changed `currentOrientation` values have no test yet. - **A slowly settled hinge.** Four reads consume twenty seconds of streams. A Duo that takes longer to settle inside `half-open` is refused. diff --git a/docs/agents/device-foldable.md b/docs/agents/device-foldable.md index 777cdb8e22..af80b8f83d 100644 --- a/docs/agents/device-foldable.md +++ b/docs/agents/device-foldable.md @@ -1,11 +1,12 @@ # Foldable Apple verification -Before you change capture behavior on a multi-panel device, read ADR 0025. -To list the panels, run `xcrun devicectl device info displays --device `. -For rotation, run `xcrun simctl [--set ] spawn ~/.agent-device/fold-helper//fold-helper panels`. +Read ADR 0025 before capture changes. +For default-set capture, run `xcrun devicectl device info displays --device `. +For scoped capture or rotation, run `xcrun simctl --set spawn ~/.agent-device/fold-helper//fold-helper panels`. +For default-set rotation, omit `--set `. - Panels: the list shows each integrated panel with `backlightState`. Only the lit panel permits capture. - A capture of the dark panel exits 0 and writes an all-black PNG. + A dark-panel capture exits 0 with a black PNG. - Input routing: after each pose change, verify a fresh control. A successful synthesis acknowledgement does not prove a hit. Examine the runner and simulator `testmanagerd`/BackBoard logs for display identity and delivery. @@ -14,13 +15,13 @@ For rotation, run `xcrun simctl [--set ] spawn ~/.agent-device/fold- The same helper reads the hinge back in `hinge` mode. After a fold, take another snapshot. Folds invalidate refs and coordinates. Verify locally with Device Hub stopped and the simulator booted through `simctl boot`. - Test these items in the default set and with `--ios-simulator-device-set`: + Test both sets (`--ios-simulator-device-set`): - All three poses. - Capture of the active panel. - An app interaction. Until GHA supports the runtime, Duo coverage remains local. After a fresh boot, a closed hinge can keep the inner panel lit for more than 30 seconds. - The lit panel is evidence only. It cannot separate `half-open` from `open`. + Panel power cannot separate `half-open` from `open`. - Touch overlays export at the captured track size. The compositor checks its output size and non-black content against the raw capture. If that check fails, `record stop` keeps the raw file. It reports `overlayWarning`. diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 371d7da78c..f1456e2b4c 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -1,4 +1,3 @@ -import fs from 'node:fs'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import { bindTestCommandHost, createLiveTestHostEnvironment } from '@agent-device/test-kit'; @@ -181,19 +180,7 @@ function fakeDuo( const [tool, ...rest] = args; if (tool !== 'simctl') { devicectl.push(args.join(' ')); - if (!device.simulatorSetPath && tool === 'devicectl') { - const displays = [panel('outer', false), panel('inner', false)]; - const rotation = rotationAtAngle(shown[lit]); - const deviceOrientation = rotation.replaceAll(/-([a-z])/g, (_, letter: string) => - letter.toUpperCase(), - ); - fs.writeFileSync( - args.at(-1)!, - JSON.stringify({ result: { displays, deviceOrientation } }), - ); - return { stdout: '', stderr: '', exitCode: 0 }; - } - throw new Error(`unexpected tool call in a scoped set: ${args.join(' ')}`); + throw new Error(`unexpected tool call: ${args.join(' ')}`); } const spawn = rest.slice(rest.indexOf('spawn')); expect(spawn.slice(0, 3)).toEqual(['spawn', device.id, '/cache/fold-helper']); From ae7a60271ffe8e114c9c4181635b3d7b0bd4ad6e Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 16:24:07 +0000 Subject: [PATCH 12/16] fix(ios): keep foldable classification on the simulator set source --- .changeset/foldable-scoped-rotation.md | 2 +- docs/adr/0025-foldable-apple-panels.md | 7 +- .../core/__tests__/display-inventory.test.ts | 22 ++- .../src/core/display-inventory.ts | 8 +- .../src/foldable/orientation.test.ts | 140 +++++++++++++----- .../src/foldable/orientation.ts | 28 ++-- packages/platform-apple/src/foldable/pose.ts | 7 +- .../src/foldable/runner-panel.test.ts | 16 +- 8 files changed, 158 insertions(+), 72 deletions(-) diff --git a/.changeset/foldable-scoped-rotation.md b/.changeset/foldable-scoped-rotation.md index 9219daaeeb..d7e106b6a7 100644 --- a/.changeset/foldable-scoped-rotation.md +++ b/.changeset/foldable-scoped-rotation.md @@ -2,6 +2,6 @@ 'agent-device': patch --- -Fix foldable iPhone simulator rotation in scoped sets and confirm the lit panel in both default and scoped sets. Both routes use the fold helper, with no CoreDevice orientation polling. An already visible target dispatches nothing. Incomplete, unknown-power and multi-lit panel tables neither confirm nor dispatch. A target the panel never shows fails with `orientation-unverified`. +Fix foldable iPhone simulator rotation in scoped sets and confirm the lit panel in both default and scoped sets. Rotation in both sets uses the fold helper, with no CoreDevice orientation polling. An already visible target dispatches nothing. Incomplete, unknown-power and multi-lit panel tables neither confirm nor dispatch. A target the panel never shows fails with `orientation-unverified`. Use the fixed iPhone Duo picker offsets measured in both sets (outer 0 degrees, inner +90 degrees), replacing runtime calibration. Inner landscape-left sends `portrait-upside-down`, mapped to the HID wire value `pud`; it remains unsupported as a user target. Delayed reads wait without resending the same picker value to the same panel. diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 88d40ccbfe..8290dab976 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -58,7 +58,8 @@ It is the only reader of panel tables, and it selects the source. For a simulator in a scoped set, the fold helper's `panels` mode is the screen authority. It emits the same entry shape from the guest sources that `dtdeviceinfod` relays to CoreDevice: `CADisplay` and `BLSBacklight`. The guest lists every integrated panel and omits only the facts it cannot read. - Foldable `fold` and `orientation` operations use the guest table in either simulator set. + Fold pose and rotation lit-panel reads explicitly request the guest table in either simulator set. + Panel-count classification keeps the set-based source: CoreDevice in the default set, guest in a scoped set. Rotation confirms the lit panel because CoreDevice orientation can match the picker while the visible panel shows another orientation. - **Completeness.** Both sources list one entry per integrated panel. A panel that the parser drops is not single-panel evidence. @@ -374,7 +375,9 @@ Rejected alternatives: `orientation` therefore rotates a multi-panel simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). The picker accepts `pud` for upside-down dispatch, which the inner panel needs to show landscape-left. The user target `portrait-upside-down` remains refused with `unsupported-foldable-orientation`. -Routing uses the `fold` admission and the panel table. A resolved panel count is a hardware fact, so the daemon probes each simulator once. +Routing uses the `fold` admission and the set-based panel-count source. A resolved panel count is a hardware fact, so the daemon probes each simulator once. +A supplied helper is a reuse handle; only an explicit guest-source request changes the default-set source. +Thus, classifying a single-panel default-set simulator does not prepare the fold helper. Verification was updated on 2026-11-12 under [#4440](https://github.com/callstack/agent-device/pull/4440). In both simulator sets, the helper reads the guest panel table and dispatches the fixed picker value for the established lit panel. diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index 86d8b15aff..39ce493a6f 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -765,7 +765,7 @@ describe('scoped simulator panel table', () => { readPanelPayload, }; - const inventory = await queryAppleDisplayInventory(device, { foldHelper }); + const inventory = await queryAppleDisplayInventory(device, { source: 'guest', foldHelper }); assert.equal(inventory.activeDisplay?.name, 'LCD-1'); assert.equal(readPanelPayload.mock.calls.length, 1); @@ -774,6 +774,26 @@ describe('scoped simulator panel table', () => { assert.equal(mockRunXcrun.mock.calls.length, 0); }); + test('a helper handle alone keeps the default-set CoreDevice source', async () => { + mockReadHostTextFile.mockResolvedValue(JSON.stringify(IPHONE_DUO_CLOSED)); + const readPanelPayload = vi.fn(async () => { + throw new Error('guest must not classify'); + }); + const foldHelper = { + setPose: vi.fn(), + rotate: vi.fn(), + readHingeAngle: vi.fn(), + readPanelPayload, + }; + const inventory = await queryAppleDisplayInventory( + { ...scopedDuo, simulatorSetPath: undefined }, + { foldHelper }, + ); + assert.equal(inventory.multiScreen, true); + assert.equal(readPanelPayload.mock.calls.length, 0); + assert.equal(mockRunXcrun.mock.calls.length, 1); + }); + test('applies the same rule to a CoreDevice table whose second panel does not parse', async () => { const [outer, inner] = IPHONE_DUO_CLOSED.result.displays; const { displayId: _displayId, ...innerWithoutId } = inner!; diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 3b835437e5..aeb192e121 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -90,7 +90,9 @@ type CoreDeviceDisplayPayload = { type AppleDisplayInventoryOptions = { timeoutMs?: number; signal?: AbortSignal; - /** Selects the guest table in either simulator set and reuses this helper. */ + /** Requests the guest table for a lit-panel read in either simulator set. */ + source?: 'guest'; + /** Reuses a helper without changing the simulator set's table source. */ foldHelper?: FoldHelper; }; @@ -99,7 +101,7 @@ const DISPLAYS_UNSUPPORTED_HINT = /** * Reads the CoreDevice display table for one Apple device, or the guest's equivalent for a - * simulator in a scoped device set or a caller that supplies its fold helper. + * simulator in a scoped device set or a caller that explicitly requests the guest table. * * Deliberately uncached: which panel is lit is exactly what changes when an * operator folds or opens the device, and a cached answer would resume capturing @@ -114,7 +116,7 @@ export async function queryAppleDisplayInventory( options: AppleDisplayInventoryOptions, ): Promise { const table = - options.foldHelper || simulatorAddressFor(device).simulatorSetPath + options.source === 'guest' || simulatorAddressFor(device).simulatorSetPath ? await readGuestPanelTable(device, options) : await readCoreDevicePanelTable(device, options); if (!table.ok) { diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index f1456e2b4c..85a1bcec9a 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs'; import { beforeEach, describe, expect, test, vi } from 'vitest'; import { bindTestCommandHost, createLiveTestHostEnvironment } from '@agent-device/test-kit'; @@ -14,9 +15,10 @@ import { AppError } from '@agent-device/kernel/errors'; import { runAppleRunnerCommand } from '../core/runner-client.ts'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/tool-provider.ts'; import { createAppleInteractor } from '../interactor.ts'; -import { readFoldablePanelTable } from './orientation.ts'; +import { isAppleFoldableSimulator, readFoldablePanelTable } from './orientation.ts'; +import { ensureFoldHelperBinary } from './fold-helper-cache.ts'; import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; -import { IPHONE_17_SINGLE_PANEL } from '../__tests__/display-fixtures.ts'; +import { IPHONE_17_SINGLE_PANEL, IPHONE_DUO_CLOSED } from '../__tests__/display-fixtures.ts'; bindTestCommandHost(); @@ -24,6 +26,7 @@ const mockRunner = vi.mocked(runAppleRunnerCommand); beforeEach(() => { mockRunner.mockClear(); + vi.mocked(ensureFoldHelperBinary).mockReset().mockResolvedValue({ path: '/cache/fold-helper' }); }); /** A fresh simulator id per test: the daemon keeps each simulator's resolved panel count. */ @@ -38,11 +41,17 @@ const SINGLE_PANEL_DISPLAYS = IPHONE_17_SINGLE_PANEL.result.displays; /** Single-panel and physical targets keep their runner rotation. */ function fakeSimulator(device: DeviceInfo, displays: readonly object[]) { const helpers: string[][] = []; - const devicectl: string[] = []; + const unexpectedTools: string[] = []; + const classificationReads: string[] = []; const provider = createLocalAppleToolProvider({ runCommand: async (_command, args) => { + if (args.slice(0, 4).join(' ') === 'devicectl device info displays') { + classificationReads.push(args.join(' ')); + fs.writeFileSync(args.at(-1)!, JSON.stringify({ result: { displays } })); + return { stdout: '', stderr: '', exitCode: 0 }; + } if (args[0] !== 'simctl') { - devicectl.push(args.join(' ')); + unexpectedTools.push(args.join(' ')); throw new Error(`unexpected tool call: ${args.join(' ')}`); } const spawn = args.slice(args.indexOf('spawn')); @@ -55,7 +64,8 @@ function fakeSimulator(device: DeviceInfo, displays: readonly object[]) { }); return { helpers, - devicectl, + unexpectedTools, + classificationReads, rotate: (rotation: DeviceRotation) => withAppleToolProvider(provider, () => createAppleInteractor(device, {}, createLiveTestHostEnvironment()).setOrientation(rotation), @@ -68,15 +78,18 @@ test('a simulator is probed for its panel count once', async () => { await rotateDuo(sim, 'landscape-left'); await rotateDuo(sim, 'portrait'); expect(sim.panelReads()).toBe(5); + expect(sim.classificationReads).toHaveLength(1); expect(sim.rotations()).toEqual(['landscape-left', 'portrait']); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); }); test('a single-panel simulator keeps the runner rotation', async () => { const device = simulator(); const sim = fakeSimulator(device, SINGLE_PANEL_DISPLAYS); await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); - expect(sim.helpers).toEqual([['panels']]); + expect(sim.helpers).toEqual([]); + expect(ensureFoldHelperBinary).not.toHaveBeenCalled(); + expect(sim.classificationReads).toHaveLength(1); expect(mockRunner).toHaveBeenCalledWith( device, expect.objectContaining({ command: 'rotate', orientation: 'landscape-left' }), @@ -84,10 +97,45 @@ test('a single-panel simulator keeps the runner rotation', async () => { ); }); +test('a cold default Duo classifies without preparing the guest helper', async () => { + const device = simulator(); + const calls: string[] = []; + vi.mocked(ensureFoldHelperBinary).mockRejectedValue( + new Error('cold helper exceeds guest budget'), + ); + const provider = createLocalAppleToolProvider({ + runCommand: async (_command, args) => { + calls.push(args.join(' ')); + expect(args.slice(0, 4)).toEqual(['devicectl', 'device', 'info', 'displays']); + fs.writeFileSync(args.at(-1)!, JSON.stringify(IPHONE_DUO_CLOSED)); + return { stdout: '', stderr: '', exitCode: 0 }; + }, + }); + await expect( + withAppleToolProvider(provider, () => isAppleFoldableSimulator(device, undefined)), + ).resolves.toBe(true); + expect(calls).toHaveLength(1); + expect(ensureFoldHelperBinary).not.toHaveBeenCalled(); +}); + +test('a cold default Duo helper failure cannot fall back to runner success', async () => { + const sim = fakeDuo(simulator(), { lit: 'outer', picker: 'portrait' }); + vi.mocked(ensureFoldHelperBinary).mockRejectedValue( + new Error('cold helper exceeds guest budget'), + ); + await expect(rotateDuo(sim, 'landscape-left')).resolves.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-unverified' }, + }); + expect(sim.classificationReads).toHaveLength(1); + expect(mockRunner).not.toHaveBeenCalled(); + expect(sim.rotations()).toEqual([]); +}); + test('a physical device keeps the runner rotation without a CoreDevice probe', async () => { const sim = fakeSimulator(IOS_DEVICE, []); await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); expect(mockRunner).toHaveBeenCalledTimes(1); }); @@ -142,7 +190,8 @@ function fakeDuo( }, ) { const helpers: string[][] = []; - const devicectl: string[] = []; + const unexpectedTools: string[] = []; + const classificationReads: string[] = []; const offsets = options.offsets ?? MEASURED_OFFSETS; let lit = options.lit; let pickerAngle = PANEL_ANGLE[options.picker]; @@ -175,11 +224,30 @@ function fakeDuo( type: { integrated: {} }, }; }; + const readPanels = () => { + panelReads += 1; + if (lagReadsLeft > 0) lagReadsLeft -= 1; + else if (pendingPicker !== undefined) { + pickerAngle = pendingPicker; + pendingPicker = undefined; + show(lit); + } + const bothLit = + bothLitReads > 0 || + (options.bothLitFromRead !== undefined && panelReads >= options.bothLitFromRead); + if (bothLitReads > 0) bothLitReads -= 1; + return [panel('outer', bothLit), panel('inner', bothLit)]; + }; const provider = createLocalAppleToolProvider({ runCommand: async (_command, args) => { const [tool, ...rest] = args; + if (args.slice(0, 4).join(' ') === 'devicectl device info displays') { + classificationReads.push(args.join(' ')); + fs.writeFileSync(args.at(-1)!, JSON.stringify({ result: { displays: readPanels() } })); + return { stdout: '', stderr: '', exitCode: 0 }; + } if (tool !== 'simctl') { - devicectl.push(args.join(' ')); + unexpectedTools.push(args.join(' ')); throw new Error(`unexpected tool call: ${args.join(' ')}`); } const spawn = rest.slice(rest.indexOf('spawn')); @@ -187,18 +255,7 @@ function fakeDuo( const mode = spawn.slice(3); helpers.push(mode); if (mode[0] === 'panels') { - panelReads += 1; - if (lagReadsLeft > 0) lagReadsLeft -= 1; - else if (pendingPicker !== undefined) { - pickerAngle = pendingPicker; - pendingPicker = undefined; - show(lit); - } - const bothLit = - bothLitReads > 0 || - (options.bothLitFromRead !== undefined && panelReads >= options.bothLitFromRead); - if (bothLitReads > 0) bothLitReads -= 1; - const displays = [panel('outer', bothLit), panel('inner', bothLit)]; + const displays = readPanels(); return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; } expect(mode[0]).toBe('rotate'); @@ -214,7 +271,8 @@ function fakeDuo( }); return { helpers, - devicectl, + unexpectedTools, + classificationReads, panelReads: () => panelReads, rotations: () => helpers.filter((mode) => mode[0] === 'rotate').map((mode) => mode[1]!), litScreen: () => `rot${shown[lit]}`, @@ -238,16 +296,32 @@ async function rotateDuo( sim: ReturnType, rotation: DeviceRotation, ): Promise { - vi.useFakeTimers(); + const prepare = vi.mocked(ensureFoldHelperBinary).getMockImplementation()!; + let clockStarted = false; + let ready!: () => void; + const helperStarted = new Promise((resolve) => { + ready = resolve; + }); + vi.mocked(ensureFoldHelperBinary).mockImplementation(async (input) => { + if (!clockStarted) { + vi.useFakeTimers(); + clockStarted = true; + ready(); + } + return prepare(input); + }); try { const outcome = sim.rotate(rotation).then( (value) => value, (error: unknown) => error, ); - await vi.advanceTimersByTimeAsync(30_000); + // CoreDevice classification performs file I/O before any settle timer exists. + await Promise.race([helperStarted, outcome]); + if (clockStarted) await vi.advanceTimersByTimeAsync(30_000); return await outcome; } finally { vi.useRealTimers(); + vi.mocked(ensureFoldHelperBinary).mockImplementation(prepare); } } @@ -266,7 +340,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu await expect(rotateDuo(sim, target)).resolves.toEqual({ orientation: target }); expect(sim.litScreen()).toBe(`rot${PANEL_ANGLE[target]}`); expect(sim.rotations()).toEqual(dispatched); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); expect(mockRunner).not.toHaveBeenCalled(); }, ); @@ -280,7 +354,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu expect(sim.litScreen()).toBe('rot0'); // The measured inner offset makes the first dispatch exact: portrait - 90 is landscape-left. expect(sim.rotations()).toEqual(['landscape-left']); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); expect(mockRunner).not.toHaveBeenCalled(); }); @@ -296,7 +370,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu }); expect(sim.litScreen()).toBe('rot270'); expect(sim.rotations()).toEqual(['portrait-upside-down']); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); }); test('the outer panel reads the picker value itself, so its first dispatch is the target', async () => { @@ -308,7 +382,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu await expect(rotateDuo(sim, rotation)).resolves.toEqual({ orientation: rotation }); expect(sim.rotations()).toEqual([rotation]); } - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); }); test('an unexpected panel response cannot calibrate another picker value', async () => { @@ -326,7 +400,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu expect(sim.litScreen()).toBe('rot0'); // An unexpected reading is not evidence that an unsupported picker value is safe to send. expect(sim.rotations()).toEqual(['landscape-right']); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); }); test('the picker survives a fold in both directions and each panel adds its own offset', async () => { @@ -463,7 +537,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu }); expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); expect(sim.rotations()).toEqual([]); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); }, ); @@ -489,7 +563,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu }); expect((outcome as AppError).details).not.toHaveProperty('observedOrientation'); expect(sim.rotations()).toEqual([]); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); }, ); @@ -526,7 +600,7 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu // The panel never moved after the one dispatch, so nothing more was // dispatched: the loop re-read until its budget ended, then refused. expect(sim.rotations()).toEqual(['landscape-left']); - expect(sim.devicectl).toEqual([]); + expect(sim.unexpectedTools).toEqual([]); }); test('a foldable still refuses portrait-upside-down as a target', async () => { diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 5d3349b6d6..d824948a9b 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -62,7 +62,6 @@ export async function isAppleFoldableSimulator( const read = ++panelTableReads; const inventory = await queryAppleDisplayInventory(device, { signal, - foldHelper: createFoldHelper(device), }); if (inventory.unresolved) return false; recordPanelTable(device, inventory, read); @@ -106,20 +105,14 @@ export async function readFoldablePanelTable( /** iPhone Duo (iOS 27.3) renders picker + offset: outer 0 degrees, inner +90. */ const PANEL_PICKERS = { outer: { - offset: 0, - values: { - portrait: 'portrait', - 'landscape-left': 'landscape-left', - 'landscape-right': 'landscape-right', - }, + portrait: 'portrait', + 'landscape-left': 'landscape-left', + 'landscape-right': 'landscape-right', }, inner: { - offset: 90, - values: { - portrait: 'landscape-left', - 'landscape-left': 'portrait-upside-down', - 'landscape-right': 'portrait', - }, + portrait: 'landscape-left', + 'landscape-left': 'portrait-upside-down', + 'landscape-right': 'portrait', }, } as const; @@ -149,7 +142,7 @@ export async function setAppleFoldableOrientation( if (reading?.rotation === rotation) return { orientation: rotation }; if (reading && reading.panel.displayId !== dispatchedDisplayId) { const mapping = reading.panel.primary ? PANEL_PICKERS.outer : PANEL_PICKERS.inner; - const value = mapping.values[rotation]; + const value = mapping[rotation]; await helper.rotate(value, signal); dispatchedDisplayId = reading.panel.displayId; emitDiagnostic({ @@ -163,7 +156,6 @@ export async function setAppleFoldableOrientation( attempt, observedOrientation: reading.rotation, litDisplay: reading.panel.name, - panelOffset: mapping.offset, }, }); } @@ -218,7 +210,11 @@ async function readLitPanel( ): Promise { let inventory: AppleDisplayInventory; try { - inventory = await queryAppleDisplayInventory(device, { signal, foldHelper: helper }); + inventory = await queryAppleDisplayInventory(device, { + signal, + source: 'guest', + foldHelper: helper, + }); } catch (error) { if (signal?.aborted) throw error; return undefined; diff --git a/packages/platform-apple/src/foldable/pose.ts b/packages/platform-apple/src/foldable/pose.ts index f80e781532..f3f4be6ad2 100644 --- a/packages/platform-apple/src/foldable/pose.ts +++ b/packages/platform-apple/src/foldable/pose.ts @@ -49,6 +49,7 @@ export async function setAppleFoldPose( const helper = createFoldHelper(device); const inventory = await queryAppleDisplayInventory(device, { signal: options.signal, + source: 'guest', foldHelper: helper, }); requireFoldableInventory(device, inventory); @@ -275,7 +276,11 @@ async function readLitPanel( foldHelper: FoldHelper, signal: AbortSignal | undefined, ): Promise { - const inventory = await queryAppleDisplayInventory(device, { signal, foldHelper }); + const inventory = await queryAppleDisplayInventory(device, { + signal, + source: 'guest', + foldHelper, + }); if (inventory.unresolved || inventory.ambiguous) return undefined; return inventory.activeDisplay; } diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index e6d3236dee..b1c76fc015 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -16,7 +16,6 @@ vi.mock('../core/devicectl.ts', async (importOriginal) => ({ })), })); -import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/tool-provider.ts'; import { runIosDevicectlJsonRequest } from '../core/devicectl.ts'; import { runAppleRunnerCommand } from '../core/runner-client.ts'; import { iosRunnerOverrides } from '../interactions.ts'; @@ -167,20 +166,7 @@ test('a resolved read lifts the back-off a failed one opened', async () => { assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); answering(IPHONE_DUO_CLOSED); // A rotation reads the table on its own route; its answer clears the back-off for surface reads. - const guest = createLocalAppleToolProvider({ - runCommand: async (_command, args) => { - assert.equal(args.at(-1), 'panels'); - return { - exitCode: 0, - stdout: JSON.stringify({ version: 1, displays: IPHONE_DUO_CLOSED.result.displays }), - stderr: '', - }; - }, - }); - assert.equal( - await withAppleToolProvider(guest, () => isAppleFoldableSimulator(device, undefined)), - true, - ); + assert.equal(await isAppleFoldableSimulator(device, undefined), true); assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); }); From 07ce5100171312635a96cd313a179ce20525485d Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 18:11:30 +0000 Subject: [PATCH 13/16] fix(ios): guard Duo picker mapping and report panel read failures --- docs/adr/0025-foldable-apple-panels.md | 5 ++-- docs/agents/device-foldable.md | 2 +- .../src/foldable/orientation.test.ts | 30 +++++++++++++++++-- .../src/foldable/orientation.ts | 26 ++++++++++------ .../platform-apple/src/foldable/pose.test.ts | 4 +++ packages/platform-apple/src/foldable/pose.ts | 3 +- .../provider-scenarios/ios-fold.test.ts | 24 +++++++++++++-- website/docs/docs/commands.md | 2 +- 8 files changed, 77 insertions(+), 19 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 8290dab976..0511038f44 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -372,20 +372,21 @@ Rejected alternatives: `XCUIDevice.orientation` has no effect on an iPhone Duo simulator, and the runner readback repeats the requested value. `devicectl device orientation set` reports success and has no effect on a simulator that `simctl boot` started. -`orientation` therefore rotates a multi-panel simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). +`orientation` therefore rotates a multi-panel iPhone Duo simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). The picker accepts `pud` for upside-down dispatch, which the inner panel needs to show landscape-left. The user target `portrait-upside-down` remains refused with `unsupported-foldable-orientation`. Routing uses the `fold` admission and the set-based panel-count source. A resolved panel count is a hardware fact, so the daemon probes each simulator once. A supplied helper is a reuse handle; only an explicit guest-source request changes the default-set source. Thus, classifying a single-panel default-set simulator does not prepare the fold helper. -Verification was updated on 2026-11-12 under [#4440](https://github.com/callstack/agent-device/pull/4440). +The picker mapping requires the discovered model to be iPhone Duo; other multi-panel models are refused. In both simulator sets, the helper reads the guest panel table and dispatches the fixed picker value for the established lit panel. The table maps the outer panel at zero offset and the inner panel at +90 degrees. Only a fresh `currentOrientation` from an established lit panel confirms the requested visible orientation. An incomplete table, unknown-power sibling, or multiple lit panels cannot confirm or dispatch. A repeated visible target dispatches nothing; delayed reads wait without resending to the same panel. A target that never appears is refused with `orientation-unverified` in either set. +An unreadable guest table reports `orientation-panel-table-unavailable` and points to the inventory diagnostic. Rejected alternatives: diff --git a/docs/agents/device-foldable.md b/docs/agents/device-foldable.md index af80b8f83d..7e06637cd7 100644 --- a/docs/agents/device-foldable.md +++ b/docs/agents/device-foldable.md @@ -1,6 +1,6 @@ # Foldable Apple verification -Read ADR 0025 before capture changes. +Before capture changes, read ADR 0025. For default-set capture, run `xcrun devicectl device info displays --device `. For scoped capture or rotation, run `xcrun simctl --set spawn ~/.agent-device/fold-helper//fold-helper panels`. For default-set rotation, omit `--set `. diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 85a1bcec9a..53a30c7f54 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -33,7 +33,12 @@ beforeEach(() => { let simulatorCount = 0; function simulator(overrides: Partial = {}): DeviceInfo { simulatorCount += 1; - return { ...IOS_SIMULATOR, id: `${IOS_SIMULATOR.id}-${simulatorCount}`, ...overrides }; + return { + ...IOS_SIMULATOR, + model: 'iPhone Duo', + id: `${IOS_SIMULATOR.id}-${simulatorCount}`, + ...overrides, + }; } const SINGLE_PANEL_DISPLAYS = IPHONE_17_SINGLE_PANEL.result.displays; @@ -125,7 +130,10 @@ test('a cold default Duo helper failure cannot fall back to runner success', asy ); await expect(rotateDuo(sim, 'landscape-left')).resolves.toMatchObject({ code: 'COMMAND_FAILED', - details: { reason: 'orientation-unverified' }, + details: { + reason: 'orientation-panel-table-unavailable', + hint: expect.stringContaining('apple_display_inventory_unresolved'), + }, }); expect(sim.classificationReads).toHaveLength(1); expect(mockRunner).not.toHaveBeenCalled(); @@ -136,6 +144,7 @@ test('a physical device keeps the runner rotation without a CoreDevice probe', a const sim = fakeSimulator(IOS_DEVICE, []); await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); expect(sim.unexpectedTools).toEqual([]); + expect(sim.classificationReads).toEqual([]); expect(mockRunner).toHaveBeenCalledTimes(1); }); @@ -652,3 +661,20 @@ test('a rotation is not routed by a pending panel-table retry', async () => { expect(sim.rotations()).toEqual(['landscape-left']); expect(mockRunner).not.toHaveBeenCalled(); }); + +test.each(['iPad Pro', 'Another Foldable', undefined])( + 'an unverified multi-panel model %s cannot use the Duo picker mapping', + async (model) => { + const sim = fakeDuo(simulator({ model, name: 'iPhone Duo' }), { + lit: 'outer', + picker: 'portrait', + }); + await expect(sim.rotate('landscape-left')).rejects.toMatchObject({ + code: 'UNSUPPORTED_OPERATION', + details: { reason: 'unsupported-foldable-model' }, + }); + expect(ensureFoldHelperBinary).not.toHaveBeenCalled(); + expect(sim.rotations()).toEqual([]); + expect(mockRunner).not.toHaveBeenCalled(); + }, +); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index d824948a9b..1f39886028 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -125,6 +125,13 @@ export async function setAppleFoldableOrientation( options: { signal?: AbortSignal } = {}, ): Promise<{ orientation: DeviceRotation }> { const { signal } = options; + if (device.model !== 'iPhone Duo') { + throw new AppError('UNSUPPORTED_OPERATION', 'Guest rotation requires an iPhone Duo model', { + deviceId: device.id, + reason: 'unsupported-foldable-model', + hint: 'The guest orientation picker mapping is verified only for iPhone Duo. Use a simulator whose discovered model is iPhone Duo.', + }); + } if (rotation === 'portrait-upside-down') { throw new AppError('UNSUPPORTED_OPERATION', `${device.name} cannot rotate to ${rotation}`, { deviceId: device.id, @@ -208,16 +215,17 @@ async function readLitPanel( helper: FoldHelper, signal: AbortSignal | undefined, ): Promise { - let inventory: AppleDisplayInventory; - try { - inventory = await queryAppleDisplayInventory(device, { - signal, - source: 'guest', - foldHelper: helper, + const inventory = await queryAppleDisplayInventory(device, { + signal, + source: 'guest', + foldHelper: helper, + }); + if (inventory.unresolved) { + throw new AppError('COMMAND_FAILED', 'The guest panel table could not be read', { + deviceId: device.id, + reason: 'orientation-panel-table-unavailable', + hint: 'The fold helper could not provide a usable display table. Check the apple_display_inventory_unresolved diagnostic for the helper or display-table failure, then retry.', }); - } catch (error) { - if (signal?.aborted) throw error; - return undefined; } const panel = establishedLitPanel(inventory); if (!panel) return undefined; diff --git a/packages/platform-apple/src/foldable/pose.test.ts b/packages/platform-apple/src/foldable/pose.test.ts index 60d3ef89f4..0b46b1b7c7 100644 --- a/packages/platform-apple/src/foldable/pose.test.ts +++ b/packages/platform-apple/src/foldable/pose.test.ts @@ -107,6 +107,10 @@ test('sends the simulator HID pose and reports the pose the guest hinge read bac screen: { display: 'LCD-1', coordinateSpace: 'native-panel', widthPt: 669, heightPt: 951 }, }); + expect(mockInventory.mock.calls.map(([, options]) => options.source)).toEqual([ + undefined, + 'guest', + ]); expect(mockSend).toHaveBeenCalledWith('open', undefined); expect(mockHinge).toHaveBeenCalledTimes(2); // Reads are spaced so two samples of a moving hinge cannot land together; the first is not delayed. diff --git a/packages/platform-apple/src/foldable/pose.ts b/packages/platform-apple/src/foldable/pose.ts index f3f4be6ad2..45aa4426bd 100644 --- a/packages/platform-apple/src/foldable/pose.ts +++ b/packages/platform-apple/src/foldable/pose.ts @@ -26,7 +26,7 @@ const FOLDABLE_REQUIRED_HINT = 'fold sets the hinge angle of a foldable simulator such as iPhone Duo; this simulator reports one integrated panel, so it has no hinge to pose.'; const INVENTORY_REQUIRED_HINT = - "fold needs the simulator's integrated displays to identify a foldable: the fold helper reads their panel table in either simulator set. The apple_display_inventory_unresolved diagnostic names why the read failed."; + "fold needs the simulator's integrated displays to identify a foldable: the default set uses CoreDevice and a scoped set uses the fold helper. The apple_display_inventory_unresolved diagnostic names why the read failed."; const DISPATCHED_UNVERIFIED_HINT = 'The pose request was sent and the device may have followed it, but the first hinge-angle read returned no sample, so fold cannot confirm it. Take a screenshot or snapshot to see which panel is lit; hingeReadFailure names why the read failed.'; @@ -49,7 +49,6 @@ export async function setAppleFoldPose( const helper = createFoldHelper(device); const inventory = await queryAppleDisplayInventory(device, { signal: options.signal, - source: 'guest', foldHelper: helper, }); requireFoldableInventory(device, inventory); diff --git a/test/integration/provider-scenarios/ios-fold.test.ts b/test/integration/provider-scenarios/ios-fold.test.ts index 87a6d1ed3c..61634d53e9 100644 --- a/test/integration/provider-scenarios/ios-fold.test.ts +++ b/test/integration/provider-scenarios/ios-fold.test.ts @@ -84,6 +84,11 @@ test('timed fold keyframes reach simulator HID through the public client and dae const ok = { stdout: '', stderr: '', exitCode: 0 }; let angle = 0; const tool = createRecordingAppleToolProvider({ + devicectl: async (args) => { + assert.deepEqual(args.slice(0, 3), ['device', 'info', 'displays']); + fs.writeFileSync(args.at(-1)!, JSON.stringify({ result: JSON.parse(guestPanelReading()) })); + return ok; + }, simctl: async (args, options) => { if (args[0] === 'list') { return { @@ -142,6 +147,7 @@ test('timed fold keyframes reach simulator HID through the public client and dae }); assert.equal(result.pose, 'half-open'); assert.equal(result.hingeAngleDegrees, 100); + assert.equal(result.screen?.display, 'LCD-1'); const replayed = await daemon.client().command.fold({ platform: 'ios', udid: PROVIDER_SCENARIO_IOS_SIMULATOR.id, @@ -209,6 +215,12 @@ function createFoldLedgerAppleToolProvider(params: { // Built on the shared recording provider so any call this route does not script (macosHelper, // macosHost, plist, or an unexpected runCommand) throws instead of being silently answered. const recording = createRecordingAppleToolProvider({ + devicectl: async (args, options) => { + await record('devicectl', args, options); + assert.deepEqual(args.slice(0, 3), ['device', 'info', 'displays']); + fs.writeFileSync(args.at(-1)!, JSON.stringify({ result: JSON.parse(guestPanelReading()) })); + return ok; + }, simctl: async (args, options) => { if (args[0] === 'list') { if (params.isTracking()) await record('simctl', args, options); @@ -344,13 +356,21 @@ function assertFoldLedgerPhaseCoverage(ledger: readonly FoldLedgerCall[]): void const isDisplayInventory = (call: FoldLedgerCall) => call.tool === 'simctl' && call.args[0] === 'spawn' && call.args[3] === 'panels'; + assert.equal( + ledger.filter( + (call) => + call.tool === 'devicectl' && call.args.slice(0, 3).join(' ') === 'device info displays', + ).length, + 1, + 'ledger must record the set-source foldable classification', + ); assert.ok(ledger.some(isBuild), 'ledger is missing the helper-build phase'); assert.ok(ledger.some(isSpawn), 'ledger is missing the HID-dispatch (simctl spawn) phase'); assert.ok(ledger.some(isHingeRead), 'ledger is missing the hinge-settle-read phase'); assert.equal( ledger.filter(isDisplayInventory).length, - 2, - 'ledger must record both the foldable-check and the lit-panel guest inventory reads', + 1, + 'ledger must record the lit-panel guest inventory read', ); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 65dabe8bbf..d5c9e365f7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -132,7 +132,7 @@ Back, orientation, and hardware buttons: - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. - An app with a left-edge drawer may open the drawer instead of going back. When a back or close control is visible, press it by its ref. - `back --system` sends system back input: the Android back keyevent, the iOS back swipe, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. -- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. A foldable iPhone simulator (iPhone Duo) refuses `portrait-upside-down`. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. +- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. In both default and scoped simulator sets, an iPhone Duo classified with more than one integrated panel refuses `portrait-upside-down` and rotates through the fold helper. It succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. An unreadable guest table reports `orientation-panel-table-unavailable`. Other multi-panel simulator models are refused. A simulator classified with one panel, or whose first classification read is unresolved, uses the standard runner rotation. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`. - `action-button` presses the iPhone or iPad Action Button once. It takes no arguments, no hold duration, and no `--settle`, and it doesn't activate the session app or drive Camera Control. Other platforms refuse it, and a model without the button (an iPhone SE, most iPad simulators) fails with `UNSUPPORTED_OPERATION`. - `action-button` reports that the press was dispatched, not what it triggered. Simulators run no Shortcuts or App Intents, so verify the outcome on a physical iPhone. Expect a press to take about five seconds on a Simulator. From 28f1ac4883affe4f57f7ff671567cf142fe6e30b Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 18:19:13 +0000 Subject: [PATCH 14/16] docs(ios): name the unreadable panel table reason in the orientation tool description Co-Authored-By: Claude Opus 5.5 --- packages/command-surface/src/system/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 5a5ccef8d2..bd57af3fb8 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -290,7 +290,7 @@ const backCommandDescription = const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = - 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified.'; + 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified, or orientation-panel-table-unavailable when the guest panel table cannot be read.'; const foldGuidance = { hingeEvent: 'simulator HID hinge event', permissions: 'Device Hub and host Accessibility permission are not required.', From c47ed58d62c08d35b319c60acfbc940771c815aa Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 18:58:31 +0000 Subject: [PATCH 15/16] fix(ios): retry transient foldable panel reads without a model guard --- .changeset/foldable-scoped-rotation.md | 2 + docs/adr/0025-foldable-apple-panels.md | 7 +- packages/command-surface/src/system/index.ts | 2 +- .../src/foldable/orientation.test.ts | 72 ++++++++++++++----- .../src/foldable/orientation.ts | 17 ++--- website/docs/docs/commands.md | 2 +- 6 files changed, 67 insertions(+), 35 deletions(-) diff --git a/.changeset/foldable-scoped-rotation.md b/.changeset/foldable-scoped-rotation.md index d7e106b6a7..f327dd3492 100644 --- a/.changeset/foldable-scoped-rotation.md +++ b/.changeset/foldable-scoped-rotation.md @@ -5,3 +5,5 @@ Fix foldable iPhone simulator rotation in scoped sets and confirm the lit panel in both default and scoped sets. Rotation in both sets uses the fold helper, with no CoreDevice orientation polling. An already visible target dispatches nothing. Incomplete, unknown-power and multi-lit panel tables neither confirm nor dispatch. A target the panel never shows fails with `orientation-unverified`. Use the fixed iPhone Duo picker offsets measured in both sets (outer 0 degrees, inner +90 degrees), replacing runtime calibration. Inner landscape-left sends `portrait-upside-down`, mapped to the HID wire value `pud`; it remains unsupported as a user target. Delayed reads wait without resending the same picker value to the same panel. + +Unresolved guest reads inside the settle loop neither confirm nor dispatch and are retried. An unresolved final read fails with `orientation-panel-table-unavailable`. Multi-panel classification admits rotation without a model-name guard; a new multi-panel model needs its own picker mapping. diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 0511038f44..e240779db2 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -372,21 +372,22 @@ Rejected alternatives: `XCUIDevice.orientation` has no effect on an iPhone Duo simulator, and the runner readback repeats the requested value. `devicectl device orientation set` reports success and has no effect on a simulator that `simctl boot` started. -`orientation` therefore rotates a multi-panel iPhone Duo simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). +`orientation` therefore rotates a multi-panel simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). The picker accepts `pud` for upside-down dispatch, which the inner panel needs to show landscape-left. The user target `portrait-upside-down` remains refused with `unsupported-foldable-orientation`. Routing uses the `fold` admission and the set-based panel-count source. A resolved panel count is a hardware fact, so the daemon probes each simulator once. A supplied helper is a reuse handle; only an explicit guest-source request changes the default-set source. Thus, classifying a single-panel default-set simulator does not prepare the fold helper. -The picker mapping requires the discovered model to be iPhone Duo; other multi-panel models are refused. +The picker offsets were measured on iPhone Duo; a new multi-panel model needs its own mapping. In both simulator sets, the helper reads the guest panel table and dispatches the fixed picker value for the established lit panel. The table maps the outer panel at zero offset and the inner panel at +90 degrees. Only a fresh `currentOrientation` from an established lit panel confirms the requested visible orientation. An incomplete table, unknown-power sibling, or multiple lit panels cannot confirm or dispatch. A repeated visible target dispatches nothing; delayed reads wait without resending to the same panel. A target that never appears is refused with `orientation-unverified` in either set. -An unreadable guest table reports `orientation-panel-table-unavailable` and points to the inventory diagnostic. +An unresolved guest read inside the settle loop neither confirms nor dispatches; the loop retries it. +An unresolved final read reports `orientation-panel-table-unavailable` and points to the inventory diagnostic. Rejected alternatives: diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index bd57af3fb8..0a3818afc7 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -290,7 +290,7 @@ const backCommandDescription = const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = - 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified, or orientation-panel-table-unavailable when the guest panel table cannot be read.'; + 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified, or orientation-panel-table-unavailable when the final guest panel table cannot be read.'; const foldGuidance = { hingeEvent: 'simulator HID hinge event', permissions: 'Device Hub and host Accessibility permission are not required.', diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 53a30c7f54..6339e716a2 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -17,6 +17,7 @@ import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/too import { createAppleInteractor } from '../interactor.ts'; import { isAppleFoldableSimulator, readFoldablePanelTable } from './orientation.ts'; import { ensureFoldHelperBinary } from './fold-helper-cache.ts'; +import { IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS } from '../core/config.ts'; import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; import { IPHONE_17_SINGLE_PANEL, IPHONE_DUO_CLOSED } from '../__tests__/display-fixtures.ts'; @@ -35,7 +36,6 @@ function simulator(overrides: Partial = {}): DeviceInfo { simulatorCount += 1; return { ...IOS_SIMULATOR, - model: 'iPhone Duo', id: `${IOS_SIMULATOR.id}-${simulatorCount}`, ...overrides, }; @@ -184,6 +184,7 @@ function fakeDuo( picker: DeviceRotation; offsets?: { inner: number; outer: number }; followsHid?: boolean; + unresolvedGuestReads?: readonly number[] | 'all'; /** Rotations a panel's foreground UI cannot show; the panel keeps its last rotation. */ cannotShow?: Partial>; /** The rotation a dark panel last showed, for the fold that lights it. */ @@ -208,6 +209,7 @@ function fakeDuo( let lagReadsLeft = 0; let bothLitReads = 0; let panelReads = 0; + let guestReads = 0; const shown: Record = { inner: PANEL_ANGLE[options.lastShown?.inner ?? 'portrait'], outer: PANEL_ANGLE[options.lastShown?.outer ?? 'portrait'], @@ -264,6 +266,13 @@ function fakeDuo( const mode = spawn.slice(3); helpers.push(mode); if (mode[0] === 'panels') { + guestReads += 1; + if ( + options.unresolvedGuestReads === 'all' || + options.unresolvedGuestReads?.includes(guestReads) + ) { + return { stdout: '', stderr: 'transient guest panel read failure', exitCode: 1 }; + } const displays = readPanels(); return { stdout: JSON.stringify({ version: 1, displays }), stderr: '', exitCode: 0 }; } @@ -354,6 +363,50 @@ describe.each([undefined, '/tmp/scoped-set'])('foldable rotation, set %s', (simu }, ); + test.each([1, 2])('retries unresolved guest read %s before confirming', async (failedRead) => { + const device = simulator({ simulatorSetPath }); + const classification = fakeDuo(device, { lit: 'outer', picker: 'portrait' }); + await expect(rotateDuo(classification, 'portrait')).resolves.toEqual({ + orientation: 'portrait', + }); + const sim = fakeDuo(device, { + lit: 'outer', + picker: 'portrait', + unresolvedGuestReads: [failedRead], + }); + await expect(rotateDuo(sim, 'landscape-left')).resolves.toEqual({ + orientation: 'landscape-left', + }); + expect(sim.helpers.map((mode) => mode[0])).toEqual( + failedRead === 1 + ? ['panels', 'panels', 'rotate', 'panels'] + : ['panels', 'rotate', 'panels', 'panels'], + ); + expect(sim.rotations()).toEqual(['landscape-left']); + expect(mockRunner).not.toHaveBeenCalled(); + }); + + test('always unresolved guest reads exhaust the loop without dispatch', async () => { + const device = simulator({ simulatorSetPath }); + const sim = fakeDuo(device, { + lit: 'outer', + picker: 'portrait', + unresolvedGuestReads: 'all', + }); + // Classify before injecting guest failures, including in a scoped set. + const classification = fakeDuo(device, { lit: 'outer', picker: 'portrait' }); + await expect(rotateDuo(classification, 'portrait')).resolves.toEqual({ + orientation: 'portrait', + }); + await expect(rotateDuo(sim, 'landscape-left')).resolves.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'orientation-panel-table-unavailable' }, + }); + expect(sim.helpers).toHaveLength(IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS + 1); + expect(sim.rotations()).toEqual([]); + expect(mockRunner).not.toHaveBeenCalled(); + }); + test('a foldable rotates on the first command, confirmed by the panel table', async () => { const scoped = simulator({ simulatorSetPath }); // The inner panel lights at rot90 while the picker sits at portrait (K = +90, measured). @@ -661,20 +714,3 @@ test('a rotation is not routed by a pending panel-table retry', async () => { expect(sim.rotations()).toEqual(['landscape-left']); expect(mockRunner).not.toHaveBeenCalled(); }); - -test.each(['iPad Pro', 'Another Foldable', undefined])( - 'an unverified multi-panel model %s cannot use the Duo picker mapping', - async (model) => { - const sim = fakeDuo(simulator({ model, name: 'iPhone Duo' }), { - lit: 'outer', - picker: 'portrait', - }); - await expect(sim.rotate('landscape-left')).rejects.toMatchObject({ - code: 'UNSUPPORTED_OPERATION', - details: { reason: 'unsupported-foldable-model' }, - }); - expect(ensureFoldHelperBinary).not.toHaveBeenCalled(); - expect(sim.rotations()).toEqual([]); - expect(mockRunner).not.toHaveBeenCalled(); - }, -); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 1f39886028..a6ec1d45b5 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -125,13 +125,6 @@ export async function setAppleFoldableOrientation( options: { signal?: AbortSignal } = {}, ): Promise<{ orientation: DeviceRotation }> { const { signal } = options; - if (device.model !== 'iPhone Duo') { - throw new AppError('UNSUPPORTED_OPERATION', 'Guest rotation requires an iPhone Duo model', { - deviceId: device.id, - reason: 'unsupported-foldable-model', - hint: 'The guest orientation picker mapping is verified only for iPhone Duo. Use a simulator whose discovered model is iPhone Duo.', - }); - } if (rotation === 'portrait-upside-down') { throw new AppError('UNSUPPORTED_OPERATION', `${device.name} cannot rotate to ${rotation}`, { deviceId: device.id, @@ -145,7 +138,7 @@ export async function setAppleFoldableOrientation( let reading: LitPanel | undefined; for (let attempt = 1; attempt <= IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS; attempt += 1) { signal?.throwIfAborted(); - reading = await readLitPanel(device, helper, signal); + reading = await readLitPanel(device, helper, { signal }); if (reading?.rotation === rotation) return { orientation: rotation }; if (reading && reading.panel.displayId !== dispatchedDisplayId) { const mapping = reading.panel.primary ? PANEL_PICKERS.outer : PANEL_PICKERS.inner; @@ -168,7 +161,7 @@ export async function setAppleFoldableOrientation( } await settleRotation(); } - reading = await readLitPanel(device, helper, signal); + reading = await readLitPanel(device, helper, { signal, requireTable: true }); if (reading?.rotation === rotation) return { orientation: rotation }; throw unverifiedError(device, rotation, reading); } @@ -213,14 +206,14 @@ function unverifiedError( async function readLitPanel( device: DeviceInfo, helper: FoldHelper, - signal: AbortSignal | undefined, + options: { signal?: AbortSignal; requireTable?: boolean }, ): Promise { const inventory = await queryAppleDisplayInventory(device, { - signal, + signal: options.signal, source: 'guest', foldHelper: helper, }); - if (inventory.unresolved) { + if (inventory.unresolved && options.requireTable) { throw new AppError('COMMAND_FAILED', 'The guest panel table could not be read', { deviceId: device.id, reason: 'orientation-panel-table-unavailable', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d5c9e365f7..cbfcf77765 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -132,7 +132,7 @@ Back, orientation, and hardware buttons: - A simulator sends the swipe as a screen-edge touch. A physical device has no such touch and sends a quick fling from the leading edge instead; if that fling does not pop the screen, `back` refuses with `back_swipe_no_change`. visionOS has no back gesture and refuses before dispatch with `UNSUPPORTED_OPERATION`. - An app with a left-edge drawer may open the drawer instead of going back. When a back or close control is visible, press it by its ref. - `back --system` sends system back input: the Android back keyevent, the iOS back swipe, or the tvOS Siri Remote Menu button. macOS has no system back, so it refuses before dispatch. -- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. In both default and scoped simulator sets, an iPhone Duo classified with more than one integrated panel refuses `portrait-upside-down` and rotates through the fold helper. It succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. An unreadable guest table reports `orientation-panel-table-unavailable`. Other multi-panel simulator models are refused. A simulator classified with one panel, or whose first classification read is unresolved, uses the standard runner rotation. +- `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. In both default and scoped simulator sets, an iPhone Duo classified with more than one integrated panel refuses `portrait-upside-down` and rotates through the fold helper. It succeeds only when the lit panel shows the requested orientation; if it never does, or the simulator never reports a single lit panel, the command fails with `orientation-unverified`. Unresolved guest reads are retried inside the settle loop; an unresolved final read reports `orientation-panel-table-unavailable`. The picker offsets were measured on iPhone Duo; a new multi-panel model needs its own mapping. A simulator classified with one panel, or whose first classification read is unresolved, uses the standard runner rotation. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`. - `action-button` presses the iPhone or iPad Action Button once. It takes no arguments, no hold duration, and no `--settle`, and it doesn't activate the session app or drive Camera Control. Other platforms refuse it, and a model without the button (an iPhone SE, most iPad simulators) fails with `UNSUPPORTED_OPERATION`. - `action-button` reports that the press was dispatched, not what it triggered. Simulators run no Shortcuts or App Intents, so verify the outcome on a physical iPhone. Expect a press to take about five seconds on a Simulator. From 92e55582098ce6d418ffd411219356e079bfc43f Mon Sep 17 00:00:00 2001 From: Sittingmongoose Date: Thu, 12 Nov 2026 19:06:42 +0000 Subject: [PATCH 16/16] docs(ios): limit the orientation tool's fold-helper route to multi-panel Duos Co-Authored-By: Claude Opus 5.5 --- packages/command-surface/src/system/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 0a3818afc7..947cf830b6 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -290,7 +290,7 @@ const backCommandDescription = const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = - 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. A foldable iPhone simulator (iPhone Duo) refuses portrait-upside-down. In both default and scoped simulator sets it rotates through the fold helper and succeeds only when the lit panel shows the requested orientation; otherwise it fails with orientation-unverified, or orientation-panel-table-unavailable when the final guest panel table cannot be read.'; + 'Set device orientation on iOS and Android. auto hands rotation back to the sensor on ADB-backed Android (local and Limrun); other backends refuse it. In both default and scoped simulator sets, an iPhone Duo classified with more than one integrated panel refuses portrait-upside-down and rotates through the fold helper; it succeeds only when the lit panel shows the requested orientation, otherwise it fails with orientation-unverified, or orientation-panel-table-unavailable when the final guest panel table cannot be read. A simulator classified with one panel, or whose first classification read is unresolved, uses the standard runner rotation.'; const foldGuidance = { hingeEvent: 'simulator HID hinge event', permissions: 'Device Hub and host Accessibility permission are not required.',