From 87374a06af8cd973a72eb5e1eb2228a920c90ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 17:54:13 +0100 Subject: [PATCH 1/2] perf(android): read screenshot rotation from the helper response The helper's screenshot response now carries the default display's rotation and the display count, so a helper capture no longer spawns adb shell dumpsys display beside it. The screencap fallback and a helper without the headers keep the dumpsys probe; a device with more than one display still reports no rotation. Co-Authored-By: Claude Opus 5.5 --- android/snapshot-helper/README.md | 4 +- .../AutomationCommandHandler.java | 35 +++++++++- .../SnapshotInstrumentation.java | 2 +- .../src/automation-helper-fallback.test.ts | 36 ++++++++++ .../platform-android/src/automation-helper.ts | 32 ++++++++- .../platform-android/src/input-actions.ts | 2 +- packages/platform-android/src/screenshot.ts | 67 ++++++++++++++----- 7 files changed, 153 insertions(+), 25 deletions(-) diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index c7bc9ca187..4e38bb75e8 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -113,7 +113,9 @@ closes that connection; the process stays alive for the next connection: - `snapshot-short-idle ` — capture with at most a 100 ms idle wait, capped by the session's configured timeout - `input ` — inject a tap, key, or point-targeted text replacement - `focus ` — read the focused window type and root package after clearing the accessibility cache -- `screenshot ` — return a PNG in the `payloadBase64` header +- `screenshot ` — return a PNG of the default display in the `payloadBase64` header, + with `displayCount` and the default display's `Surface.ROTATION_*` index in `displayRotation`. + `displayRotation` is omitted when the rotation changed during the capture. - `viewport ` — return interactive-window viewport bounds - `gesture ` — inject a planned touch gesture (same payload as the one-shot `gesture` mode) diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java index 76be54dce0..03d83cdc15 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java @@ -1,11 +1,14 @@ package com.callstack.agentdevice.snapshothelper; import android.app.UiAutomation; +import android.content.Context; import android.graphics.Bitmap; import android.graphics.Rect; +import android.hardware.display.DisplayManager; import android.os.Bundle; import android.os.SystemClock; import android.util.Base64; +import android.view.Display; import android.view.InputDevice; import android.view.KeyCharacterMap; import android.view.KeyEvent; @@ -23,7 +26,12 @@ final class AutomationCommandHandler { private AutomationCommandHandler() {} static void writeSessionResponse( - OutputStream output, String requestId, String action, String payload, UiAutomation automation) + OutputStream output, + String requestId, + String action, + String payload, + UiAutomation automation, + Context context) throws IOException { Bundle result = SessionResponseWriter.sessionResponseBundle(requestId); InputDispatch dispatch = new InputDispatch(); @@ -31,7 +39,7 @@ static void writeSessionResponse( if ("app".equals(action)) { ApplicationCommandHandler.execute(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8)), result); } else if ("screenshot".equals(action)) { - screenshot(result, automation); + screenshot(result, automation, context); } else if ("focus".equals(action)) { focus(result, automation); } else { @@ -176,7 +184,15 @@ private static AccessibilityNodeInfo editableAtPoint(AccessibilityNodeInfo node, } } - private static void screenshot(Bundle result, UiAutomation automation) { + /** + * {@code UiAutomation.takeScreenshot()} captures the default display in its current rotation, so + * that display's rotation pairs with the image. It is published only when one read before and one + * after the capture agree, so a rotation landing mid-capture reports none. The display count lets + * the host keep its rule that a device with more than one display reports no rotation. + */ + private static void screenshot(Bundle result, UiAutomation automation, Context context) { + DisplayManager displays = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + int rotationBefore = defaultDisplayRotation(displays); Bitmap image = automation.takeScreenshot(); if (image == null) throw new IllegalStateException("Screen capture was unavailable"); try { @@ -188,6 +204,19 @@ private static void screenshot(Bundle result, UiAutomation automation) { } finally { image.recycle(); } + if (displays == null) return; + int rotationAfter = defaultDisplayRotation(displays); + result.putString("displayCount", Integer.toString(displays.getDisplays().length)); + if (rotationBefore >= 0 && rotationBefore == rotationAfter) { + result.putString("displayRotation", Integer.toString(rotationAfter)); + } + } + + /** The default display's {@code Surface.ROTATION_*} index, or -1 when it cannot be read. */ + private static int defaultDisplayRotation(DisplayManager displays) { + if (displays == null) return -1; + Display display = displays.getDisplay(Display.DEFAULT_DISPLAY); + return display == null ? -1 : display.getRotation(); } private static void focus(Bundle result, UiAutomation automation) throws Exception { diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index c28361a666..c7fb436485 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -210,7 +210,7 @@ private void runSnapshotSession( if ("input".equals(action) || "screenshot".equals(action) || "focus".equals(action) || "app".equals(action)) { AutomationCommandHandler.writeSessionResponse( socket.getOutputStream(), requestId, action, - parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked()); + parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked(), getContext()); continue; } if ("viewport".equals(action)) { diff --git a/packages/platform-android/src/automation-helper-fallback.test.ts b/packages/platform-android/src/automation-helper-fallback.test.ts index 73ea363f5a..f040578cb6 100644 --- a/packages/platform-android/src/automation-helper-fallback.test.ts +++ b/packages/platform-android/src/automation-helper-fallback.test.ts @@ -119,6 +119,42 @@ test.each>([ }, ); +test.each<[string, Record, object, number]>([ + [ + 'one display', + { displayCount: '1', displayRotation: '1' }, + { displayRotation: 'landscape-left' }, + 0, + ], + ['two displays', { displayCount: '2', displayRotation: '1' }, {}, 0], + ['a rotation mid-capture', { displayCount: '1' }, {}, 0], + ['an older helper', {}, { displayRotation: 'landscape-right' }, 1], +])( + 'a helper screenshot from %s reports its rotation without screencap', + async (_name, displays, facts, dumpsysReads) => { + const f = fixture({ ok: 'true', payloadBase64: png.toString('base64'), ...displays }); + const exec = f.provider.exec; + f.provider.exec = async (args, options) => { + if (args[0] === 'shell' && args[1] === 'dumpsys' && args[2] === 'display') { + f.calls.push(args); + return { exitCode: 0, stdout: ' mCurrentOrientation=3\n', stderr: '' }; + } + return await exec(args, options); + }; + const dir = await mkdtempForTest('android-helper-screenshot-rotation-'); + const output = path.join(dir, 'helper.png'); + await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { + await readAndroidFillTargetBeforeMutation(device, 10, 10, helper); + expect(await screenshotAndroid(device, output)).toEqual(facts); + }); + expect(await fs.readFile(output)).toEqual(png); + expect(f.calls.filter((args) => args[1] === 'dumpsys' && args[2] === 'display')).toHaveLength( + dumpsysReads, + ); + expect(f.calls.some((args) => args[0] === 'exec-out')).toBe(false); + }, +); + test('a cached ended helper is retired before a pointer command takes the host path', async () => { const f = fixture({ ok: 'true' }); await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { diff --git a/packages/platform-android/src/automation-helper.ts b/packages/platform-android/src/automation-helper.ts index 4040d6e6a7..453ddd2b7b 100644 --- a/packages/platform-android/src/automation-helper.ts +++ b/packages/platform-android/src/automation-helper.ts @@ -24,6 +24,14 @@ type AndroidAutomationRequest = | { action: 'focus' }; type AndroidHelperFocus = { focusedPackage?: string; windowType?: string }; +/** + * `displays` is absent when the helper predates the display headers. `rotationIndex` is the default + * display's `Surface.ROTATION_*` index, absent when the display turned during the capture. + */ +export type AndroidHelperScreenshot = { + png: Buffer; + displays?: { count: number; rotationIndex?: number }; +}; type AndroidHelperAppResult = | { kind: 'component-unavailable' } | { kind: 'launched'; output: string }; @@ -43,7 +51,7 @@ export function runAndroidAutomationHelper( export function runAndroidAutomationHelper( device: DeviceInfo, request: Extract, -): Promise; +): Promise; export function runAndroidAutomationHelper( device: DeviceInfo, request: Extract, @@ -55,7 +63,9 @@ export function runAndroidAutomationHelper( export async function runAndroidAutomationHelper( device: DeviceInfo, request: AndroidAutomationRequest, -): Promise { +): Promise< + true | AndroidHelperAppResult | AndroidHelperScreenshot | AndroidHelperFocus | undefined +> { const { action } = request; const deviceKey = getAndroidSnapshotHelperSessionDeviceKey(device); if (!(await getLiveAndroidSnapshotHelperSession(deviceKey))?.commands.has(action)) @@ -77,7 +87,10 @@ export async function runAndroidAutomationHelper( case 'input': return true; case 'screenshot': - return decodeHelperPayload(headers, 'payloadBase64', false); + return { + png: decodeHelperPayload(headers, 'payloadBase64', false), + displays: decodeHelperDisplays(headers), + }; case 'focus': return { focusedPackage: headers.focusedPackage, windowType: headers.windowType }; case 'app': @@ -107,6 +120,19 @@ export async function runAndroidAutomationHelper( } } +function decodeHelperDisplays( + headers: Record, +): AndroidHelperScreenshot['displays'] { + const count = decodeHelperIndex(headers.displayCount); + if (count === undefined) return undefined; + const rotationIndex = decodeHelperIndex(headers.displayRotation); + return rotationIndex === undefined ? { count } : { count, rotationIndex }; +} + +function decodeHelperIndex(value: string | undefined): number | undefined { + return value !== undefined && /^\d+$/.test(value) ? Number(value) : undefined; +} + function decodeHelperPayload( headers: Record, field: string, diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 2d0b5656d9..2f4affd9b4 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -238,7 +238,7 @@ async function readAndroidDisplayRotationIndices( */ export async function probeAndroidDisplayRotation( device: DeviceInfo, - options: { timeoutMs: number; signal: AbortSignal }, + options: { timeoutMs: number; signal?: AbortSignal }, ): Promise { try { const indices = await readAndroidDisplayRotationIndices(device, options); diff --git a/packages/platform-android/src/screenshot.ts b/packages/platform-android/src/screenshot.ts index a7c9e4728e..07ddef8f69 100644 --- a/packages/platform-android/src/screenshot.ts +++ b/packages/platform-android/src/screenshot.ts @@ -1,3 +1,7 @@ +import { + deviceRotationFromSurfaceIndex, + type DeviceRotation, +} from '@agent-device/contracts/device'; import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor-types'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -6,14 +10,17 @@ import { runAndroidExecOut, runAndroidShell, sleep } from './adb.ts'; import { emitAndroidAdbDiagnostic, requireAndroidAdbHost } from './adb-host.ts'; import { androidAdbResultError } from './adb-executor.ts'; import { probeAndroidDisplayRotation } from './input-actions.ts'; +import type { AndroidHelperScreenshot } from './automation-helper.ts'; // PNG file signature: 0x89 P N G \r \n 0x1A \n const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const ANDROID_SCREENSHOT_SETTLE_DELAY_MS = 1_000; +// The `dumpsys` rotation probe serves only captures whose helper reported no display facts: the +// `screencap` fallback and a helper that predates the display headers. const ANDROID_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS = 2_000; -// The rotation probe is optional metadata: one still running once the image is written gets this -// long, then is aborted, so a slow `dumpsys` adds at most this much to a screenshot and no adb -// work outlives it. +// Beside `screencap` the probe is optional metadata: one still running once the image is written +// gets this long, then is aborted, so a slow `dumpsys` adds at most this much to a screenshot and +// no adb work outlives it. const ANDROID_SCREENSHOT_ROTATION_GRACE_MS = 250; export type AndroidScreenshotOptions = { @@ -108,6 +115,39 @@ async function disableAndroidDemoMode(device: DeviceInfo): Promise { async function captureAndroidScreenshotWithRotation( device: DeviceInfo, outPath: string, +): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); + const helper = await runAndroidAutomationHelper(device, { action: 'screenshot' }); + if (!helper) return await captureAndroidScreencapWithRotation(device, outPath); + await writeAndroidPng(helper.png, outPath); + if (helper.displays) return rotationFacts(helperDisplayRotation(helper.displays)); + // A helper that predates the display headers learns nothing about rotation until it answers. + return rotationFacts( + await probeAndroidDisplayRotation(device, { + timeoutMs: ANDROID_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS, + }), + ); +} + +/** + * The helper captures the default display, but a device with more than one display reports no + * rotation on every capture path, so the helper's reading follows the `dumpsys` probe's rule. + */ +function helperDisplayRotation( + displays: NonNullable, +): DeviceRotation | undefined { + return displays.count === 1 && displays.rotationIndex !== undefined + ? deviceRotationFromSurfaceIndex(displays.rotationIndex) + : undefined; +} + +function rotationFacts(displayRotation: DeviceRotation | undefined): ScreenshotCaptureFacts { + return displayRotation ? { displayRotation } : {}; +} + +async function captureAndroidScreencapWithRotation( + device: DeviceInfo, + outPath: string, ): Promise { const probeController = new AbortController(); const probe = probeAndroidDisplayRotation(device, { @@ -115,9 +155,13 @@ async function captureAndroidScreenshotWithRotation( signal: probeController.signal, }); try { - await captureAndroidScreenshot(device, outPath); - const displayRotation = await settledWithin(probe, ANDROID_SCREENSHOT_ROTATION_GRACE_MS); - return displayRotation ? { displayRotation } : {}; + const bytes = (await runAndroidExecOut(device, ['screencap', '-p'], { binaryStdout: true })) + .stdoutBuffer; + if (!bytes) { + throw new AppError('COMMAND_FAILED', 'Failed to capture screenshot'); + } + await writeAndroidPng(bytes, outPath); + return rotationFacts(await settledWithin(probe, ANDROID_SCREENSHOT_ROTATION_GRACE_MS)); } finally { probeController.abort(); } @@ -137,16 +181,7 @@ async function settledWithin(pending: Promise, graceMs: number): Promise { - const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); - const helper = await runAndroidAutomationHelper(device, { action: 'screenshot' }); - const bytes = helper - ? helper - : (await runAndroidExecOut(device, ['screencap', '-p'], { binaryStdout: true })).stdoutBuffer; - if (!bytes) { - throw new AppError('COMMAND_FAILED', 'Failed to capture screenshot'); - } - +async function writeAndroidPng(bytes: Buffer, outPath: string): Promise { // On multi-display devices (e.g. Galaxy Z Fold), adb screencap may write a // warning to stdout before the PNG data. Strip any leading garbage by // locating the PNG signature and discarding everything before it. From 04a204582736c2840dba0911e7ffd6d2ed4bf8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 17:59:46 +0100 Subject: [PATCH 2/2] fix(android): guard the helper's display reads and document their semantics Co-Authored-By: Claude Opus 5.5 --- android/snapshot-helper/README.md | 6 ++- .../AutomationCommandHandler.java | 31 +++++------ .../snapshothelper/ScreenshotDisplay.java | 52 +++++++++++++++++++ .../snapshothelper/ScreenshotDisplayTest.java | 28 ++++++++++ .../SnapshotHelperTestSuite.java | 1 + .../src/automation-helper-fallback.test.ts | 2 +- .../platform-android/src/automation-helper.ts | 5 +- packages/platform-android/src/screenshot.ts | 13 +++-- 8 files changed, 109 insertions(+), 29 deletions(-) create mode 100644 android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplay.java create mode 100644 android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplayTest.java diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index 4e38bb75e8..8660c1cbe1 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -114,8 +114,10 @@ closes that connection; the process stays alive for the next connection: - `input ` — inject a tap, key, or point-targeted text replacement - `focus ` — read the focused window type and root package after clearing the accessibility cache - `screenshot ` — return a PNG of the default display in the `payloadBase64` header, - with `displayCount` and the default display's `Surface.ROTATION_*` index in `displayRotation`. - `displayRotation` is omitted when the rotation changed during the capture. + with the number of logical displays the helper can see in `displayCount` and the default + display's logical `Surface.ROTATION_*` index in `displayRotation`. `displayRotation` is omitted + when the rotation changed during the capture or could not be read. Both are omitted when the + helper could not read its displays. - `viewport ` — return interactive-window viewport bounds - `gesture ` — inject a planned touch gesture (same payload as the one-shot `gesture` mode) diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java index 03d83cdc15..4b1868f965 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java @@ -8,7 +8,6 @@ import android.os.Bundle; import android.os.SystemClock; import android.util.Base64; -import android.view.Display; import android.view.InputDevice; import android.view.KeyCharacterMap; import android.view.KeyEvent; @@ -185,14 +184,13 @@ private static AccessibilityNodeInfo editableAtPoint(AccessibilityNodeInfo node, } /** - * {@code UiAutomation.takeScreenshot()} captures the default display in its current rotation, so - * that display's rotation pairs with the image. It is published only when one read before and one - * after the capture agree, so a rotation landing mid-capture reports none. The display count lets - * the host keep its rule that a device with more than one display reports no rotation. + * The display count lets the host keep its rule that a device with more than one display reports + * no rotation. The rotation is published only when a read before and a read after the capture + * agree, so a rotation landing mid-capture reports none. */ private static void screenshot(Bundle result, UiAutomation automation, Context context) { - DisplayManager displays = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); - int rotationBefore = defaultDisplayRotation(displays); + DisplayManager displays = ScreenshotDisplay.displays(context); + int rotationBefore = ScreenshotDisplay.rotation(displays); Bitmap image = automation.takeScreenshot(); if (image == null) throw new IllegalStateException("Screen capture was unavailable"); try { @@ -204,21 +202,16 @@ private static void screenshot(Bundle result, UiAutomation automation, Context c } finally { image.recycle(); } - if (displays == null) return; - int rotationAfter = defaultDisplayRotation(displays); - result.putString("displayCount", Integer.toString(displays.getDisplays().length)); - if (rotationBefore >= 0 && rotationBefore == rotationAfter) { - result.putString("displayRotation", Integer.toString(rotationAfter)); + int count = ScreenshotDisplay.count(displays); + if (count == ScreenshotDisplay.UNKNOWN) return; + result.putString("displayCount", Integer.toString(count)); + int rotation = + ScreenshotDisplay.whenBothAgree(rotationBefore, ScreenshotDisplay.rotation(displays)); + if (rotation != ScreenshotDisplay.UNKNOWN) { + result.putString("displayRotation", Integer.toString(rotation)); } } - /** The default display's {@code Surface.ROTATION_*} index, or -1 when it cannot be read. */ - private static int defaultDisplayRotation(DisplayManager displays) { - if (displays == null) return -1; - Display display = displays.getDisplay(Display.DEFAULT_DISPLAY); - return display == null ? -1 : display.getRotation(); - } - private static void focus(Bundle result, UiAutomation automation) throws Exception { AccessibilityTreeCapture.prepare(automation, 0, 0); AccessibilityTreeCapture.clearAccessibilityCache(automation); diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplay.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplay.java new file mode 100644 index 0000000000..1183d3c774 --- /dev/null +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplay.java @@ -0,0 +1,52 @@ +package com.callstack.agentdevice.snapshothelper; + +import android.content.Context; +import android.hardware.display.DisplayManager; +import android.view.Display; + +/** + * Display facts published beside a session screenshot. {@code UiAutomation.takeScreenshot()} + * captures the default display in its current rotation, so that display's logical rotation pairs + * with the image. A read that fails answers {@link #UNKNOWN}, and the response then omits the fact + * instead of failing the capture. + */ +final class ScreenshotDisplay { + static final int UNKNOWN = -1; + + private ScreenshotDisplay() {} + + static DisplayManager displays(Context context) { + try { + return (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + } catch (Throwable error) { + return null; + } + } + + /** The default display's {@code Surface.ROTATION_*} index. */ + static int rotation(DisplayManager displays) { + try { + Display display = displays.getDisplay(Display.DEFAULT_DISPLAY); + return display == null ? UNKNOWN : display.getRotation(); + } catch (Throwable error) { + return UNKNOWN; + } + } + + /** The number of logical displays this process can see. */ + static int count(DisplayManager displays) { + try { + return displays.getDisplays().length; + } catch (Throwable error) { + return UNKNOWN; + } + } + + /** + * The rotation read before and after the capture agreed on, or {@link #UNKNOWN} when either read + * failed or the display turned during the capture. + */ + static int whenBothAgree(int before, int after) { + return before >= 0 && before == after ? before : UNKNOWN; + } +} diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplayTest.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplayTest.java new file mode 100644 index 0000000000..4b81c966d6 --- /dev/null +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplayTest.java @@ -0,0 +1,28 @@ +package com.callstack.agentdevice.snapshothelper; + +public final class ScreenshotDisplayTest { + private ScreenshotDisplayTest() {} + + static void run() { + assertRotation(ScreenshotDisplay.whenBothAgree(1, 1), 1, "both reads of one landscape display"); + assertRotation(ScreenshotDisplay.whenBothAgree(0, 0), 0, "both reads of one portrait display"); + // A rotation landing inside the capture would pair one rotation with the other's pixels. + assertRotation( + ScreenshotDisplay.whenBothAgree(0, 1), ScreenshotDisplay.UNKNOWN, "display turned mid-capture"); + assertRotation( + ScreenshotDisplay.whenBothAgree(ScreenshotDisplay.UNKNOWN, 0), + ScreenshotDisplay.UNKNOWN, + "read before the capture failed"); + assertRotation( + ScreenshotDisplay.whenBothAgree(0, ScreenshotDisplay.UNKNOWN), + ScreenshotDisplay.UNKNOWN, + "read after the capture failed"); + } + + private static void assertRotation(int actual, int expected, String label) { + if (actual != expected) { + throw new AssertionError( + "wrong rotation for " + label + ": expected " + expected + " but got " + actual); + } + } +} diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java index ad86760372..faca37a485 100644 --- a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java @@ -11,5 +11,6 @@ public static void main(String[] args) throws Exception { BoundedUiAutomationConnectionTest.run(); GestureViewportReaderTest.run(); DisplayExtentTest.run(); + ScreenshotDisplayTest.run(); } } diff --git a/packages/platform-android/src/automation-helper-fallback.test.ts b/packages/platform-android/src/automation-helper-fallback.test.ts index f040578cb6..4b79ba1c31 100644 --- a/packages/platform-android/src/automation-helper-fallback.test.ts +++ b/packages/platform-android/src/automation-helper-fallback.test.ts @@ -130,7 +130,7 @@ test.each<[string, Record, object, number]>([ ['a rotation mid-capture', { displayCount: '1' }, {}, 0], ['an older helper', {}, { displayRotation: 'landscape-right' }, 1], ])( - 'a helper screenshot from %s reports its rotation without screencap', + 'a helper screenshot from %s reports its rotation facts without screencap', async (_name, displays, facts, dumpsysReads) => { const f = fixture({ ok: 'true', payloadBase64: png.toString('base64'), ...displays }); const exec = f.provider.exec; diff --git a/packages/platform-android/src/automation-helper.ts b/packages/platform-android/src/automation-helper.ts index 453ddd2b7b..10040b1cd1 100644 --- a/packages/platform-android/src/automation-helper.ts +++ b/packages/platform-android/src/automation-helper.ts @@ -25,8 +25,9 @@ type AndroidAutomationRequest = type AndroidHelperFocus = { focusedPackage?: string; windowType?: string }; /** - * `displays` is absent when the helper predates the display headers. `rotationIndex` is the default - * display's `Surface.ROTATION_*` index, absent when the display turned during the capture. + * `displays` is absent when the helper predates the display headers or could not read its displays. + * `rotationIndex` is the default display's logical `Surface.ROTATION_*` index, absent when the + * display turned during the capture or the read failed. */ export type AndroidHelperScreenshot = { png: Buffer; diff --git a/packages/platform-android/src/screenshot.ts b/packages/platform-android/src/screenshot.ts index 07ddef8f69..99833c0411 100644 --- a/packages/platform-android/src/screenshot.ts +++ b/packages/platform-android/src/screenshot.ts @@ -15,8 +15,9 @@ import type { AndroidHelperScreenshot } from './automation-helper.ts'; // PNG file signature: 0x89 P N G \r \n 0x1A \n const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const ANDROID_SCREENSHOT_SETTLE_DELAY_MS = 1_000; -// The `dumpsys` rotation probe serves only captures whose helper reported no display facts: the -// `screencap` fallback and a helper that predates the display headers. +// The `dumpsys` rotation probe serves only captures without helper display facts: the `screencap` +// fallback, and a helper that predates the display headers or could not read its displays. For +// that helper the probe runs after the image, so it can add up to this budget to a screenshot. const ANDROID_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS = 2_000; // Beside `screencap` the probe is optional metadata: one still running once the image is written // gets this long, then is aborted, so a slow `dumpsys` adds at most this much to a screenshot and @@ -121,7 +122,7 @@ async function captureAndroidScreenshotWithRotation( if (!helper) return await captureAndroidScreencapWithRotation(device, outPath); await writeAndroidPng(helper.png, outPath); if (helper.displays) return rotationFacts(helperDisplayRotation(helper.displays)); - // A helper that predates the display headers learns nothing about rotation until it answers. + // A helper without display facts is only known once it answers, so the probe cannot overlap it. return rotationFacts( await probeAndroidDisplayRotation(device, { timeoutMs: ANDROID_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS, @@ -130,8 +131,10 @@ async function captureAndroidScreenshotWithRotation( } /** - * The helper captures the default display, but a device with more than one display reports no - * rotation on every capture path, so the helper's reading follows the `dumpsys` probe's rule. + * The helper captures the default display, so its rotation pairs with the image. A device with more + * than one display still reports no rotation, as on the `dumpsys` path. The helper counts the + * logical displays it can see, so an inactive foldable panel or another app's private virtual + * display does not count, while `dumpsys` counts every display device. */ function helperDisplayRotation( displays: NonNullable,