diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index c7bc9ca187..8660c1cbe1 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -113,7 +113,11 @@ closes that connection; the process stays alive for the next connection: - `snapshot-short-idle ` — capture with at most a 100 ms idle wait, capped by the session's configured timeout - `input ` — inject a tap, key, or point-targeted text replacement - `focus ` — read the focused window type and root package after clearing the accessibility cache -- `screenshot ` — return a PNG in the `payloadBase64` header +- `screenshot ` — return a PNG of the default display in the `payloadBase64` header, + with the number of logical displays the helper can see in `displayCount` and the default + display's logical `Surface.ROTATION_*` index in `displayRotation`. `displayRotation` is omitted + when the rotation changed during the capture or could not be read. Both are omitted when the + helper could not read its displays. - `viewport ` — return interactive-window viewport bounds - `gesture ` — inject a planned touch gesture (same payload as the one-shot `gesture` mode) diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java index 76be54dce0..4b1868f965 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AutomationCommandHandler.java @@ -1,8 +1,10 @@ package com.callstack.agentdevice.snapshothelper; import android.app.UiAutomation; +import android.content.Context; import android.graphics.Bitmap; import android.graphics.Rect; +import android.hardware.display.DisplayManager; import android.os.Bundle; import android.os.SystemClock; import android.util.Base64; @@ -23,7 +25,12 @@ final class AutomationCommandHandler { private AutomationCommandHandler() {} static void writeSessionResponse( - OutputStream output, String requestId, String action, String payload, UiAutomation automation) + OutputStream output, + String requestId, + String action, + String payload, + UiAutomation automation, + Context context) throws IOException { Bundle result = SessionResponseWriter.sessionResponseBundle(requestId); InputDispatch dispatch = new InputDispatch(); @@ -31,7 +38,7 @@ static void writeSessionResponse( if ("app".equals(action)) { ApplicationCommandHandler.execute(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8)), result); } else if ("screenshot".equals(action)) { - screenshot(result, automation); + screenshot(result, automation, context); } else if ("focus".equals(action)) { focus(result, automation); } else { @@ -176,7 +183,14 @@ private static AccessibilityNodeInfo editableAtPoint(AccessibilityNodeInfo node, } } - private static void screenshot(Bundle result, UiAutomation automation) { + /** + * The display count lets the host keep its rule that a device with more than one display reports + * no rotation. The rotation is published only when a read before and a read after the capture + * agree, so a rotation landing mid-capture reports none. + */ + private static void screenshot(Bundle result, UiAutomation automation, Context context) { + DisplayManager displays = ScreenshotDisplay.displays(context); + int rotationBefore = ScreenshotDisplay.rotation(displays); Bitmap image = automation.takeScreenshot(); if (image == null) throw new IllegalStateException("Screen capture was unavailable"); try { @@ -188,6 +202,14 @@ private static void screenshot(Bundle result, UiAutomation automation) { } finally { image.recycle(); } + int count = ScreenshotDisplay.count(displays); + if (count == ScreenshotDisplay.UNKNOWN) return; + result.putString("displayCount", Integer.toString(count)); + int rotation = + ScreenshotDisplay.whenBothAgree(rotationBefore, ScreenshotDisplay.rotation(displays)); + if (rotation != ScreenshotDisplay.UNKNOWN) { + result.putString("displayRotation", Integer.toString(rotation)); + } } private static void focus(Bundle result, UiAutomation automation) throws Exception { diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplay.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplay.java new file mode 100644 index 0000000000..1183d3c774 --- /dev/null +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplay.java @@ -0,0 +1,52 @@ +package com.callstack.agentdevice.snapshothelper; + +import android.content.Context; +import android.hardware.display.DisplayManager; +import android.view.Display; + +/** + * Display facts published beside a session screenshot. {@code UiAutomation.takeScreenshot()} + * captures the default display in its current rotation, so that display's logical rotation pairs + * with the image. A read that fails answers {@link #UNKNOWN}, and the response then omits the fact + * instead of failing the capture. + */ +final class ScreenshotDisplay { + static final int UNKNOWN = -1; + + private ScreenshotDisplay() {} + + static DisplayManager displays(Context context) { + try { + return (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + } catch (Throwable error) { + return null; + } + } + + /** The default display's {@code Surface.ROTATION_*} index. */ + static int rotation(DisplayManager displays) { + try { + Display display = displays.getDisplay(Display.DEFAULT_DISPLAY); + return display == null ? UNKNOWN : display.getRotation(); + } catch (Throwable error) { + return UNKNOWN; + } + } + + /** The number of logical displays this process can see. */ + static int count(DisplayManager displays) { + try { + return displays.getDisplays().length; + } catch (Throwable error) { + return UNKNOWN; + } + } + + /** + * The rotation read before and after the capture agreed on, or {@link #UNKNOWN} when either read + * failed or the display turned during the capture. + */ + static int whenBothAgree(int before, int after) { + return before >= 0 && before == after ? before : UNKNOWN; + } +} diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index c28361a666..c7fb436485 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -210,7 +210,7 @@ private void runSnapshotSession( if ("input".equals(action) || "screenshot".equals(action) || "focus".equals(action) || "app".equals(action)) { AutomationCommandHandler.writeSessionResponse( socket.getOutputStream(), requestId, action, - parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked()); + parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked(), getContext()); continue; } if ("viewport".equals(action)) { diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplayTest.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplayTest.java new file mode 100644 index 0000000000..4b81c966d6 --- /dev/null +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/ScreenshotDisplayTest.java @@ -0,0 +1,28 @@ +package com.callstack.agentdevice.snapshothelper; + +public final class ScreenshotDisplayTest { + private ScreenshotDisplayTest() {} + + static void run() { + assertRotation(ScreenshotDisplay.whenBothAgree(1, 1), 1, "both reads of one landscape display"); + assertRotation(ScreenshotDisplay.whenBothAgree(0, 0), 0, "both reads of one portrait display"); + // A rotation landing inside the capture would pair one rotation with the other's pixels. + assertRotation( + ScreenshotDisplay.whenBothAgree(0, 1), ScreenshotDisplay.UNKNOWN, "display turned mid-capture"); + assertRotation( + ScreenshotDisplay.whenBothAgree(ScreenshotDisplay.UNKNOWN, 0), + ScreenshotDisplay.UNKNOWN, + "read before the capture failed"); + assertRotation( + ScreenshotDisplay.whenBothAgree(0, ScreenshotDisplay.UNKNOWN), + ScreenshotDisplay.UNKNOWN, + "read after the capture failed"); + } + + private static void assertRotation(int actual, int expected, String label) { + if (actual != expected) { + throw new AssertionError( + "wrong rotation for " + label + ": expected " + expected + " but got " + actual); + } + } +} diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java index ad86760372..faca37a485 100644 --- a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java @@ -11,5 +11,6 @@ public static void main(String[] args) throws Exception { BoundedUiAutomationConnectionTest.run(); GestureViewportReaderTest.run(); DisplayExtentTest.run(); + ScreenshotDisplayTest.run(); } } diff --git a/packages/platform-android/src/automation-helper-fallback.test.ts b/packages/platform-android/src/automation-helper-fallback.test.ts index 73ea363f5a..4b79ba1c31 100644 --- a/packages/platform-android/src/automation-helper-fallback.test.ts +++ b/packages/platform-android/src/automation-helper-fallback.test.ts @@ -119,6 +119,42 @@ test.each>([ }, ); +test.each<[string, Record, object, number]>([ + [ + 'one display', + { displayCount: '1', displayRotation: '1' }, + { displayRotation: 'landscape-left' }, + 0, + ], + ['two displays', { displayCount: '2', displayRotation: '1' }, {}, 0], + ['a rotation mid-capture', { displayCount: '1' }, {}, 0], + ['an older helper', {}, { displayRotation: 'landscape-right' }, 1], +])( + 'a helper screenshot from %s reports its rotation facts without screencap', + async (_name, displays, facts, dumpsysReads) => { + const f = fixture({ ok: 'true', payloadBase64: png.toString('base64'), ...displays }); + const exec = f.provider.exec; + f.provider.exec = async (args, options) => { + if (args[0] === 'shell' && args[1] === 'dumpsys' && args[2] === 'display') { + f.calls.push(args); + return { exitCode: 0, stdout: ' mCurrentOrientation=3\n', stderr: '' }; + } + return await exec(args, options); + }; + const dir = await mkdtempForTest('android-helper-screenshot-rotation-'); + const output = path.join(dir, 'helper.png'); + await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { + await readAndroidFillTargetBeforeMutation(device, 10, 10, helper); + expect(await screenshotAndroid(device, output)).toEqual(facts); + }); + expect(await fs.readFile(output)).toEqual(png); + expect(f.calls.filter((args) => args[1] === 'dumpsys' && args[2] === 'display')).toHaveLength( + dumpsysReads, + ); + expect(f.calls.some((args) => args[0] === 'exec-out')).toBe(false); + }, +); + test('a cached ended helper is retired before a pointer command takes the host path', async () => { const f = fixture({ ok: 'true' }); await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => { diff --git a/packages/platform-android/src/automation-helper.ts b/packages/platform-android/src/automation-helper.ts index 4040d6e6a7..10040b1cd1 100644 --- a/packages/platform-android/src/automation-helper.ts +++ b/packages/platform-android/src/automation-helper.ts @@ -24,6 +24,15 @@ type AndroidAutomationRequest = | { action: 'focus' }; type AndroidHelperFocus = { focusedPackage?: string; windowType?: string }; +/** + * `displays` is absent when the helper predates the display headers or could not read its displays. + * `rotationIndex` is the default display's logical `Surface.ROTATION_*` index, absent when the + * display turned during the capture or the read failed. + */ +export type AndroidHelperScreenshot = { + png: Buffer; + displays?: { count: number; rotationIndex?: number }; +}; type AndroidHelperAppResult = | { kind: 'component-unavailable' } | { kind: 'launched'; output: string }; @@ -43,7 +52,7 @@ export function runAndroidAutomationHelper( export function runAndroidAutomationHelper( device: DeviceInfo, request: Extract, -): Promise; +): Promise; export function runAndroidAutomationHelper( device: DeviceInfo, request: Extract, @@ -55,7 +64,9 @@ export function runAndroidAutomationHelper( export async function runAndroidAutomationHelper( device: DeviceInfo, request: AndroidAutomationRequest, -): Promise { +): Promise< + true | AndroidHelperAppResult | AndroidHelperScreenshot | AndroidHelperFocus | undefined +> { const { action } = request; const deviceKey = getAndroidSnapshotHelperSessionDeviceKey(device); if (!(await getLiveAndroidSnapshotHelperSession(deviceKey))?.commands.has(action)) @@ -77,7 +88,10 @@ export async function runAndroidAutomationHelper( case 'input': return true; case 'screenshot': - return decodeHelperPayload(headers, 'payloadBase64', false); + return { + png: decodeHelperPayload(headers, 'payloadBase64', false), + displays: decodeHelperDisplays(headers), + }; case 'focus': return { focusedPackage: headers.focusedPackage, windowType: headers.windowType }; case 'app': @@ -107,6 +121,19 @@ export async function runAndroidAutomationHelper( } } +function decodeHelperDisplays( + headers: Record, +): AndroidHelperScreenshot['displays'] { + const count = decodeHelperIndex(headers.displayCount); + if (count === undefined) return undefined; + const rotationIndex = decodeHelperIndex(headers.displayRotation); + return rotationIndex === undefined ? { count } : { count, rotationIndex }; +} + +function decodeHelperIndex(value: string | undefined): number | undefined { + return value !== undefined && /^\d+$/.test(value) ? Number(value) : undefined; +} + function decodeHelperPayload( headers: Record, field: string, diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 2d0b5656d9..2f4affd9b4 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -238,7 +238,7 @@ async function readAndroidDisplayRotationIndices( */ export async function probeAndroidDisplayRotation( device: DeviceInfo, - options: { timeoutMs: number; signal: AbortSignal }, + options: { timeoutMs: number; signal?: AbortSignal }, ): Promise { try { const indices = await readAndroidDisplayRotationIndices(device, options); diff --git a/packages/platform-android/src/screenshot.ts b/packages/platform-android/src/screenshot.ts index a7c9e4728e..99833c0411 100644 --- a/packages/platform-android/src/screenshot.ts +++ b/packages/platform-android/src/screenshot.ts @@ -1,3 +1,7 @@ +import { + deviceRotationFromSurfaceIndex, + type DeviceRotation, +} from '@agent-device/contracts/device'; import type { ScreenshotCaptureFacts } from '@agent-device/contracts/interactor-types'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -6,14 +10,18 @@ import { runAndroidExecOut, runAndroidShell, sleep } from './adb.ts'; import { emitAndroidAdbDiagnostic, requireAndroidAdbHost } from './adb-host.ts'; import { androidAdbResultError } from './adb-executor.ts'; import { probeAndroidDisplayRotation } from './input-actions.ts'; +import type { AndroidHelperScreenshot } from './automation-helper.ts'; // PNG file signature: 0x89 P N G \r \n 0x1A \n const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const ANDROID_SCREENSHOT_SETTLE_DELAY_MS = 1_000; +// The `dumpsys` rotation probe serves only captures without helper display facts: the `screencap` +// fallback, and a helper that predates the display headers or could not read its displays. For +// that helper the probe runs after the image, so it can add up to this budget to a screenshot. const ANDROID_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS = 2_000; -// The rotation probe is optional metadata: one still running once the image is written gets this -// long, then is aborted, so a slow `dumpsys` adds at most this much to a screenshot and no adb -// work outlives it. +// Beside `screencap` the probe is optional metadata: one still running once the image is written +// gets this long, then is aborted, so a slow `dumpsys` adds at most this much to a screenshot and +// no adb work outlives it. const ANDROID_SCREENSHOT_ROTATION_GRACE_MS = 250; export type AndroidScreenshotOptions = { @@ -108,6 +116,41 @@ async function disableAndroidDemoMode(device: DeviceInfo): Promise { async function captureAndroidScreenshotWithRotation( device: DeviceInfo, outPath: string, +): Promise { + const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); + const helper = await runAndroidAutomationHelper(device, { action: 'screenshot' }); + if (!helper) return await captureAndroidScreencapWithRotation(device, outPath); + await writeAndroidPng(helper.png, outPath); + if (helper.displays) return rotationFacts(helperDisplayRotation(helper.displays)); + // A helper without display facts is only known once it answers, so the probe cannot overlap it. + return rotationFacts( + await probeAndroidDisplayRotation(device, { + timeoutMs: ANDROID_SCREENSHOT_ROTATION_PROBE_TIMEOUT_MS, + }), + ); +} + +/** + * The helper captures the default display, so its rotation pairs with the image. A device with more + * than one display still reports no rotation, as on the `dumpsys` path. The helper counts the + * logical displays it can see, so an inactive foldable panel or another app's private virtual + * display does not count, while `dumpsys` counts every display device. + */ +function helperDisplayRotation( + displays: NonNullable, +): DeviceRotation | undefined { + return displays.count === 1 && displays.rotationIndex !== undefined + ? deviceRotationFromSurfaceIndex(displays.rotationIndex) + : undefined; +} + +function rotationFacts(displayRotation: DeviceRotation | undefined): ScreenshotCaptureFacts { + return displayRotation ? { displayRotation } : {}; +} + +async function captureAndroidScreencapWithRotation( + device: DeviceInfo, + outPath: string, ): Promise { const probeController = new AbortController(); const probe = probeAndroidDisplayRotation(device, { @@ -115,9 +158,13 @@ async function captureAndroidScreenshotWithRotation( signal: probeController.signal, }); try { - await captureAndroidScreenshot(device, outPath); - const displayRotation = await settledWithin(probe, ANDROID_SCREENSHOT_ROTATION_GRACE_MS); - return displayRotation ? { displayRotation } : {}; + const bytes = (await runAndroidExecOut(device, ['screencap', '-p'], { binaryStdout: true })) + .stdoutBuffer; + if (!bytes) { + throw new AppError('COMMAND_FAILED', 'Failed to capture screenshot'); + } + await writeAndroidPng(bytes, outPath); + return rotationFacts(await settledWithin(probe, ANDROID_SCREENSHOT_ROTATION_GRACE_MS)); } finally { probeController.abort(); } @@ -137,16 +184,7 @@ async function settledWithin(pending: Promise, graceMs: number): Promise { - const { runAndroidAutomationHelper } = await import('./automation-helper.ts'); - const helper = await runAndroidAutomationHelper(device, { action: 'screenshot' }); - const bytes = helper - ? helper - : (await runAndroidExecOut(device, ['screencap', '-p'], { binaryStdout: true })).stdoutBuffer; - if (!bytes) { - throw new AppError('COMMAND_FAILED', 'Failed to capture screenshot'); - } - +async function writeAndroidPng(bytes: Buffer, outPath: string): Promise { // On multi-display devices (e.g. Galaxy Z Fold), adb screencap may write a // warning to stdout before the PNG data. Strip any leading garbage by // locating the PNG signature and discarding everything before it.