Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion android/snapshot-helper/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,11 @@ closes that connection; the process stays alive for the next connection:
- `snapshot-short-idle <requestId>` — capture with at most a 100 ms idle wait, capped by the session's configured timeout
- `input <requestId> <payloadBase64>` — inject a tap, key, or point-targeted text replacement
- `focus <requestId>` — read the focused window type and root package after clearing the accessibility cache
- `screenshot <requestId>` — return a PNG in the `payloadBase64` header
- `screenshot <requestId>` — return a PNG of the default display in the `payloadBase64` header,
with the number of logical displays the helper can see in `displayCount` and the default
display's logical `Surface.ROTATION_*` index in `displayRotation`. `displayRotation` is omitted
when the rotation changed during the capture or could not be read. Both are omitted when the
helper could not read its displays.
- `viewport <requestId>` — return interactive-window viewport bounds
- `gesture <requestId> <payloadBase64>` — inject a planned touch gesture (same payload as the
one-shot `gesture` mode)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package com.callstack.agentdevice.snapshothelper;

import android.app.UiAutomation;
import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.Rect;
import android.hardware.display.DisplayManager;
import android.os.Bundle;
import android.os.SystemClock;
import android.util.Base64;
Expand All @@ -23,15 +25,20 @@ final class AutomationCommandHandler {
private AutomationCommandHandler() {}

static void writeSessionResponse(
OutputStream output, String requestId, String action, String payload, UiAutomation automation)
OutputStream output,
String requestId,
String action,
String payload,
UiAutomation automation,
Context context)
throws IOException {
Bundle result = SessionResponseWriter.sessionResponseBundle(requestId);
InputDispatch dispatch = new InputDispatch();
try {
if ("app".equals(action)) {
ApplicationCommandHandler.execute(automation, new JSONObject(new String(Base64.decode(payload, Base64.DEFAULT), StandardCharsets.UTF_8)), result);
} else if ("screenshot".equals(action)) {
screenshot(result, automation);
screenshot(result, automation, context);
} else if ("focus".equals(action)) {
focus(result, automation);
} else {
Expand Down Expand Up @@ -176,7 +183,14 @@ private static AccessibilityNodeInfo editableAtPoint(AccessibilityNodeInfo node,
}
}

private static void screenshot(Bundle result, UiAutomation automation) {
/**
* The display count lets the host keep its rule that a device with more than one display reports
* no rotation. The rotation is published only when a read before and a read after the capture
* agree, so a rotation landing mid-capture reports none.
*/
private static void screenshot(Bundle result, UiAutomation automation, Context context) {
DisplayManager displays = ScreenshotDisplay.displays(context);
int rotationBefore = ScreenshotDisplay.rotation(displays);
Bitmap image = automation.takeScreenshot();
if (image == null) throw new IllegalStateException("Screen capture was unavailable");
try {
Expand All @@ -188,6 +202,14 @@ private static void screenshot(Bundle result, UiAutomation automation) {
} finally {
image.recycle();
}
int count = ScreenshotDisplay.count(displays);
if (count == ScreenshotDisplay.UNKNOWN) return;
result.putString("displayCount", Integer.toString(count));
int rotation =
ScreenshotDisplay.whenBothAgree(rotationBefore, ScreenshotDisplay.rotation(displays));
if (rotation != ScreenshotDisplay.UNKNOWN) {
result.putString("displayRotation", Integer.toString(rotation));
}
}

private static void focus(Bundle result, UiAutomation automation) throws Exception {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package com.callstack.agentdevice.snapshothelper;

import android.content.Context;
import android.hardware.display.DisplayManager;
import android.view.Display;

/**
* Display facts published beside a session screenshot. {@code UiAutomation.takeScreenshot()}
* captures the default display in its current rotation, so that display's logical rotation pairs
* with the image. A read that fails answers {@link #UNKNOWN}, and the response then omits the fact
* instead of failing the capture.
*/
final class ScreenshotDisplay {
static final int UNKNOWN = -1;

private ScreenshotDisplay() {}

static DisplayManager displays(Context context) {
try {
return (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE);
} catch (Throwable error) {
return null;
}
}

/** The default display's {@code Surface.ROTATION_*} index. */
static int rotation(DisplayManager displays) {
try {
Display display = displays.getDisplay(Display.DEFAULT_DISPLAY);
return display == null ? UNKNOWN : display.getRotation();
} catch (Throwable error) {
return UNKNOWN;
}
}

/** The number of logical displays this process can see. */
static int count(DisplayManager displays) {
try {
return displays.getDisplays().length;
} catch (Throwable error) {
return UNKNOWN;
}
}

/**
* The rotation read before and after the capture agreed on, or {@link #UNKNOWN} when either read
* failed or the display turned during the capture.
*/
static int whenBothAgree(int before, int after) {
return before >= 0 && before == after ? before : UNKNOWN;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ private void runSnapshotSession(
if ("input".equals(action) || "screenshot".equals(action) || "focus".equals(action) || "app".equals(action)) {
AutomationCommandHandler.writeSessionResponse(
socket.getOutputStream(), requestId, action,
parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked());
parts.length > 2 ? parts[2] : "", getConnectedUiAutomationUnchecked(), getContext());
continue;
}
if ("viewport".equals(action)) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package com.callstack.agentdevice.snapshothelper;

public final class ScreenshotDisplayTest {
private ScreenshotDisplayTest() {}

static void run() {
assertRotation(ScreenshotDisplay.whenBothAgree(1, 1), 1, "both reads of one landscape display");
assertRotation(ScreenshotDisplay.whenBothAgree(0, 0), 0, "both reads of one portrait display");
// A rotation landing inside the capture would pair one rotation with the other's pixels.
assertRotation(
ScreenshotDisplay.whenBothAgree(0, 1), ScreenshotDisplay.UNKNOWN, "display turned mid-capture");
assertRotation(
ScreenshotDisplay.whenBothAgree(ScreenshotDisplay.UNKNOWN, 0),
ScreenshotDisplay.UNKNOWN,
"read before the capture failed");
assertRotation(
ScreenshotDisplay.whenBothAgree(0, ScreenshotDisplay.UNKNOWN),
ScreenshotDisplay.UNKNOWN,
"read after the capture failed");
}

private static void assertRotation(int actual, int expected, String label) {
if (actual != expected) {
throw new AssertionError(
"wrong rotation for " + label + ": expected " + expected + " but got " + actual);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,6 @@ public static void main(String[] args) throws Exception {
BoundedUiAutomationConnectionTest.run();
GestureViewportReaderTest.run();
DisplayExtentTest.run();
ScreenshotDisplayTest.run();
}
}
36 changes: 36 additions & 0 deletions packages/platform-android/src/automation-helper-fallback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,42 @@ test.each<Record<string, string>>([
},
);

test.each<[string, Record<string, string>, object, number]>([
[
'one display',
{ displayCount: '1', displayRotation: '1' },
{ displayRotation: 'landscape-left' },
0,
],
['two displays', { displayCount: '2', displayRotation: '1' }, {}, 0],
['a rotation mid-capture', { displayCount: '1' }, {}, 0],
['an older helper', {}, { displayRotation: 'landscape-right' }, 1],
])(
'a helper screenshot from %s reports its rotation facts without screencap',
async (_name, displays, facts, dumpsysReads) => {
const f = fixture({ ok: 'true', payloadBase64: png.toString('base64'), ...displays });
const exec = f.provider.exec;
f.provider.exec = async (args, options) => {
if (args[0] === 'shell' && args[1] === 'dumpsys' && args[2] === 'display') {
f.calls.push(args);
return { exitCode: 0, stdout: ' mCurrentOrientation=3\n', stderr: '' };
}
return await exec(args, options);
};
const dir = await mkdtempForTest('android-helper-screenshot-rotation-');
const output = path.join(dir, 'helper.png');
await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => {
await readAndroidFillTargetBeforeMutation(device, 10, 10, helper);
expect(await screenshotAndroid(device, output)).toEqual(facts);
});
expect(await fs.readFile(output)).toEqual(png);
expect(f.calls.filter((args) => args[1] === 'dumpsys' && args[2] === 'display')).toHaveLength(
dumpsysReads,
);
expect(f.calls.some((args) => args[0] === 'exec-out')).toBe(false);
},
);

test('a cached ended helper is retired before a pointer command takes the host path', async () => {
const f = fixture({ ok: 'true' });
await withAndroidAdbProvider(f.provider, { serial: device.id }, async () => {
Expand Down
33 changes: 30 additions & 3 deletions packages/platform-android/src/automation-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@ type AndroidAutomationRequest =
| { action: 'focus' };

type AndroidHelperFocus = { focusedPackage?: string; windowType?: string };
/**
* `displays` is absent when the helper predates the display headers or could not read its displays.
* `rotationIndex` is the default display's logical `Surface.ROTATION_*` index, absent when the
* display turned during the capture or the read failed.
*/
export type AndroidHelperScreenshot = {
png: Buffer;
displays?: { count: number; rotationIndex?: number };
};
type AndroidHelperAppResult =
| { kind: 'component-unavailable' }
| { kind: 'launched'; output: string };
Expand All @@ -43,7 +52,7 @@ export function runAndroidAutomationHelper(
export function runAndroidAutomationHelper(
device: DeviceInfo,
request: Extract<AndroidAutomationRequest, { action: 'screenshot' }>,
): Promise<Buffer | undefined>;
): Promise<AndroidHelperScreenshot | undefined>;
export function runAndroidAutomationHelper(
device: DeviceInfo,
request: Extract<AndroidAutomationRequest, { action: 'focus' }>,
Expand All @@ -55,7 +64,9 @@ export function runAndroidAutomationHelper(
export async function runAndroidAutomationHelper(
device: DeviceInfo,
request: AndroidAutomationRequest,
): Promise<true | AndroidHelperAppResult | Buffer | AndroidHelperFocus | undefined> {
): Promise<
true | AndroidHelperAppResult | AndroidHelperScreenshot | AndroidHelperFocus | undefined
> {
const { action } = request;
const deviceKey = getAndroidSnapshotHelperSessionDeviceKey(device);
if (!(await getLiveAndroidSnapshotHelperSession(deviceKey))?.commands.has(action))
Expand All @@ -77,7 +88,10 @@ export async function runAndroidAutomationHelper(
case 'input':
return true;
case 'screenshot':
return decodeHelperPayload(headers, 'payloadBase64', false);
return {
png: decodeHelperPayload(headers, 'payloadBase64', false),
displays: decodeHelperDisplays(headers),
};
case 'focus':
return { focusedPackage: headers.focusedPackage, windowType: headers.windowType };
case 'app':
Expand Down Expand Up @@ -107,6 +121,19 @@ export async function runAndroidAutomationHelper(
}
}

function decodeHelperDisplays(
headers: Record<string, string>,
): AndroidHelperScreenshot['displays'] {
const count = decodeHelperIndex(headers.displayCount);
if (count === undefined) return undefined;
const rotationIndex = decodeHelperIndex(headers.displayRotation);
return rotationIndex === undefined ? { count } : { count, rotationIndex };
}

function decodeHelperIndex(value: string | undefined): number | undefined {
return value !== undefined && /^\d+$/.test(value) ? Number(value) : undefined;
}

function decodeHelperPayload(
headers: Record<string, string>,
field: string,
Expand Down
2 changes: 1 addition & 1 deletion packages/platform-android/src/input-actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ async function readAndroidDisplayRotationIndices(
*/
export async function probeAndroidDisplayRotation(
device: DeviceInfo,
options: { timeoutMs: number; signal: AbortSignal },
options: { timeoutMs: number; signal?: AbortSignal },
): Promise<DeviceRotation | undefined> {
try {
const indices = await readAndroidDisplayRotationIndices(device, options);
Expand Down
Loading
Loading