Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/device-target-readiness.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@e2e-dev/mobile": patch
---

Add opt-in Android device target readiness for deterministic taps when agent-device advertises its bounded geometry guard.
1 change: 1 addition & 0 deletions apps/mobile-benchmark/e2e.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ export const ios = mobile({
export const android = mobile({
platform: 'android',
session: 'e2e-mobile-benchmark-android',
...(process.env.E2E_MOBILE_DEVICE_READINESS === '1' ? { targetReadiness: 'device' as const } : {}),
// A control that arrived with the last action is found again once this
// budget has passed, so the budget only has to outlast the transition: the
// Modal Flow's modal slides in from below for about 700 ms on the emulator,
Expand Down
9 changes: 9 additions & 0 deletions docs/reference/mobile.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ The app under test is the target's `app`, not an engine option; see
| [`session`](#session) | `string` | `e2e-<target name>` |
| [`snapshot`](#snapshot) | `'full' \| 'interactive'` | `'full'` |
| [`settle`](#settle) | `number \| false` | `150` |
| [`targetReadiness`](#targetreadiness) | `'device'` | adapter transition policy |
| [`transition`](#transition) | `number` | `500` |
| [`videoTouches`](#videotouches) | `boolean` | `true` |

Expand Down Expand Up @@ -209,6 +210,14 @@ readonly settle?: number | false | undefined;
mobile({ platform: 'ios', settle: 300 }); // default: 150
```

#### `targetReadiness`

Set `targetReadiness: 'device'` on Android to let agent-device check target geometry before deterministic taps, double taps, focus, and toggle presses on controls with a test ID. Startup, tree-less device commands, controls without a test ID, and rows that dispatch an inner control retain the adapter transition check. Delegation becomes eligible after a completed target press. Initialization requires an advertised bounded press-readiness capability; an older agent-device fails with `INVALID_CONFIG`. Other actions keep the `transition` budget. The default keeps adapter readiness checks.

```ts
mobile({ platform: 'android', targetReadiness: 'device' });
```

#### `transition`

```ts
Expand Down
2 changes: 2 additions & 0 deletions packages/mobile/src/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ export interface MobileOptions {
* transitions.
*/
readonly transition?: number | undefined;
/** Delegate deterministic tap readiness to an advertised device guard. Android only. */
readonly targetReadiness?: 'device' | undefined;
/**
* Whether a video recording draws agent-device's touch indicator where a
* test tapped. Default `true`. `false` records the plain screen, and skips
Expand Down
39 changes: 32 additions & 7 deletions packages/mobile/src/surface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ const MOBILE_OPTION_KEYS: readonly string[] = Object.keys({
snapshot: true,
settle: true,
transition: true,
targetReadiness: true,
videoTouches: true,
} satisfies Record<keyof MobileOptions, true>);

Expand Down Expand Up @@ -396,6 +397,7 @@ export class AgentDeviceSurface {
readonly settleOptions: SettleOptions;
/** When this surface last acted on the device (an input or an app launch); the screen may be in transition for a while after. */
private lastActionAt = 0;
private canDelegatePressReadiness = false;
/** The screen as last projected before that action, to tell controls that were already there from ones that came with it. */
private indexBeforeAction: readonly ProjectedNode[] | undefined;
/** The most recent projection of the screen, from any observe or locate. */
Expand All @@ -415,6 +417,9 @@ export class AgentDeviceSurface {
private readonly createClient: ClientFactory,
) {
rejectUnknownKeys('mobile()', options, MOBILE_OPTION_KEYS);
if (options.targetReadiness !== undefined && (options.targetReadiness !== 'device' || options.platform !== 'android')) {
throw new ConfigurationError('INVALID_CONFIG', 'mobile: `targetReadiness` accepts device on Android only');
}
this.pool = new DevicePool(options, createClient);
this.settleOptions = settleOptions(options.settle);
this.transitionMs = transitionMs(options.transition);
Expand Down Expand Up @@ -451,7 +456,8 @@ export class AgentDeviceSurface {
* its failure. Contributed-fixture methods route through here too, so the
* device fixture never carries its own error mapping.
*/
async command<T>(label: string, run: (client: AgentDeviceClient) => Promise<T>, signal?: AbortSignal): Promise<T> {
async command<T>(label: string, run: (client: AgentDeviceClient) => Promise<T>, signal?: AbortSignal, options: { preservePressReadiness?: boolean } = {}): Promise<T> {
if (!options.preservePressReadiness) this.canDelegatePressReadiness = false;
const client = this.requireClient();
return runCommand(label, () => this.track(run(client)), signal ?? new AbortController().signal, this.where);
}
Expand Down Expand Up @@ -524,6 +530,14 @@ export class AgentDeviceSurface {
? undefined
: { provider, lease, runId: info.runId, targetName: info.targetName, env: info.env };
await this.command('boot', (client) => client.devices.boot(this.selection()), info.signal);
if (this.options.targetReadiness === 'device') {
const capabilities = await this.command('capabilities', (client) => client.devices.capabilities(this.selection()), info.signal);
const readiness = capabilities !== null && typeof capabilities === 'object' && 'targetReadiness' in capabilities ? capabilities.targetReadiness : undefined;
if (readiness === null || typeof readiness !== 'object' || !('press' in readiness) || readiness.press !== 'bounded-geometry') {
throw new ConfigurationError('INVALID_CONFIG', 'mobile: targetReadiness device requires agent-device with bounded press geometry readiness');
}
}

// Nothing is installed here: a device provider that installed the build
// from `appPath` says so on the binding, and otherwise the suite installs
// it where it wants to, with `device.installApp()`.
Expand Down Expand Up @@ -910,7 +924,7 @@ export class AgentDeviceSurface {
*/
private async capture(signal: AbortSignal, interactiveOnly: boolean): Promise<RawSnapshot> {
const take = (): Promise<RawSnapshot> =>
this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal);
this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal, { preservePressReadiness: true });
try {
return await take();
} catch (cause) {
Expand Down Expand Up @@ -1160,11 +1174,14 @@ export class AgentDeviceSurface {
* return, since the input lands late in the command's own round trip.
*/
private markAction(before: readonly ProjectedNode[] | undefined): void {
this.canDelegatePressReadiness = false;
this.indexBeforeAction = before;
this.lastActionAt = Date.now();
}

async perform(ref: NodeRef, action: LocatorAction, operation: OperationContext, input: InputBudget): Promise<void> {
const priorPress = this.canDelegatePressReadiness;
this.canDelegatePressReadiness = false;
// A test's step verifies its outcome with `expect`, so it never waits for
// the screen to settle afterwards; it only lets a control that came with
// the last action finish arriving. The agent reads the screen right after
Expand Down Expand Up @@ -1195,20 +1212,27 @@ export class AgentDeviceSurface {
return;
}
const before = this.latestIndex;
let pressed = false;
const press = async (target: NodeBinding, control = false, count?: number): Promise<unknown> => {
const result = await input.send(() => client.interactions.press({ ...this.actionTarget(target, control), ...(count === undefined ? {} : { count }), ...settle }));
pressed = true;
return result;
};
const run = async (): Promise<unknown> => {
const target = deterministic ? await this.settled(entry, operation) : entry;
const deviceReady = priorPress && this.options.targetReadiness === 'device' && entry.node.testId !== undefined && entry.controlRef === entry.ref && ['tap', 'focus', 'doubleTap', 'check', 'uncheck'].includes(action.kind);
const target = deterministic && !deviceReady ? await this.settled(entry, operation) : entry;
switch (action.kind) {
case 'tap':
return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle }));
return press(target, true);
case 'focus':
// A touch surface focuses by tapping, and a tap on anything but an
// editable field activates it; focus is offered for fields only.
if (target.node.role !== 'textbox') {
throw unsupported(`agent-device can only focus editable fields; node ${target.id} is ${target.node.role ?? 'unknown'}`);
}
return input.send(() => client.interactions.press({ ...this.actionTarget(target), ...settle }));
return press(target);
case 'doubleTap':
return input.send(() => client.interactions.press({ ...this.actionTarget(target), count: 2, ...settle }));
return press(target, false, 2);
case 'longPress':
return input.send(
() => client.interactions.longPress({ ...this.actionTarget(target), ...settle, durationMs: action.durationMs ?? DEFAULT_LONG_PRESS_MS }),
Expand All @@ -1232,7 +1256,7 @@ export class AgentDeviceSurface {
throw unsupported(`agent-device cannot read whether node ${target.id} is checked; tap it instead`);
}
if (checked === wanted) return undefined;
return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle }));
return press(target, true);
}
case 'press':
return this.pressKey(client, target, action.key, settle, input);
Expand Down Expand Up @@ -1263,6 +1287,7 @@ export class AgentDeviceSurface {
throw staleOr(cause, label, this.where);
}
this.markAction(before);
this.canDelegatePressReadiness = pressed;
}

/**
Expand Down
4 changes: 4 additions & 0 deletions packages/mobile/tests/types/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,7 @@ const farm = {
({ daemon: { baseUrl: 'http://10.0.0.7:4700' } }) satisfies DeviceLease;
// @ts-expect-error a lease names its daemon by URL.
({ id: 'lease-1', daemon: { baseUrl: 4700 } }) satisfies DeviceLease;

({ platform: 'android', targetReadiness: 'device' }) satisfies MobileOptions;
// @ts-expect-error Only device readiness is supported.
({ platform: 'android', targetReadiness: 'instant' }) satisfies MobileOptions;
145 changes: 145 additions & 0 deletions packages/mobile/tests/unit/device-readiness.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
import { setImmediate } from 'node:timers/promises';
import { afterEach, expect, it, vi } from 'vitest';
import type { OperationContext } from 'e2e/engine';
import { boot, harness, type Harness } from '../helpers/harness.ts';

const operation: OperationContext = { signal: new AbortController().signal, timeoutMs: 5000, runId: 'r', attemptId: 'a', origin: 'test' };
afterEach(() => { vi.useRealTimers(); });

async function seedPress(h: Harness): Promise<void> {
h.fake.respond('capture.snapshot', () => ({ nodes: [
{ ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{ ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Seed', identifier: 'seed', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true },
] }));
const snapshot = await h.engine.observe!(operation);
const seed = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Seed');
expect(seed).toBeDefined();
const pending = h.engine.perform!(seed!.ref, { kind: 'tap' }, operation);
await setImmediate();
await vi.advanceTimersByTimeAsync(1500);
await pending;
h.fake.calls.length = 0;
}

it('requires advertised bounded press readiness before delegating test taps', async () => {
const h = harness({ platform: 'android', targetReadiness: 'device' });
h.fake.respond('devices.capabilities', () => ({ availableCommands: ['press'] }));
await expect(boot(h, 'android')).rejects.toMatchObject({ code: 'INVALID_CONFIG', message: expect.stringContaining('bounded press geometry') });
expect(h.fake.methods()).not.toContain('interactions.press');
});

it('delegates a newly arrived test tap without the adapter transition timer or another capture', async () => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 });
h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } }));
await boot(h, 'android');
await h.engine.session!.restart!(operation);
await seedPress(h);
h.fake.respond('capture.snapshot', () => ({ nodes: [
{ ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{ ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Submit', identifier: 'submit', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true },
] }));
const snapshot = await h.engine.observe!(operation);
const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit');
expect(button).toBeDefined();
const before = h.fake.methods().filter((method) => method === 'capture.snapshot').length;
const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation);
await setImmediate();
expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e2' });
expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(before);
expect(vi.getTimerCount()).toBe(0);
await pending;
});

it('retains the adapter transition wait for a target without a test ID', async () => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 });
h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } }));
await boot(h, 'android');
await h.engine.session!.restart!(operation);
await seedPress(h);
h.fake.respond('capture.snapshot', () => ({ nodes: [
{ ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{ ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Submit', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true },
] }));
const snapshot = await h.engine.observe!(operation);
const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit');
expect(button).toBeDefined();
const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation);
await setImmediate();
expect(h.fake.methods()).not.toContain('interactions.press');
expect(vi.getTimerCount()).toBe(1);
await vi.advanceTimersByTimeAsync(1500);
await pending;
});

it('retains adapter readiness when an identified row dispatches an anonymous inner control', async () => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 });
h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } }));
await boot(h, 'android');
await h.engine.session!.restart!(operation);
await seedPress(h);
h.fake.respond('capture.snapshot', () => ({ nodes: [
{ ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{ ref: '@e2', index: 1, parentIndex: 0, type: 'android.widget.CheckBox', label: 'Submit', identifier: 'submit', checked: false, rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true },
{ ref: '@e3', index: 2, parentIndex: 1, type: 'android.widget.CheckBox', checked: false, rect: { x: 100, y: 100, width: 40, height: 40 }, hittable: true },
] }));
const snapshot = await h.engine.observe!(operation);
const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit');
expect(button).toBeDefined();
const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation);
await setImmediate();
expect(h.fake.methods()).not.toContain('interactions.press');
expect(vi.getTimerCount()).toBe(1);
await vi.advanceTimersByTimeAsync(1500);
await pending;
});

it('keeps the default path free of capability negotiation', async () => {
const h = harness({ platform: 'android' });
await boot(h, 'android');
expect(h.fake.methods()).not.toContain('devices.capabilities');
});

it('refuses device target readiness on platforms without the advertised guard', () => {
expect(() => harness({ platform: 'ios', targetReadiness: 'device' })).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG' }));
});

it.each(['startup', 'back', 'failed mutation', 'no-op toggle'] as const)('retains adapter readiness after %s', async (boundary) => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 });
h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } }));
await boot(h, 'android');
await h.engine.session!.restart!(operation);
if (boundary !== 'startup') await seedPress(h);
if (boundary === 'back') await h.engine.session!.back!(operation);
if (boundary === 'failed mutation') {
h.fake.respond('command.back', () => { throw new Error('back failed'); });
await expect(h.engine.session!.back!(operation)).rejects.toThrow();
}
if (boundary === 'no-op toggle') {
h.fake.respond('capture.snapshot', () => ({ nodes: [
{ ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{ ref: '@e2', index: 1, parentIndex: 0, type: 'android.widget.CheckBox', label: 'Checked', identifier: 'checked', checked: true, rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true },
] }));
const current = await h.engine.observe!(operation);
const toggle = current.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Checked');
expect(toggle).toBeDefined();
await h.engine.perform!(toggle!.ref, { kind: 'check' }, operation);
}
h.fake.calls.length = 0;
h.fake.respond('capture.snapshot', () => ({ nodes: [
{ ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{ ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Submit', identifier: 'submit', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true },
] }));
const snapshot = await h.engine.observe!(operation);
const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit');
expect(button).toBeDefined();
const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation);
await setImmediate();
expect(h.fake.methods()).not.toContain('interactions.press');
expect(vi.getTimerCount()).toBe(1);
await vi.advanceTimersByTimeAsync(1500);
await pending;
});
Loading