diff --git a/.changeset/device-target-readiness.md b/.changeset/device-target-readiness.md new file mode 100644 index 0000000000..28e32c030a --- /dev/null +++ b/.changeset/device-target-readiness.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/mobile": patch +--- + +Add opt-in Android device target readiness for deterministic taps when agent-device advertises its bounded geometry guard. diff --git a/apps/mobile-benchmark/e2e.config.ts b/apps/mobile-benchmark/e2e.config.ts index a5ee78d874..343b3fdd26 100644 --- a/apps/mobile-benchmark/e2e.config.ts +++ b/apps/mobile-benchmark/e2e.config.ts @@ -45,6 +45,7 @@ export const ios = mobile({ export const android = mobile({ platform: 'android', session: 'e2e-mobile-benchmark-android', + ...(process.env.E2E_MOBILE_DEVICE_READINESS === '1' ? { targetReadiness: 'device' as const } : {}), // A control that arrived with the last action is found again once this // budget has passed, so the budget only has to outlast the transition: the // Modal Flow's modal slides in from below for about 700 ms on the emulator, diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 558b842293..87e6b3ffcb 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -86,6 +86,7 @@ The app under test is the target's `app`, not an engine option; see | [`session`](#session) | `string` | `e2e-` | | [`snapshot`](#snapshot) | `'full' \| 'interactive'` | `'full'` | | [`settle`](#settle) | `number \| false` | `150` | +| [`targetReadiness`](#targetreadiness) | `'device'` | adapter transition policy | | [`transition`](#transition) | `number` | `500` | | [`videoTouches`](#videotouches) | `boolean` | `true` | @@ -209,6 +210,14 @@ readonly settle?: number | false | undefined; mobile({ platform: 'ios', settle: 300 }); // default: 150 ``` +#### `targetReadiness` + +Set `targetReadiness: 'device'` on Android to let agent-device check target geometry before deterministic taps, double taps, focus, and toggle presses on controls with a test ID. Startup, tree-less device commands, controls without a test ID, and rows that dispatch an inner control retain the adapter transition check. Delegation becomes eligible after a completed target press. Initialization requires an advertised bounded press-readiness capability; an older agent-device fails with `INVALID_CONFIG`. Other actions keep the `transition` budget. The default keeps adapter readiness checks. + +```ts +mobile({ platform: 'android', targetReadiness: 'device' }); +``` + #### `transition` ```ts diff --git a/packages/mobile/src/options.ts b/packages/mobile/src/options.ts index cad25450c2..922cac7c74 100644 --- a/packages/mobile/src/options.ts +++ b/packages/mobile/src/options.ts @@ -109,6 +109,8 @@ export interface MobileOptions { * transitions. */ readonly transition?: number | undefined; + /** Delegate deterministic tap readiness to an advertised device guard. Android only. */ + readonly targetReadiness?: 'device' | undefined; /** * Whether a video recording draws agent-device's touch indicator where a * test tapped. Default `true`. `false` records the plain screen, and skips diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index f512510b09..0ba733213f 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -300,6 +300,7 @@ const MOBILE_OPTION_KEYS: readonly string[] = Object.keys({ snapshot: true, settle: true, transition: true, + targetReadiness: true, videoTouches: true, } satisfies Record); @@ -396,6 +397,7 @@ export class AgentDeviceSurface { readonly settleOptions: SettleOptions; /** When this surface last acted on the device (an input or an app launch); the screen may be in transition for a while after. */ private lastActionAt = 0; + private canDelegatePressReadiness = false; /** The screen as last projected before that action, to tell controls that were already there from ones that came with it. */ private indexBeforeAction: readonly ProjectedNode[] | undefined; /** The most recent projection of the screen, from any observe or locate. */ @@ -415,6 +417,9 @@ export class AgentDeviceSurface { private readonly createClient: ClientFactory, ) { rejectUnknownKeys('mobile()', options, MOBILE_OPTION_KEYS); + if (options.targetReadiness !== undefined && (options.targetReadiness !== 'device' || options.platform !== 'android')) { + throw new ConfigurationError('INVALID_CONFIG', 'mobile: `targetReadiness` accepts device on Android only'); + } this.pool = new DevicePool(options, createClient); this.settleOptions = settleOptions(options.settle); this.transitionMs = transitionMs(options.transition); @@ -451,7 +456,8 @@ export class AgentDeviceSurface { * its failure. Contributed-fixture methods route through here too, so the * device fixture never carries its own error mapping. */ - async command(label: string, run: (client: AgentDeviceClient) => Promise, signal?: AbortSignal): Promise { + async command(label: string, run: (client: AgentDeviceClient) => Promise, signal?: AbortSignal, options: { preservePressReadiness?: boolean } = {}): Promise { + if (!options.preservePressReadiness) this.canDelegatePressReadiness = false; const client = this.requireClient(); return runCommand(label, () => this.track(run(client)), signal ?? new AbortController().signal, this.where); } @@ -524,6 +530,14 @@ export class AgentDeviceSurface { ? undefined : { provider, lease, runId: info.runId, targetName: info.targetName, env: info.env }; await this.command('boot', (client) => client.devices.boot(this.selection()), info.signal); + if (this.options.targetReadiness === 'device') { + const capabilities = await this.command('capabilities', (client) => client.devices.capabilities(this.selection()), info.signal); + const readiness = capabilities !== null && typeof capabilities === 'object' && 'targetReadiness' in capabilities ? capabilities.targetReadiness : undefined; + if (readiness === null || typeof readiness !== 'object' || !('press' in readiness) || readiness.press !== 'bounded-geometry') { + throw new ConfigurationError('INVALID_CONFIG', 'mobile: targetReadiness device requires agent-device with bounded press geometry readiness'); + } + } + // Nothing is installed here: a device provider that installed the build // from `appPath` says so on the binding, and otherwise the suite installs // it where it wants to, with `device.installApp()`. @@ -910,7 +924,7 @@ export class AgentDeviceSurface { */ private async capture(signal: AbortSignal, interactiveOnly: boolean): Promise { const take = (): Promise => - this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal); + this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal, { preservePressReadiness: true }); try { return await take(); } catch (cause) { @@ -1160,11 +1174,14 @@ export class AgentDeviceSurface { * return, since the input lands late in the command's own round trip. */ private markAction(before: readonly ProjectedNode[] | undefined): void { + this.canDelegatePressReadiness = false; this.indexBeforeAction = before; this.lastActionAt = Date.now(); } async perform(ref: NodeRef, action: LocatorAction, operation: OperationContext, input: InputBudget): Promise { + const priorPress = this.canDelegatePressReadiness; + this.canDelegatePressReadiness = false; // A test's step verifies its outcome with `expect`, so it never waits for // the screen to settle afterwards; it only lets a control that came with // the last action finish arriving. The agent reads the screen right after @@ -1195,20 +1212,27 @@ export class AgentDeviceSurface { return; } const before = this.latestIndex; + let pressed = false; + const press = async (target: NodeBinding, control = false, count?: number): Promise => { + const result = await input.send(() => client.interactions.press({ ...this.actionTarget(target, control), ...(count === undefined ? {} : { count }), ...settle })); + pressed = true; + return result; + }; const run = async (): Promise => { - const target = deterministic ? await this.settled(entry, operation) : entry; + const deviceReady = priorPress && this.options.targetReadiness === 'device' && entry.node.testId !== undefined && entry.controlRef === entry.ref && ['tap', 'focus', 'doubleTap', 'check', 'uncheck'].includes(action.kind); + const target = deterministic && !deviceReady ? await this.settled(entry, operation) : entry; switch (action.kind) { case 'tap': - return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle })); + return press(target, true); case 'focus': // A touch surface focuses by tapping, and a tap on anything but an // editable field activates it; focus is offered for fields only. if (target.node.role !== 'textbox') { throw unsupported(`agent-device can only focus editable fields; node ${target.id} is ${target.node.role ?? 'unknown'}`); } - return input.send(() => client.interactions.press({ ...this.actionTarget(target), ...settle })); + return press(target); case 'doubleTap': - return input.send(() => client.interactions.press({ ...this.actionTarget(target), count: 2, ...settle })); + return press(target, false, 2); case 'longPress': return input.send( () => client.interactions.longPress({ ...this.actionTarget(target), ...settle, durationMs: action.durationMs ?? DEFAULT_LONG_PRESS_MS }), @@ -1232,7 +1256,7 @@ export class AgentDeviceSurface { throw unsupported(`agent-device cannot read whether node ${target.id} is checked; tap it instead`); } if (checked === wanted) return undefined; - return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle })); + return press(target, true); } case 'press': return this.pressKey(client, target, action.key, settle, input); @@ -1263,6 +1287,7 @@ export class AgentDeviceSurface { throw staleOr(cause, label, this.where); } this.markAction(before); + this.canDelegatePressReadiness = pressed; } /** diff --git a/packages/mobile/tests/types/options.ts b/packages/mobile/tests/types/options.ts index a4237ac4ac..2e4d458bf8 100644 --- a/packages/mobile/tests/types/options.ts +++ b/packages/mobile/tests/types/options.ts @@ -35,3 +35,7 @@ const farm = { ({ daemon: { baseUrl: 'http://10.0.0.7:4700' } }) satisfies DeviceLease; // @ts-expect-error a lease names its daemon by URL. ({ id: 'lease-1', daemon: { baseUrl: 4700 } }) satisfies DeviceLease; + +({ platform: 'android', targetReadiness: 'device' }) satisfies MobileOptions; +// @ts-expect-error Only device readiness is supported. +({ platform: 'android', targetReadiness: 'instant' }) satisfies MobileOptions; diff --git a/packages/mobile/tests/unit/device-readiness.test.ts b/packages/mobile/tests/unit/device-readiness.test.ts new file mode 100644 index 0000000000..7483c9d4ac --- /dev/null +++ b/packages/mobile/tests/unit/device-readiness.test.ts @@ -0,0 +1,145 @@ +import { setImmediate } from 'node:timers/promises'; +import { afterEach, expect, it, vi } from 'vitest'; +import type { OperationContext } from 'e2e/engine'; +import { boot, harness, type Harness } from '../helpers/harness.ts'; + +const operation: OperationContext = { signal: new AbortController().signal, timeoutMs: 5000, runId: 'r', attemptId: 'a', origin: 'test' }; +afterEach(() => { vi.useRealTimers(); }); + +async function seedPress(h: Harness): Promise { + h.fake.respond('capture.snapshot', () => ({ nodes: [ + { ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Seed', identifier: 'seed', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true }, + ] })); + const snapshot = await h.engine.observe!(operation); + const seed = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Seed'); + expect(seed).toBeDefined(); + const pending = h.engine.perform!(seed!.ref, { kind: 'tap' }, operation); + await setImmediate(); + await vi.advanceTimersByTimeAsync(1500); + await pending; + h.fake.calls.length = 0; +} + +it('requires advertised bounded press readiness before delegating test taps', async () => { + const h = harness({ platform: 'android', targetReadiness: 'device' }); + h.fake.respond('devices.capabilities', () => ({ availableCommands: ['press'] })); + await expect(boot(h, 'android')).rejects.toMatchObject({ code: 'INVALID_CONFIG', message: expect.stringContaining('bounded press geometry') }); + expect(h.fake.methods()).not.toContain('interactions.press'); +}); + +it('delegates a newly arrived test tap without the adapter transition timer or another capture', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 }); + h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } })); + await boot(h, 'android'); + await h.engine.session!.restart!(operation); + await seedPress(h); + h.fake.respond('capture.snapshot', () => ({ nodes: [ + { ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Submit', identifier: 'submit', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true }, + ] })); + const snapshot = await h.engine.observe!(operation); + const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit'); + expect(button).toBeDefined(); + const before = h.fake.methods().filter((method) => method === 'capture.snapshot').length; + const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation); + await setImmediate(); + expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e2' }); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(before); + expect(vi.getTimerCount()).toBe(0); + await pending; +}); + +it('retains the adapter transition wait for a target without a test ID', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 }); + h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } })); + await boot(h, 'android'); + await h.engine.session!.restart!(operation); + await seedPress(h); + h.fake.respond('capture.snapshot', () => ({ nodes: [ + { ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Submit', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true }, + ] })); + const snapshot = await h.engine.observe!(operation); + const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit'); + expect(button).toBeDefined(); + const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation); + await setImmediate(); + expect(h.fake.methods()).not.toContain('interactions.press'); + expect(vi.getTimerCount()).toBe(1); + await vi.advanceTimersByTimeAsync(1500); + await pending; +}); + +it('retains adapter readiness when an identified row dispatches an anonymous inner control', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 }); + h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } })); + await boot(h, 'android'); + await h.engine.session!.restart!(operation); + await seedPress(h); + h.fake.respond('capture.snapshot', () => ({ nodes: [ + { ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: '@e2', index: 1, parentIndex: 0, type: 'android.widget.CheckBox', label: 'Submit', identifier: 'submit', checked: false, rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true }, + { ref: '@e3', index: 2, parentIndex: 1, type: 'android.widget.CheckBox', checked: false, rect: { x: 100, y: 100, width: 40, height: 40 }, hittable: true }, + ] })); + const snapshot = await h.engine.observe!(operation); + const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit'); + expect(button).toBeDefined(); + const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation); + await setImmediate(); + expect(h.fake.methods()).not.toContain('interactions.press'); + expect(vi.getTimerCount()).toBe(1); + await vi.advanceTimersByTimeAsync(1500); + await pending; +}); + +it('keeps the default path free of capability negotiation', async () => { + const h = harness({ platform: 'android' }); + await boot(h, 'android'); + expect(h.fake.methods()).not.toContain('devices.capabilities'); +}); + +it('refuses device target readiness on platforms without the advertised guard', () => { + expect(() => harness({ platform: 'ios', targetReadiness: 'device' })).toThrow(expect.objectContaining({ code: 'INVALID_CONFIG' })); +}); + +it.each(['startup', 'back', 'failed mutation', 'no-op toggle'] as const)('retains adapter readiness after %s', async (boundary) => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + const h = harness({ platform: 'android', targetReadiness: 'device', transition: 1500 }); + h.fake.respond('devices.capabilities', () => ({ targetReadiness: { press: 'bounded-geometry' } })); + await boot(h, 'android'); + await h.engine.session!.restart!(operation); + if (boundary !== 'startup') await seedPress(h); + if (boundary === 'back') await h.engine.session!.back!(operation); + if (boundary === 'failed mutation') { + h.fake.respond('command.back', () => { throw new Error('back failed'); }); + await expect(h.engine.session!.back!(operation)).rejects.toThrow(); + } + if (boundary === 'no-op toggle') { + h.fake.respond('capture.snapshot', () => ({ nodes: [ + { ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: '@e2', index: 1, parentIndex: 0, type: 'android.widget.CheckBox', label: 'Checked', identifier: 'checked', checked: true, rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true }, + ] })); + const current = await h.engine.observe!(operation); + const toggle = current.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Checked'); + expect(toggle).toBeDefined(); + await h.engine.perform!(toggle!.ref, { kind: 'check' }, operation); + } + h.fake.calls.length = 0; + h.fake.respond('capture.snapshot', () => ({ nodes: [ + { ref: '@e1', index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { ref: '@e2', index: 1, parentIndex: 0, type: 'Button', label: 'Submit', identifier: 'submit', rect: { x: 20, y: 100, width: 120, height: 40 }, hittable: true }, + ] })); + const snapshot = await h.engine.observe!(operation); + const button = snapshot.root.children?.flatMap((node) => node.children ?? [node]).find((node) => node.name === 'Submit'); + expect(button).toBeDefined(); + const pending = h.engine.perform!(button!.ref, { kind: 'tap' }, operation); + await setImmediate(); + expect(h.fake.methods()).not.toContain('interactions.press'); + expect(vi.getTimerCount()).toBe(1); + await vi.advanceTimersByTimeAsync(1500); + await pending; +});