diff --git a/.changeset/mobile-keyboard-transitions.md b/.changeset/mobile-keyboard-transitions.md new file mode 100644 index 0000000000..e4a89e21bc --- /dev/null +++ b/.changeset/mobile-keyboard-transitions.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/mobile": patch +--- + +A test step on a control the keyboard moves now waits for it to land instead of tapping where it was. Moving-target refusals are retried when agent-device sent no input. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 8a6b44e464..f0fd7bfdd4 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -223,7 +223,9 @@ mobile({ platform: 'ios', settle: 300 }); // default: 150 * control from a landed one (iOS reports the final frame at once, Android * the frame in flight), so the control is found again in a fresh snapshot * once the wait is over and acted on where it landed. Unmoved controls are - * acted on immediately. Raise this for slower transitions. + * acted on immediately only when keyboard visibility is unchanged. Opening + * or closing the keyboard also uses the remaining budget. Raise it for + * slower transitions. */ readonly transition?: number | undefined; ``` diff --git a/packages/mobile/README.md b/packages/mobile/README.md index 1acbd83779..5ff63588f1 100644 --- a/packages/mobile/README.md +++ b/packages/mobile/README.md @@ -73,7 +73,7 @@ the other old app options are `INVALID_CONFIG` naming their key under `app`. | `session` | agent-device session name, before the worker slot: slot `n` drives its device under `-`, `e2e--` by default. One run per session at a time. | | `snapshot` | `'full'` (default, includes static text) or `'interactive'` (actionable nodes only). | | `settle` | For agent actions: milliseconds the UI must hold still after an action before the agent observes again, default `150`; `false` skips the wait. A test's own steps never settle; `expect` verifies their outcome. | -| `transition` | For a test's steps: milliseconds a control that appeared or moved with the last action gets to finish arriving before it is acted on, default `500`. Controls already in place before the action are acted on at once. | +| `transition` | For a test's steps: milliseconds a control that appeared or moved with the last action gets to finish arriving before it is acted on, default `500`. Unmoved controls are acted on immediately only when keyboard visibility is unchanged. Opening or closing the keyboard also uses the remaining budget. | Every optional `mobile()` value also accepts `undefined`, so a config passes `device: process.env.E2E_DEVICE` straight through, with no conditional diff --git a/packages/mobile/src/errors.ts b/packages/mobile/src/errors.ts index 7860d4477b..3bbaaf4eb9 100644 --- a/packages/mobile/src/errors.ts +++ b/packages/mobile/src/errors.ts @@ -8,7 +8,8 @@ * | SESSION_NOT_FOUND, or a session-or-device-selector refusal | INVALID_STATE | * | UNSUPPORTED_OPERATION, NOT_IMPLEMENTED, UNSUPPORTED_PLATFORM | UNSUPPORTED_CAPABILITY | * | "not supported on this device" under any code | UNSUPPORTED_CAPABILITY | - * | a stale-ref refusal, by `details.reason` (action paths only) | NODE_STALE (retryable) | + * | a stale-ref refusal by `details.reason` (action paths only) | NODE_STALE (retryable) | + * | target_still_moving with `details.dispatched: no` | NODE_STALE (retryable) | * | the iOS runner busy, wedged, or past its watchdog | ENGINE_FAILURE, naming the | * | a snapshot the iOS runner acquired but could not present | runner and the recovery | * | anything else, agent-device's own timeouts included | ENGINE_FAILURE with the text | @@ -272,15 +273,16 @@ export async function runCommand(label: string, work: () => Promise, signa /** * Like translateError, but a ref agent-device refused as stale becomes * retryable `NODE_STALE`: nothing was dispatched, so the harness may - * re-observe and re-resolve the node instead of failing the action. Keyed on - * the reason alone: agent-device marks a drag's stale ref `dispatched: - * 'unknown'` after Android blocking-dialog recovery, although it refuses it - * before the gesture. + * re-observe and re-resolve the node instead of failing the action. Existing + * stale-ref reasons are sufficient on their own: Android dialog recovery can + * mark a refused drag as `dispatched: unknown`. A `target_still_moving` refusal + * additionally requires `dispatched: no`; missing or unknown dispatch is not retryable. */ export function staleOr(cause: unknown, operation: string, where?: string): Error { if (isClassified(cause)) return cause; - const { reason } = details(cause); - if (typeof reason === 'string' && STALE_REF_REASONS.has(reason)) { + const { reason, dispatched } = details(cause); + if ((typeof reason === 'string' && STALE_REF_REASONS.has(reason)) || + (reason === 'target_still_moving' && dispatched === 'no')) { return new EngineError('NODE_STALE', `${operation}: ${message(cause)}`, { retryable: true, cause }); } return translateError(cause, operation, where); diff --git a/packages/mobile/src/options.ts b/packages/mobile/src/options.ts index 8eff2d1c98..53037c6e4d 100644 --- a/packages/mobile/src/options.ts +++ b/packages/mobile/src/options.ts @@ -104,9 +104,9 @@ export interface MobileOptions { * 500, the length of a modal or screen transition. Accessibility frames * report a control's final position from the first frame of a transition, * so this budget is the only thing that keeps a test from tapping a point - * the control has not reached; controls already on screen at the same - * place before the action are acted on at once. Raise it for slower - * transitions. + * the control has not reached. Unmoved controls are acted on immediately + * only when keyboard visibility is unchanged. Opening or closing the + * keyboard also uses the remaining budget. Raise it for slower transitions. */ readonly transition?: number | undefined; /** diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 35e81b2a29..250c35a965 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -159,6 +159,9 @@ function keyboardTop(raw: RawSnapshot, viewport: ViewportSize): number { return Math.min(viewport.height, ...keys.map((node) => node.rect!.y)); } +/** One projection and the keyboard state captured with it, kept together across an action. */ +type ProjectedScreen = ProjectedSnapshot & { readonly keyboardVisible: boolean }; + /** * Whether the capture's producer measured the soft keyboard on screen: * undefined when it did not look or could not. @@ -196,6 +199,7 @@ interface NodeBinding { readonly ref: string; readonly node: SemanticNode; readonly controlRef: string; + readonly keyboardVisible: boolean; } @@ -207,8 +211,8 @@ interface NodeBinding { * keeps sliding for about half a second after the action that opened it and * a tap aimed from either frame lands on whatever is behind the control. * Once the budget has passed the control is found again in a fresh - * snapshot. Controls that were already on screen at the same place before - * the action are acted on at once. + * snapshot. Unmoved controls are immediate only when keyboard state also + * stayed the same; a visible keyboard can precede the app's relayout. */ const DEFAULT_TRANSITION_MS = 500; @@ -419,9 +423,9 @@ export class AgentDeviceSurface { /** When this surface last acted on the device (an input or an app launch); the screen may be in transition for a while after. */ private lastActionAt = 0; /** The screen as last projected before that action, to tell controls that were already there from ones that came with it. */ - private indexBeforeAction: readonly ProjectedNode[] | undefined; + private screenBeforeAction: ProjectedScreen | undefined; /** The most recent projection of the screen, from any observe or locate. */ - private latestIndex: readonly ProjectedNode[] | undefined; + private latestScreen: ProjectedScreen | undefined; /** Budget a control that came with the last action gets to finish arriving; see DEFAULT_TRANSITION_MS. */ private readonly transitionMs: number; /** @@ -489,7 +493,7 @@ export class AgentDeviceSurface { * not yet arrived. */ async screenCommand(label: string, run: (client: AgentDeviceClient) => Promise, signal: AbortSignal): Promise { - const before = this.latestIndex; + const before = this.latestScreen; const result = await this.command(label, run, signal); this.markAction(before); return result; @@ -985,15 +989,18 @@ export class AgentDeviceSurface { * observation's `viewportFor` checks it against the device before it * replaces the known one. */ - private project(raw: RawSnapshot): ProjectedSnapshot { - const projected = projectSnapshot(raw.nodes ?? [], { - mintId: () => { - this.idCounter += 1; - return `n${this.idCounter}`; - }, - viewport: raw.viewport, - }); - this.latestIndex = projected.index; + private project(raw: RawSnapshot): ProjectedScreen { + const projected = { + ...projectSnapshot(raw.nodes ?? [], { + mintId: () => { + this.idCounter += 1; + return `n${this.idCounter}`; + }, + viewport: raw.viewport, + }), + keyboardVisible: keyboardShowing(raw), + }; + this.latestScreen = projected; const read = projected.viewport; if (read !== undefined && (this.knownViewport === undefined || sameScreen(read, this.knownViewport))) this.knownViewport = read; return projected; @@ -1045,7 +1052,7 @@ export class AgentDeviceSurface { async observe(operation: OperationContext, options?: EngineObserveOptions): Promise { const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive'); const projected = this.project(raw); - this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index)])); + this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected)])); const viewport = await this.viewportFor(projected, operation.signal); const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected)); const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined; @@ -1067,7 +1074,7 @@ export class AgentDeviceSurface { const projected = this.project(raw); this.traceScreen(raw, projected); const matches = resolveExpression(expression, projected.index, this.options.platform); - for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected.index)); + for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected)); for (const oldest of this.located.keys()) { if (this.located.size <= MAX_LOCATED_REFS) break; this.located.delete(oldest); @@ -1133,10 +1140,10 @@ export class AgentDeviceSurface { } /** Copies the fields actions use and pre-resolves a toggle's inner control. */ - private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[]): NodeBinding { + private bind(entry: ProjectedNode, snapshot: ProjectedScreen): NodeBinding { const { children: _children, ...node } = entry.node; - const control = this.controlOf(entry, snapshot); - return { id: entry.id, ref: entry.ref, node, controlRef: control.ref }; + const control = this.controlOf(entry, snapshot.index); + return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, keyboardVisible: snapshot.keyboardVisible }; } /** @@ -1171,7 +1178,8 @@ export class AgentDeviceSurface { * Lets a control that came with the last action finish arriving before a * test acts on it, and answers the binding to act on. A control already * present at the same place in the snapshot the last action was resolved - * from is not in transition and is acted on at once; anything else waits + * from is acted on at once only when keyboard state also stayed the same. + * A keyboard change can precede the app's relayout, so anything else waits * out the remainder of the transition budget since that action and is then * found again in a fresh snapshot. The budget is the only signal that the * transition is over, since the tree cannot tell a sliding control from a @@ -1184,9 +1192,9 @@ export class AgentDeviceSurface { const since = Date.now() - this.lastActionAt; const remaining = this.transitionMs - since; if (remaining <= 0) return entry; - const before = this.indexBeforeAction; - if (before !== undefined) { - const prior = this.refind(entry, before); + const before = this.screenBeforeAction; + if (before !== undefined && before.keyboardVisible === entry.keyboardVisible) { + const prior = this.refind(entry, before.index); if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return entry; } await sleep(remaining, operation.signal); @@ -1202,7 +1210,7 @@ export class AgentDeviceSurface { private async relocated(entry: NodeBinding, operation: OperationContext): Promise { const projected = this.project(await this.snapshotOrEmpty(operation, false)); const found = this.refind(entry, projected.index); - return found === undefined ? entry : this.bind(found, projected.index); + return found === undefined ? entry : this.bind(found, projected); } /** @@ -1210,8 +1218,8 @@ export class AgentDeviceSurface { * before it, and when it returned. The transition budget counts from the * return, since the input lands late in the command's own round trip. */ - private markAction(before: readonly ProjectedNode[] | undefined): void { - this.indexBeforeAction = before; + private markAction(before: ProjectedScreen | undefined): void { + this.screenBeforeAction = before; this.lastActionAt = Date.now(); } @@ -1232,7 +1240,7 @@ export class AgentDeviceSurface { // chrome flow that way. Only the settle wait varies here, by origin. if (ref.id === ROOT_ID) { if (action.kind !== 'swipe') throw notActionable(`the screen root takes swipe only, not ${action.kind}; act on a node`); - const before = this.latestIndex; + const before = this.latestScreen; await this.command('swipe', (client) => input.send(() => client.interactions.scroll({ direction: action.direction, ...settle })), operation.signal); this.markAction(before); return; @@ -1245,7 +1253,7 @@ export class AgentDeviceSurface { if ((action.kind === 'check' || action.kind === 'uncheck') && entry.node.states?.checked === (action.kind === 'check')) { return; } - const before = this.latestIndex; + const before = this.latestScreen; const run = async (): Promise => { const target = deterministic ? await this.settled(entry, operation) : entry; switch (action.kind) { @@ -1353,7 +1361,7 @@ export class AgentDeviceSurface { */ async performAt(point: ViewportPoint, action: PointerAction, operation: OperationContext, input: InputBudget): Promise { const settle = operation.origin === 'test' ? {} : this.settleOptions; - const before = this.latestIndex; + const before = this.latestScreen; const holdMs = action.kind === 'longPress' ? (action.durationMs ?? DEFAULT_LONG_PRESS_MS) : action.kind === 'swipeTo' ? action.durationMs : undefined; await this.command(`${action.kind} at point`, (client) => input.send(() => pointerInteraction(client, point, action, settle), holdMs), operation.signal); this.markAction(before); @@ -1369,7 +1377,7 @@ export class AgentDeviceSurface { if (options.replace) { throw unsupported('agent-device cannot clear the focused field without a node; type into a listed field by id to replace its value'); } - const before = this.latestIndex; + const before = this.latestScreen; await this.command('keyboard.type', (client) => input.send(() => client.interactions.type({ text })), operation.signal); this.markAction(before); } @@ -1389,7 +1397,7 @@ export class AgentDeviceSurface { if (parsed.modifiers.length > 0) { throw unsupported(`agent-device cannot hold ${parsed.modifiers.join('+')} on a device surface; press the key alone`); } - const before = this.latestIndex; + const before = this.latestScreen; if (parsed.key.kind === 'named' && parsed.key.name === 'Enter') { await this.command('keyboard.press', (client) => input.send(() => client.command.keyboard({ action: 'enter' })), operation.signal); } else if (parsed.key.kind === 'char' || parsed.key.name === 'Space') { @@ -1410,7 +1418,7 @@ export class AgentDeviceSurface { * in {@link dismissKeyboardByGesture}. */ async dismissKeyboard(signal: AbortSignal, input?: InputBudget): Promise { - const before = this.latestIndex; + const before = this.latestScreen; try { await this.command('keyboard.dismiss', (client) => sendInput(input, () => client.command.keyboard({ action: 'dismiss' })), signal); } catch (cause) { @@ -1489,7 +1497,7 @@ export class AgentDeviceSurface { async back(operation: OperationContext, input: InputBudget): Promise { const settle = operation.origin === 'test' ? {} : this.settleOptions; - const before = this.latestIndex; + const before = this.latestScreen; await this.command('back', (client) => input.send(() => client.command.back({ ...settle })), operation.signal); this.markAction(before); } diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index a3aee922db..de7c21b0e7 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2456,6 +2456,71 @@ describe('deterministic actions', () => { expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); }); + it.each( + (['tap', 'focus', 'fill', 'clear', 'type', 'press character', 'focused character'] as const).flatMap(action => + (['measured', 'tree'] as const).map(signal => [action, signal] as const)), + )('reobserves after %s when %s keyboard state changes before the control moves', async (action, signal) => { + const h = harness({ transition: 120 }); + await openAttempt(h); + await sleep(150); + const visible = signal === 'measured' + ? { ...SETTINGS_SNAPSHOT, keyboard: { kind: 'visible' as const, frame: { x: 0, y: 580, width: 390, height: 264 } } } + : { ...SETTINGS_SNAPSHOT, nodes: [...SETTINGS_NODES, { index: 11, ref: '@keyboard', type: 'Keyboard', rect: { x: 0, y: 580, width: 390, height: 264 } }] }; + const search = await observed(h, 'Search'); + if (action === 'type') await h.engine.keyboard!.type('blue', { replace: false }, test()); + else if (action === 'focused character') await h.engine.keyboard!.press('a', test()); + else if (action === 'press character') await h.engine.perform!(search.ref, { kind: 'press', key: 'a' }, test()); + else if (action === 'fill') await h.engine.perform!(search.ref, { kind: 'fill', value: 'blue', sensitive: false }, test()); + else if (action === 'clear') await h.engine.perform!(search.ref, { kind: 'clear' }, test()); + else await h.engine.perform!(search.ref, { kind: action }, test()); + // The first stale layout already reports the keyboard, as in the live probe. + h.fake.respond('capture.snapshot', () => visible); + const stale = await observed(h, 'About'); + h.fake.respond('capture.snapshot', () => ({ + ...visible, + nodes: visible.nodes.map(node => node.identifier === 'ABOUT' || node.label === 'About' + ? { ...node, ref: '@e42', rect: { x: 0, y: 200, width: 390, height: 44 } } : node), + })); + await h.engine.perform!(stale.ref, { kind: 'tap' }, test()); + expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); + }); + + it.each(['Enter', 'focused Enter', 'dismiss'] as const)('reobserves after %s hides the keyboard before the control moves', async action => { + const h = harness({ transition: 120 }); + await openAttempt(h); + await sleep(150); + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'visible', frame: { x: 0, y: 580, width: 390, height: 264 } } })); + const search = await observed(h, 'Search'); + if (action === 'Enter') await h.engine.perform!(search.ref, { kind: 'press', key: 'Enter' }, test()); + else if (action === 'focused Enter') await h.engine.keyboard!.press('Enter', test()); + else await h.surface.dismissKeyboard(test().signal); + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'absent' } })); + const stale = await observed(h, 'About'); + h.fake.respond('capture.snapshot', () => ({ + ...SETTINGS_SNAPSHOT, keyboard: { kind: 'absent' }, + nodes: SETTINGS_NODES.map(node => node.identifier === 'ABOUT' || node.label === 'About' + ? { ...node, ref: '@e42', rect: { x: 0, y: 200, width: 390, height: 44 } } : node), + })); + await h.engine.perform!(stale.ref, { kind: 'tap' }, test()); + expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); + }); + + it.each(['type', 'fill', 'clear'] as const)('keeps unchanged controls immediate after %s when keyboard state is unchanged', async action => { + for (const visible of [false, true]) { + const h = harness({ transition: 120 }); + await openAttempt(h); + await sleep(150); + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, + keyboard: visible ? { kind: 'visible', frame: { x: 0, y: 580, width: 390, height: 264 } } : { kind: 'absent' }, + })); + const search = await observed(h, 'Search'); + if (action === 'type') await h.engine.keyboard!.type('blue', { replace: false }, test()); + else if (action === 'fill') await h.engine.perform!(search.ref, { kind: 'fill', value: 'blue', sensitive: false }, test()); + else await h.engine.perform!(search.ref, { kind: 'clear' }, test()); + await tapsAtOnce(h, await observed(h, 'About'), { ref: '@e4' }); + } + }); + it('acts on the duplicate nearest where the control was when a fresh snapshot lists several alike', async () => { const h = harness({ transition: 120 }); await openAttempt(h); diff --git a/packages/mobile/tests/unit/errors.test.ts b/packages/mobile/tests/unit/errors.test.ts index 4abe753229..23d7fe9700 100644 --- a/packages/mobile/tests/unit/errors.test.ts +++ b/packages/mobile/tests/unit/errors.test.ts @@ -86,6 +86,18 @@ describe('error translation', () => { expect(translateError(refused, 'observe')).toMatchObject({ code: 'ENGINE_FAILURE' }); }); + it.each(['no', 'unknown', undefined] as const)('retries a moving target only when no input was dispatched (%s)', (dispatched) => { + const cause = new AppError('COMMAND_FAILED', 'Target position was not confirmed', { + reason: 'target_still_moving', + ...(dispatched === undefined ? {} : { dispatched }), + }); + expect(staleOr(cause, 'perform tap')).toMatchObject({ + code: dispatched === 'no' ? 'NODE_STALE' : 'ENGINE_FAILURE', + retryable: dispatched === 'no', + cause, + }); + }); + it('maps the rest to ENGINE_FAILURE with the text, a ref named in a message without a stale reason included', () => { expect(staleOr(new Error('ref @e12 not found in the current snapshot'), 'perform tap')).toMatchObject({ code: 'ENGINE_FAILURE', diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index e76d07bceb..172ab0d40e 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -376,8 +376,9 @@ export default { exactly, TestMu AI installs `app` (omit `app.appPath`), and it reads `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. -- Only a control that appeared or moved with the previous action waits out - `transition` (default 500 ms); agent actions settle `settle` ms (default +- A control that appeared or moved with the previous action waits out + `transition` (default 500 ms). Opening or closing the keyboard also uses + the remaining budget. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index e76d07bceb..172ab0d40e 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -376,8 +376,9 @@ export default { exactly, TestMu AI installs `app` (omit `app.appPath`), and it reads `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. -- Only a control that appeared or moved with the previous action waits out - `transition` (default 500 ms); agent actions settle `settle` ms (default +- A control that appeared or moved with the previous action waits out + `transition` (default 500 ms). Opening or closing the keyboard also uses + the remaining budget. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`,