From 33ee1e0fb2a0355a4305eb651b675db493f5e6c5 Mon Sep 17 00:00:00 2001 From: Tim Shamilov Date: Tue, 20 Oct 2026 13:49:39 -0400 Subject: [PATCH 1/6] fix(mobile): reobserve targets after keyboard transitions --- .changeset/mobile-keyboard-transitions.md | 5 +++++ docs/reference/mobile.mdx | 4 +++- packages/mobile/src/errors.ts | 6 +++-- packages/mobile/src/options.ts | 5 +++-- packages/mobile/src/surface.ts | 7 +++--- packages/mobile/tests/unit/engine.test.ts | 26 ++++++++++++++++++++++ packages/mobile/tests/unit/errors.test.ts | 12 ++++++++++ plugins/e2e/skills/e2e/references/setup.md | 6 +++-- skills/e2e/references/setup.md | 6 +++-- 9 files changed, 65 insertions(+), 12 deletions(-) create mode 100644 .changeset/mobile-keyboard-transitions.md diff --git a/.changeset/mobile-keyboard-transitions.md b/.changeset/mobile-keyboard-transitions.md new file mode 100644 index 0000000000..02b935ba98 --- /dev/null +++ b/.changeset/mobile-keyboard-transitions.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/mobile": patch +--- + +Reobserve controls after the transition budget following text entry, even when the first snapshot has unchanged bounds, so a delayed keyboard animation does not send the next tap to the old position. Retry moving-target refusals only when agent-device confirms no input was dispatched. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 8a6b44e464..54b772b58d 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -223,7 +223,9 @@ mobile({ platform: 'ios', settle: 300 }); // default: 150 * control from a landed one (iOS reports the final frame at once, Android * the frame in flight), so the control is found again in a fresh snapshot * once the wait is over and acted on where it landed. Unmoved controls are - * acted on immediately. Raise this for slower transitions. + * acted on immediately, except after typing, filling, or clearing: the + * keyboard can move them after an initially unchanged snapshot. These actions + * always give the next control the remaining budget. Raise it for slower transitions. */ readonly transition?: number | undefined; ``` diff --git a/packages/mobile/src/errors.ts b/packages/mobile/src/errors.ts index 7860d4477b..39aa0ce9a9 100644 --- a/packages/mobile/src/errors.ts +++ b/packages/mobile/src/errors.ts @@ -279,8 +279,10 @@ export async function runCommand(label: string, work: () => Promise, signa */ export function staleOr(cause: unknown, operation: string, where?: string): Error { if (isClassified(cause)) return cause; - const { reason } = details(cause); - if (typeof reason === 'string' && STALE_REF_REASONS.has(reason)) { + const { reason, dispatched } = details(cause); + // A moving-target refusal is safe to retry only when no input was sent. + if ((typeof reason === 'string' && STALE_REF_REASONS.has(reason)) || + (reason === 'target_still_moving' && dispatched === 'no')) { return new EngineError('NODE_STALE', `${operation}: ${message(cause)}`, { retryable: true, cause }); } return translateError(cause, operation, where); diff --git a/packages/mobile/src/options.ts b/packages/mobile/src/options.ts index 8eff2d1c98..bbde294ab6 100644 --- a/packages/mobile/src/options.ts +++ b/packages/mobile/src/options.ts @@ -105,8 +105,9 @@ export interface MobileOptions { * report a control's final position from the first frame of a transition, * so this budget is the only thing that keeps a test from tapping a point * the control has not reached; controls already on screen at the same - * place before the action are acted on at once. Raise it for slower - * transitions. + * place before the action are acted on at once, except after typing, filling, + * or clearing, when the keyboard may move them later. Those actions give + * the next control the remaining budget. Raise it for slower transitions. */ readonly transition?: number | undefined; /** diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 35e81b2a29..d7eff0df41 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -1313,7 +1313,8 @@ export class AgentDeviceSurface { } catch (cause) { throw staleOr(cause, label, this.where); } - this.markAction(before); + // Text entry can open the keyboard after the first unchanged snapshot. + this.markAction(action.kind === 'fill' || action.kind === 'clear' ? undefined : before); } /** @@ -1369,9 +1370,9 @@ export class AgentDeviceSurface { if (options.replace) { throw unsupported('agent-device cannot clear the focused field without a node; type into a listed field by id to replace its value'); } - const before = this.latestIndex; await this.command('keyboard.type', (client) => input.send(() => client.interactions.type({ text })), operation.signal); - this.markAction(before); + // Typing may return before the keyboard moves surrounding controls. + this.markAction(undefined); } /** diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index a3aee922db..3eabc7506d 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2456,6 +2456,32 @@ describe('deterministic actions', () => { expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); }); + it.each(['type', 'fill', 'clear'] as const)('reobserves after %s when the keyboard moves a control after an unchanged snapshot', async (action) => { + autoAdvanceTimers(); + const h = harness({ transition: 120 }); + await openAttempt(h); + await sleep(150); + const search = await observed(h, 'Search'); + if (action === 'type') { + await h.engine.keyboard!.type('blue', { replace: false }, test()); + } else { + await h.engine.perform!(search.ref, action === 'fill' ? { kind: 'fill', value: 'blue', sensitive: false } : { kind: 'clear' }, test()); + } + // The first post-entry snapshot still has the old bounds. The keyboard + // arrives later, moving an existing control before the next tap lands. + const beforeKeyboard = await observed(h, 'About'); + h.fake.respond('capture.snapshot', () => ({ + ...SETTINGS_SNAPSHOT, + nodes: SETTINGS_NODES.map((node) => + node.identifier === 'ABOUT' || node.label === 'About' + ? { ...node, ref: '@e42', rect: { x: 0, y: 200, width: 390, height: 44 } } + : node, + ), + })); + await h.engine.perform!(beforeKeyboard.ref, { kind: 'tap' }, test()); + expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); + }); + it('acts on the duplicate nearest where the control was when a fresh snapshot lists several alike', async () => { const h = harness({ transition: 120 }); await openAttempt(h); diff --git a/packages/mobile/tests/unit/errors.test.ts b/packages/mobile/tests/unit/errors.test.ts index 4abe753229..23d7fe9700 100644 --- a/packages/mobile/tests/unit/errors.test.ts +++ b/packages/mobile/tests/unit/errors.test.ts @@ -86,6 +86,18 @@ describe('error translation', () => { expect(translateError(refused, 'observe')).toMatchObject({ code: 'ENGINE_FAILURE' }); }); + it.each(['no', 'unknown', undefined] as const)('retries a moving target only when no input was dispatched (%s)', (dispatched) => { + const cause = new AppError('COMMAND_FAILED', 'Target position was not confirmed', { + reason: 'target_still_moving', + ...(dispatched === undefined ? {} : { dispatched }), + }); + expect(staleOr(cause, 'perform tap')).toMatchObject({ + code: dispatched === 'no' ? 'NODE_STALE' : 'ENGINE_FAILURE', + retryable: dispatched === 'no', + cause, + }); + }); + it('maps the rest to ENGINE_FAILURE with the text, a ref named in a message without a stale reason included', () => { expect(staleOr(new Error('ref @e12 not found in the current snapshot'), 'perform tap')).toMatchObject({ code: 'ENGINE_FAILURE', diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index 7fa4f97406..f4f40bffa3 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -376,8 +376,10 @@ export default { exactly, TestMu AI installs `app` (omit `app.appPath`), and it reads `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. -- Only a control that appeared or moved with the previous action waits out - `transition` (default 500 ms); agent actions settle `settle` ms (default +- A control that appeared or moved with the previous action waits out + `transition` (default 500 ms). After typing, filling, or clearing, even an + unchanged control waits out the remaining budget because the keyboard may + move it later. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 7fa4f97406..f4f40bffa3 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -376,8 +376,10 @@ export default { exactly, TestMu AI installs `app` (omit `app.appPath`), and it reads `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. -- Only a control that appeared or moved with the previous action waits out - `transition` (default 500 ms); agent actions settle `settle` ms (default +- A control that appeared or moved with the previous action waits out + `transition` (default 500 ms). After typing, filling, or clearing, even an + unchanged control waits out the remaining budget because the keyboard may + move it later. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, From ae496a92bd1edadd7285e19bd25bccd8c9121726 Mon Sep 17 00:00:00 2001 From: Tim Shamilov Date: Sun, 25 Oct 2026 22:53:40 -0400 Subject: [PATCH 2/6] fix(mobile): detect keyboard transitions from captured state --- .changeset/mobile-keyboard-transitions.md | 2 +- docs/reference/mobile.mdx | 5 +- packages/mobile/src/errors.ts | 12 ++-- packages/mobile/src/options.ts | 5 +- packages/mobile/src/surface.ts | 79 ++++++++++++---------- packages/mobile/tests/unit/engine.test.ts | 73 +++++++++++++++----- plugins/e2e/skills/e2e/references/setup.md | 5 +- skills/e2e/references/setup.md | 5 +- 8 files changed, 114 insertions(+), 72 deletions(-) diff --git a/.changeset/mobile-keyboard-transitions.md b/.changeset/mobile-keyboard-transitions.md index 02b935ba98..2b1d5d366d 100644 --- a/.changeset/mobile-keyboard-transitions.md +++ b/.changeset/mobile-keyboard-transitions.md @@ -2,4 +2,4 @@ "@e2e-dev/mobile": patch --- -Reobserve controls after the transition budget following text entry, even when the first snapshot has unchanged bounds, so a delayed keyboard animation does not send the next tap to the old position. Retry moving-target refusals only when agent-device confirms no input was dispatched. +Reobserve a control after the remaining transition budget when captured keyboard state changes, even if its first bounds are unchanged. Unchanged controls keep the immediate path when keyboard state stays the same. Retry moving-target refusals only when agent-device confirms no input was dispatched. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 54b772b58d..5ef32ae7aa 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -223,9 +223,8 @@ mobile({ platform: 'ios', settle: 300 }); // default: 150 * control from a landed one (iOS reports the final frame at once, Android * the frame in flight), so the control is found again in a fresh snapshot * once the wait is over and acted on where it landed. Unmoved controls are - * acted on immediately, except after typing, filling, or clearing: the - * keyboard can move them after an initially unchanged snapshot. These actions - * always give the next control the remaining budget. Raise it for slower transitions. + * acted on immediately. A control the keyboard moved also waits out the + * budget. Raise it for slower transitions. */ readonly transition?: number | undefined; ``` diff --git a/packages/mobile/src/errors.ts b/packages/mobile/src/errors.ts index 39aa0ce9a9..3bbaaf4eb9 100644 --- a/packages/mobile/src/errors.ts +++ b/packages/mobile/src/errors.ts @@ -8,7 +8,8 @@ * | SESSION_NOT_FOUND, or a session-or-device-selector refusal | INVALID_STATE | * | UNSUPPORTED_OPERATION, NOT_IMPLEMENTED, UNSUPPORTED_PLATFORM | UNSUPPORTED_CAPABILITY | * | "not supported on this device" under any code | UNSUPPORTED_CAPABILITY | - * | a stale-ref refusal, by `details.reason` (action paths only) | NODE_STALE (retryable) | + * | a stale-ref refusal by `details.reason` (action paths only) | NODE_STALE (retryable) | + * | target_still_moving with `details.dispatched: no` | NODE_STALE (retryable) | * | the iOS runner busy, wedged, or past its watchdog | ENGINE_FAILURE, naming the | * | a snapshot the iOS runner acquired but could not present | runner and the recovery | * | anything else, agent-device's own timeouts included | ENGINE_FAILURE with the text | @@ -272,15 +273,14 @@ export async function runCommand(label: string, work: () => Promise, signa /** * Like translateError, but a ref agent-device refused as stale becomes * retryable `NODE_STALE`: nothing was dispatched, so the harness may - * re-observe and re-resolve the node instead of failing the action. Keyed on - * the reason alone: agent-device marks a drag's stale ref `dispatched: - * 'unknown'` after Android blocking-dialog recovery, although it refuses it - * before the gesture. + * re-observe and re-resolve the node instead of failing the action. Existing + * stale-ref reasons are sufficient on their own: Android dialog recovery can + * mark a refused drag as `dispatched: unknown`. A `target_still_moving` refusal + * additionally requires `dispatched: no`; missing or unknown dispatch is not retryable. */ export function staleOr(cause: unknown, operation: string, where?: string): Error { if (isClassified(cause)) return cause; const { reason, dispatched } = details(cause); - // A moving-target refusal is safe to retry only when no input was sent. if ((typeof reason === 'string' && STALE_REF_REASONS.has(reason)) || (reason === 'target_still_moving' && dispatched === 'no')) { return new EngineError('NODE_STALE', `${operation}: ${message(cause)}`, { retryable: true, cause }); diff --git a/packages/mobile/src/options.ts b/packages/mobile/src/options.ts index bbde294ab6..6be00cec04 100644 --- a/packages/mobile/src/options.ts +++ b/packages/mobile/src/options.ts @@ -105,9 +105,8 @@ export interface MobileOptions { * report a control's final position from the first frame of a transition, * so this budget is the only thing that keeps a test from tapping a point * the control has not reached; controls already on screen at the same - * place before the action are acted on at once, except after typing, filling, - * or clearing, when the keyboard may move them later. Those actions give - * the next control the remaining budget. Raise it for slower transitions. + * place before the action are acted on at once. A control the keyboard + * moved also waits out the budget. Raise it for slower transitions. */ readonly transition?: number | undefined; /** diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index d7eff0df41..250c35a965 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -159,6 +159,9 @@ function keyboardTop(raw: RawSnapshot, viewport: ViewportSize): number { return Math.min(viewport.height, ...keys.map((node) => node.rect!.y)); } +/** One projection and the keyboard state captured with it, kept together across an action. */ +type ProjectedScreen = ProjectedSnapshot & { readonly keyboardVisible: boolean }; + /** * Whether the capture's producer measured the soft keyboard on screen: * undefined when it did not look or could not. @@ -196,6 +199,7 @@ interface NodeBinding { readonly ref: string; readonly node: SemanticNode; readonly controlRef: string; + readonly keyboardVisible: boolean; } @@ -207,8 +211,8 @@ interface NodeBinding { * keeps sliding for about half a second after the action that opened it and * a tap aimed from either frame lands on whatever is behind the control. * Once the budget has passed the control is found again in a fresh - * snapshot. Controls that were already on screen at the same place before - * the action are acted on at once. + * snapshot. Unmoved controls are immediate only when keyboard state also + * stayed the same; a visible keyboard can precede the app's relayout. */ const DEFAULT_TRANSITION_MS = 500; @@ -419,9 +423,9 @@ export class AgentDeviceSurface { /** When this surface last acted on the device (an input or an app launch); the screen may be in transition for a while after. */ private lastActionAt = 0; /** The screen as last projected before that action, to tell controls that were already there from ones that came with it. */ - private indexBeforeAction: readonly ProjectedNode[] | undefined; + private screenBeforeAction: ProjectedScreen | undefined; /** The most recent projection of the screen, from any observe or locate. */ - private latestIndex: readonly ProjectedNode[] | undefined; + private latestScreen: ProjectedScreen | undefined; /** Budget a control that came with the last action gets to finish arriving; see DEFAULT_TRANSITION_MS. */ private readonly transitionMs: number; /** @@ -489,7 +493,7 @@ export class AgentDeviceSurface { * not yet arrived. */ async screenCommand(label: string, run: (client: AgentDeviceClient) => Promise, signal: AbortSignal): Promise { - const before = this.latestIndex; + const before = this.latestScreen; const result = await this.command(label, run, signal); this.markAction(before); return result; @@ -985,15 +989,18 @@ export class AgentDeviceSurface { * observation's `viewportFor` checks it against the device before it * replaces the known one. */ - private project(raw: RawSnapshot): ProjectedSnapshot { - const projected = projectSnapshot(raw.nodes ?? [], { - mintId: () => { - this.idCounter += 1; - return `n${this.idCounter}`; - }, - viewport: raw.viewport, - }); - this.latestIndex = projected.index; + private project(raw: RawSnapshot): ProjectedScreen { + const projected = { + ...projectSnapshot(raw.nodes ?? [], { + mintId: () => { + this.idCounter += 1; + return `n${this.idCounter}`; + }, + viewport: raw.viewport, + }), + keyboardVisible: keyboardShowing(raw), + }; + this.latestScreen = projected; const read = projected.viewport; if (read !== undefined && (this.knownViewport === undefined || sameScreen(read, this.knownViewport))) this.knownViewport = read; return projected; @@ -1045,7 +1052,7 @@ export class AgentDeviceSurface { async observe(operation: OperationContext, options?: EngineObserveOptions): Promise { const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive'); const projected = this.project(raw); - this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index)])); + this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected)])); const viewport = await this.viewportFor(projected, operation.signal); const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected)); const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined; @@ -1067,7 +1074,7 @@ export class AgentDeviceSurface { const projected = this.project(raw); this.traceScreen(raw, projected); const matches = resolveExpression(expression, projected.index, this.options.platform); - for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected.index)); + for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected)); for (const oldest of this.located.keys()) { if (this.located.size <= MAX_LOCATED_REFS) break; this.located.delete(oldest); @@ -1133,10 +1140,10 @@ export class AgentDeviceSurface { } /** Copies the fields actions use and pre-resolves a toggle's inner control. */ - private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[]): NodeBinding { + private bind(entry: ProjectedNode, snapshot: ProjectedScreen): NodeBinding { const { children: _children, ...node } = entry.node; - const control = this.controlOf(entry, snapshot); - return { id: entry.id, ref: entry.ref, node, controlRef: control.ref }; + const control = this.controlOf(entry, snapshot.index); + return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, keyboardVisible: snapshot.keyboardVisible }; } /** @@ -1171,7 +1178,8 @@ export class AgentDeviceSurface { * Lets a control that came with the last action finish arriving before a * test acts on it, and answers the binding to act on. A control already * present at the same place in the snapshot the last action was resolved - * from is not in transition and is acted on at once; anything else waits + * from is acted on at once only when keyboard state also stayed the same. + * A keyboard change can precede the app's relayout, so anything else waits * out the remainder of the transition budget since that action and is then * found again in a fresh snapshot. The budget is the only signal that the * transition is over, since the tree cannot tell a sliding control from a @@ -1184,9 +1192,9 @@ export class AgentDeviceSurface { const since = Date.now() - this.lastActionAt; const remaining = this.transitionMs - since; if (remaining <= 0) return entry; - const before = this.indexBeforeAction; - if (before !== undefined) { - const prior = this.refind(entry, before); + const before = this.screenBeforeAction; + if (before !== undefined && before.keyboardVisible === entry.keyboardVisible) { + const prior = this.refind(entry, before.index); if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return entry; } await sleep(remaining, operation.signal); @@ -1202,7 +1210,7 @@ export class AgentDeviceSurface { private async relocated(entry: NodeBinding, operation: OperationContext): Promise { const projected = this.project(await this.snapshotOrEmpty(operation, false)); const found = this.refind(entry, projected.index); - return found === undefined ? entry : this.bind(found, projected.index); + return found === undefined ? entry : this.bind(found, projected); } /** @@ -1210,8 +1218,8 @@ export class AgentDeviceSurface { * before it, and when it returned. The transition budget counts from the * return, since the input lands late in the command's own round trip. */ - private markAction(before: readonly ProjectedNode[] | undefined): void { - this.indexBeforeAction = before; + private markAction(before: ProjectedScreen | undefined): void { + this.screenBeforeAction = before; this.lastActionAt = Date.now(); } @@ -1232,7 +1240,7 @@ export class AgentDeviceSurface { // chrome flow that way. Only the settle wait varies here, by origin. if (ref.id === ROOT_ID) { if (action.kind !== 'swipe') throw notActionable(`the screen root takes swipe only, not ${action.kind}; act on a node`); - const before = this.latestIndex; + const before = this.latestScreen; await this.command('swipe', (client) => input.send(() => client.interactions.scroll({ direction: action.direction, ...settle })), operation.signal); this.markAction(before); return; @@ -1245,7 +1253,7 @@ export class AgentDeviceSurface { if ((action.kind === 'check' || action.kind === 'uncheck') && entry.node.states?.checked === (action.kind === 'check')) { return; } - const before = this.latestIndex; + const before = this.latestScreen; const run = async (): Promise => { const target = deterministic ? await this.settled(entry, operation) : entry; switch (action.kind) { @@ -1313,8 +1321,7 @@ export class AgentDeviceSurface { } catch (cause) { throw staleOr(cause, label, this.where); } - // Text entry can open the keyboard after the first unchanged snapshot. - this.markAction(action.kind === 'fill' || action.kind === 'clear' ? undefined : before); + this.markAction(before); } /** @@ -1354,7 +1361,7 @@ export class AgentDeviceSurface { */ async performAt(point: ViewportPoint, action: PointerAction, operation: OperationContext, input: InputBudget): Promise { const settle = operation.origin === 'test' ? {} : this.settleOptions; - const before = this.latestIndex; + const before = this.latestScreen; const holdMs = action.kind === 'longPress' ? (action.durationMs ?? DEFAULT_LONG_PRESS_MS) : action.kind === 'swipeTo' ? action.durationMs : undefined; await this.command(`${action.kind} at point`, (client) => input.send(() => pointerInteraction(client, point, action, settle), holdMs), operation.signal); this.markAction(before); @@ -1370,9 +1377,9 @@ export class AgentDeviceSurface { if (options.replace) { throw unsupported('agent-device cannot clear the focused field without a node; type into a listed field by id to replace its value'); } + const before = this.latestScreen; await this.command('keyboard.type', (client) => input.send(() => client.interactions.type({ text })), operation.signal); - // Typing may return before the keyboard moves surrounding controls. - this.markAction(undefined); + this.markAction(before); } /** @@ -1390,7 +1397,7 @@ export class AgentDeviceSurface { if (parsed.modifiers.length > 0) { throw unsupported(`agent-device cannot hold ${parsed.modifiers.join('+')} on a device surface; press the key alone`); } - const before = this.latestIndex; + const before = this.latestScreen; if (parsed.key.kind === 'named' && parsed.key.name === 'Enter') { await this.command('keyboard.press', (client) => input.send(() => client.command.keyboard({ action: 'enter' })), operation.signal); } else if (parsed.key.kind === 'char' || parsed.key.name === 'Space') { @@ -1411,7 +1418,7 @@ export class AgentDeviceSurface { * in {@link dismissKeyboardByGesture}. */ async dismissKeyboard(signal: AbortSignal, input?: InputBudget): Promise { - const before = this.latestIndex; + const before = this.latestScreen; try { await this.command('keyboard.dismiss', (client) => sendInput(input, () => client.command.keyboard({ action: 'dismiss' })), signal); } catch (cause) { @@ -1490,7 +1497,7 @@ export class AgentDeviceSurface { async back(operation: OperationContext, input: InputBudget): Promise { const settle = operation.origin === 'test' ? {} : this.settleOptions; - const before = this.latestIndex; + const before = this.latestScreen; await this.command('back', (client) => input.send(() => client.command.back({ ...settle })), operation.signal); this.markAction(before); } diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 3eabc7506d..de7c21b0e7 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2456,32 +2456,71 @@ describe('deterministic actions', () => { expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); }); - it.each(['type', 'fill', 'clear'] as const)('reobserves after %s when the keyboard moves a control after an unchanged snapshot', async (action) => { - autoAdvanceTimers(); + it.each( + (['tap', 'focus', 'fill', 'clear', 'type', 'press character', 'focused character'] as const).flatMap(action => + (['measured', 'tree'] as const).map(signal => [action, signal] as const)), + )('reobserves after %s when %s keyboard state changes before the control moves', async (action, signal) => { const h = harness({ transition: 120 }); await openAttempt(h); await sleep(150); + const visible = signal === 'measured' + ? { ...SETTINGS_SNAPSHOT, keyboard: { kind: 'visible' as const, frame: { x: 0, y: 580, width: 390, height: 264 } } } + : { ...SETTINGS_SNAPSHOT, nodes: [...SETTINGS_NODES, { index: 11, ref: '@keyboard', type: 'Keyboard', rect: { x: 0, y: 580, width: 390, height: 264 } }] }; const search = await observed(h, 'Search'); - if (action === 'type') { - await h.engine.keyboard!.type('blue', { replace: false }, test()); - } else { - await h.engine.perform!(search.ref, action === 'fill' ? { kind: 'fill', value: 'blue', sensitive: false } : { kind: 'clear' }, test()); - } - // The first post-entry snapshot still has the old bounds. The keyboard - // arrives later, moving an existing control before the next tap lands. - const beforeKeyboard = await observed(h, 'About'); + if (action === 'type') await h.engine.keyboard!.type('blue', { replace: false }, test()); + else if (action === 'focused character') await h.engine.keyboard!.press('a', test()); + else if (action === 'press character') await h.engine.perform!(search.ref, { kind: 'press', key: 'a' }, test()); + else if (action === 'fill') await h.engine.perform!(search.ref, { kind: 'fill', value: 'blue', sensitive: false }, test()); + else if (action === 'clear') await h.engine.perform!(search.ref, { kind: 'clear' }, test()); + else await h.engine.perform!(search.ref, { kind: action }, test()); + // The first stale layout already reports the keyboard, as in the live probe. + h.fake.respond('capture.snapshot', () => visible); + const stale = await observed(h, 'About'); h.fake.respond('capture.snapshot', () => ({ - ...SETTINGS_SNAPSHOT, - nodes: SETTINGS_NODES.map((node) => - node.identifier === 'ABOUT' || node.label === 'About' - ? { ...node, ref: '@e42', rect: { x: 0, y: 200, width: 390, height: 44 } } - : node, - ), + ...visible, + nodes: visible.nodes.map(node => node.identifier === 'ABOUT' || node.label === 'About' + ? { ...node, ref: '@e42', rect: { x: 0, y: 200, width: 390, height: 44 } } : node), + })); + await h.engine.perform!(stale.ref, { kind: 'tap' }, test()); + expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); + }); + + it.each(['Enter', 'focused Enter', 'dismiss'] as const)('reobserves after %s hides the keyboard before the control moves', async action => { + const h = harness({ transition: 120 }); + await openAttempt(h); + await sleep(150); + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'visible', frame: { x: 0, y: 580, width: 390, height: 264 } } })); + const search = await observed(h, 'Search'); + if (action === 'Enter') await h.engine.perform!(search.ref, { kind: 'press', key: 'Enter' }, test()); + else if (action === 'focused Enter') await h.engine.keyboard!.press('Enter', test()); + else await h.surface.dismissKeyboard(test().signal); + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, keyboard: { kind: 'absent' } })); + const stale = await observed(h, 'About'); + h.fake.respond('capture.snapshot', () => ({ + ...SETTINGS_SNAPSHOT, keyboard: { kind: 'absent' }, + nodes: SETTINGS_NODES.map(node => node.identifier === 'ABOUT' || node.label === 'About' + ? { ...node, ref: '@e42', rect: { x: 0, y: 200, width: 390, height: 44 } } : node), })); - await h.engine.perform!(beforeKeyboard.ref, { kind: 'tap' }, test()); + await h.engine.perform!(stale.ref, { kind: 'tap' }, test()); expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e42' }); }); + it.each(['type', 'fill', 'clear'] as const)('keeps unchanged controls immediate after %s when keyboard state is unchanged', async action => { + for (const visible of [false, true]) { + const h = harness({ transition: 120 }); + await openAttempt(h); + await sleep(150); + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, + keyboard: visible ? { kind: 'visible', frame: { x: 0, y: 580, width: 390, height: 264 } } : { kind: 'absent' }, + })); + const search = await observed(h, 'Search'); + if (action === 'type') await h.engine.keyboard!.type('blue', { replace: false }, test()); + else if (action === 'fill') await h.engine.perform!(search.ref, { kind: 'fill', value: 'blue', sensitive: false }, test()); + else await h.engine.perform!(search.ref, { kind: 'clear' }, test()); + await tapsAtOnce(h, await observed(h, 'About'), { ref: '@e4' }); + } + }); + it('acts on the duplicate nearest where the control was when a fresh snapshot lists several alike', async () => { const h = harness({ transition: 120 }); await openAttempt(h); diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index f4f40bffa3..6597298477 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -377,9 +377,8 @@ export default { `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. - A control that appeared or moved with the previous action waits out - `transition` (default 500 ms). After typing, filling, or clearing, even an - unchanged control waits out the remaining budget because the keyboard may - move it later. Agent actions settle `settle` ms (default + `transition` (default 500 ms). A control the keyboard moved also waits out + the budget. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index f4f40bffa3..6597298477 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -377,9 +377,8 @@ export default { `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. - A control that appeared or moved with the previous action waits out - `transition` (default 500 ms). After typing, filling, or clearing, even an - unchanged control waits out the remaining budget because the keyboard may - move it later. Agent actions settle `settle` ms (default + `transition` (default 500 ms). A control the keyboard moved also waits out + the budget. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, From 62031b0849f47b0b058528475949e5fe71bc1ff0 Mon Sep 17 00:00:00 2001 From: Tim Shamilov Date: Sun, 25 Oct 2026 23:02:14 -0400 Subject: [PATCH 3/6] docs(mobile): record public keyboard-state diagnostic evidence --- .../tests/fixtures/keyboard-signal/README.md | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 packages/mobile/tests/fixtures/keyboard-signal/README.md diff --git a/packages/mobile/tests/fixtures/keyboard-signal/README.md b/packages/mobile/tests/fixtures/keyboard-signal/README.md new file mode 100644 index 0000000000..a318fd2648 --- /dev/null +++ b/packages/mobile/tests/fixtures/keyboard-signal/README.md @@ -0,0 +1,97 @@ +# Keyboard signal before delayed app layout + +Local diagnostic, iPhone 15 Pro simulator on iOS 27.0, agent-device 0.21.28. +This is a controlled focus-driven relayout, not a reproduction of the original +consumer's exact timing. Its original raw capture was not retained. + +The native keyboard appears while JavaScript handles a bounded 300 ms workload. +The app then moves an existing button. The first post-focus snapshot already has +`Keyboard`/`Key` nodes, so the engine's existing `keyboardShowing` predicate is true, +although the button still has its pre-focus rectangle. The key plane was +`(0, 568, 393, 226)`, inside the 393 × 852 viewport, and a screenshot confirmed +the software keyboard was visible. No measured keyboard fact +was returned by this SDK capture; the predicate uses its existing node fallback. + +| Capture received after focus started | Keyboard nodes | Button y | +| --- | ---: | ---: | +| Before focus | 0 | 418.67 | +| 483 ms | 38 | 418.67 | +| 562 ms | 38 | 418.67 | +| 711 ms | 38 | 418.67 | +| 785 ms | 38 | 258.67 | + +This establishes the signal needed by the new condition. It does not establish +that the missing original capture contained the same signal. + +## Diagnostic component + +In an Expo host, use this as a separate entry point. The workload deliberately +makes the timing window observable; it is not an application fix or a new wait +in the engine. + +```tsx +import { registerRootComponent } from 'expo'; +import React, { useState } from 'react'; +import { Pressable, Text, TextInput, View } from 'react-native'; + +function Probe() { + const [count, setCount] = useState(0); + const [shift, setShift] = useState(0); + return + + { + const until = performance.now() + 300; + while (performance.now() < until) {} + setShift(160); + }} onBlur={() => setShift(0)} /> + setCount(n => n + 1)} + style={{ padding: 12, backgroundColor: '#ddd' }}> + Probe action + + Actions: {count} + + ; +} +registerRootComponent(Probe); +``` + +The local native host came from `apps/mobile-benchmark`, with the scenarios +replaced only in a separate diagnostic bundle. The generated host initially +trapped at `UIApplicationEvaluateRuntimeIssueForNoSceneLifecycleAdoption` under +Xcode 27.1/iOS 27.0. Its ignored native files were adapted to create a +`UIWindow(windowScene:)` in a `UIWindowSceneDelegate`, then start the existing +React Native factory there. A scene configuration was returned from the app +delegate, and `UIApplicationSceneManifest` declared single-scene support. +No benchmark scenario or committed native application source was changed. +This is not an unmodified mobile benchmark suite pass. + +Build the host following `apps/mobile-benchmark/README.md`. Save the component +as `keyboard-signal-probe.local.tsx` there and export its diagnostic bundle: + +```sh +pnpm --filter @e2e-dev/mobile-benchmark exec expo export:embed \ + --entry-file keyboard-signal-probe.local.tsx --platform ios --dev false \ + --bundle-output /tmp/probe.jsbundle --max-workers 2 +``` + +Copy the native `.app`, replace its `main.jsbundle` with that bundle, ad-hoc sign, +and install it on an owned simulator. Capture the initial tree through +`client.capture.snapshot({raw: true})`, press the returned `probe-input` ref, +and capture again serially. Record keyboard/key-node presence and +`probe-action.rect` from each same capture, with receipt times. + +## Verification + +- Same focused scripted-client regressions on PR base `93f77d8`, tests only: + **17 failed, 4 passed**. Candidate: **21 passed**. All **366** mobile unit + tests and `pnpm check` pass. +- Built CLI, one worker, zero retries, `transition: 500`, on the same diagnostic + binary: base passed 1/1, candidate passed 1/1. Test bodies took 3.71 s and + 4.24 s respectively. These single runs are compatibility observations, + not a performance benchmark or proof of a deterministic native failure. +- The unchanged-keyboard tests use a stopped fake clock and verify no pending + transition timer after type/fill/clear. The new rule adds no unconditional + text-entry wait. From 530e53046a9d790f8891629443b9c383568e873f Mon Sep 17 00:00:00 2001 From: Tim Shamilov Date: Sun, 25 Oct 2026 23:36:59 -0400 Subject: [PATCH 4/6] docs(mobile): record verification after rebasing onto main --- packages/mobile/tests/fixtures/keyboard-signal/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/mobile/tests/fixtures/keyboard-signal/README.md b/packages/mobile/tests/fixtures/keyboard-signal/README.md index a318fd2648..412799400d 100644 --- a/packages/mobile/tests/fixtures/keyboard-signal/README.md +++ b/packages/mobile/tests/fixtures/keyboard-signal/README.md @@ -85,13 +85,15 @@ and capture again serially. Record keyboard/key-node presence and ## Verification -- Same focused scripted-client regressions on PR base `93f77d8`, tests only: - **17 failed, 4 passed**. Candidate: **21 passed**. All **366** mobile unit +- Same focused scripted-client regressions on current main `d8c597b`, tests only: + **17 failed, 4 passed**. Candidate: **21 passed**. All **374** mobile unit tests and `pnpm check` pass. - Built CLI, one worker, zero retries, `transition: 500`, on the same diagnostic - binary: base passed 1/1, candidate passed 1/1. Test bodies took 3.71 s and + binary before rebase: base `93f77d8` passed 1/1, candidate `828b1e8` passed 1/1. Test bodies took 3.71 s and 4.24 s respectively. These single runs are compatibility observations, not a performance benchmark or proof of a deterministic native failure. +- The rebased candidate also passed the same built CLI check with agent-device + 0.21.29, one worker and zero retries. - The unchanged-keyboard tests use a stopped fake clock and verify no pending transition timer after type/fill/clear. The new rule adds no unconditional text-entry wait. From 4a092a550f689ce3f3ba77619f897eeb4bc71eb3 Mon Sep 17 00:00:00 2001 From: Tim Shamilov Date: Mon, 26 Oct 2026 00:02:20 -0400 Subject: [PATCH 5/6] docs(mobile): clarify keyboard transition budget --- docs/reference/mobile.mdx | 5 +++-- packages/mobile/README.md | 2 +- packages/mobile/src/options.ts | 6 +++--- plugins/e2e/skills/e2e/references/setup.md | 4 ++-- skills/e2e/references/setup.md | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 5ef32ae7aa..f0fd7bfdd4 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -223,8 +223,9 @@ mobile({ platform: 'ios', settle: 300 }); // default: 150 * control from a landed one (iOS reports the final frame at once, Android * the frame in flight), so the control is found again in a fresh snapshot * once the wait is over and acted on where it landed. Unmoved controls are - * acted on immediately. A control the keyboard moved also waits out the - * budget. Raise it for slower transitions. + * acted on immediately only when keyboard visibility is unchanged. Opening + * or closing the keyboard also uses the remaining budget. Raise it for + * slower transitions. */ readonly transition?: number | undefined; ``` diff --git a/packages/mobile/README.md b/packages/mobile/README.md index 1acbd83779..5ff63588f1 100644 --- a/packages/mobile/README.md +++ b/packages/mobile/README.md @@ -73,7 +73,7 @@ the other old app options are `INVALID_CONFIG` naming their key under `app`. | `session` | agent-device session name, before the worker slot: slot `n` drives its device under `-`, `e2e--` by default. One run per session at a time. | | `snapshot` | `'full'` (default, includes static text) or `'interactive'` (actionable nodes only). | | `settle` | For agent actions: milliseconds the UI must hold still after an action before the agent observes again, default `150`; `false` skips the wait. A test's own steps never settle; `expect` verifies their outcome. | -| `transition` | For a test's steps: milliseconds a control that appeared or moved with the last action gets to finish arriving before it is acted on, default `500`. Controls already in place before the action are acted on at once. | +| `transition` | For a test's steps: milliseconds a control that appeared or moved with the last action gets to finish arriving before it is acted on, default `500`. Unmoved controls are acted on immediately only when keyboard visibility is unchanged. Opening or closing the keyboard also uses the remaining budget. | Every optional `mobile()` value also accepts `undefined`, so a config passes `device: process.env.E2E_DEVICE` straight through, with no conditional diff --git a/packages/mobile/src/options.ts b/packages/mobile/src/options.ts index 6be00cec04..53037c6e4d 100644 --- a/packages/mobile/src/options.ts +++ b/packages/mobile/src/options.ts @@ -104,9 +104,9 @@ export interface MobileOptions { * 500, the length of a modal or screen transition. Accessibility frames * report a control's final position from the first frame of a transition, * so this budget is the only thing that keeps a test from tapping a point - * the control has not reached; controls already on screen at the same - * place before the action are acted on at once. A control the keyboard - * moved also waits out the budget. Raise it for slower transitions. + * the control has not reached. Unmoved controls are acted on immediately + * only when keyboard visibility is unchanged. Opening or closing the + * keyboard also uses the remaining budget. Raise it for slower transitions. */ readonly transition?: number | undefined; /** diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index 6597298477..8116790411 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -377,8 +377,8 @@ export default { `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. - A control that appeared or moved with the previous action waits out - `transition` (default 500 ms). A control the keyboard moved also waits out - the budget. Agent actions settle `settle` ms (default + `transition` (default 500 ms). Opening or closing the keyboard also uses + the remaining budget. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 6597298477..8116790411 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -377,8 +377,8 @@ export default { `LT_USERNAME` and `LT_ACCESS_KEY`. An attempt's video links TestMu AI's recording of the whole session. - A control that appeared or moved with the previous action waits out - `transition` (default 500 ms). A control the keyboard moved also waits out - the budget. Agent actions settle `settle` ms (default + `transition` (default 500 ms). Opening or closing the keyboard also uses + the remaining budget. Agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, From 17b7d3106204e64869ec034d7ab64a099fc1ae1e Mon Sep 17 00:00:00 2001 From: Tim Shamilov Date: Mon, 26 Oct 2026 18:29:56 -0400 Subject: [PATCH 6/6] docs(mobile): trim keyboard transition review evidence --- .changeset/mobile-keyboard-transitions.md | 2 +- .../tests/fixtures/keyboard-signal/README.md | 99 ------------------- 2 files changed, 1 insertion(+), 100 deletions(-) delete mode 100644 packages/mobile/tests/fixtures/keyboard-signal/README.md diff --git a/.changeset/mobile-keyboard-transitions.md b/.changeset/mobile-keyboard-transitions.md index 2b1d5d366d..e4a89e21bc 100644 --- a/.changeset/mobile-keyboard-transitions.md +++ b/.changeset/mobile-keyboard-transitions.md @@ -2,4 +2,4 @@ "@e2e-dev/mobile": patch --- -Reobserve a control after the remaining transition budget when captured keyboard state changes, even if its first bounds are unchanged. Unchanged controls keep the immediate path when keyboard state stays the same. Retry moving-target refusals only when agent-device confirms no input was dispatched. +A test step on a control the keyboard moves now waits for it to land instead of tapping where it was. Moving-target refusals are retried when agent-device sent no input. diff --git a/packages/mobile/tests/fixtures/keyboard-signal/README.md b/packages/mobile/tests/fixtures/keyboard-signal/README.md deleted file mode 100644 index 412799400d..0000000000 --- a/packages/mobile/tests/fixtures/keyboard-signal/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# Keyboard signal before delayed app layout - -Local diagnostic, iPhone 15 Pro simulator on iOS 27.0, agent-device 0.21.28. -This is a controlled focus-driven relayout, not a reproduction of the original -consumer's exact timing. Its original raw capture was not retained. - -The native keyboard appears while JavaScript handles a bounded 300 ms workload. -The app then moves an existing button. The first post-focus snapshot already has -`Keyboard`/`Key` nodes, so the engine's existing `keyboardShowing` predicate is true, -although the button still has its pre-focus rectangle. The key plane was -`(0, 568, 393, 226)`, inside the 393 × 852 viewport, and a screenshot confirmed -the software keyboard was visible. No measured keyboard fact -was returned by this SDK capture; the predicate uses its existing node fallback. - -| Capture received after focus started | Keyboard nodes | Button y | -| --- | ---: | ---: | -| Before focus | 0 | 418.67 | -| 483 ms | 38 | 418.67 | -| 562 ms | 38 | 418.67 | -| 711 ms | 38 | 418.67 | -| 785 ms | 38 | 258.67 | - -This establishes the signal needed by the new condition. It does not establish -that the missing original capture contained the same signal. - -## Diagnostic component - -In an Expo host, use this as a separate entry point. The workload deliberately -makes the timing window observable; it is not an application fix or a new wait -in the engine. - -```tsx -import { registerRootComponent } from 'expo'; -import React, { useState } from 'react'; -import { Pressable, Text, TextInput, View } from 'react-native'; - -function Probe() { - const [count, setCount] = useState(0); - const [shift, setShift] = useState(0); - return - - { - const until = performance.now() + 300; - while (performance.now() < until) {} - setShift(160); - }} onBlur={() => setShift(0)} /> - setCount(n => n + 1)} - style={{ padding: 12, backgroundColor: '#ddd' }}> - Probe action - - Actions: {count} - - ; -} -registerRootComponent(Probe); -``` - -The local native host came from `apps/mobile-benchmark`, with the scenarios -replaced only in a separate diagnostic bundle. The generated host initially -trapped at `UIApplicationEvaluateRuntimeIssueForNoSceneLifecycleAdoption` under -Xcode 27.1/iOS 27.0. Its ignored native files were adapted to create a -`UIWindow(windowScene:)` in a `UIWindowSceneDelegate`, then start the existing -React Native factory there. A scene configuration was returned from the app -delegate, and `UIApplicationSceneManifest` declared single-scene support. -No benchmark scenario or committed native application source was changed. -This is not an unmodified mobile benchmark suite pass. - -Build the host following `apps/mobile-benchmark/README.md`. Save the component -as `keyboard-signal-probe.local.tsx` there and export its diagnostic bundle: - -```sh -pnpm --filter @e2e-dev/mobile-benchmark exec expo export:embed \ - --entry-file keyboard-signal-probe.local.tsx --platform ios --dev false \ - --bundle-output /tmp/probe.jsbundle --max-workers 2 -``` - -Copy the native `.app`, replace its `main.jsbundle` with that bundle, ad-hoc sign, -and install it on an owned simulator. Capture the initial tree through -`client.capture.snapshot({raw: true})`, press the returned `probe-input` ref, -and capture again serially. Record keyboard/key-node presence and -`probe-action.rect` from each same capture, with receipt times. - -## Verification - -- Same focused scripted-client regressions on current main `d8c597b`, tests only: - **17 failed, 4 passed**. Candidate: **21 passed**. All **374** mobile unit - tests and `pnpm check` pass. -- Built CLI, one worker, zero retries, `transition: 500`, on the same diagnostic - binary before rebase: base `93f77d8` passed 1/1, candidate `828b1e8` passed 1/1. Test bodies took 3.71 s and - 4.24 s respectively. These single runs are compatibility observations, - not a performance benchmark or proof of a deterministic native failure. -- The rebased candidate also passed the same built CLI check with agent-device - 0.21.29, one worker and zero retries. -- The unchanged-keyboard tests use a stopped fake clock and verify no pending - transition timer after type/fill/clear. The new rule adds no unconditional - text-entry wait.