From b10ea45464287c45ffe41ef5b9bf2f759b0518cb Mon Sep 17 00:00:00 2001 From: bheemreddy181 <101338506+bheemreddy181@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:57:46 -0500 Subject: [PATCH 1/6] fix(mobile): revalidate input bindings before fill and clear --- .changeset/fresh-mobile-inputs.md | 5 + docs/reference/mobile.mdx | 12 ++ .../integration/agent-input-stale.test.ts | 103 +++++++++++++++ packages/mobile/src/surface.ts | 38 +++++- packages/mobile/tests/unit/engine.test.ts | 122 ++++++++++++++++++ plugins/e2e/skills/e2e/references/setup.md | 5 + skills/e2e/references/setup.md | 5 + 7 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 .changeset/fresh-mobile-inputs.md create mode 100644 packages/e2e/tests/integration/agent-input-stale.test.ts diff --git a/.changeset/fresh-mobile-inputs.md b/.changeset/fresh-mobile-inputs.md new file mode 100644 index 0000000000..3afe445f66 --- /dev/null +++ b/.changeset/fresh-mobile-inputs.md @@ -0,0 +1,5 @@ +--- +'@e2e-dev/mobile': patch +--- + +Revalidate mobile fill and clear targets with a fresh accessibility snapshot before mutation, including cached replay. Changed geometry, identity, or security facts return NODE_STALE so the harness re-observes and reruns secret authorization instead of dispatching an old binding. Coordinate-only backends can still race movement after this check. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 8a6b44e464..acc46a1d37 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -237,6 +237,18 @@ mobile({ platform: 'ios', transition: 800 }); // default: 500 budget the way a tap does. A control that arrives with the screen they reveal waits out the budget before the next action lands on it. +`fill` and `clear` also take a fresh accessibility snapshot before mutation, +including agent actions and cached replay, even after this budget expires. +The input must have one matching role, test id, name, text, and placeholder +in both snapshots. Changed bounds or disabled, hidden, secure, or password-purpose +facts return `NODE_STALE` before input is sent. Missing inputs, duplicate +identities, missing bounds, and incomplete fresh trees also return `NODE_STALE`. +The harness re-observes before retrying; an agent secret fill reruns authorization. +Use distinct test ids when fields otherwise share an identity. + +This checks accessibility evidence, not rendered stability. A coordinate-only +backend can still race movement after the check or receive stale platform bounds. + #### `videoTouches` ```ts diff --git a/packages/e2e/tests/integration/agent-input-stale.test.ts b/packages/e2e/tests/integration/agent-input-stale.test.ts new file mode 100644 index 0000000000..b98e06bb5c --- /dev/null +++ b/packages/e2e/tests/integration/agent-input-stale.test.ts @@ -0,0 +1,103 @@ +/** A stale fill must re-enter secret authorization, on live actions and cached replay alike. */ + +import { describe, expect, it } from 'vitest'; +import type { StepExecutor } from '../../src/agent/executor.ts'; +import { EngineError, type SemanticNode } from 'e2e/engine'; +import { createFakeEngine } from '../helpers/fake-engine.ts'; +import { nodeIdFor } from '../helpers/fake-loop-model.ts'; +import { createProject, resultByTitle, runExisting } from '../helpers/run-project.ts'; + +const SUITE = `import { test, credentials, expect } from 'e2e'; +test('fills the account password', async ({ agent, screen }) => { + await agent.act('fill the password', { params: { password: credentials.user('member').password } }); + await expect(screen.getByRole('status')).toBeVisible(); +}); +`; + +describe('secret fill stale retry', () => { + for (const replay of [false, true]) { + it.each([false, true])(`reauthorizes a changed sink with disabled=%s during ${replay ? 'replay' : 'live execution'}`, async (disabled) => { + const field: SemanticNode = { + ref: { id: 'password', revision: '' }, role: 'textbox', name: 'Password', + testId: 'password', inputPurpose: 'password', states: { secure: true }, + rect: { x: 40, y: 237, width: 322, height: 49 }, + }; + const tree = { ref: { id: 'root', revision: '' }, role: 'document', children: [field] }; + let stale = false; + let dispatched = 0; + const origins: string[] = []; + const pixels: boolean[] = []; + const fake = createFakeEngine({ + tree, + onStartAttempt: () => { tree.children = [field]; }, + locate: () => tree.children.filter((node) => node.role === 'status'), + perform: (_ref, action, operation) => { + if (action.kind !== 'fill') throw new Error('expected fill'); + origins.push(operation.origin); + if (stale) { + stale = false; + tree.children = [{ ...field, states: { secure: true, disabled }, rect: { x: 40, y: 126, width: 322, height: 49 } }]; + throw new EngineError('NODE_STALE', 'input moved before mutation', { retryable: true }); + } + dispatched += 1; + tree.children.push({ ref: { id: 'saved', revision: '' }, role: 'status', name: 'Saved' }); + }, + // No pixel payload is needed: this records any request after authorization. + pixels: () => { pixels.push(true); return undefined; }, + }); + const taint: boolean[] = []; + const executor: StepExecutor = { + name: 'stale-secret', + async runStep(context) { + const observation = await context.observe(); + try { + await context.actions.typeSecret({ id: nodeIdFor(observation.text, /textbox "Password"/) }, 'member.password'); + } finally { + taint.push(context.pixelsTainted); + expect((await context.observe({ pixels: true })).pixels).toBeUndefined(); + } + return { status: 'passed', summary: 'filled' }; + }, + }; + const project = createProject({ 'tests/input.e2e.ts': SUITE }); + const options = { + config: { + targets: [{ name: 'fake', platform: 'fake', engine: fake.engine }], + agents: { default: { executor } }, + credentials: { member: { username: 'example', password: 'private-fixture-value' } }, + cache: 'read-write' as const, + }, + }; + try { + if (replay) { + const recorded = await runExisting(project, options); + const recording = resultByTitle(recorded, 'fills the account password'); + expect(recording.status).toBe('passed'); + expect(recording.attempts[0]?.steps.find((entry) => entry.api === 'agent.act')?.cache).toMatchObject({ write: 'saved' }); + dispatched = 0; + origins.length = 0; + pixels.length = 0; + taint.length = 0; + } + stale = true; + const outcome = await runExisting(project, options); + const result = resultByTitle(outcome, 'fills the account password'); + const step = result.attempts[0]!.steps.find((entry) => entry.api === 'agent.act')!; + expect(result.status, JSON.stringify(result.attempts[0]?.error)).toBe(disabled ? 'failed' : 'passed'); + expect(dispatched).toBe(disabled ? 0 : 1); + expect(origins).toEqual(Array(disabled ? 1 : 2).fill(replay ? 'test' : 'agent')); + // A denied replay hands off to the executor, whose own denied action also counts. + expect(step.metrics?.actionSteps).toBe(replay && disabled ? 2 : 1); + expect(step.events.filter((event) => event.kind === 'observation' && event.name === 'relocate')).toHaveLength(1); + expect(step.events.filter((event) => event.kind === 'policy' && event.name === 'secret.fill' && event.status === 'passed')).toHaveLength(disabled ? 1 : 2); + if (disabled) expect(JSON.stringify(step)).toContain('POLICY_DENIED'); + if (!replay || disabled) expect(taint).toEqual([true]); + else expect(step.cache).toMatchObject({ mode: 'self-finalized', replayedActions: 1 }); + expect(pixels).toEqual([]); + expect(JSON.stringify(outcome)).not.toContain('private-fixture-value'); + } finally { + project.cleanup(); + } + }, 60_000); + } +}); diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 35e81b2a29..3646711a2a 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -196,6 +196,7 @@ interface NodeBinding { readonly ref: string; readonly node: SemanticNode; readonly controlRef: string; + readonly uniqueInput: boolean; } @@ -235,6 +236,12 @@ function sameRect(a: Rect | undefined, b: Rect | undefined): boolean { return Math.abs(a.x - b.x) < 1 && Math.abs(a.y - b.y) < 1 && Math.abs(a.width - b.width) < 1 && Math.abs(a.height - b.height) < 1; } +/** The semantic identity a fill must retain across its preflight capture. */ +function sameInputIdentity(a: SemanticNode, b: SemanticNode): boolean { + return a.role === b.role && a.testId === b.testId && a.name === b.name && a.text === b.text && + a.attributes?.placeholder === b.attributes?.placeholder; +} + const MAX_LOCATED_REFS = 2048; interface Attempt { @@ -1136,7 +1143,9 @@ export class AgentDeviceSurface { private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[]): NodeBinding { const { children: _children, ...node } = entry.node; const control = this.controlOf(entry, snapshot); - return { id: entry.id, ref: entry.ref, node, controlRef: control.ref }; + const uniqueInput = (node.role === 'textbox' || node.role === 'searchbox' || node.role === 'combobox') && + snapshot.filter((candidate) => sameInputIdentity(node, candidate.node)).length === 1; + return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, uniqueInput }; } /** @@ -1205,6 +1214,28 @@ export class AgentDeviceSurface { return found === undefined ? entry : this.bind(found, projected.index); } + /** + * Revalidates an input immediately before mutation. A changed binding must + * go through the harness's stale retry so secret authorization runs against + * the new observation, rather than silently following a moved or changed + * field. This narrows snapshot age, not the backend's check/use gap. + */ + private async freshInput(entry: NodeBinding, operation: OperationContext): Promise { + const raw = await this.snapshotOrEmpty(operation, false); + const projected = this.project(raw); + const matches = projected.index.filter((candidate) => sameInputIdentity(entry.node, candidate.node)); + const found = matches.length === 1 ? matches[0] : undefined; + if (!entry.uniqueInput || isTruncated(raw) || found === undefined || + !sameRect(entry.node.rect, found.node.rect) || + entry.node.inputPurpose !== found.node.inputPurpose || + entry.node.states?.disabled !== found.node.states?.disabled || + entry.node.states?.secure !== found.node.states?.secure || + entry.node.states?.hidden !== found.node.states?.hidden) { + throw new EngineError('NODE_STALE', `input node ${entry.id} changed before mutation; observe and authorize it again`, { retryable: true }); + } + return this.bind(found, projected.index); + } + /** * Records an action the device received: the screen as it was projected * before it, and when it returned. The transition budget counts from the @@ -1247,7 +1278,10 @@ export class AgentDeviceSurface { } const before = this.latestIndex; const run = async (): Promise => { - const target = deterministic ? await this.settled(entry, operation) : entry; + let target = deterministic ? await this.settled(entry, operation) : entry; + if (action.kind === 'fill' || action.kind === 'clear') { + target = await this.freshInput(entry, operation); + } switch (action.kind) { case 'tap': return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle })); diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index a3aee922db..90f17c7489 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -1162,7 +1162,9 @@ describe('perform', () => { ['interactions.longPress', { ref: '@e4', settle: true, settleQuietMs: 150, durationMs: 1000 }], ['interactions.press', { ref: '@e7', settle: true, settleQuietMs: 150 }], ['interactions.hover', { ref: '@e4' }], + ['capture.snapshot', { interactiveOnly: false }], ['interactions.fill', { ref: '@e7', text: 'blue', settle: true, settleQuietMs: 150 }], + ['capture.snapshot', { interactiveOnly: false }], ['interactions.fill', { ref: '@e7', text: '', settle: true, settleQuietMs: 150 }], ['interactions.press', { ref: '@e6', settle: true, settleQuietMs: 150 }], ['command.keyboard', { action: 'enter' }], @@ -2371,6 +2373,126 @@ describe('video', () => { }); }); +describe('input freshness', () => { + beforeEach(autoAdvanceTimers); + + /** A uniquely identified field and its neighbor, after a slow accessibility capture. */ + function form(y = 237, changes: Partial = {}, duplicates = false) { + const field: RawNode = { + ref: '@e20', index: 1, parentIndex: 0, depth: 1, type: 'secure-text-field', + label: 'Account', identifier: 'account', rect: { x: 40, y, width: 322, height: 49 }, ...changes, + }; + return { + ...SETTINGS_SNAPSHOT, + nodes: [ + SETTINGS_NODES[0]!, field, + { ref: '@e21', index: 2, parentIndex: 0, depth: 1, type: 'text-field', label: 'Username', identifier: 'username', rect: { x: 40, y: 218, width: 322, height: 49 } }, + ...(duplicates ? [{ ...field, ref: '@e22', index: 3 }] : []), + ], + }; + } + + for (const origin of ['test', 'agent'] as const) { + for (const kind of ['fill', 'clear'] as const) { + it(`rejects moved geometry before ${origin} ${kind}, even when capture consumed the transition budget`, async () => { + const h = harness(); + await openAttempt(h); + h.fake.respond('capture.snapshot', async () => { await sleep(600); return form(); }); + const old = await observed(h, 'Account'); + h.fake.respond('capture.snapshot', () => form(126)); + const action = kind === 'fill' ? { kind, value: 'example', sensitive: true } : { kind }; + await expect(h.engine.perform!(old.ref, action, { ...operation(), origin })).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); + expect(h.fake.methods()).not.toContain('interactions.fill'); + // The runner's stale retry re-observes before retrying the mutation. + const fresh = await observed(h, 'Account'); + await h.engine.perform!(fresh.ref, action, { ...operation(), origin }); + expect(h.fake.lastArgs('interactions.fill')).toMatchObject({ ref: '@e20', text: kind === 'fill' ? 'example' : '' }); + expect(h.fake.methods().filter((method) => method === 'interactions.fill')).toHaveLength(1); + }); + } + } + + it.each([ + ['missing', () => ({ ...form(), nodes: [SETTINGS_NODES[0]!] })], + ['ambiguous', () => form(237, {}, true)], + ['renamed', () => form(237, { label: 'Other account' })], + ['different identifier', () => form(237, { identifier: 'other' })], + ['different placeholder', () => form(237, { placeholder: 'Other account' })], + ['truncated', () => ({ ...form(), truncated: true })], + ['non-editable', () => form(237, { type: 'button' })], + ['no longer secure', () => form(237, { type: 'text-field' })], + ['disabled', () => form(237, { enabled: false })], + ['hidden', () => form(237, { visibleToUser: false })], + ['unmeasurable', () => { + const result = form(); + return { ...result, nodes: result.nodes.map(({ rect: _rect, ...node }) => node) }; + }], + ] as const)('rejects a %s target without dispatching or exposing the value', async (_name, snapshot) => { + const h = harness({ transition: 0 }); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form()); + const old = await observed(h, 'Account'); + h.fake.respond('capture.snapshot', snapshot); + const failure = h.engine.perform!(old.ref, { kind: 'fill', value: 'not-for-diagnostics', sensitive: true }, operation()); + await expect(failure).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); + await expect(failure).rejects.not.toThrow('not-for-diagnostics'); + expect(h.fake.methods()).not.toContain('interactions.fill'); + }); + + it('rejects an originally ambiguous identity even if only one duplicate remains', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form(237, {}, true)); + const old = await observed(h, 'Account'); + h.fake.respond('capture.snapshot', () => form()); + await expect(h.engine.perform!(old.ref, { kind: 'clear' }, operation())).rejects.toMatchObject({ code: 'NODE_STALE' }); + expect(h.fake.methods()).not.toContain('interactions.fill'); + }); + + it('uses the newest backend ref for an unchanged uniquely identified input without taking pixels', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form()); + const old = await observed(h, 'Account'); + h.fake.respond('capture.snapshot', () => form(237, { ref: '@e42', focused: true })); + await h.engine.perform!(old.ref, { kind: 'fill', value: 'example', sensitive: true }, operation()); + expect(h.fake.lastArgs('interactions.fill')).toMatchObject({ ref: '@e42', text: 'example' }); + expect(h.fake.methods()).not.toContain('capture.screenshot'); + expect(h.fake.lastArgs('capture.snapshot')).toEqual({ interactiveOnly: false }); + }); + + it('does not silently accept changed security facts from the transition relocation', async () => { + const h = harness(); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form()); + const old = await observed(h, 'Account'); + h.fake.respond('capture.snapshot', () => form(237, { enabled: false })); + await expect(h.engine.perform!(old.ref, { kind: 'fill', value: 'example', sensitive: true }, { ...operation(), origin: 'test' })).rejects.toMatchObject({ code: 'NODE_STALE' }); + expect(h.fake.methods()).not.toContain('interactions.fill'); + }); + + it('does not send input if the preflight consumed the input budget', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form()); + const old = await observed(h, 'Account'); + h.fake.respond('capture.snapshot', async () => { await sleep(600); return form(); }); + await expect(h.engine.perform!(old.ref, { kind: 'clear' }, { ...operation(), timeoutMs: 500 })).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + expect(h.fake.methods()).not.toContain('interactions.fill'); + }); + + it('does not dispatch when aborted during the preflight capture', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form()); + const old = await observed(h, 'Account'); + const controller = new AbortController(); + h.fake.respond('capture.snapshot', async () => { controller.abort(); return form(); }); + await expect(h.engine.perform!(old.ref, { kind: 'clear' }, operation(controller.signal))).rejects.toMatchObject({ code: 'CANCELLED' }); + expect(h.fake.methods()).not.toContain('interactions.fill'); + }); +}); + describe('deterministic actions', () => { const test = (): OperationContext => ({ ...operation(), origin: 'test' }); diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index e76d07bceb..4b03feb5fe 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -379,6 +379,11 @@ export default { - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. +- Mobile `fill` and `clear` always preflight a fresh accessibility tree, + replay included. A missing, ambiguous, moved, or security-changed input + returns `NODE_STALE` before mutation; retry re-observes and secret fills + rerun authorization. Give otherwise identical fields distinct test ids. + This is not atomic coordinate dispatch or proof of rendered stability. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index e76d07bceb..4b03feb5fe 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -379,6 +379,11 @@ export default { - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. +- Mobile `fill` and `clear` always preflight a fresh accessibility tree, + replay included. A missing, ambiguous, moved, or security-changed input + returns `NODE_STALE` before mutation; retry re-observes and secret fills + rerun authorization. Give otherwise identical fields distinct test ids. + This is not atomic coordinate dispatch or proof of rendered stability. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` From 19735a95b8a695d5708c438eaa250189358b0bd4 Mon Sep 17 00:00:00 2001 From: bheemreddy181 <101338506+bheemreddy181@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:45:12 -0500 Subject: [PATCH 2/6] fix(mobile): reject non-input fills before stale retries --- .changeset/fresh-mobile-inputs.md | 2 +- docs/reference/mobile.mdx | 4 +++- packages/mobile/src/surface.ts | 11 ++++++++++- packages/mobile/tests/unit/engine.test.ts | 14 ++++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.changeset/fresh-mobile-inputs.md b/.changeset/fresh-mobile-inputs.md index 3afe445f66..46cfa04a3f 100644 --- a/.changeset/fresh-mobile-inputs.md +++ b/.changeset/fresh-mobile-inputs.md @@ -2,4 +2,4 @@ '@e2e-dev/mobile': patch --- -Revalidate mobile fill and clear targets with a fresh accessibility snapshot before mutation, including cached replay. Changed geometry, identity, or security facts return NODE_STALE so the harness re-observes and reruns secret authorization instead of dispatching an old binding. Coordinate-only backends can still race movement after this check. +Revalidate mobile fill and clear targets with a fresh accessibility snapshot before mutation, including cached replay. Changed geometry, identity, or security facts return NODE_STALE so the harness re-observes and reruns secret authorization instead of dispatching an old binding. Initially non-input targets fail immediately with UNSUPPORTED_CAPABILITY instead of entering stale retries. Coordinate-only backends can still race movement after this check. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index acc46a1d37..897387fd41 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -237,7 +237,9 @@ mobile({ platform: 'ios', transition: 800 }); // default: 500 budget the way a tap does. A control that arrives with the screen they reveal waits out the budget before the next action lands on it. -`fill` and `clear` also take a fresh accessibility snapshot before mutation, +`fill` and `clear` reject an initially non-input role with +`UNSUPPORTED_CAPABILITY`, without waiting or taking another snapshot. +For inputs they take a fresh accessibility snapshot before mutation, including agent actions and cached replay, even after this budget expires. The input must have one matching role, test id, name, text, and placeholder in both snapshots. Changed bounds or disabled, hidden, secure, or password-purpose diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 3646711a2a..1ee6650909 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -236,6 +236,11 @@ function sameRect(a: Rect | undefined, b: Rect | undefined): boolean { return Math.abs(a.x - b.x) < 1 && Math.abs(a.y - b.y) < 1 && Math.abs(a.width - b.width) < 1 && Math.abs(a.height - b.height) < 1; } +/** Roles the mobile tree projects as editable inputs. */ +function isInputRole(role: SemanticNode['role']): boolean { + return role === 'textbox' || role === 'searchbox' || role === 'combobox'; +} + /** The semantic identity a fill must retain across its preflight capture. */ function sameInputIdentity(a: SemanticNode, b: SemanticNode): boolean { return a.role === b.role && a.testId === b.testId && a.name === b.name && a.text === b.text && @@ -1143,7 +1148,7 @@ export class AgentDeviceSurface { private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[]): NodeBinding { const { children: _children, ...node } = entry.node; const control = this.controlOf(entry, snapshot); - const uniqueInput = (node.role === 'textbox' || node.role === 'searchbox' || node.role === 'combobox') && + const uniqueInput = isInputRole(node.role) && snapshot.filter((candidate) => sameInputIdentity(node, candidate.node)).length === 1; return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, uniqueInput }; } @@ -1269,6 +1274,9 @@ export class AgentDeviceSurface { return; } const entry = this.resolveRef(ref); + if ((action.kind === 'fill' || action.kind === 'clear') && !isInputRole(entry.node.role)) { + throw unsupported(`agent-device can only ${action.kind} editable fields; node ${entry.id} is ${entry.node.role ?? 'unknown'}`); + } const label = `perform ${action.kind}`; const client = this.requireClient(); // A toggle already in the wanted state sends nothing, so it neither waits @@ -1280,6 +1288,7 @@ export class AgentDeviceSurface { const run = async (): Promise => { let target = deterministic ? await this.settled(entry, operation) : entry; if (action.kind === 'fill' || action.kind === 'clear') { + // Settling can change security facts; compare with the node the caller authorized. target = await this.freshInput(entry, operation); } switch (action.kind) { diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 90f17c7489..8ef4240066 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2409,6 +2409,20 @@ describe('input freshness', () => { expect(h.fake.lastArgs('interactions.fill')).toMatchObject({ ref: '@e20', text: kind === 'fill' ? 'example' : '' }); expect(h.fake.methods().filter((method) => method === 'interactions.fill')).toHaveLength(1); }); + + it(`rejects an originally non-input target before waiting or capturing for ${origin} ${kind}`, async () => { + const h = harness(); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form(237, { type: 'button' })); + const button = await observed(h, 'Account'); + const captures = h.fake.methods().filter((method) => method === 'capture.snapshot').length; + const startedAt = Date.now(); + const action = kind === 'fill' ? { kind, value: 'example', sensitive: false } : { kind }; + await expect(h.engine.perform!(button.ref, action, { ...operation(), origin })).rejects.toMatchObject({ code: 'UNSUPPORTED_CAPABILITY', retryable: false }); + expect(Date.now()).toBe(startedAt); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(captures); + expect(h.fake.methods()).not.toContain('interactions.fill'); + }); } } From 8d5621364ff36a14ad1055301ba95947e8b11753 Mon Sep 17 00:00:00 2001 From: bheemreddy181 <101338506+bheemreddy181@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:57:52 -0500 Subject: [PATCH 3/6] test(mobile): cover recovery from incomplete input preflights --- docs/reference/mobile.mdx | 4 ++-- packages/mobile/tests/unit/engine.test.ts | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 897387fd41..65634eec0e 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -242,8 +242,8 @@ waits out the budget before the next action lands on it. For inputs they take a fresh accessibility snapshot before mutation, including agent actions and cached replay, even after this budget expires. The input must have one matching role, test id, name, text, and placeholder -in both snapshots. Changed bounds or disabled, hidden, secure, or password-purpose -facts return `NODE_STALE` before input is sent. Missing inputs, duplicate +in both snapshots. Any change to bounds, disabled state, hidden state, secure +state, or password purpose returns `NODE_STALE` before input is sent. Missing inputs, duplicate identities, missing bounds, and incomplete fresh trees also return `NODE_STALE`. The harness re-observes before retrying; an agent secret fill reruns authorization. Use distinct test ids when fields otherwise share an identity. diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 8ef4240066..3d0a2666ee 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2461,6 +2461,23 @@ describe('input freshness', () => { h.fake.respond('capture.snapshot', () => form()); await expect(h.engine.perform!(old.ref, { kind: 'clear' }, operation())).rejects.toMatchObject({ code: 'NODE_STALE' }); expect(h.fake.methods()).not.toContain('interactions.fill'); + const fresh = await observed(h, 'Account'); + await h.engine.perform!(fresh.ref, { kind: 'clear' }, operation()); + expect(h.fake.methods().filter((method) => method === 'interactions.fill')).toHaveLength(1); + }); + + it.each(['ambiguous', 'truncated'] as const)('recovers from a transient %s preflight after re-observation', async (state) => { + const h = harness({ transition: 0 }); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form()); + const old = await observed(h, 'Account'); + h.fake.respond('capture.snapshot', () => state === 'ambiguous' ? form(237, {}, true) : { ...form(), truncated: true }); + await expect(h.engine.perform!(old.ref, { kind: 'clear' }, operation())).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); + expect(h.fake.methods()).not.toContain('interactions.fill'); + h.fake.respond('capture.snapshot', () => form()); + const fresh = await observed(h, 'Account'); + await h.engine.perform!(fresh.ref, { kind: 'clear' }, operation()); + expect(h.fake.methods().filter((method) => method === 'interactions.fill')).toHaveLength(1); }); it('uses the newest backend ref for an unchanged uniquely identified input without taking pixels', async () => { From c6f49aaa9d2ae96c67a52ba5e47cc3c34ca1ad95 Mon Sep 17 00:00:00 2001 From: bheemreddy181 <101338506+bheemreddy181@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:54:56 -0500 Subject: [PATCH 4/6] fix(mobile): relocate inputs once without geometry guards --- .changeset/fresh-mobile-inputs.md | 2 +- docs/reference/mobile.mdx | 16 +--- packages/mobile/src/nodes.ts | 3 +- packages/mobile/src/surface.ts | 65 +++++--------- packages/mobile/tests/unit/engine.test.ts | 99 ++++++++++++++-------- plugins/e2e/skills/e2e/references/setup.md | 8 +- skills/e2e/references/setup.md | 8 +- 7 files changed, 96 insertions(+), 105 deletions(-) diff --git a/.changeset/fresh-mobile-inputs.md b/.changeset/fresh-mobile-inputs.md index 46cfa04a3f..dafb68f8de 100644 --- a/.changeset/fresh-mobile-inputs.md +++ b/.changeset/fresh-mobile-inputs.md @@ -2,4 +2,4 @@ '@e2e-dev/mobile': patch --- -Revalidate mobile fill and clear targets with a fresh accessibility snapshot before mutation, including cached replay. Changed geometry, identity, or security facts return NODE_STALE so the harness re-observes and reruns secret authorization instead of dispatching an old binding. Initially non-input targets fail immediately with UNSUPPORTED_CAPABILITY instead of entering stale retries. Coordinate-only backends can still race movement after this check. +Re-read mobile fill and clear targets before typing, including cached replay. Missing inputs, changed security state, or incomplete captures retry after a fresh observation. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 65634eec0e..e4540c59e0 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -237,19 +237,9 @@ mobile({ platform: 'ios', transition: 800 }); // default: 500 budget the way a tap does. A control that arrives with the screen they reveal waits out the budget before the next action lands on it. -`fill` and `clear` reject an initially non-input role with -`UNSUPPORTED_CAPABILITY`, without waiting or taking another snapshot. -For inputs they take a fresh accessibility snapshot before mutation, -including agent actions and cached replay, even after this budget expires. -The input must have one matching role, test id, name, text, and placeholder -in both snapshots. Any change to bounds, disabled state, hidden state, secure -state, or password purpose returns `NODE_STALE` before input is sent. Missing inputs, duplicate -identities, missing bounds, and incomplete fresh trees also return `NODE_STALE`. -The harness re-observes before retrying; an agent secret fill reruns authorization. -Use distinct test ids when fields otherwise share an identity. - -This checks accessibility evidence, not rendered stability. A coordinate-only -backend can still race movement after the check or receive stale platform bounds. +`fill` and `clear` re-read the field before typing; missing fields, changed +security state, or incomplete captures trigger a fresh-observation retry. +A coordinate-only backend can still race movement after this read. #### `videoTouches` diff --git a/packages/mobile/src/nodes.ts b/packages/mobile/src/nodes.ts index 3ba4cfebf1..7ae8b2b08f 100644 --- a/packages/mobile/src/nodes.ts +++ b/packages/mobile/src/nodes.ts @@ -94,7 +94,8 @@ const ROLE_MAP: Readonly> = { }; const SECURE_KINDS = new Set(['secure-text-field', 'securetextfield', 'password-field']); -const TEXT_INPUT_ROLES = new Set(['textbox', 'searchbox', 'combobox']); +/** Semantic roles that accept text input. */ +export const TEXT_INPUT_ROLES: ReadonlySet = new Set(['textbox', 'searchbox', 'combobox']); const CHECKABLE_ROLES = new Set(['switch', 'checkbox']); const CHECKED_VALUES = new Set(['1', 'on', 'true', 'checked', 'selected']); const UNCHECKED_VALUES = new Set(['0', 'off', 'false', 'unchecked']); diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 1ee6650909..983fedea82 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -52,6 +52,7 @@ import { normalizeKind, projectSnapshot, ROOT_ID, + TEXT_INPUT_ROLES, screenRoot, screenTitle, type ProjectedNode, @@ -196,7 +197,6 @@ interface NodeBinding { readonly ref: string; readonly node: SemanticNode; readonly controlRef: string; - readonly uniqueInput: boolean; } @@ -236,17 +236,6 @@ function sameRect(a: Rect | undefined, b: Rect | undefined): boolean { return Math.abs(a.x - b.x) < 1 && Math.abs(a.y - b.y) < 1 && Math.abs(a.width - b.width) < 1 && Math.abs(a.height - b.height) < 1; } -/** Roles the mobile tree projects as editable inputs. */ -function isInputRole(role: SemanticNode['role']): boolean { - return role === 'textbox' || role === 'searchbox' || role === 'combobox'; -} - -/** The semantic identity a fill must retain across its preflight capture. */ -function sameInputIdentity(a: SemanticNode, b: SemanticNode): boolean { - return a.role === b.role && a.testId === b.testId && a.name === b.name && a.text === b.text && - a.attributes?.placeholder === b.attributes?.placeholder; -} - const MAX_LOCATED_REFS = 2048; interface Attempt { @@ -1148,9 +1137,7 @@ export class AgentDeviceSurface { private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[]): NodeBinding { const { children: _children, ...node } = entry.node; const control = this.controlOf(entry, snapshot); - const uniqueInput = isInputRole(node.role) && - snapshot.filter((candidate) => sameInputIdentity(node, candidate.node)).length === 1; - return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, uniqueInput }; + return { id: entry.id, ref: entry.ref, node, controlRef: control.ref }; } /** @@ -1192,16 +1179,17 @@ export class AgentDeviceSurface { * landed one; the fresh snapshot is where the control landed. Android * reports a window's frames while it slides in, so a ref resolved * mid-flight would send the action where the control was: below the - * screen, for a modal rising from the bottom. + * screen, for a modal rising from the bottom. Input mutations also refresh + * when no transition wait was needed, using this one capture for dispatch. */ - private async settled(entry: NodeBinding, operation: OperationContext): Promise { + private async settled(entry: NodeBinding, operation: OperationContext, refreshInput = false): Promise { const since = Date.now() - this.lastActionAt; const remaining = this.transitionMs - since; - if (remaining <= 0) return entry; + if (remaining <= 0) return refreshInput ? this.relocated(entry, operation, true) : entry; const before = this.indexBeforeAction; if (before !== undefined) { const prior = this.refind(entry, before); - if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return entry; + if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return refreshInput ? this.relocated(entry, operation, true) : entry; } await sleep(remaining, operation.signal); // The sleep resolves on abort; the action behind it must not go out once @@ -1209,36 +1197,22 @@ export class AgentDeviceSurface { if (operation.signal.aborted) { throw new EngineError('CANCELLED', 'transition wait cancelled', { retryable: false }); } - return this.relocated(entry, operation); - } - - /** The same control in a fresh snapshot; the binding as it was when the snapshot no longer lists it. */ - private async relocated(entry: NodeBinding, operation: OperationContext): Promise { - const projected = this.project(await this.snapshotOrEmpty(operation, false)); - const found = this.refind(entry, projected.index); - return found === undefined ? entry : this.bind(found, projected.index); + return this.relocated(entry, operation, refreshInput); } - /** - * Revalidates an input immediately before mutation. A changed binding must - * go through the harness's stale retry so secret authorization runs against - * the new observation, rather than silently following a moved or changed - * field. This narrows snapshot age, not the backend's check/use gap. - */ - private async freshInput(entry: NodeBinding, operation: OperationContext): Promise { + /** The same control in a fresh snapshot; input mutations refuse a missing target or changed security facts. */ + private async relocated(entry: NodeBinding, operation: OperationContext, input = false): Promise { const raw = await this.snapshotOrEmpty(operation, false); const projected = this.project(raw); - const matches = projected.index.filter((candidate) => sameInputIdentity(entry.node, candidate.node)); - const found = matches.length === 1 ? matches[0] : undefined; - if (!entry.uniqueInput || isTruncated(raw) || found === undefined || - !sameRect(entry.node.rect, found.node.rect) || + const found = this.refind(entry, projected.index); + if (input && (isTruncated(raw) || found === undefined || entry.node.inputPurpose !== found.node.inputPurpose || entry.node.states?.disabled !== found.node.states?.disabled || entry.node.states?.secure !== found.node.states?.secure || - entry.node.states?.hidden !== found.node.states?.hidden) { + entry.node.states?.hidden !== found.node.states?.hidden)) { throw new EngineError('NODE_STALE', `input node ${entry.id} changed before mutation; observe and authorize it again`, { retryable: true }); } - return this.bind(found, projected.index); + return found === undefined ? entry : this.bind(found, projected.index); } /** @@ -1274,7 +1248,8 @@ export class AgentDeviceSurface { return; } const entry = this.resolveRef(ref); - if ((action.kind === 'fill' || action.kind === 'clear') && !isInputRole(entry.node.role)) { + const filling = action.kind === 'fill' || action.kind === 'clear'; + if (filling && (entry.node.role === undefined || !TEXT_INPUT_ROLES.has(entry.node.role))) { throw unsupported(`agent-device can only ${action.kind} editable fields; node ${entry.id} is ${entry.node.role ?? 'unknown'}`); } const label = `perform ${action.kind}`; @@ -1286,11 +1261,9 @@ export class AgentDeviceSurface { } const before = this.latestIndex; const run = async (): Promise => { - let target = deterministic ? await this.settled(entry, operation) : entry; - if (action.kind === 'fill' || action.kind === 'clear') { - // Settling can change security facts; compare with the node the caller authorized. - target = await this.freshInput(entry, operation); - } + const target = deterministic + ? await this.settled(entry, operation, filling) + : filling ? await this.relocated(entry, operation, true) : entry; switch (action.kind) { case 'tap': return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle })); diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 3d0a2666ee..377c49ea0c 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2377,7 +2377,7 @@ describe('input freshness', () => { beforeEach(autoAdvanceTimers); /** A uniquely identified field and its neighbor, after a slow accessibility capture. */ - function form(y = 237, changes: Partial = {}, duplicates = false) { + function form(y = 237, changes: Partial = {}) { const field: RawNode = { ref: '@e20', index: 1, parentIndex: 0, depth: 1, type: 'secure-text-field', label: 'Account', identifier: 'account', rect: { x: 40, y, width: 322, height: 49 }, ...changes, @@ -2387,29 +2387,41 @@ describe('input freshness', () => { nodes: [ SETTINGS_NODES[0]!, field, { ref: '@e21', index: 2, parentIndex: 0, depth: 1, type: 'text-field', label: 'Username', identifier: 'username', rect: { x: 40, y: 218, width: 322, height: 49 } }, - ...(duplicates ? [{ ...field, ref: '@e22', index: 3 }] : []), ], }; } for (const origin of ['test', 'agent'] as const) { for (const kind of ['fill', 'clear'] as const) { - it(`rejects moved geometry before ${origin} ${kind}, even when capture consumed the transition budget`, async () => { + it(`uses one fresh binding after a slow capture for ${origin} ${kind}`, async () => { const h = harness(); await openAttempt(h); h.fake.respond('capture.snapshot', async () => { await sleep(600); return form(); }); const old = await observed(h, 'Account'); - h.fake.respond('capture.snapshot', () => form(126)); + const captures = h.fake.methods().filter((method) => method === 'capture.snapshot').length; + h.fake.respond('capture.snapshot', () => form(126, { ref: '@e42' })); const action = kind === 'fill' ? { kind, value: 'example', sensitive: true } : { kind }; - await expect(h.engine.perform!(old.ref, action, { ...operation(), origin })).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); - expect(h.fake.methods()).not.toContain('interactions.fill'); - // The runner's stale retry re-observes before retrying the mutation. - const fresh = await observed(h, 'Account'); - await h.engine.perform!(fresh.ref, action, { ...operation(), origin }); - expect(h.fake.lastArgs('interactions.fill')).toMatchObject({ ref: '@e20', text: kind === 'fill' ? 'example' : '' }); + await h.engine.perform!(old.ref, action, { ...operation(), origin }); + expect(h.fake.lastArgs('interactions.fill')).toMatchObject({ ref: '@e42', text: kind === 'fill' ? 'example' : '' }); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(captures + 1); expect(h.fake.methods().filter((method) => method === 'interactions.fill')).toHaveLength(1); }); + it(`acts on a sliding input with one capture for ${origin} ${kind}`, async () => { + const h = harness(); + await openAttempt(h); + h.fake.respond('capture.snapshot', () => form(2000)); + const sliding = await observed(h, 'Account'); + const captures = h.fake.methods().filter((method) => method === 'capture.snapshot').length; + h.fake.respond('capture.snapshot', () => form(126, { ref: '@e42' })); + const startedAt = Date.now(); + const action = kind === 'fill' ? { kind, value: 'example', sensitive: false } : { kind }; + await h.engine.perform!(sliding.ref, action, { ...operation(), origin }); + expect(h.fake.lastArgs('interactions.fill')).toMatchObject({ ref: '@e42', text: kind === 'fill' ? 'example' : '' }); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(captures + 1); + if (origin === 'test') expect(Date.now() - startedAt).toBeGreaterThanOrEqual(500); + }); + it(`rejects an originally non-input target before waiting or capturing for ${origin} ${kind}`, async () => { const h = harness(); await openAttempt(h); @@ -2426,21 +2438,29 @@ describe('input freshness', () => { } } + it('refreshes an unchanged input even when the transition shortcut skips the wait', async () => { + const h = harness(); + await openAttempt(h); + await h.engine.perform!((await observed(h, 'Back')).ref, { kind: 'tap' }, { ...operation(), origin: 'test' }); + const search = await observed(h, 'Search'); + const captures = h.fake.methods().filter((method) => method === 'capture.snapshot').length; + h.fake.respond('capture.snapshot', () => ({ ...SETTINGS_SNAPSHOT, nodes: SETTINGS_NODES.map((node) => node.label === 'Search' ? { ...node, ref: '@e42' } : node) })); + const startedAt = Date.now(); + await h.engine.perform!(search.ref, { kind: 'clear' }, { ...operation(), origin: 'test' }); + expect(Date.now()).toBe(startedAt); + expect(h.fake.lastArgs('interactions.fill')).toEqual({ ref: '@e42', text: '' }); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(captures + 1); + }); + it.each([ ['missing', () => ({ ...form(), nodes: [SETTINGS_NODES[0]!] })], - ['ambiguous', () => form(237, {}, true)], ['renamed', () => form(237, { label: 'Other account' })], ['different identifier', () => form(237, { identifier: 'other' })], - ['different placeholder', () => form(237, { placeholder: 'Other account' })], ['truncated', () => ({ ...form(), truncated: true })], ['non-editable', () => form(237, { type: 'button' })], ['no longer secure', () => form(237, { type: 'text-field' })], ['disabled', () => form(237, { enabled: false })], ['hidden', () => form(237, { visibleToUser: false })], - ['unmeasurable', () => { - const result = form(); - return { ...result, nodes: result.nodes.map(({ rect: _rect, ...node }) => node) }; - }], ] as const)('rejects a %s target without dispatching or exposing the value', async (_name, snapshot) => { const h = harness({ transition: 0 }); await openAttempt(h); @@ -2453,30 +2473,39 @@ describe('input freshness', () => { expect(h.fake.methods()).not.toContain('interactions.fill'); }); - it('rejects an originally ambiguous identity even if only one duplicate remains', async () => { - const h = harness({ transition: 0 }); - await openAttempt(h); - h.fake.respond('capture.snapshot', () => form(237, {}, true)); - const old = await observed(h, 'Account'); - h.fake.respond('capture.snapshot', () => form()); - await expect(h.engine.perform!(old.ref, { kind: 'clear' }, operation())).rejects.toMatchObject({ code: 'NODE_STALE' }); - expect(h.fake.methods()).not.toContain('interactions.fill'); - const fresh = await observed(h, 'Account'); - await h.engine.perform!(fresh.ref, { kind: 'clear' }, operation()); - expect(h.fake.methods().filter((method) => method === 'interactions.fill')).toHaveLength(1); - }); + for (const kind of ['fill', 'clear'] as const) { + it.each([0, 1])(`can ${kind} identical OTP input %i with the existing nearest-node matching`, async (selected) => { + const h = harness(); + await openAttempt(h); + const digits = (refs: readonly string[], shift: number) => ({ + ...SETTINGS_SNAPSHOT, + nodes: [ + SETTINGS_NODES[0]!, + ...refs.map((ref, index) => ({ ref, index: index + 1, parentIndex: 0, depth: 1, type: 'text-field', label: 'Digit', rect: { x: 40, y: 100 + index * 100 + shift, width: 49, height: 49 } })), + ], + }); + h.fake.respond('capture.snapshot', () => digits(['@e20', '@e21'], 0)); + const snapshot = await h.engine.observe!(operation()); + const target = [...walk(snapshot.root)].filter((node) => node.name === 'Digit')[selected]!; + h.fake.respond('capture.snapshot', () => digits(['@e41', '@e42'], 10)); + const captures = h.fake.methods().filter((method) => method === 'capture.snapshot').length; + const action = kind === 'fill' ? { kind, value: '7', sensitive: false } : { kind }; + await h.engine.perform!(target.ref, action, { ...operation(), origin: 'test' }); + expect(h.fake.lastArgs('interactions.fill')).toMatchObject({ ref: selected === 0 ? '@e41' : '@e42' }); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(captures + 1); + }); + } - it.each(['ambiguous', 'truncated'] as const)('recovers from a transient %s preflight after re-observation', async (state) => { + it('recovers from a truncated capture after re-observation', async () => { const h = harness({ transition: 0 }); await openAttempt(h); h.fake.respond('capture.snapshot', () => form()); const old = await observed(h, 'Account'); - h.fake.respond('capture.snapshot', () => state === 'ambiguous' ? form(237, {}, true) : { ...form(), truncated: true }); + h.fake.respond('capture.snapshot', () => ({ ...form(), truncated: true })); await expect(h.engine.perform!(old.ref, { kind: 'clear' }, operation())).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); expect(h.fake.methods()).not.toContain('interactions.fill'); h.fake.respond('capture.snapshot', () => form()); - const fresh = await observed(h, 'Account'); - await h.engine.perform!(fresh.ref, { kind: 'clear' }, operation()); + await h.engine.perform!((await observed(h, 'Account')).ref, { kind: 'clear' }, operation()); expect(h.fake.methods().filter((method) => method === 'interactions.fill')).toHaveLength(1); }); @@ -2492,14 +2521,16 @@ describe('input freshness', () => { expect(h.fake.lastArgs('capture.snapshot')).toEqual({ interactiveOnly: false }); }); - it('does not silently accept changed security facts from the transition relocation', async () => { + it('does not silently accept changed security facts during a transition', async () => { const h = harness(); await openAttempt(h); h.fake.respond('capture.snapshot', () => form()); const old = await observed(h, 'Account'); - h.fake.respond('capture.snapshot', () => form(237, { enabled: false })); + h.fake.respond('capture.snapshot', () => form(126, { enabled: false })); + const captures = h.fake.methods().filter((method) => method === 'capture.snapshot').length; await expect(h.engine.perform!(old.ref, { kind: 'fill', value: 'example', sensitive: true }, { ...operation(), origin: 'test' })).rejects.toMatchObject({ code: 'NODE_STALE' }); expect(h.fake.methods()).not.toContain('interactions.fill'); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(captures + 1); }); it('does not send input if the preflight consumed the input budget', async () => { diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index 4b03feb5fe..118d3b51e5 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -379,11 +379,9 @@ export default { - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. -- Mobile `fill` and `clear` always preflight a fresh accessibility tree, - replay included. A missing, ambiguous, moved, or security-changed input - returns `NODE_STALE` before mutation; retry re-observes and secret fills - rerun authorization. Give otherwise identical fields distinct test ids. - This is not atomic coordinate dispatch or proof of rendered stability. +- Mobile `fill` and `clear` re-read the field before typing, replay included; + missing fields, changed security state, or incomplete captures trigger a + fresh-observation retry. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 4b03feb5fe..118d3b51e5 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -379,11 +379,9 @@ export default { - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. -- Mobile `fill` and `clear` always preflight a fresh accessibility tree, - replay included. A missing, ambiguous, moved, or security-changed input - returns `NODE_STALE` before mutation; retry re-observes and secret fills - rerun authorization. Give otherwise identical fields distinct test ids. - This is not atomic coordinate dispatch or proof of rendered stability. +- Mobile `fill` and `clear` re-read the field before typing, replay included; + missing fields, changed security state, or incomplete captures trigger a + fresh-observation retry. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` From 2dbda1cb8cefdaa393ff56517dc55037977cd09e Mon Sep 17 00:00:00 2001 From: bheemreddy181 <101338506+bheemreddy181@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:10:10 -0500 Subject: [PATCH 5/6] docs(mobile): clarify stale retries and coordinate limits --- docs/reference/mobile.mdx | 8 +++++--- plugins/e2e/skills/e2e/references/setup.md | 8 +++++--- skills/e2e/references/setup.md | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index e4540c59e0..d65be7c331 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -237,9 +237,11 @@ mobile({ platform: 'ios', transition: 800 }); // default: 500 budget the way a tap does. A control that arrives with the screen they reveal waits out the budget before the next action lands on it. -`fill` and `clear` re-read the field before typing; missing fields, changed -security state, or incomplete captures trigger a fresh-observation retry. -A coordinate-only backend can still race movement after this read. +`fill` and `clear` re-read the field before typing. Missing fields, changed +security state, or incomplete captures return `NODE_STALE` before mutation, +then retry after a fresh observation. Non-input targets return +`UNSUPPORTED_CAPABILITY`. A coordinate-only backend can still race movement +after this read. #### `videoTouches` diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index 118d3b51e5..14f1b48ed7 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -379,9 +379,11 @@ export default { - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. -- Mobile `fill` and `clear` re-read the field before typing, replay included; - missing fields, changed security state, or incomplete captures trigger a - fresh-observation retry. +- Mobile `fill` and `clear` re-read the field before typing, replay included. + Missing fields, changed security state, or incomplete captures return + `NODE_STALE` before mutation, then retry after a fresh observation. + Non-input targets return `UNSUPPORTED_CAPABILITY`. A fresh read does not + make coordinate dispatch atomic. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 118d3b51e5..14f1b48ed7 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -379,9 +379,11 @@ export default { - Only a control that appeared or moved with the previous action waits out `transition` (default 500 ms); agent actions settle `settle` ms (default 150) before the next observation, `settle: false` skips it. -- Mobile `fill` and `clear` re-read the field before typing, replay included; - missing fields, changed security state, or incomplete captures trigger a - fresh-observation retry. +- Mobile `fill` and `clear` re-read the field before typing, replay included. + Missing fields, changed security state, or incomplete captures return + `NODE_STALE` before mutation, then retry after a fresh observation. + Non-input targets return `UNSUPPORTED_CAPABILITY`. A fresh read does not + make coordinate dispatch atomic. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` From 7d710946a1d7175e5742b4167a97dec59aa97f6c Mon Sep 17 00:00:00 2001 From: bheemreddy181 <101338506+bheemreddy181@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:36:57 -0500 Subject: [PATCH 6/6] refactor(mobile): separate transition waits from input refresh --- docs/reference/mobile.mdx | 3 +- packages/mobile/src/surface.ts | 38 +++++++++++----------- packages/mobile/tests/unit/engine.test.ts | 14 ++++++++ plugins/e2e/skills/e2e/references/setup.md | 3 +- skills/e2e/references/setup.md | 3 +- 5 files changed, 36 insertions(+), 25 deletions(-) diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index d65be7c331..bd46e1d9fd 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -240,8 +240,7 @@ waits out the budget before the next action lands on it. `fill` and `clear` re-read the field before typing. Missing fields, changed security state, or incomplete captures return `NODE_STALE` before mutation, then retry after a fresh observation. Non-input targets return -`UNSUPPORTED_CAPABILITY`. A coordinate-only backend can still race movement -after this read. +`UNSUPPORTED_CAPABILITY`. #### `videoTouches` diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 983fedea82..af2c7a5d15 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -1169,27 +1169,17 @@ export class AgentDeviceSurface { } /** - * Lets a control that came with the last action finish arriving before a - * test acts on it, and answers the binding to act on. A control already - * present at the same place in the snapshot the last action was resolved - * from is not in transition and is acted on at once; anything else waits - * out the remainder of the transition budget since that action and is then - * found again in a fresh snapshot. The budget is the only signal that the - * transition is over, since the tree cannot tell a sliding control from a - * landed one; the fresh snapshot is where the control landed. Android - * reports a window's frames while it slides in, so a ref resolved - * mid-flight would send the action where the control was: below the - * screen, for a modal rising from the bottom. Input mutations also refresh - * when no transition wait was needed, using this one capture for dispatch. + * Waits for a control that arrived or moved with the last action, returning + * whether it waited. Controls already present at the same place need no wait. */ - private async settled(entry: NodeBinding, operation: OperationContext, refreshInput = false): Promise { + private async waitForTransition(entry: NodeBinding, operation: OperationContext): Promise { const since = Date.now() - this.lastActionAt; const remaining = this.transitionMs - since; - if (remaining <= 0) return refreshInput ? this.relocated(entry, operation, true) : entry; + if (remaining <= 0) return false; const before = this.indexBeforeAction; if (before !== undefined) { const prior = this.refind(entry, before); - if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return refreshInput ? this.relocated(entry, operation, true) : entry; + if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return false; } await sleep(remaining, operation.signal); // The sleep resolves on abort; the action behind it must not go out once @@ -1197,7 +1187,13 @@ export class AgentDeviceSurface { if (operation.signal.aborted) { throw new EngineError('CANCELLED', 'transition wait cancelled', { retryable: false }); } - return this.relocated(entry, operation, refreshInput); + return true; + } + + /** Refreshes a control after a transition wait, keeping the binding when no wait was needed. */ + private async settled(entry: NodeBinding, operation: OperationContext): Promise { + if (await this.waitForTransition(entry, operation)) return this.relocated(entry, operation); + return entry; } /** The same control in a fresh snapshot; input mutations refuse a missing target or changed security facts. */ @@ -1261,9 +1257,13 @@ export class AgentDeviceSurface { } const before = this.latestIndex; const run = async (): Promise => { - const target = deterministic - ? await this.settled(entry, operation, filling) - : filling ? await this.relocated(entry, operation, true) : entry; + let target = entry; + if (filling) { + if (deterministic) await this.waitForTransition(entry, operation); + target = await this.relocated(entry, operation, true); + } else if (deterministic) { + target = await this.settled(entry, operation); + } switch (action.kind) { case 'tap': return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle })); diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 377c49ea0c..3e685ef9e6 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2543,6 +2543,20 @@ describe('input freshness', () => { expect(h.fake.methods()).not.toContain('interactions.fill'); }); + it.each(['fill', 'clear'] as const)('does not capture or send %s after its transition wait is cancelled', async (kind) => { + const h = harness({ transition: 300 }); + await openAttempt(h); + const search = await observed(h, 'Search'); + const captures = h.fake.methods().filter((method) => method === 'capture.snapshot').length; + const controller = new AbortController(); + const action = kind === 'fill' ? { kind, value: 'example', sensitive: false } : { kind }; + const pending = h.engine.perform!(search.ref, action, { ...operation(controller.signal), origin: 'test' }); + setTimeout(() => controller.abort(), 20); + await expect(pending).rejects.toMatchObject({ code: 'CANCELLED' }); + expect(h.fake.methods().filter((method) => method === 'capture.snapshot')).toHaveLength(captures); + expect(h.fake.methods()).not.toContain('interactions.fill'); + }); + it('does not dispatch when aborted during the preflight capture', async () => { const h = harness({ transition: 0 }); await openAttempt(h); diff --git a/plugins/e2e/skills/e2e/references/setup.md b/plugins/e2e/skills/e2e/references/setup.md index 14f1b48ed7..1011c10de9 100644 --- a/plugins/e2e/skills/e2e/references/setup.md +++ b/plugins/e2e/skills/e2e/references/setup.md @@ -382,8 +382,7 @@ export default { - Mobile `fill` and `clear` re-read the field before typing, replay included. Missing fields, changed security state, or incomplete captures return `NODE_STALE` before mutation, then retry after a fresh observation. - Non-input targets return `UNSUPPORTED_CAPABILITY`. A fresh read does not - make coordinate dispatch atomic. + Non-input targets return `UNSUPPORTED_CAPABILITY`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold` diff --git a/skills/e2e/references/setup.md b/skills/e2e/references/setup.md index 14f1b48ed7..1011c10de9 100644 --- a/skills/e2e/references/setup.md +++ b/skills/e2e/references/setup.md @@ -382,8 +382,7 @@ export default { - Mobile `fill` and `clear` re-read the field before typing, replay included. Missing fields, changed security state, or incomplete captures return `NODE_STALE` before mutation, then retry after a fresh observation. - Non-input targets return `UNSUPPORTED_CAPABILITY`. A fresh read does not - make coordinate dispatch atomic. + Non-input targets return `UNSUPPORTED_CAPABILITY`. - `screen`, `expect`, `app`, `agent` work unchanged; `test` from `@e2e-dev/mobile` types the `device` fixture (`installApp`, `openLink`, `setPermission`, `setNetwork`, `setAppearance`, `clearKeychain`, `fold`