From 8c8fc99f513bab4d1756f371855b6317451647c6 Mon Sep 17 00:00:00 2001 From: Samanyu Das Date: Wed, 28 Oct 2026 02:20:57 +1030 Subject: [PATCH 1/5] test(mobile): pin ref actions to their snapshot generation Fails on main: taps send plain @eN refs, so agent-device cannot refuse a ref from a superseded capture (#1406). Co-Authored-By: Claude Opus 5.5 --- packages/mobile/tests/unit/engine.test.ts | 65 +++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index a3aee922db..339a8e68ac 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2227,6 +2227,71 @@ describe('reference lifetime and cancellation', () => { }); }); +describe('snapshot generations', () => { + /** + * agent-device's ref admission: every capture starts a new generation, and + * a ref pinned with `~s` is refused unless it names the newest. + */ + function generations(h: Harness): void { + let generation = 6; + h.fake.respond('capture.snapshot', () => { + generation += 1; + return { ...SETTINGS_SNAPSHOT, refsGeneration: generation }; + }); + const admit = (ref: string): void => { + const pinned = /~s(\d+)$/.exec(ref); + if (pinned !== null && Number(pinned[1]) !== generation) { + throw new AppError('COMMAND_FAILED', `Ref ${ref} was minted from a superseded snapshot generation`, { reason: 'ref_generation_mismatch', dispatched: 'no' }); + } + }; + h.fake.respond('interactions.press', (args) => admit((args as { ref: string }).ref)); + h.fake.respond('interactions.drag', (args) => { + const { source, destination } = args as { source: string; destination: string }; + admit(source); + admit(destination); + }); + } + + it('pins every ref action to the capture that issued its ref', async () => { + const h = harness(); + await openAttempt(h); + generations(h); + const op = operation(); + const { root } = await h.engine.observe!(op); + const [about, toggle, back] = ['About', 'Airplane Mode', 'Back'].map((name) => named(root, name)); + const before = h.fake.calls.length; + await h.engine.perform!(about!.ref, { kind: 'tap' }, op); + await h.engine.perform!(toggle!.ref, { kind: 'check' }, op); + await h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, op); + expect(h.fake.calls.slice(before).map((call) => [call.method, call.args])).toEqual([ + ['interactions.press', { ref: '@e4~s7', settle: true, settleQuietMs: 150 }], + ['interactions.press', { ref: '@e6~s7', settle: true, settleQuietMs: 150 }], + ['interactions.drag', { source: '@e4~s7', destination: '@e3~s7' }], + ]); + }); + + it('refuses a ref from a superseded capture as stale instead of acting on the newer one', async () => { + const h = harness(); + await openAttempt(h); + generations(h); + const about = await observed(h, 'About'); + await h.engine.locate!(testIdQuery('BackButton'), operation()); + await expect(h.engine.perform!(about.ref, { kind: 'tap' }, operation())).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); + expect(h.fake.lastArgs('interactions.press')).toMatchObject({ ref: '@e4~s7' }); + }); + + it('drags between nodes located in separate captures from the newest one', async () => { + const h = harness(); + await openAttempt(h); + generations(h); + const step: OperationContext = { ...operation(), origin: 'test' }; + const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); + const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); + await h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step); + expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s8', destination: '@e3~s8' }); + }); +}); + describe('video', () => { const recorder = (h: Harness, finalizeAt?: string) => { let requested: string | undefined; From dbcd6a1ff8fc4a50cd1a23e5c891fdcd1fdce10f Mon Sep 17 00:00:00 2001 From: Samanyu Das Date: Wed, 28 Oct 2026 02:24:39 +1030 Subject: [PATCH 2/5] fix(mobile): pin each action's ref to the snapshot it came from agent-device reports the capture its refs were issued from as refsGeneration. The engine dropped it and sent plain @eN, which the daemon resolves against its newest snapshot, so a ref from an older capture could tap a different element. Each binding now keeps its capture's generation and every ref action sends @eN~sG, so the daemon refuses a superseded ref (ref_generation_mismatch, already NODE_STALE) instead. A test's dragTo locates its drop target in a second capture after the source's, so both ends are found again in the newest capture before the drag. A daemon that reports no generation still gets plain refs. Fixes #1406 Co-Authored-By: Claude Opus 5.5 --- .changeset/mobile-ref-generation.md | 5 +++ docs/reference/mobile.mdx | 2 +- packages/mobile/src/surface.ts | 44 +++++++++++++++++------ packages/mobile/tests/unit/engine.test.ts | 3 +- 4 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 .changeset/mobile-ref-generation.md diff --git a/.changeset/mobile-ref-generation.md b/.changeset/mobile-ref-generation.md new file mode 100644 index 0000000000..b12a7537f2 --- /dev/null +++ b/.changeset/mobile-ref-generation.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/mobile": patch +--- + +Mobile actions name the snapshot each node came from (`@e4~s7`), so agent-device refuses a node that a newer snapshot replaced instead of tapping whatever holds its ref now. The action finds the node again, as it does for any stale node. A `dragTo` sends both nodes from the newest snapshot. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 8a6b44e464..592cc2ef7d 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -349,7 +349,7 @@ running on the device. The next attempt waits for it to settle. | Contract member | On a device | | --- | --- | | `observe` | The accessibility tree with rects. iOS: `Button` is `button`, `TextField` and `SearchField` are `textbox`, `SecureTextField` is a secure `textbox`, `Cell` is `listitem`, `StaticText` is `text`, `Switch` is `switch` with `checked`, `TabBar` and `SegmentedControl` are `tablist` and the buttons inside them `tab`, `ProgressIndicator` is `progressbar`, `ActivityIndicator` is `status`, `Stepper` is `spinbutton`, `Toolbar`, `Menu`, and `RadioGroup` are `toolbar`, `menu`, and `radiogroup`. Android: `TextView` is `text`, `EditText` is `textbox` and one UIAutomator flags as `password` is a secure `textbox` like `SecureTextField`, `Switch` and `CheckBox` keep their roles and `RadioButton` is `radio`, each with the `checked` state the device reports, layouts are `group`, `RecyclerView` and `AbsListView` (React Native's `FlatList`) are `list`, `TabLayout` and `BottomNavigationView` are `tablist` with their items as `tab`, `ProgressBar` is `progressbar`, `NumberPicker` is `spinbutton`, `Toolbar` and `RadioGroup` keep their names. Any other type is kept verbatim in lower case (`picker-wheel`), so it answers no vocabulary role, unless the platform reports it editable, which makes it a `textbox`. React Native on iOS has no trait for `checkbox`, `radio`, and a few other roles and spells the role and state into the accessibility value instead (`checkbox, unchecked`, `radio button, checked`); such a node takes that role, `checked` and `expanded` from the state words, and reports only what follows them as its value. An iOS field keeps a value equal to its label (an unlabeled field is named by its text); Android's echo of a text view's label as its value is dropped. The accessibility identifier (iOS) or resource id (Android) is the node's `testId`. An Android node is named by the content description its app set (React Native's `accessibilityLabel`), else by its text, and its `text` is what it shows, so a labeled text view or a filled field answers `getByLabel` by its label and `getByText` by its text; a view flagged as a heading is a `heading`, and the role description React Native writes for a role Android has no class for (`Tab`, `Tab List`, `Radio Group`) gives the node that role. A field's hint is its `placeholder` attribute, showing or not. An empty field has no value: Android flags a showing hint, and XCTest reports an empty iOS field's placeholder as its value, so an iOS field whose value equals its placeholder (trimmed) reads empty, text typed to match the placeholder exactly included. One Android fact is agent-device's, not the platform's: `rect` and every point are the screen's physical pixels; the display density it reports is not applied yet. Android's status bar and navigation bar, the full-width `com.android.systemui` windows along the screen's edges and the narrow status icons (clock, signal, battery) agent-device reports on their own inside that edge band, are left out; the notification shade, system dialogs, and a heads-up notification or other popup inset from the sides stay. Pixels on request, with every secure node's bounds painted black. | -| `perform` | `tap`, `doubleTap`, `longPress`, `fill`, `clear`, `hover`, `check`, `uncheck`, `press` (`Enter`, `Space`, or one character), `swipe`, `dragTo`, and `focus` on editable fields. A tap on a `switch` or `checkbox` lands on its innermost control, since UIKit reports a settings row as one labelled switch with the real toggle as a child. `check` and `uncheck` need a `checked` state, which Android switches lack and a React Native checkbox or radio on iOS has. `doubleTap` is two presses, 284 to 285 ms apart on iOS (the cadence of agent-device's XCTest tap): agent-device's own double-tap gesture reaches a React Native `Pressable` as one press. A detector with a window of 300 ms or less can read the pair as two single taps. `longPress` holds for one second unless the test passes `duration`, past a React Native `Pressable`'s `delayLongPress` of 500 ms and the longer thresholds apps set; agent-device's own default hold is shorter and registers as a tap on those. `selectOption`, `setInputFiles`, `scrollIntoView`, and `secondaryTap` are not declared, and neither is `tapModifiers`, so `tap` or `doubleTap` with `modifiers` is `UNSUPPORTED_CAPABILITY`. | +| `perform` | `tap`, `doubleTap`, `longPress`, `fill`, `clear`, `hover`, `check`, `uncheck`, `press` (`Enter`, `Space`, or one character), `swipe`, `dragTo`, and `focus` on editable fields. A tap on a `switch` or `checkbox` lands on its innermost control, since UIKit reports a settings row as one labelled switch with the real toggle as a child. `check` and `uncheck` need a `checked` state, which Android switches lack and a React Native checkbox or radio on iOS has. `doubleTap` is two presses, 284 to 285 ms apart on iOS (the cadence of agent-device's XCTest tap): agent-device's own double-tap gesture reaches a React Native `Pressable` as one press. A detector with a window of 300 ms or less can read the pair as two single taps. `longPress` holds for one second unless the test passes `duration`, past a React Native `Pressable`'s `delayLongPress` of 500 ms and the longer thresholds apps set; agent-device's own default hold is shorter and registers as a tap on those. `selectOption`, `setInputFiles`, `scrollIntoView`, and `secondaryTap` are not declared, and neither is `tapModifiers`, so `tap` or `doubleTap` with `modifiers` is `UNSUPPORTED_CAPABILITY`. Each action names the snapshot its node came from (`@e4~s7`), so agent-device refuses a node that a newer snapshot replaced instead of acting on whatever holds its ref now, and the action finds it again as `NODE_STALE`. A `dragTo` sends both nodes from the newest snapshot. | | `performAt` | `tap`, `doubleTap`, `longPress`, and `swipeTo` at a bare screen point, so `screen.tapAt`, `tap({ position })`, `screen.swipe({ from, to })`, and the agent's `tap_at` reach a control the tree does not list. No `hover`, `secondaryTap`, or `dragTo` at a point: touch has none without a node, so `hover_at` needs a listed control. | | `keyboard` | `type` and `press` into the focused field, the agent's `type` and `press` without a target and the fallback of `type_at` and `press_at`, and `dismiss`, the agent's `dismiss_keyboard`. | | `locate` | Every `screen` query. Text and label queries answer with the innermost match, since a device echoes both up the tree (a React Native input is a host view and the field inside it, both labelled). On Android a test id query, a string or a RegExp, also matches a resource id without its `:id/` prefix, for any package but the framework's `android`, when nothing on the screen matches the full id. `screen.locator('id=…')` is agent-device's selector and takes the full id. | diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 35e81b2a29..73bb5dfc95 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -130,9 +130,10 @@ export interface OpenAppOptions { /** * The snapshot fields this engine reads off agent-device's response. `keyboard` * is the band the capture's producer measured: `visible` with its frame, - * `absent`, or that it could not look. + * `absent`, or that it could not look. `refsGeneration` is the capture its + * refs were issued from. */ -type RawSnapshot = Partial>; +type RawSnapshot = Partial>; /** Platform element types that are the soft keyboard or one of its keys, as agent-device names them. */ const KEYBOARD_TYPES: ReadonlySet = new Set(['keyboard', 'key']); @@ -196,6 +197,8 @@ interface NodeBinding { readonly ref: string; readonly node: SemanticNode; readonly controlRef: string; + /** The capture `ref` and `controlRef` came from, pinned on every action so agent-device refuses them once a newer capture supersedes it. */ + readonly refsGeneration: number | undefined; } @@ -422,6 +425,8 @@ export class AgentDeviceSurface { private indexBeforeAction: readonly ProjectedNode[] | undefined; /** The most recent projection of the screen, from any observe or locate. */ private latestIndex: readonly ProjectedNode[] | undefined; + /** The capture generation of `latestIndex`. */ + private latestGeneration: number | undefined; /** Budget a control that came with the last action gets to finish arriving; see DEFAULT_TRANSITION_MS. */ private readonly transitionMs: number; /** @@ -994,6 +999,7 @@ export class AgentDeviceSurface { viewport: raw.viewport, }); this.latestIndex = projected.index; + this.latestGeneration = raw.refsGeneration; const read = projected.viewport; if (read !== undefined && (this.knownViewport === undefined || sameScreen(read, this.knownViewport))) this.knownViewport = read; return projected; @@ -1045,7 +1051,7 @@ export class AgentDeviceSurface { async observe(operation: OperationContext, options?: EngineObserveOptions): Promise { const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive'); const projected = this.project(raw); - this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index)])); + this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index, raw.refsGeneration)])); const viewport = await this.viewportFor(projected, operation.signal); const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected)); const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined; @@ -1067,7 +1073,7 @@ export class AgentDeviceSurface { const projected = this.project(raw); this.traceScreen(raw, projected); const matches = resolveExpression(expression, projected.index, this.options.platform); - for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected.index)); + for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected.index, raw.refsGeneration)); for (const oldest of this.located.keys()) { if (this.located.size <= MAX_LOCATED_REFS) break; this.located.delete(oldest); @@ -1106,7 +1112,7 @@ export class AgentDeviceSurface { private actionTarget(entry: NodeBinding, control = false): { ref: string } { const ref = control ? entry.controlRef : entry.ref; if (ref === '') throw notActionable(`node ${entry.id} has no agent-device ref to act on`); - return { ref: `@${ref}` }; + return { ref: entry.refsGeneration === undefined ? `@${ref}` : `@${ref}~s${entry.refsGeneration}` }; } /** @@ -1133,10 +1139,22 @@ export class AgentDeviceSurface { } /** Copies the fields actions use and pre-resolves a toggle's inner control. */ - private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[]): NodeBinding { + private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[], refsGeneration: number | undefined): NodeBinding { const { children: _children, ...node } = entry.node; const control = this.controlOf(entry, snapshot); - return { id: entry.id, ref: entry.ref, node, controlRef: control.ref }; + return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, refsGeneration }; + } + + /** + * The same control in the newest capture: the binding itself when it came + * from that capture, else found there again, else as it was, for + * agent-device to refuse as stale. + */ + private current(entry: NodeBinding): NodeBinding { + const index = this.latestIndex; + if (index === undefined || entry.refsGeneration === this.latestGeneration) return entry; + const found = this.refind(entry, index); + return found === undefined ? entry : this.bind(found, index, this.latestGeneration); } /** @@ -1200,9 +1218,10 @@ export class AgentDeviceSurface { /** The same control in a fresh snapshot; the binding as it was when the snapshot no longer lists it. */ private async relocated(entry: NodeBinding, operation: OperationContext): Promise { - const projected = this.project(await this.snapshotOrEmpty(operation, false)); + const raw = await this.snapshotOrEmpty(operation, false); + const projected = this.project(raw); const found = this.refind(entry, projected.index); - return found === undefined ? entry : this.bind(found, projected.index); + return found === undefined ? entry : this.bind(found, projected.index, raw.refsGeneration); } /** @@ -1293,10 +1312,13 @@ export class AgentDeviceSurface { return input.send(() => client.interactions.swipe(swipeWithin(rect, action.direction, action.momentum))); } case 'dragTo': { - const destination = this.resolveRef(action.target); + // A test locates the drop target in a capture of its own, after the + // source's; agent-device takes both refs from the newest one only. + const source = this.current(target); + const destination = this.current(this.resolveRef(action.target)); return input.send(() => client.interactions.drag({ - source: this.actionTarget(target).ref, + source: this.actionTarget(source).ref, destination: this.actionTarget(destination).ref, }), ); diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 339a8e68ac..495b27f68f 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2288,7 +2288,8 @@ describe('snapshot generations', () => { const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); await h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step); - expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s8', destination: '@e3~s8' }); + // The launch leaves the source in transition, so it is found again in a third capture. + expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s9', destination: '@e3~s9' }); }); }); From da77b8ebe15f4912d9faf99df1d89f222d60759e Mon Sep 17 00:00:00 2001 From: Samanyu Das Date: Wed, 28 Oct 2026 02:30:26 +1030 Subject: [PATCH 3/5] fix(mobile): re-find a drag end only where it was current() matched a drag end by role, test id, name, and text, then took the closest candidate, so a source that disappeared between the two locates could be swapped for an identical sibling. It now takes the re-found node only at the same rect and otherwise keeps the old pin for agent-device to refuse as stale. Co-Authored-By: Claude Opus 5.5 --- packages/mobile/src/surface.ts | 6 +++--- packages/mobile/tests/unit/engine.test.ts | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 73bb5dfc95..250c74350d 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -1147,14 +1147,14 @@ export class AgentDeviceSurface { /** * The same control in the newest capture: the binding itself when it came - * from that capture, else found there again, else as it was, for - * agent-device to refuse as stale. + * from that capture, else found there again where it was, else as it was, + * for agent-device to refuse as stale. */ private current(entry: NodeBinding): NodeBinding { const index = this.latestIndex; if (index === undefined || entry.refsGeneration === this.latestGeneration) return entry; const found = this.refind(entry, index); - return found === undefined ? entry : this.bind(found, index, this.latestGeneration); + return found === undefined || !sameRect(found.node.rect, entry.node.rect) ? entry : this.bind(found, index, this.latestGeneration); } /** diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 495b27f68f..4f55d5b423 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2232,11 +2232,11 @@ describe('snapshot generations', () => { * agent-device's ref admission: every capture starts a new generation, and * a ref pinned with `~s` is refused unless it names the newest. */ - function generations(h: Harness): void { + function generations(h: Harness, nodes: () => readonly RawNode[] = () => SETTINGS_NODES): void { let generation = 6; h.fake.respond('capture.snapshot', () => { generation += 1; - return { ...SETTINGS_SNAPSHOT, refsGeneration: generation }; + return { ...SETTINGS_SNAPSHOT, nodes: nodes(), refsGeneration: generation }; }); const admit = (ref: string): void => { const pinned = /~s(\d+)$/.exec(ref); @@ -2291,6 +2291,20 @@ describe('snapshot generations', () => { // The launch leaves the source in transition, so it is found again in a third capture. expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s9', destination: '@e3~s9' }); }); + + it('keeps a drag source pinned to its own capture when a newer one lists it nowhere it was', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + const twin: RawNode = { ref: '@e11', index: 11, parentIndex: 0, depth: 1, type: 'cell', label: 'About', identifier: 'ABOUT', rect: { x: 0, y: 600, width: 390, height: 44 } }; + let nodes: readonly RawNode[] = [...SETTINGS_NODES, twin]; + generations(h, () => nodes); + const step: OperationContext = { ...operation(), origin: 'test' }; + const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); + nodes = [...SETTINGS_NODES.filter((node) => node.label !== 'About'), twin]; + const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); + await expect(h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step)).rejects.toMatchObject({ code: 'NODE_STALE' }); + expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s7', destination: '@e3~s8' }); + }); }); describe('video', () => { From 8aa5753290c823c8968747f876143e97b0921352 Mon Sep 17 00:00:00 2001 From: Samanyu Das Date: Wed, 28 Oct 2026 02:49:17 +1030 Subject: [PATCH 4/5] fix(mobile): refuse a moved drag end before sending it Review follow-up. current() now throws a retryable NODE_STALE when a drag end is gone or moved, instead of sending a ref agent-device will refuse. refsGeneration lives on ProjectedSnapshot, so one latest projection replaces the two parallel fields and bind() reads it from the projection. samePlace() holds the refind-at-the-same-rect rule current() and settled() share. Tests cover the drag with no transition, a tap found again after a transition, and a vanished source. Docs and changeset say what users see. Co-Authored-By: Claude Opus 5.5 --- .changeset/mobile-ref-generation.md | 2 +- docs/reference/mobile.mdx | 2 +- packages/mobile/src/nodes.ts | 6 ++- packages/mobile/src/surface.ts | 65 +++++++++++------------ packages/mobile/tests/unit/engine.test.ts | 27 ++++++++-- 5 files changed, 62 insertions(+), 40 deletions(-) diff --git a/.changeset/mobile-ref-generation.md b/.changeset/mobile-ref-generation.md index b12a7537f2..a361bee6ed 100644 --- a/.changeset/mobile-ref-generation.md +++ b/.changeset/mobile-ref-generation.md @@ -2,4 +2,4 @@ "@e2e-dev/mobile": patch --- -Mobile actions name the snapshot each node came from (`@e4~s7`), so agent-device refuses a node that a newer snapshot replaced instead of tapping whatever holds its ref now. The action finds the node again, as it does for any stale node. A `dragTo` sends both nodes from the newest snapshot. +A mobile action on a node that a newer snapshot replaced finds the node again instead of acting on a different element. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index 592cc2ef7d..2a9a42cdbc 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -349,7 +349,7 @@ running on the device. The next attempt waits for it to settle. | Contract member | On a device | | --- | --- | | `observe` | The accessibility tree with rects. iOS: `Button` is `button`, `TextField` and `SearchField` are `textbox`, `SecureTextField` is a secure `textbox`, `Cell` is `listitem`, `StaticText` is `text`, `Switch` is `switch` with `checked`, `TabBar` and `SegmentedControl` are `tablist` and the buttons inside them `tab`, `ProgressIndicator` is `progressbar`, `ActivityIndicator` is `status`, `Stepper` is `spinbutton`, `Toolbar`, `Menu`, and `RadioGroup` are `toolbar`, `menu`, and `radiogroup`. Android: `TextView` is `text`, `EditText` is `textbox` and one UIAutomator flags as `password` is a secure `textbox` like `SecureTextField`, `Switch` and `CheckBox` keep their roles and `RadioButton` is `radio`, each with the `checked` state the device reports, layouts are `group`, `RecyclerView` and `AbsListView` (React Native's `FlatList`) are `list`, `TabLayout` and `BottomNavigationView` are `tablist` with their items as `tab`, `ProgressBar` is `progressbar`, `NumberPicker` is `spinbutton`, `Toolbar` and `RadioGroup` keep their names. Any other type is kept verbatim in lower case (`picker-wheel`), so it answers no vocabulary role, unless the platform reports it editable, which makes it a `textbox`. React Native on iOS has no trait for `checkbox`, `radio`, and a few other roles and spells the role and state into the accessibility value instead (`checkbox, unchecked`, `radio button, checked`); such a node takes that role, `checked` and `expanded` from the state words, and reports only what follows them as its value. An iOS field keeps a value equal to its label (an unlabeled field is named by its text); Android's echo of a text view's label as its value is dropped. The accessibility identifier (iOS) or resource id (Android) is the node's `testId`. An Android node is named by the content description its app set (React Native's `accessibilityLabel`), else by its text, and its `text` is what it shows, so a labeled text view or a filled field answers `getByLabel` by its label and `getByText` by its text; a view flagged as a heading is a `heading`, and the role description React Native writes for a role Android has no class for (`Tab`, `Tab List`, `Radio Group`) gives the node that role. A field's hint is its `placeholder` attribute, showing or not. An empty field has no value: Android flags a showing hint, and XCTest reports an empty iOS field's placeholder as its value, so an iOS field whose value equals its placeholder (trimmed) reads empty, text typed to match the placeholder exactly included. One Android fact is agent-device's, not the platform's: `rect` and every point are the screen's physical pixels; the display density it reports is not applied yet. Android's status bar and navigation bar, the full-width `com.android.systemui` windows along the screen's edges and the narrow status icons (clock, signal, battery) agent-device reports on their own inside that edge band, are left out; the notification shade, system dialogs, and a heads-up notification or other popup inset from the sides stay. Pixels on request, with every secure node's bounds painted black. | -| `perform` | `tap`, `doubleTap`, `longPress`, `fill`, `clear`, `hover`, `check`, `uncheck`, `press` (`Enter`, `Space`, or one character), `swipe`, `dragTo`, and `focus` on editable fields. A tap on a `switch` or `checkbox` lands on its innermost control, since UIKit reports a settings row as one labelled switch with the real toggle as a child. `check` and `uncheck` need a `checked` state, which Android switches lack and a React Native checkbox or radio on iOS has. `doubleTap` is two presses, 284 to 285 ms apart on iOS (the cadence of agent-device's XCTest tap): agent-device's own double-tap gesture reaches a React Native `Pressable` as one press. A detector with a window of 300 ms or less can read the pair as two single taps. `longPress` holds for one second unless the test passes `duration`, past a React Native `Pressable`'s `delayLongPress` of 500 ms and the longer thresholds apps set; agent-device's own default hold is shorter and registers as a tap on those. `selectOption`, `setInputFiles`, `scrollIntoView`, and `secondaryTap` are not declared, and neither is `tapModifiers`, so `tap` or `doubleTap` with `modifiers` is `UNSUPPORTED_CAPABILITY`. Each action names the snapshot its node came from (`@e4~s7`), so agent-device refuses a node that a newer snapshot replaced instead of acting on whatever holds its ref now, and the action finds it again as `NODE_STALE`. A `dragTo` sends both nodes from the newest snapshot. | +| `perform` | `tap`, `doubleTap`, `longPress`, `fill`, `clear`, `hover`, `check`, `uncheck`, `press` (`Enter`, `Space`, or one character), `swipe`, `dragTo`, and `focus` on editable fields. A tap on a `switch` or `checkbox` lands on its innermost control, since UIKit reports a settings row as one labelled switch with the real toggle as a child. `check` and `uncheck` need a `checked` state, which Android switches lack and a React Native checkbox or radio on iOS has. `doubleTap` is two presses, 284 to 285 ms apart on iOS (the cadence of agent-device's XCTest tap): agent-device's own double-tap gesture reaches a React Native `Pressable` as one press. A detector with a window of 300 ms or less can read the pair as two single taps. `longPress` holds for one second unless the test passes `duration`, past a React Native `Pressable`'s `delayLongPress` of 500 ms and the longer thresholds apps set; agent-device's own default hold is shorter and registers as a tap on those. `selectOption`, `setInputFiles`, `scrollIntoView`, and `secondaryTap` are not declared, and neither is `tapModifiers`, so `tap` or `doubleTap` with `modifiers` is `UNSUPPORTED_CAPABILITY`. An action on a node that a newer snapshot replaced finds the node again instead of acting on a different element. | | `performAt` | `tap`, `doubleTap`, `longPress`, and `swipeTo` at a bare screen point, so `screen.tapAt`, `tap({ position })`, `screen.swipe({ from, to })`, and the agent's `tap_at` reach a control the tree does not list. No `hover`, `secondaryTap`, or `dragTo` at a point: touch has none without a node, so `hover_at` needs a listed control. | | `keyboard` | `type` and `press` into the focused field, the agent's `type` and `press` without a target and the fallback of `type_at` and `press_at`, and `dismiss`, the agent's `dismiss_keyboard`. | | `locate` | Every `screen` query. Text and label queries answer with the innermost match, since a device echoes both up the tree (a React Native input is a host view and the field inside it, both labelled). On Android a test id query, a string or a RegExp, also matches a resource id without its `:id/` prefix, for any package but the framework's `android`, when nothing on the screen matches the full id. `screen.locator('id=…')` is agent-device's selector and takes the full id. | diff --git a/packages/mobile/src/nodes.ts b/packages/mobile/src/nodes.ts index 3ba4cfebf1..e9de1cfe07 100644 --- a/packages/mobile/src/nodes.ts +++ b/packages/mobile/src/nodes.ts @@ -35,6 +35,8 @@ export interface ProjectedSnapshot { /** Every node in document order, parents before children. */ readonly index: readonly ProjectedNode[]; readonly viewport: ViewportSize | undefined; + /** The agent-device capture the refs were issued from; undefined from one that reports none. */ + readonly refsGeneration: number | undefined; } /** @@ -406,7 +408,7 @@ function isSystemBar(node: RawNode, screen: ViewportSize | undefined, displayOnl */ export function projectSnapshot( raw: readonly RawNode[], - options: { readonly mintId: () => string; readonly viewport?: ViewportSize | undefined }, + options: { readonly mintId: () => string; readonly viewport?: ViewportSize | undefined; readonly refsGeneration?: number | undefined }, ): ProjectedSnapshot { const viewport = options.viewport ?? viewportOf(raw); const parents = parentPositions(raw); @@ -506,7 +508,7 @@ export function projectSnapshot( const rootNodes = roots .filter((position) => !isSystemBar(raw[position] as RawNode, viewport, () => displayOnly(position))) .map((position) => build(position, undefined)); - return { roots: rootNodes, index, viewport }; + return { roots: rootNodes, index, viewport, refsGeneration: options.refsGeneration }; } /** Quotes one selector term value the way agent-device's parser reads it back. */ diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 250c74350d..c762781e76 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -424,9 +424,7 @@ export class AgentDeviceSurface { /** The screen as last projected before that action, to tell controls that were already there from ones that came with it. */ private indexBeforeAction: readonly ProjectedNode[] | undefined; /** The most recent projection of the screen, from any observe or locate. */ - private latestIndex: readonly ProjectedNode[] | undefined; - /** The capture generation of `latestIndex`. */ - private latestGeneration: number | undefined; + private latest: ProjectedSnapshot | undefined; /** Budget a control that came with the last action gets to finish arriving; see DEFAULT_TRANSITION_MS. */ private readonly transitionMs: number; /** @@ -494,7 +492,7 @@ export class AgentDeviceSurface { * not yet arrived. */ async screenCommand(label: string, run: (client: AgentDeviceClient) => Promise, signal: AbortSignal): Promise { - const before = this.latestIndex; + const before = this.latest?.index; const result = await this.command(label, run, signal); this.markAction(before); return result; @@ -997,9 +995,9 @@ export class AgentDeviceSurface { return `n${this.idCounter}`; }, viewport: raw.viewport, + refsGeneration: raw.refsGeneration, }); - this.latestIndex = projected.index; - this.latestGeneration = raw.refsGeneration; + this.latest = projected; const read = projected.viewport; if (read !== undefined && (this.knownViewport === undefined || sameScreen(read, this.knownViewport))) this.knownViewport = read; return projected; @@ -1051,7 +1049,7 @@ export class AgentDeviceSurface { async observe(operation: OperationContext, options?: EngineObserveOptions): Promise { const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive'); const projected = this.project(raw); - this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index, raw.refsGeneration)])); + this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected)])); const viewport = await this.viewportFor(projected, operation.signal); const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected)); const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined; @@ -1073,7 +1071,7 @@ export class AgentDeviceSurface { const projected = this.project(raw); this.traceScreen(raw, projected); const matches = resolveExpression(expression, projected.index, this.options.platform); - for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected.index, raw.refsGeneration)); + for (const entry of matches) this.located.set(entry.id, this.bind(entry, projected)); for (const oldest of this.located.keys()) { if (this.located.size <= MAX_LOCATED_REFS) break; this.located.delete(oldest); @@ -1139,22 +1137,27 @@ export class AgentDeviceSurface { } /** Copies the fields actions use and pre-resolves a toggle's inner control. */ - private bind(entry: ProjectedNode, snapshot: readonly ProjectedNode[], refsGeneration: number | undefined): NodeBinding { + private bind(entry: ProjectedNode, projected: ProjectedSnapshot): NodeBinding { const { children: _children, ...node } = entry.node; - const control = this.controlOf(entry, snapshot); - return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, refsGeneration }; + const control = this.controlOf(entry, projected.index); + return { id: entry.id, ref: entry.ref, node, controlRef: control.ref, refsGeneration: projected.refsGeneration }; } - /** - * The same control in the newest capture: the binding itself when it came - * from that capture, else found there again where it was, else as it was, - * for agent-device to refuse as stale. - */ + /** The same control in the newest capture: the binding itself when it came from that capture, else found there again where it was. */ private current(entry: NodeBinding): NodeBinding { - const index = this.latestIndex; - if (index === undefined || entry.refsGeneration === this.latestGeneration) return entry; + const latest = this.latest; + if (latest === undefined || entry.refsGeneration === latest.refsGeneration) return entry; + const found = this.samePlace(entry, latest.index); + if (found === undefined) { + throw new EngineError('NODE_STALE', `node ${entry.id} changed since it was located; re-resolve`, { retryable: true }); + } + return this.bind(found, latest); + } + + /** The node a binding stands for in a snapshot, when it is still where it was. */ + private samePlace(entry: NodeBinding, index: readonly ProjectedNode[]): ProjectedNode | undefined { const found = this.refind(entry, index); - return found === undefined || !sameRect(found.node.rect, entry.node.rect) ? entry : this.bind(found, index, this.latestGeneration); + return found !== undefined && sameRect(found.node.rect, entry.node.rect) ? found : undefined; } /** @@ -1203,10 +1206,7 @@ export class AgentDeviceSurface { const remaining = this.transitionMs - since; if (remaining <= 0) return entry; const before = this.indexBeforeAction; - if (before !== undefined) { - const prior = this.refind(entry, before); - if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return entry; - } + if (before !== undefined && this.samePlace(entry, before) !== undefined) return entry; await sleep(remaining, operation.signal); // The sleep resolves on abort; the action behind it must not go out once // the caller has already been told the operation was cancelled. @@ -1218,10 +1218,9 @@ export class AgentDeviceSurface { /** The same control in a fresh snapshot; the binding as it was when the snapshot no longer lists it. */ private async relocated(entry: NodeBinding, operation: OperationContext): Promise { - const raw = await this.snapshotOrEmpty(operation, false); - const projected = this.project(raw); + const projected = this.project(await this.snapshotOrEmpty(operation, false)); const found = this.refind(entry, projected.index); - return found === undefined ? entry : this.bind(found, projected.index, raw.refsGeneration); + return found === undefined ? entry : this.bind(found, projected); } /** @@ -1251,7 +1250,7 @@ export class AgentDeviceSurface { // chrome flow that way. Only the settle wait varies here, by origin. if (ref.id === ROOT_ID) { if (action.kind !== 'swipe') throw notActionable(`the screen root takes swipe only, not ${action.kind}; act on a node`); - const before = this.latestIndex; + const before = this.latest?.index; await this.command('swipe', (client) => input.send(() => client.interactions.scroll({ direction: action.direction, ...settle })), operation.signal); this.markAction(before); return; @@ -1264,7 +1263,7 @@ export class AgentDeviceSurface { if ((action.kind === 'check' || action.kind === 'uncheck') && entry.node.states?.checked === (action.kind === 'check')) { return; } - const before = this.latestIndex; + const before = this.latest?.index; const run = async (): Promise => { const target = deterministic ? await this.settled(entry, operation) : entry; switch (action.kind) { @@ -1375,7 +1374,7 @@ export class AgentDeviceSurface { */ async performAt(point: ViewportPoint, action: PointerAction, operation: OperationContext, input: InputBudget): Promise { const settle = operation.origin === 'test' ? {} : this.settleOptions; - const before = this.latestIndex; + const before = this.latest?.index; const holdMs = action.kind === 'longPress' ? (action.durationMs ?? DEFAULT_LONG_PRESS_MS) : action.kind === 'swipeTo' ? action.durationMs : undefined; await this.command(`${action.kind} at point`, (client) => input.send(() => pointerInteraction(client, point, action, settle), holdMs), operation.signal); this.markAction(before); @@ -1391,7 +1390,7 @@ export class AgentDeviceSurface { if (options.replace) { throw unsupported('agent-device cannot clear the focused field without a node; type into a listed field by id to replace its value'); } - const before = this.latestIndex; + const before = this.latest?.index; await this.command('keyboard.type', (client) => input.send(() => client.interactions.type({ text })), operation.signal); this.markAction(before); } @@ -1411,7 +1410,7 @@ export class AgentDeviceSurface { if (parsed.modifiers.length > 0) { throw unsupported(`agent-device cannot hold ${parsed.modifiers.join('+')} on a device surface; press the key alone`); } - const before = this.latestIndex; + const before = this.latest?.index; if (parsed.key.kind === 'named' && parsed.key.name === 'Enter') { await this.command('keyboard.press', (client) => input.send(() => client.command.keyboard({ action: 'enter' })), operation.signal); } else if (parsed.key.kind === 'char' || parsed.key.name === 'Space') { @@ -1432,7 +1431,7 @@ export class AgentDeviceSurface { * in {@link dismissKeyboardByGesture}. */ async dismissKeyboard(signal: AbortSignal, input?: InputBudget): Promise { - const before = this.latestIndex; + const before = this.latest?.index; try { await this.command('keyboard.dismiss', (client) => sendInput(input, () => client.command.keyboard({ action: 'dismiss' })), signal); } catch (cause) { @@ -1511,7 +1510,7 @@ export class AgentDeviceSurface { async back(operation: OperationContext, input: InputBudget): Promise { const settle = operation.origin === 'test' ? {} : this.settleOptions; - const before = this.latestIndex; + const before = this.latest?.index; await this.command('back', (client) => input.send(() => client.command.back({ ...settle })), operation.signal); this.markAction(before); } diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 4f55d5b423..ce73893312 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2281,6 +2281,27 @@ describe('snapshot generations', () => { }); it('drags between nodes located in separate captures from the newest one', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + generations(h); + const step: OperationContext = { ...operation(), origin: 'test' }; + const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); + const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); + await h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step); + expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s8', destination: '@e3~s8' }); + }); + + it('pins a control found again after a transition to the capture it was found in', async () => { + const h = harness(); + await openAttempt(h); + generations(h); + const step: OperationContext = { ...operation(), origin: 'test' }; + const about = await observed(h, 'About'); + await h.engine.perform!(about.ref, { kind: 'tap' }, step); + expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e4~s8' }); + }); + + it('drags from a source found again after a transition to a target located before it', async () => { const h = harness(); await openAttempt(h); generations(h); @@ -2292,7 +2313,7 @@ describe('snapshot generations', () => { expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s9', destination: '@e3~s9' }); }); - it('keeps a drag source pinned to its own capture when a newer one lists it nowhere it was', async () => { + it('refuses a drag whose source a newer capture lists nowhere it was, before sending it', async () => { const h = harness({ transition: 0 }); await openAttempt(h); const twin: RawNode = { ref: '@e11', index: 11, parentIndex: 0, depth: 1, type: 'cell', label: 'About', identifier: 'ABOUT', rect: { x: 0, y: 600, width: 390, height: 44 } }; @@ -2302,8 +2323,8 @@ describe('snapshot generations', () => { const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); nodes = [...SETTINGS_NODES.filter((node) => node.label !== 'About'), twin]; const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); - await expect(h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step)).rejects.toMatchObject({ code: 'NODE_STALE' }); - expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s7', destination: '@e3~s8' }); + await expect(h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step)).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); + expect(h.fake.methods()).not.toContain('interactions.drag'); }); }); From 422bdb16b18bd11c87b8bf49f85633294cbdbf29 Mon Sep 17 00:00:00 2001 From: Samanyu Das Date: Wed, 28 Oct 2026 02:56:42 +1030 Subject: [PATCH 5/5] test(mobile): fake the clock in the transition generation tests Both tests relied on setup finishing inside the 500 ms transition budget, so a slow worker could skip the re-capture and fail them. Co-Authored-By: Claude Opus 5.5 --- packages/mobile/tests/unit/engine.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index ce73893312..f817e4e2ab 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2292,6 +2292,7 @@ describe('snapshot generations', () => { }); it('pins a control found again after a transition to the capture it was found in', async () => { + autoAdvanceTimers(); const h = harness(); await openAttempt(h); generations(h); @@ -2302,6 +2303,7 @@ describe('snapshot generations', () => { }); it('drags from a source found again after a transition to a target located before it', async () => { + autoAdvanceTimers(); const h = harness(); await openAttempt(h); generations(h);