diff --git a/.changeset/mobile-ref-generation.md b/.changeset/mobile-ref-generation.md new file mode 100644 index 0000000000..a361bee6ed --- /dev/null +++ b/.changeset/mobile-ref-generation.md @@ -0,0 +1,5 @@ +--- +"@e2e-dev/mobile": patch +--- + +A mobile action on a node that a newer snapshot replaced finds the node again instead of acting on a different element. diff --git a/docs/reference/mobile.mdx b/docs/reference/mobile.mdx index cf8bd76ffa..05e1666916 100644 --- a/docs/reference/mobile.mdx +++ b/docs/reference/mobile.mdx @@ -358,7 +358,7 @@ running on the device. The next attempt waits for it to settle. | Contract member | On a device | | --- | --- | | `observe` | The accessibility tree with rects. iOS: `Button` is `button`, `TextField` and `SearchField` are `textbox`, `SecureTextField` is a secure `textbox`, `Cell` is `listitem`, `StaticText` is `text`, `Switch` is `switch` with `checked`, `TabBar` and `SegmentedControl` are `tablist` and the buttons inside them `tab`, `ProgressIndicator` is `progressbar`, `ActivityIndicator` is `status`, `Stepper` is `spinbutton`, `Toolbar`, `Menu`, and `RadioGroup` are `toolbar`, `menu`, and `radiogroup`. Android: `TextView` is `text`, `EditText` is `textbox` and one UIAutomator flags as `password` is a secure `textbox` like `SecureTextField`, `Switch` and `CheckBox` keep their roles and `RadioButton` is `radio`, each with the `checked` state the device reports, layouts are `group`, `RecyclerView` and `AbsListView` (React Native's `FlatList`) are `list`, `TabLayout` and `BottomNavigationView` are `tablist` with their items as `tab`, `ProgressBar` is `progressbar`, `NumberPicker` is `spinbutton`, `Toolbar` and `RadioGroup` keep their names. Any other type is kept verbatim in lower case (`picker-wheel`), so it answers no vocabulary role, unless the platform reports it editable, which makes it a `textbox`. React Native on iOS has no trait for `checkbox`, `radio`, and a few other roles and spells the role and state into the accessibility value instead (`checkbox, unchecked`, `radio button, checked`); such a node takes that role, `checked` and `expanded` from the state words, and reports only what follows them as its value. An iOS field keeps a value equal to its label (an unlabeled field is named by its text); Android's echo of a text view's label as its value is dropped. The accessibility identifier (iOS) or resource id (Android) is the node's `testId`. An Android node is named by the content description its app set (React Native's `accessibilityLabel`), else by its text, and its `text` is what it shows, so a labeled text view or a filled field answers `getByLabel` by its label and `getByText` by its text; a view flagged as a heading is a `heading`, and the role description React Native writes for a role Android has no class for (`Tab`, `Tab List`, `Radio Group`) gives the node that role. A field's hint is its `placeholder` attribute, showing or not. Compose Multiplatform reports no placeholder on iOS: an empty field appends the text it draws to its label (`Name, Ada Lovelace`) over one `StaticText` child, so that text is the field's `placeholder` and the child is left out. An empty field has no value: Android flags a showing hint, and XCTest reports an empty iOS field's placeholder as its value, so an iOS field whose value equals its placeholder (trimmed) reads empty, text typed to match the placeholder exactly included. One Android fact is agent-device's, not the platform's: `rect` and every point are the screen's physical pixels; the display density it reports is not applied yet. Android's status bar and navigation bar, the full-width `com.android.systemui` windows along the screen's edges and the narrow status icons (clock, signal, battery) agent-device reports on their own inside that edge band, are left out; the notification shade, system dialogs, and a heads-up notification or other popup inset from the sides stay. Pixels on request, with every secure node's bounds painted black. | -| `perform` | `tap`, `doubleTap`, `longPress`, `fill`, `clear`, `hover`, `check`, `uncheck`, `press` (`Enter`, `Space`, or one character), `swipe`, `dragTo`, and `focus` on editable fields. A tap on a `switch` or `checkbox` lands on its innermost control, since UIKit reports a settings row as one labelled switch with the real toggle as a child. `check` and `uncheck` need a `checked` state, which Android switches lack and a React Native checkbox or radio on iOS has. `doubleTap` is two presses, 284 to 285 ms apart on iOS (the cadence of agent-device's XCTest tap): agent-device's own double-tap gesture reaches a React Native `Pressable` as one press. A detector with a window of 300 ms or less can read the pair as two single taps. `longPress` holds for one second unless the test passes `duration`, past a React Native `Pressable`'s `delayLongPress` of 500 ms and the longer thresholds apps set; agent-device's own default hold is shorter and registers as a tap on those. `selectOption`, `setInputFiles`, `scrollIntoView`, and `secondaryTap` are not declared, and neither is `tapModifiers`, so `tap` or `doubleTap` with `modifiers` is `UNSUPPORTED_CAPABILITY`. | +| `perform` | `tap`, `doubleTap`, `longPress`, `fill`, `clear`, `hover`, `check`, `uncheck`, `press` (`Enter`, `Space`, or one character), `swipe`, `dragTo`, and `focus` on editable fields. A tap on a `switch` or `checkbox` lands on its innermost control, since UIKit reports a settings row as one labelled switch with the real toggle as a child. `check` and `uncheck` need a `checked` state, which Android switches lack and a React Native checkbox or radio on iOS has. `doubleTap` is two presses, 284 to 285 ms apart on iOS (the cadence of agent-device's XCTest tap): agent-device's own double-tap gesture reaches a React Native `Pressable` as one press. A detector with a window of 300 ms or less can read the pair as two single taps. `longPress` holds for one second unless the test passes `duration`, past a React Native `Pressable`'s `delayLongPress` of 500 ms and the longer thresholds apps set; agent-device's own default hold is shorter and registers as a tap on those. `selectOption`, `setInputFiles`, `scrollIntoView`, and `secondaryTap` are not declared, and neither is `tapModifiers`, so `tap` or `doubleTap` with `modifiers` is `UNSUPPORTED_CAPABILITY`. An action on a node that a newer snapshot replaced finds the node again instead of acting on a different element. | | `performAt` | `tap`, `doubleTap`, `longPress`, and `swipeTo` at a bare screen point, so `screen.tapAt`, `tap({ position })`, `screen.swipe({ from, to })`, and the agent's `tap_at` reach a control the tree does not list. No `hover`, `secondaryTap`, or `dragTo` at a point: touch has none without a node, so `hover_at` needs a listed control. | | `keyboard` | `type` and `press` into the focused field, the agent's `type` and `press` without a target and the fallback of `type_at` and `press_at`, and `dismiss`, the agent's `dismiss_keyboard`. | | `locate` | Every `screen` query. Text and label queries answer with the innermost match, since a device echoes both up the tree (a React Native input is a host view and the field inside it, both labelled). On Android a test id query, a string or a RegExp, also matches a resource id without its `:id/` prefix, for any package but the framework's `android`, when nothing on the screen matches the full id. `screen.locator('id=…')` is agent-device's selector and takes the full id. | diff --git a/packages/mobile/src/nodes.ts b/packages/mobile/src/nodes.ts index 3022e8c6a9..575ff19d58 100644 --- a/packages/mobile/src/nodes.ts +++ b/packages/mobile/src/nodes.ts @@ -35,6 +35,8 @@ export interface ProjectedSnapshot { /** Every node in document order, parents before children. */ readonly index: readonly ProjectedNode[]; readonly viewport: ViewportSize | undefined; + /** The agent-device capture the refs were issued from; undefined from one that reports none. */ + readonly refsGeneration: number | undefined; } /** @@ -450,7 +452,7 @@ function isSystemBar(node: RawNode, screen: ViewportSize | undefined, displayOnl */ export function projectSnapshot( raw: readonly RawNode[], - options: { readonly mintId: () => string; readonly viewport?: ViewportSize | undefined }, + options: { readonly mintId: () => string; readonly viewport?: ViewportSize | undefined; readonly refsGeneration?: number | undefined }, ): ProjectedSnapshot { const viewport = options.viewport ?? viewportOf(raw); const parents = parentPositions(raw); @@ -553,7 +555,7 @@ export function projectSnapshot( const rootNodes = roots .filter((position) => !isSystemBar(raw[position] as RawNode, viewport, () => displayOnly(position))) .map((position) => build(position, undefined)); - return { roots: rootNodes, index, viewport }; + return { roots: rootNodes, index, viewport, refsGeneration: options.refsGeneration }; } /** Quotes one selector term value the way agent-device's parser reads it back. */ diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 86e117a233..7e08d39500 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -131,9 +131,10 @@ export interface OpenAppOptions { /** * The snapshot fields this engine reads off agent-device's response. `keyboard` * is the band the capture's producer measured: `visible` with its frame, - * `absent`, or that it could not look. + * `absent`, or that it could not look. `refsGeneration` is the capture its + * refs were issued from. */ -type RawSnapshot = Partial>; +type RawSnapshot = Partial>; /** Platform element types that are the soft keyboard or one of its keys, as agent-device names them. */ const KEYBOARD_TYPES: ReadonlySet = new Set(['keyboard', 'key']); @@ -200,6 +201,8 @@ interface NodeBinding { readonly ref: string; readonly node: SemanticNode; readonly controlRef: string; + /** The capture `ref` and `controlRef` came from, pinned on every action so agent-device refuses them once a newer capture supersedes it. */ + readonly refsGeneration: number | undefined; readonly keyboardVisible: boolean; /** Whether `fill` and `clear` may type into the node, judged with the children `node` leaves out. */ readonly takesText: boolean; @@ -1017,6 +1020,7 @@ export class AgentDeviceSurface { return `n${this.idCounter}`; }, viewport: raw.viewport, + refsGeneration: raw.refsGeneration, }), keyboardVisible: keyboardShowing(raw), }; @@ -1133,7 +1137,7 @@ export class AgentDeviceSurface { private actionTarget(entry: NodeBinding, control = false): { ref: string } { const ref = control ? entry.controlRef : entry.ref; if (ref === '') throw notActionable(`node ${entry.id} has no agent-device ref to act on`); - return { ref: `@${ref}` }; + return { ref: entry.refsGeneration === undefined ? `@${ref}` : `@${ref}~s${entry.refsGeneration}` }; } /** @@ -1170,9 +1174,27 @@ export class AgentDeviceSurface { controlRef: control.ref, keyboardVisible: snapshot.keyboardVisible, takesText: mayTakeText(entry.node), + refsGeneration: snapshot.refsGeneration, }; } + /** The same control in the newest capture: the binding itself when it came from that capture, else found there again where it was. */ + private current(entry: NodeBinding): NodeBinding { + const latest = this.latestScreen; + if (latest === undefined || entry.refsGeneration === latest.refsGeneration) return entry; + const found = this.samePlace(entry, latest.index); + if (found === undefined) { + throw new EngineError('NODE_STALE', `node ${entry.id} changed since it was located; re-resolve`, { retryable: true }); + } + return this.bind(found, latest); + } + + /** The node a binding stands for in a snapshot, when it is still where it was. */ + private samePlace(entry: NodeBinding, index: readonly ProjectedNode[]): ProjectedNode | undefined { + const found = this.refind(entry, index); + return found !== undefined && sameRect(found.node.rect, entry.node.rect) ? found : undefined; + } + /** * The node a binding stands for, in a fresh snapshot: same role, same test * id, same name and text, and of those the one closest to where it was. @@ -1211,10 +1233,7 @@ export class AgentDeviceSurface { const remaining = this.transitionMs - since; if (remaining <= 0) return false; const before = this.screenBeforeAction; - if (before !== undefined && before.keyboardVisible === entry.keyboardVisible) { - const prior = this.refind(entry, before.index); - if (prior !== undefined && sameRect(prior.node.rect, entry.node.rect)) return false; - } + if (before !== undefined && before.keyboardVisible === entry.keyboardVisible && this.samePlace(entry, before.index) !== undefined) return false; await sleep(remaining, operation.signal); // The sleep resolves on abort; the action behind it must not go out once // the caller has already been told the operation was cancelled. @@ -1344,10 +1363,13 @@ export class AgentDeviceSurface { return input.send(() => client.interactions.swipe(swipeWithin(rect, action.direction, action.momentum))); } case 'dragTo': { - const destination = this.resolveRef(action.target); + // A test locates the drop target in a capture of its own, after the + // source's; agent-device takes both refs from the newest one only. + const source = this.current(target); + const destination = this.current(this.resolveRef(action.target)); return input.send(() => client.interactions.drag({ - source: this.actionTarget(target).ref, + source: this.actionTarget(source).ref, destination: this.actionTarget(destination).ref, }), ); diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index a6c0adabca..49fd84d1d5 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -2288,6 +2288,109 @@ describe('reference lifetime and cancellation', () => { }); }); +describe('snapshot generations', () => { + /** + * agent-device's ref admission: every capture starts a new generation, and + * a ref pinned with `~s` is refused unless it names the newest. + */ + function generations(h: Harness, nodes: () => readonly RawNode[] = () => SETTINGS_NODES): void { + let generation = 6; + h.fake.respond('capture.snapshot', () => { + generation += 1; + return { ...SETTINGS_SNAPSHOT, nodes: nodes(), refsGeneration: generation }; + }); + const admit = (ref: string): void => { + const pinned = /~s(\d+)$/.exec(ref); + if (pinned !== null && Number(pinned[1]) !== generation) { + throw new AppError('COMMAND_FAILED', `Ref ${ref} was minted from a superseded snapshot generation`, { reason: 'ref_generation_mismatch', dispatched: 'no' }); + } + }; + h.fake.respond('interactions.press', (args) => admit((args as { ref: string }).ref)); + h.fake.respond('interactions.drag', (args) => { + const { source, destination } = args as { source: string; destination: string }; + admit(source); + admit(destination); + }); + } + + it('pins every ref action to the capture that issued its ref', async () => { + const h = harness(); + await openAttempt(h); + generations(h); + const op = operation(); + const { root } = await h.engine.observe!(op); + const [about, toggle, back] = ['About', 'Airplane Mode', 'Back'].map((name) => named(root, name)); + const before = h.fake.calls.length; + await h.engine.perform!(about!.ref, { kind: 'tap' }, op); + await h.engine.perform!(toggle!.ref, { kind: 'check' }, op); + await h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, op); + expect(h.fake.calls.slice(before).map((call) => [call.method, call.args])).toEqual([ + ['interactions.press', { ref: '@e4~s7', settle: true, settleQuietMs: 150 }], + ['interactions.press', { ref: '@e6~s7', settle: true, settleQuietMs: 150 }], + ['interactions.drag', { source: '@e4~s7', destination: '@e3~s7' }], + ]); + }); + + it('refuses a ref from a superseded capture as stale instead of acting on the newer one', async () => { + const h = harness(); + await openAttempt(h); + generations(h); + const about = await observed(h, 'About'); + await h.engine.locate!(testIdQuery('BackButton'), operation()); + await expect(h.engine.perform!(about.ref, { kind: 'tap' }, operation())).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); + expect(h.fake.lastArgs('interactions.press')).toMatchObject({ ref: '@e4~s7' }); + }); + + it('drags between nodes located in separate captures from the newest one', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + generations(h); + const step: OperationContext = { ...operation(), origin: 'test' }; + const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); + const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); + await h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step); + expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s8', destination: '@e3~s8' }); + }); + + it('pins a control found again after a transition to the capture it was found in', async () => { + autoAdvanceTimers(); + const h = harness(); + await openAttempt(h); + generations(h); + const step: OperationContext = { ...operation(), origin: 'test' }; + const about = await observed(h, 'About'); + await h.engine.perform!(about.ref, { kind: 'tap' }, step); + expect(h.fake.lastArgs('interactions.press')).toEqual({ ref: '@e4~s8' }); + }); + + it('drags from a source found again after a transition to a target located before it', async () => { + autoAdvanceTimers(); + const h = harness(); + await openAttempt(h); + generations(h); + const step: OperationContext = { ...operation(), origin: 'test' }; + const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); + const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); + await h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step); + // The launch leaves the source in transition, so it is found again in a third capture. + expect(h.fake.lastArgs('interactions.drag')).toEqual({ source: '@e4~s9', destination: '@e3~s9' }); + }); + + it('refuses a drag whose source a newer capture lists nowhere it was, before sending it', async () => { + const h = harness({ transition: 0 }); + await openAttempt(h); + const twin: RawNode = { ref: '@e11', index: 11, parentIndex: 0, depth: 1, type: 'cell', label: 'About', identifier: 'ABOUT', rect: { x: 0, y: 600, width: 390, height: 44 } }; + let nodes: readonly RawNode[] = [...SETTINGS_NODES, twin]; + generations(h, () => nodes); + const step: OperationContext = { ...operation(), origin: 'test' }; + const [about] = await h.engine.locate!(testIdQuery('ABOUT'), step); + nodes = [...SETTINGS_NODES.filter((node) => node.label !== 'About'), twin]; + const [back] = await h.engine.locate!(testIdQuery('BackButton'), step); + await expect(h.engine.perform!(about!.ref, { kind: 'dragTo', target: back!.ref }, step)).rejects.toMatchObject({ code: 'NODE_STALE', retryable: true }); + expect(h.fake.methods()).not.toContain('interactions.drag'); + }); +}); + describe('video', () => { const recorder = (h: Harness, finalizeAt?: string) => { let requested: string | undefined;