From 5d093a1d5c81079358271798f54f1df2aaa1b879 Mon Sep 17 00:00:00 2001 From: Totopo27 <35828647+Totopo27@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:25:49 -0600 Subject: [PATCH 01/12] fix(mobile): bound reads by operation budget with engine readLatency tolerance (#1099) --- .changeset/mobile-reads-budget.md | 6 ++++ docs/mobile.mdx | 6 ++-- docs/reference/engine.mdx | 6 ++++ packages/e2e/src/engine/index.ts | 7 ++++ packages/e2e/src/engine/manifest.ts | 5 +++ packages/e2e/src/engine/session.ts | 1 + packages/e2e/src/engine/surface.ts | 2 ++ packages/e2e/src/expect/async.ts | 1 + packages/e2e/src/internal/time.ts | 18 +++++----- packages/e2e/src/locator/engine.ts | 6 ++-- packages/e2e/src/locator/screen.ts | 3 +- packages/e2e/tests/types/sdk-types.ts | 1 + .../e2e/tests/unit/locator-engine.test.ts | 21 +++++++++++- packages/e2e/tests/unit/time.test.ts | 24 ++++++++++++++ packages/mobile/src/engine.ts | 4 +++ packages/mobile/src/support.ts | 33 +++++++++++++++++-- packages/mobile/src/surface.ts | 9 ++++- packages/mobile/tests/unit/engine.test.ts | 9 ++--- 18 files changed, 140 insertions(+), 22 deletions(-) create mode 100644 .changeset/mobile-reads-budget.md diff --git a/.changeset/mobile-reads-budget.md b/.changeset/mobile-reads-budget.md new file mode 100644 index 0000000000..29c527f1e0 --- /dev/null +++ b/.changeset/mobile-reads-budget.md @@ -0,0 +1,6 @@ +--- +'e2e': patch +'@e2e-dev/mobile': patch +--- + +Bound mobile reads (snapshots and screenshots) by their operation budget without cutting off a slow last poll into a timeout. Engines may now declare an estimated `readLatency` in milliseconds (the mobile engine sets 2,000 ms), which poll loops and `pollCondition` grant past the deadline to distinguish a slow read from a hung one. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index c3c8ac4bdb..eaef0e8217 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -672,8 +672,10 @@ message and hint, never `LAUNCH_TIMEOUT`, `ACTION_FAILED`, or `back` gets its `timeout` (default `actionTimeout`). If the device doesn't answer in time, the step fails with `ACTION_FAILED` and isn't retried: the input may have landed. A `longPress` that holds at least as long as the -time left is refused before it's sent; raise its `timeout`. Finding the -element isn't cut off, so a missing one still fails as `LOCATOR_NOT_FOUND`. +time left is refused before it's sent; raise its `timeout`. Device reads +(snapshots and screenshots) are bounded by their operation timeout, and slow +reads past the deadline are granted the engine's read latency so a missing +element still fails as `LOCATOR_NOT_FOUND`. ```bash npm diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 7bc204f0f8..de4e027eb8 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -85,6 +85,12 @@ export interface Engine { * higher. Omit when the engine has no concurrency limit. */ readonly workers?: number; + /** + * Expected latency in ms for a single observation or locator snapshot read. + * A slow engine declares this so poll loops (like locator engine or pollCondition) + * grant the last read past the deadline instead of treating it as hung. + */ + readonly readLatency?: number; /** capability: observation. */ observe?(context: OperationContext, options?: EngineObserveOptions): Promise; /** diff --git a/packages/e2e/src/engine/index.ts b/packages/e2e/src/engine/index.ts index 9185b5549c..8e6415c760 100644 --- a/packages/e2e/src/engine/index.ts +++ b/packages/e2e/src/engine/index.ts @@ -680,6 +680,13 @@ export interface Engine { * targets without being over-subscribed. Omit when there is no such bound. */ readonly workers?: number; + /** + * Expected latency in ms for a single observation or locator snapshot read. + * A slow engine (e.g. mobile iOS snapshot taking 1-2 s) declares this so + * poll loops (like locator engine or pollCondition) can grant the last read + * past the deadline instead of treating it as hung with budget to spare. + */ + readonly readLatency?: number; /** * The `secrets.get()` handles the engine's options hold, such as a * basic-auth password. The config load checks each names a configured diff --git a/packages/e2e/src/engine/manifest.ts b/packages/e2e/src/engine/manifest.ts index 305e007dd6..ff80be0b1a 100644 --- a/packages/e2e/src/engine/manifest.ts +++ b/packages/e2e/src/engine/manifest.ts @@ -20,6 +20,7 @@ const KNOWN_KEYS = [ 'spiVersion', 'platform', 'workers', + 'readLatency', 'observe', 'locate', 'perform', @@ -201,6 +202,9 @@ export function defineEngine(spec: Engine): EngineHandle { if (spec.workers !== undefined && (!Number.isSafeInteger(spec.workers) || spec.workers < 1)) { throw invalid(name, 'workers must be a positive safe integer: the most workers the engine serves per target'); } + if (spec.readLatency !== undefined && (!Number.isSafeInteger(spec.readLatency) || spec.readLatency < 0)) { + throw invalid(name, 'readLatency must be a non-negative safe integer'); + } const capabilities = new Set(); if (spec.observe !== undefined) capabilities.add('observation'); @@ -248,6 +252,7 @@ export function defineEngine(spec: Engine): EngineHandle { spiVersion: spec.spiVersion, platform: spec.platform, workers: spec.workers, + ...(spec.readLatency === undefined ? {} : { readLatency: spec.readLatency }), }); for (const member of FUNCTION_MEMBERS) { const fn = spec[member]; diff --git a/packages/e2e/src/engine/session.ts b/packages/e2e/src/engine/session.ts index e248a8a4e9..3e2ac6508c 100644 --- a/packages/e2e/src/engine/session.ts +++ b/packages/e2e/src/engine/session.ts @@ -323,6 +323,7 @@ export function createEngineSession(options: EngineSessionOptions): TargetSessio actions, pointerActions, tapModifiers, + ...(engine?.readLatency === undefined ? {} : { readLatency: engine.readLatency }), app, artifacts, appLog: createAppLogRoute(), diff --git a/packages/e2e/src/engine/surface.ts b/packages/e2e/src/engine/surface.ts index fe5815c9ea..149daf419a 100644 --- a/packages/e2e/src/engine/surface.ts +++ b/packages/e2e/src/engine/surface.ts @@ -165,6 +165,8 @@ export interface TargetSession { readonly pointerActions: ReadonlySet; /** Whether the engine declared `tapModifiers`, so the tap actions may carry `modifiers`. */ readonly tapModifiers: boolean; + /** Expected latency in ms for a single observation or locator snapshot read, if declared by the engine. */ + readonly readLatency?: number; /** Captures one atomic agent observation; the harness redacts it downstream. */ observe(operation: OperationContext, options?: EngineObserveOptions): Promise; /** Resolves immediately; the runner owns query polling and strictness. */ diff --git a/packages/e2e/src/expect/async.ts b/packages/e2e/src/expect/async.ts index 90b233066f..022e9d04f8 100644 --- a/packages/e2e/src/expect/async.ts +++ b/packages/e2e/src/expect/async.ts @@ -177,6 +177,7 @@ class AsyncExpectationImpl implements AsyncExpectation { deadline, signal: engine.signal, negated: this.negated, + ...(engine.session.readLatency === undefined ? {} : { readLatency: engine.session.readLatency }), evaluate: async () => { const sample = await this.sample(spec, deadline); lastSample = sample; diff --git a/packages/e2e/src/internal/time.ts b/packages/e2e/src/internal/time.ts index a8df7100f9..06d2bee8f4 100644 --- a/packages/e2e/src/internal/time.ts +++ b/packages/e2e/src/internal/time.ts @@ -66,14 +66,14 @@ export const POLL_INTERVAL_MS = 100; /** * Whether `cause` is a read a wait's deadline cut off: an operation timeout - * of a read that started with less than one poll tick of the wait left, so - * it timed out because the wait did, not because the app stopped answering. - * A read that started with more and still timed out hung, and its timeout is - * the failure. Reads the engine error a locator failure wraps as well as a - * bare one. + * of a read that started with less than one poll tick (or the engine's read + * latency, whichever is larger) of the wait left, so it timed out because the + * wait did, not because the app stopped answering. A read that started with + * more and still timed out hung, and its timeout is the failure. Reads the + * engine error a locator failure wraps as well as a bare one. */ -export function cutOffAtDeadline(cause: unknown, startedWithMs: number): boolean { - return startedWithMs < POLL_INTERVAL_MS && isOperationTimeout(cause); +export function cutOffAtDeadline(cause: unknown, startedWithMs: number, latencyMs = 0): boolean { + return startedWithMs < Math.max(POLL_INTERVAL_MS, latencyMs) && isOperationTimeout(cause); } /** Whether `cause` is the engine's `OPERATION_TIMEOUT`, bare or wrapped by the locator failure it was translated into. */ @@ -89,6 +89,8 @@ export interface PollConditionOptions { readonly deadline: Deadline; readonly signal: AbortSignal; readonly negated: boolean; + /** Expected latency in ms for a single read, if declared by the engine. */ + readonly readLatency?: number; /** * Evaluates the positive condition once, every read bounded by * `deadline`: a read on a longer budget outlasts the poll. Returns @@ -132,7 +134,7 @@ export async function pollCondition(options: PollConditionOptions): Promise { const { node } = await engine.tryRead(this.expression, deadline, ABSENCE_STATES.has(state) ? 'empty' : 'wait'); samples.add(node === null ? 'absent' : isNodeVisible(node) ? 'visible' : 'hidden'); diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts index f94cd4c76f..c2cc6d141c 100644 --- a/packages/e2e/tests/types/sdk-types.ts +++ b/packages/e2e/tests/types/sdk-types.ts @@ -145,6 +145,7 @@ engineSnapshot.treeUnavailable satisfies true | undefined; ({ targets: [{ engine, url: 'http://localhost:3000' }] }) satisfies E2EConfig; // @ts-expect-error an engine only drives the app: the target declares it ({ name: 'toy', version: '1.0.0', spiVersion: 1, app: { url: 'http://localhost:3000' } }) satisfies Engine; +({ name: 'toy', version: '1.0.0', spiVersion: 1, readLatency: 2_000 }) satisfies Engine; ({ name: 'toy', version: '1.0.0', spiVersion: 1, validateApp: (app, { targetName }) => void [app.bundleId, targetName] }) satisfies Engine; // @ts-expect-error a target declares its app; there is no services list in this version ({ targets: [{ engine, app: { url: 'http://localhost:3000' }, services: [] }] }) satisfies E2EConfig; diff --git a/packages/e2e/tests/unit/locator-engine.test.ts b/packages/e2e/tests/unit/locator-engine.test.ts index 63bfdb0904..d91789dceb 100644 --- a/packages/e2e/tests/unit/locator-engine.test.ts +++ b/packages/e2e/tests/unit/locator-engine.test.ts @@ -51,7 +51,7 @@ function foreignStale(retryable: boolean): Error { return error; } -function makeEngine(script: ScreenScript, options: { actionTimeout?: number } = {}) { +function makeEngine(script: ScreenScript, options: { actionTimeout?: number; readLatency?: number } = {}) { const calls = { resolve: 0, read: 0, perform: 0 }; const origins: (string | undefined)[] = []; const resolveTimeouts: number[] = []; @@ -61,6 +61,7 @@ function makeEngine(script: ScreenScript, options: { actionTimeout?: number } = return step; }; const session = { + ...(options.readLatency === undefined ? {} : { readLatency: options.readLatency }), async locate(_expression: LocatorExpression, operation: OperationContext) { resolveTimeouts.push(operation.timeoutMs); const step = next(script.resolve, 'resolve'); @@ -214,6 +215,24 @@ describe('LocatorEngine resolve retry contract', () => { await expect(engine.resolveExactlyOne(EXPRESSION, deadline)).rejects.toMatchObject({ code: 'ACTION_FAILED' }); }); + it('treats a slow last resolve within declared readLatency as missing, not a timeout', async () => { + const deadline = new Deadline(200); + let calls = 0; + const { engine } = makeEngine( + { + resolve: Array.from({ length: 50 }, () => async () => { + calls += 1; + if (calls === 1) return [] as readonly NodeRef[]; + // Second round starts with 100ms left, taking full 100ms and timing out + await new Promise((resolve) => setTimeout(resolve, deadline.remaining())); + throw new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); + }) as unknown as Array<() => readonly NodeRef[]>, + }, + { readLatency: 1_500 }, + ); + await expect(engine.resolveExactlyOne(EXPRESSION, deadline)).rejects.toMatchObject({ code: 'LOCATOR_NOT_FOUND' }); + }); + it('ends a frame wait the deadline cut off on the missing frame, as LOCATOR_NOT_FOUND', async () => { const deadline = new Deadline(350); const { engine } = makeEngine({ diff --git a/packages/e2e/tests/unit/time.test.ts b/packages/e2e/tests/unit/time.test.ts index fc73a6f3bc..3ec2a71d56 100644 --- a/packages/e2e/tests/unit/time.test.ts +++ b/packages/e2e/tests/unit/time.test.ts @@ -351,6 +351,30 @@ describe('pollCondition', () => { await expect(early).rejects.toBe(cut); }); + it('honors readLatency in pollCondition: treats a last read within readLatency as cut off, not hung', async () => { + vi.useFakeTimers(); + const cut = new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); + const deadline = new Deadline(600); + let calls = 0; + const promise = pollCondition({ + deadline, + signal: new AbortController().signal, + negated: false, + readLatency: 1_500, + evaluate: async () => { + calls += 1; + if (calls === 1) return false; + // Started with ~500ms remaining, which is < readLatency (1500ms) but >= POLL_INTERVAL_MS (100ms) + throw cut; + }, + onTimeout: () => new Error('poll timed out'), + }); + const assertion = expect(promise).rejects.toThrow('poll timed out'); + await vi.advanceTimersByTimeAsync(1_000); + await assertion; + expect(calls).toBe(2); + }); + it('keeps a read that hung most of the wait a failure, and never lets a negation pass on it', async () => { vi.useFakeTimers(); const cut = new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); diff --git a/packages/mobile/src/engine.ts b/packages/mobile/src/engine.ts index 3757306c38..3d54ba29a2 100644 --- a/packages/mobile/src/engine.ts +++ b/packages/mobile/src/engine.ts @@ -17,6 +17,9 @@ import { AgentDeviceSurface, assertPermissions } from './surface.ts'; const surfaces = new WeakMap(); +/** Expected latency for a device snapshot read, granting slow reads past the deadline. */ +const MOBILE_READ_LATENCY_MS = 2_000; + /** Assembles the manifest for one surface. Exported for tests that script the client. */ export function buildEngine(surface: AgentDeviceSurface): EngineHandle { const handle = defineEngine({ @@ -24,6 +27,7 @@ export function buildEngine(surface: AgentDeviceSurface): EngineHandle { version: ownVersion(), spiVersion: 1, platform: surface.options.platform, + readLatency: MOBILE_READ_LATENCY_MS, ...(surface.pool.size === undefined ? {} : { workers: surface.pool.size }), prepare: (info) => surface.pool.prepare(info), finish: (info) => surface.pool.finish(info), diff --git a/packages/mobile/src/support.ts b/packages/mobile/src/support.ts index 8b75f86edf..43591e5c29 100644 --- a/packages/mobile/src/support.ts +++ b/packages/mobile/src/support.ts @@ -28,8 +28,7 @@ const MIN_ANSWER_MS = 250; /** * One input operation's budget (`actionTimeout`, or the call's `timeout`), * which the harness signal does not enforce. Only the device commands that - * send input run under it; reads stay unbounded, so a locator poll keeps its - * last snapshot and a target settling before a tap is relocated in full. + * send input run under it. */ export class InputBudget { private readonly endsAt: number; @@ -84,6 +83,36 @@ export class InputBudget { } } +/** + * Bounds a device read (snapshot or screenshot) by the operation's budget. + * When the budget expires before or during the read, it fails with + * `OPERATION_TIMEOUT` instead of holding the test until the test timeout. + */ +export class ReadBudget { + private readonly endsAt: number; + + constructor( + private readonly label: string, + timeoutMs: number, + private readonly track: (pending: Promise) => Promise, + ) { + this.endsAt = Date.now() + timeoutMs; + } + + send(dispatch: () => Promise): Promise { + const remaining = this.endsAt - Date.now(); + if (remaining <= 0) { + return Promise.reject( + new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }), + ); + } + const pending = this.track(dispatch()); + return withTimeout(pending, remaining, () => + new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before the device answered`, { retryable: false }), + ); + } +} + export function invalidState(text: string): EngineError { return new EngineError('INVALID_STATE', text, { retryable: false }); } diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index e15cc8dd53..374b927fbe 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -75,6 +75,7 @@ import { type Rect, type Point, InputBudget, + ReadBudget, } from './support.ts'; /** Hinge poses `device.fold` accepts: `closed` lights the outer display, `half-open` and `open` the inner one. */ @@ -470,6 +471,11 @@ export class AgentDeviceSurface { return new InputBudget(label, timeoutMs, (pending) => this.track(pending)); } + /** The budget one read runs under; its device commands stay in flight until they settle. */ + readBudget(label: string, timeoutMs: number): ReadBudget { + return new ReadBudget(label, timeoutMs, (pending) => this.track(pending)); + } + /** Registers one device command as in flight until it settles. */ private track(pending: Promise): Promise { this.inflight.add(pending); @@ -881,8 +887,9 @@ export class AgentDeviceSurface { * move is the open tool, and failing the step would take that move away. */ private async snapshotOrEmpty(operation: OperationContext, interactiveOnly: boolean): Promise { + const budget = this.readBudget('snapshot', operation.timeoutMs); try { - return await this.snapshot(operation.signal, interactiveOnly); + return await budget.send(() => this.snapshot(operation.signal, interactiveOnly)); } catch (cause) { if (!this.managesApp && cause instanceof EngineError && cause.code === 'INVALID_STATE') return { nodes: [] }; throw cause; diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 1fa75f2320..6017d3ee8c 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -892,15 +892,16 @@ describe('observation', () => { expect(settled).toBe(true); }); - it('lets a locate outlive its budget: a poll keeps its last read, so a slow screen reads as missing, not as a timeout', async () => { + it('bounds a snapshot by the operation budget', async () => { const h = harness(); await openAttempt(h); vi.useFakeTimers(); try { - h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve(SETTINGS_SNAPSHOT), 3_000))); + h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve(SETTINGS_SNAPSHOT), 5_000))); const read = h.engine.locate!({ kind: 'query', query: { kind: 'text', value: { kind: 'string', value: 'About', exact: true } } }, { ...operation(), timeoutMs: 1_000 }); - await vi.advanceTimersByTimeAsync(3_000); - await expect(read).resolves.toHaveLength(1); + const assertion = expect(read).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + await vi.advanceTimersByTimeAsync(1_000); + await assertion; } finally { vi.useRealTimers(); } From 1d99afb85405fc592916b29c3db8b59f7559ff8e Mon Sep 17 00:00:00 2001 From: Totopo27 <35828647+Totopo27@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:59:13 -0600 Subject: [PATCH 02/12] fix(mobile): bound screenshot reads, interrupt sparse retries on timeout, and refine latency docs (#1099) --- docs/mobile.mdx | 7 +-- docs/reference/engine.mdx | 8 ++-- packages/e2e/src/internal/time.ts | 6 +-- packages/mobile/src/support.ts | 10 ++++ packages/mobile/src/surface.ts | 57 +++++++++++++---------- packages/mobile/tests/unit/engine.test.ts | 39 ++++++++++++++++ 6 files changed, 94 insertions(+), 33 deletions(-) diff --git a/docs/mobile.mdx b/docs/mobile.mdx index eaef0e8217..4f394a3a7e 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -673,9 +673,10 @@ message and hint, never `LAUNCH_TIMEOUT`, `ACTION_FAILED`, or answer in time, the step fails with `ACTION_FAILED` and isn't retried: the input may have landed. A `longPress` that holds at least as long as the time left is refused before it's sent; raise its `timeout`. Device reads -(snapshots and screenshots) are bounded by their operation timeout, and slow -reads past the deadline are granted the engine's read latency so a missing -element still fails as `LOCATOR_NOT_FOUND`. +(snapshots and screenshots) are bounded by their operation timeout, and reads +cut off near the deadline honor the engine's read latency tolerance so missing +elements end with `LOCATOR_NOT_FOUND` on locator actions and reads, or with +`ASSERTION_FAILED` on assertion polls. ```bash npm diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index de4e027eb8..3a43cdc3b9 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -88,7 +88,8 @@ export interface Engine { /** * Expected latency in ms for a single observation or locator snapshot read. * A slow engine declares this so poll loops (like locator engine or pollCondition) - * grant the last read past the deadline instead of treating it as hung. + * classify a read timing out near the deadline as a normal deadline cut-off + * rather than a hung read. */ readonly readLatency?: number; /** capability: observation. */ @@ -986,7 +987,8 @@ export const RETRYABLE_ENGINE_ERROR_CODES: ReadonlySet = new Se | `INVALID_STATE` | `APP_NOT_OPEN` | no | | `ENGINE_FAILURE`, any non-`EngineError` | infrastructure `ENGINE_FAILURE` | no | -An `OPERATION_TIMEOUT` on a locate or read that started with under 100 ms of +An `OPERATION_TIMEOUT` on a locate or read that started with under 100 ms +(or the engine's declared `readLatency`, whichever is larger) of a waiting call's deadline left, after one that completed, ends the wait instead: an action's or `waitFor`'s `LOCATOR_NOT_FOUND`, or an assertion's `ASSERTION_FAILED`, with the timeout as its cause. A negated assertion passes @@ -1031,7 +1033,7 @@ Use these helpers to keep behavior consistent across engines: | `matchesText(actual, pattern)`, `toTextPattern(match, { exact? })`, `describePattern(pattern)` | `TextPattern` matching for `locate` and text matchers; `toTextPattern` throws `INVALID_ARGUMENT` for a `match` that is neither a string nor a `RegExp` | | `resolveExpression(expression, nodes, { selector? })` | The reference `locate`. Resolves one `LocatorExpression` over a semantic tree in document order, with `text` and `label` answering the innermost match when a container echoes a descendant's text. An engine whose tree is the whole truth (a device) calls it; a browser engine reproduces it. A `selector` needs the hook, a `frame` is `FRAME_NOT_FOUND` | | `urlMatches(current, expected, baseHref, ignoreCase?)` | `toHaveURL`-style comparison, `ignoreCase` as on the locator matchers; an `expected` that is neither a string nor a `RegExp` throws `INVALID_ARGUMENT`, as in `toTextPattern` | -| `pollCondition({ deadline, signal, negated, evaluate, onTimeout })`, `Deadline` | Assertion polling and the negation grace window. An `OPERATION_TIMEOUT` from an `evaluate` that started with under 100 ms of the deadline left, after one that completed, ends the poll: `onTimeout(cause)` builds the failure with it, and a negation passes if it had held long enough by the time that read was cut off. Bound each read `evaluate` makes by `deadline` (`context.operation(deadline.remaining())`): one on the action timeout can outlast the assertion's own | +| `pollCondition({ deadline, signal, negated, evaluate, onTimeout, readLatency? })`, `Deadline` | Assertion polling and the negation grace window. An `OPERATION_TIMEOUT` from an `evaluate` that started with under 100 ms (or `readLatency`, whichever is larger) of the deadline left, after one that completed, ends the poll: `onTimeout(cause)` builds the failure with it, and a negation passes if it had held long enough by the time that read was cut off. Bound each read `evaluate` makes by `deadline` (`context.operation(deadline.remaining())`): one on the action timeout can outlast the assertion's own | | `validateJsonValue(value, label, rules?)` | Checks a value is plain JSON: a top-level `undefined` passes; a non-finite number, a cycle, an object that is not a plain object or array, or a function, symbol, bigint, or `undefined` inside is `INVALID_ARGUMENT` naming `label`. A `Secret` is `POLICY_DENIED` unless `rules.onSecret` throws its own; `rules.maxDepth` caps the nesting | | `rejectUnknownKeys(label, value, keys, code?)` | Throws `INVALID_CONFIG` for the first own key of `value` outside `keys`, naming the nearest known key when one is a plausible typo (`web() has unknown key "viewprt"; did you mean "viewport"?`) and the known keys otherwise: the check the runner runs on its own config objects, for an engine factory's options. Pass `'INVALID_ARGUMENT'` as `code` for an object a test passes to a fixture method | | `rejectUnknownOptions(api, options, known, code?)` | `INVALID_ARGUMENT` (or `code`, `INVALID_LOCATOR` for a locator option) for an option key a fixture method or matcher does not take, so a JavaScript caller's unsupported option fails instead of running on the default. `options` must be a plain object or `undefined`; a class instance or array is refused, and non-enumerable own keys count as keys | diff --git a/packages/e2e/src/internal/time.ts b/packages/e2e/src/internal/time.ts index 06d2bee8f4..04eef2a9e7 100644 --- a/packages/e2e/src/internal/time.ts +++ b/packages/e2e/src/internal/time.ts @@ -115,9 +115,9 @@ export interface PollConditionOptions { * past the deadline has no budget left. A read the deadline cut off (see * `cutOffAtDeadline`) after an earlier one completed saw nothing: the poll * ends there. A negation then passes if it has held long enough by the - * time the read is cut off: the read left less than one poll tick unseen, no - * more than the gap between any two reads, so a short budget still passes at - * its deadline on what it saw. + * time the read is cut off: the read left less than one poll tick (or the + * declared read latency) unseen, no more than the gap between any two reads, + * so a short budget still passes at its deadline on what it saw. */ export async function pollCondition(options: PollConditionOptions): Promise { const { deadline, negated } = options; diff --git a/packages/mobile/src/support.ts b/packages/mobile/src/support.ts index 43591e5c29..3597af0cbf 100644 --- a/packages/mobile/src/support.ts +++ b/packages/mobile/src/support.ts @@ -99,6 +99,16 @@ export class ReadBudget { this.endsAt = Date.now() + timeoutMs; } + remainingMs(): number { + return Math.max(0, this.endsAt - Date.now()); + } + + check(): void { + if (this.endsAt - Date.now() <= 0) { + throw new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }); + } + } + send(dispatch: () => Promise): Promise { const remaining = this.endsAt - Date.now(); if (remaining <= 0) { diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 374b927fbe..02b6e49a06 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -850,10 +850,14 @@ export class AgentDeviceSurface { return installed; } - private async snapshot(signal: AbortSignal, interactiveOnly: boolean): Promise { + private async snapshot(signal: AbortSignal, interactiveOnly: boolean, budget?: ReadBudget): Promise { let last: RawSnapshot = {}; for (const backoffMs of SPARSE_RETRY_BACKOFF_MS) { - if (backoffMs > 0) await sleep(backoffMs, signal); + if (backoffMs > 0) { + budget?.check(); + await sleep(backoffMs, signal); + } + budget?.check(); if (signal.aborted) throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false }); last = await this.capture(signal, interactiveOnly); if (last.appBundleId !== undefined || last.appName !== undefined) { @@ -889,7 +893,7 @@ export class AgentDeviceSurface { private async snapshotOrEmpty(operation: OperationContext, interactiveOnly: boolean): Promise { const budget = this.readBudget('snapshot', operation.timeoutMs); try { - return await budget.send(() => this.snapshot(operation.signal, interactiveOnly)); + return await budget.send(() => this.snapshot(operation.signal, interactiveOnly, budget)); } catch (cause) { if (!this.managesApp && cause instanceof EngineError && cause.code === 'INVALID_STATE') return { nodes: [] }; throw cause; @@ -916,10 +920,10 @@ export class AgentDeviceSurface { * the harness measures every rect and point against the viewport, so an * invented one would misplace every tap. */ - private async viewportFor(projected: ProjectedSnapshot, signal: AbortSignal): Promise { + private async viewportFor(projected: ProjectedSnapshot, signal: AbortSignal, timeoutMs?: number): Promise { const known = projected.viewport ?? this.knownViewport; if (known !== undefined) return known; - const probed = await this.probeViewport(signal); + const probed = await this.probeViewport(signal, timeoutMs); if (probed === undefined) { throw new EngineError( 'ENGINE_FAILURE', @@ -932,8 +936,8 @@ export class AgentDeviceSurface { } /** The device's logical screen size as its screenshot reports it; undefined when it reports none. */ - private async probeViewport(signal: AbortSignal): Promise { - const result = await this.captureScreenshot(signal, async (shot) => shot); + private async probeViewport(signal: AbortSignal, timeoutMs?: number): Promise { + const result = await this.captureScreenshot(signal, async (shot) => shot, {}, timeoutMs); return logicalScreenSize(result); } @@ -941,7 +945,7 @@ export class AgentDeviceSurface { const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive'); const projected = this.project(raw); this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index)])); - const viewport = await this.viewportFor(projected, operation.signal); + const viewport = await this.viewportFor(projected, operation.signal, operation.timeoutMs); const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected)); const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined; return { @@ -1431,7 +1435,7 @@ export class AgentDeviceSurface { async screenshot(label: string | undefined, operation: OperationContext): Promise { const attempt = this.attempt; if (attempt === undefined) throw invalidState('screenshot outside an attempt'); - const masked = await this.maskedScreenshot(operation.signal); + const masked = await this.maskedScreenshot(operation.signal, operation.timeoutMs); attempt.screenshots += 1; const name = `${String(attempt.screenshots).padStart(3, '0')}-${sanitizeFilename(label ?? 'screenshot')}.png`; const relative = path.join('screenshots', name); @@ -1441,8 +1445,8 @@ export class AgentDeviceSurface { } /** Raw device pixels; cleanup follows the capture even when its caller abandons it. */ - private rawScreenshot(signal?: AbortSignal, capture: ScreenshotCapture = {}): Promise { - return this.captureScreenshot(signal, async (shot, file) => new Uint8Array(readFileSync(shot.path ?? file)), capture); + private rawScreenshot(signal?: AbortSignal, capture: ScreenshotCapture = {}, timeoutMs?: number): Promise { + return this.captureScreenshot(signal, async (shot, file) => new Uint8Array(readFileSync(shot.path ?? file)), capture, timeoutMs); } /** @@ -1454,16 +1458,21 @@ export class AgentDeviceSurface { signal: AbortSignal | undefined, read: (shot: RawScreenshotResult, file: string) => Promise, capture: ScreenshotCapture = {}, + timeoutMs?: number, ): Promise { - return this.command('screenshot', async (client) => { - const directory = mkdtempSync(path.join(tmpdir(), 'e2e-agent-device-')); - const file = path.join(directory, 'screenshot.png'); - try { - return await read(await client.capture.screenshot({ path: file, ...(capture.normalizeStatusBar === true ? { normalizeStatusBar: true } : {}) }), file); - } finally { - rmSync(directory, { recursive: true, force: true }); - } - }, signal); + const run = () => + this.command('screenshot', async (client) => { + const directory = mkdtempSync(path.join(tmpdir(), 'e2e-agent-device-')); + const file = path.join(directory, 'screenshot.png'); + try { + return await read(await client.capture.screenshot({ path: file, ...(capture.normalizeStatusBar === true ? { normalizeStatusBar: true } : {}) }), file); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, signal); + if (timeoutMs === undefined) return run(); + const budget = this.readBudget('screenshot', timeoutMs); + return budget.send(run); } /** @@ -1472,16 +1481,16 @@ export class AgentDeviceSurface { * throws when a secure field cannot be covered, because an image that may * hold a credential must not leave the engine. */ - private async maskedScreenshot(signal?: AbortSignal): Promise<{ data: Uint8Array; masked: number }> { + private async maskedScreenshot(signal?: AbortSignal, timeoutMs = 30_000): Promise<{ data: Uint8Array; masked: number }> { const operation: OperationContext = { signal: signal ?? new AbortController().signal, - timeoutMs: 30_000, + timeoutMs, runId: '', attemptId: '', origin: 'test', }; const projected = this.project(await this.snapshotOrEmpty(operation, false)); - const data = await this.rawScreenshot(signal); + const data = await this.rawScreenshot(signal, {}, timeoutMs); const masked = redactSecure(data, projected, projected.viewport ?? this.knownViewport); if (masked === undefined) { throw new EngineError('ENGINE_FAILURE', 'a secure field on screen could not be masked; screenshot withheld', { @@ -1503,7 +1512,7 @@ export class AgentDeviceSurface { ): Promise<{ pixels: ObservationPixels; masked: number } | undefined> { let raw: Uint8Array; try { - raw = await this.rawScreenshot(operation.signal, capture); + raw = await this.rawScreenshot(operation.signal, capture, operation.timeoutMs); } catch { return undefined; } diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 6017d3ee8c..030a307ac9 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -907,6 +907,45 @@ describe('observation', () => { } }); + it('stops sparse retries when the read budget expires', async () => { + const h = harness(); + await openAttempt(h); + vi.useFakeTimers(); + try { + let snapshots = 0; + h.fake.respond('capture.snapshot', () => { + snapshots += 1; + return Promise.resolve({ nodes: [], snapshotQuality: { state: 'sparse' } }); + }); + const read = h.engine.locate!( + { kind: 'query', query: { kind: 'text', value: { kind: 'string', value: 'About', exact: true } } }, + { ...operation(), timeoutMs: 500 }, + ); + const assertion = expect(read).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + // At 500ms, first backoff (1200ms) has not completed, budget expires + await vi.advanceTimersByTimeAsync(1_500); + await assertion; + expect(snapshots).toBe(1); + } finally { + vi.useRealTimers(); + } + }); + + it('bounds screenshot by operation budget', async () => { + const h = harness(); + await openAttempt(h); + vi.useFakeTimers(); + try { + h.fake.respond('capture.screenshot', () => new Promise((resolve) => setTimeout(() => resolve({ path: '' }), 5_000))); + const shot = h.engine.artifacts!.screenshot('hung shot', { ...operation(), timeoutMs: 1_000 }); + const assertion = expect(shot).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + await vi.advanceTimersByTimeAsync(1_000); + await assertion; + } finally { + vi.useRealTimers(); + } + }); + it('cancels a snapshot when the operation aborts', async () => { const h = harness(); h.fake.respond('capture.snapshot', () => new Promise(() => undefined)); From 41cd4a6d651c97c8a405348856b9fffaa6a56a74 Mon Sep 17 00:00:00 2001 From: Totopo27 <35828647+Totopo27@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:26:49 -0600 Subject: [PATCH 03/12] fix(mobile): rethrow screenshot timeouts in capturePixels and align readLatency docs --- docs/reference/engine.mdx | 1 + packages/e2e/src/engine/index.ts | 6 +++--- packages/e2e/tests/unit/engine.test.ts | 9 +++++++++ packages/mobile/src/surface.ts | 3 ++- packages/mobile/tests/unit/engine.test.ts | 15 +++++++++++++++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 3a43cdc3b9..2f2a2f3cf8 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -26,6 +26,7 @@ Validation fails with `INVALID_CONFIG` for: - a spec that is not an object, or a missing or empty `name`; - a missing or empty `version`, or a `spiVersion` other than `1`; - an empty `platform`, or a `workers` that is not a positive safe integer; +- a `readLatency` that is not a non-negative safe integer; - a hook (`observe`, `locate`, `perform`, `performAt`, `prepare`, `finish`, `init`, `startAttempt`, `settleAttempt`, `endAttempt`, `dispose`, `validateApp`) that is not a function; diff --git a/packages/e2e/src/engine/index.ts b/packages/e2e/src/engine/index.ts index 8e6415c760..84bb4aa502 100644 --- a/packages/e2e/src/engine/index.ts +++ b/packages/e2e/src/engine/index.ts @@ -682,9 +682,9 @@ export interface Engine { readonly workers?: number; /** * Expected latency in ms for a single observation or locator snapshot read. - * A slow engine (e.g. mobile iOS snapshot taking 1-2 s) declares this so - * poll loops (like locator engine or pollCondition) can grant the last read - * past the deadline instead of treating it as hung with budget to spare. + * A slow engine declares this so poll loops (like locator engine or pollCondition) + * classify a read timing out near the deadline as a normal deadline cut-off + * rather than a hung read. */ readonly readLatency?: number; /** diff --git a/packages/e2e/tests/unit/engine.test.ts b/packages/e2e/tests/unit/engine.test.ts index fafdc5eca7..03fe2376d3 100644 --- a/packages/e2e/tests/unit/engine.test.ts +++ b/packages/e2e/tests/unit/engine.test.ts @@ -211,6 +211,15 @@ describe('defineEngine', () => { } }); + it('copies a declared readLatency bound and rejects one that is not a non-negative safe integer', () => { + expect(defineEngine(observingEngine({ readLatency: 1_500 })).readLatency).toBe(1_500); + expect(defineEngine(observingEngine({ readLatency: 0 })).readLatency).toBe(0); + expect(defineEngine(observingEngine()).readLatency).toBeUndefined(); + for (const readLatency of [-1, 1.5, Number.NaN, '1500' as never]) { + expect(() => defineEngine(observingEngine({ readLatency }))).toThrow(/readLatency must be a non-negative safe integer/); + } + }); + it('requires a version: provenance and the trace cache key depend on it', () => { expect(() => defineEngine({ ...observingEngine(), version: '' })).toThrow(/version/); expect(() => diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 02b6e49a06..ceb5428999 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -1513,7 +1513,8 @@ export class AgentDeviceSurface { let raw: Uint8Array; try { raw = await this.rawScreenshot(operation.signal, capture, operation.timeoutMs); - } catch { + } catch (cause) { + if (cause instanceof EngineError && cause.code === 'OPERATION_TIMEOUT') throw cause; return undefined; } const redacted = redactSecure(raw, projected, viewport); diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index 030a307ac9..c037904e01 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -946,6 +946,21 @@ describe('observation', () => { } }); + it('rethrows OPERATION_TIMEOUT from capturePixels during observe', async () => { + const h = harness(); + await openAttempt(h); + vi.useFakeTimers(); + try { + h.fake.respond('capture.screenshot', () => new Promise((resolve) => setTimeout(() => resolve({ path: '' }), 5_000))); + const obs = h.engine.observe!({ ...operation(), timeoutMs: 1_000 }, { pixels: true }); + const assertion = expect(obs).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + await vi.advanceTimersByTimeAsync(1_000); + await assertion; + } finally { + vi.useRealTimers(); + } + }); + it('cancels a snapshot when the operation aborts', async () => { const h = harness(); h.fake.respond('capture.snapshot', () => new Promise(() => undefined)); From da823fe5ebca9ced08345a27d1b407da8cc3cc43 Mon Sep 17 00:00:00 2001 From: Totopo27 <35828647+Totopo27@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:13:29 -0600 Subject: [PATCH 04/12] docs(mobile): note that screenshot timeouts propagate from capturePixels --- packages/mobile/src/surface.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index ceb5428999..c6b51acdd3 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -1502,7 +1502,8 @@ export class AgentDeviceSurface { /** * Viewport pixels for an observation. Best-effort: a screenshot that cannot - * be produced or redacted costs the observation its image, not the step. + * be produced or redacted costs the observation its image, not the step; + * timeouts propagate so an expired read budget fails the observation. */ private async capturePixels( operation: OperationContext, From edd367ac22aa880d1a2c9d4f53318c6390315af9 Mon Sep 17 00:00:00 2001 From: Totopo27 <35828647+Totopo27@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:51:47 -0600 Subject: [PATCH 05/12] fix(mobile): measure read latency dynamically and fold read budget into input budget (#1099) --- .changeset/mobile-reads-budget.md | 2 +- docs/mobile.mdx | 4 +- docs/reference/engine.mdx | 18 +-- packages/e2e/src/engine/index.ts | 7 -- packages/e2e/src/engine/manifest.ts | 5 - packages/e2e/src/engine/session.ts | 1 - packages/e2e/src/engine/surface.ts | 2 - packages/e2e/src/expect/async.ts | 1 - packages/e2e/src/internal/time.ts | 9 +- packages/e2e/src/locator/engine.ts | 37 ++++-- packages/e2e/src/locator/screen.ts | 9 +- packages/e2e/tests/types/sdk-types.ts | 1 - packages/e2e/tests/unit/engine.test.ts | 9 -- .../e2e/tests/unit/locator-engine.test.ts | 106 +++++++++++++----- packages/e2e/tests/unit/time.test.ts | 10 +- packages/mobile/src/engine.ts | 10 +- packages/mobile/src/support.ts | 39 ++----- packages/mobile/src/surface.ts | 83 +++++++------- packages/mobile/tests/unit/engine.test.ts | 37 ++++++ packages/mobile/tests/unit/support.test.ts | 22 ++++ 20 files changed, 235 insertions(+), 177 deletions(-) diff --git a/.changeset/mobile-reads-budget.md b/.changeset/mobile-reads-budget.md index 29c527f1e0..178390828d 100644 --- a/.changeset/mobile-reads-budget.md +++ b/.changeset/mobile-reads-budget.md @@ -3,4 +3,4 @@ '@e2e-dev/mobile': patch --- -Bound mobile reads (snapshots and screenshots) by their operation budget without cutting off a slow last poll into a timeout. Engines may now declare an estimated `readLatency` in milliseconds (the mobile engine sets 2,000 ms), which poll loops and `pollCondition` grant past the deadline to distinguish a slow read from a hung one. +The mobile engine now bounds reads (snapshots and screenshots) by their operation budget so a hung device does not hold the step until the test timeout. The locator poll measures how long reads take and grants the last read extra time past the deadline so a slow read still finishes, avoiding timeouts when elements appear late. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index 4f394a3a7e..9023e7b165 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -673,8 +673,8 @@ message and hint, never `LAUNCH_TIMEOUT`, `ACTION_FAILED`, or answer in time, the step fails with `ACTION_FAILED` and isn't retried: the input may have landed. A `longPress` that holds at least as long as the time left is refused before it's sent; raise its `timeout`. Device reads -(snapshots and screenshots) are bounded by their operation timeout, and reads -cut off near the deadline honor the engine's read latency tolerance so missing +(snapshots and screenshots) are bounded by their operation timeout, and +locator polls grant slow reads extra time past the deadline so missing elements end with `LOCATOR_NOT_FOUND` on locator actions and reads, or with `ASSERTION_FAILED` on assertion polls. diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 2f2a2f3cf8..999f5a2c83 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -26,7 +26,6 @@ Validation fails with `INVALID_CONFIG` for: - a spec that is not an object, or a missing or empty `name`; - a missing or empty `version`, or a `spiVersion` other than `1`; - an empty `platform`, or a `workers` that is not a positive safe integer; -- a `readLatency` that is not a non-negative safe integer; - a hook (`observe`, `locate`, `perform`, `performAt`, `prepare`, `finish`, `init`, `startAttempt`, `settleAttempt`, `endAttempt`, `dispose`, `validateApp`) that is not a function; @@ -86,13 +85,6 @@ export interface Engine { * higher. Omit when the engine has no concurrency limit. */ readonly workers?: number; - /** - * Expected latency in ms for a single observation or locator snapshot read. - * A slow engine declares this so poll loops (like locator engine or pollCondition) - * classify a read timing out near the deadline as a normal deadline cut-off - * rather than a hung read. - */ - readonly readLatency?: number; /** capability: observation. */ observe?(context: OperationContext, options?: EngineObserveOptions): Promise; /** @@ -234,10 +226,7 @@ export interface Engine { ```ts /** What every engine call carries: budget, ids, and who is acting. */ export interface OperationContext { - /** - * Aborts when the attempt, or the hook running the call, ends. It does not - * abort at `timeoutMs`: bound each call by that budget yourself. - */ + /** Aborts with the attempt or when the operation budget is spent. */ readonly signal: AbortSignal; /** Remaining operation budget when the call starts. */ readonly timeoutMs: number; @@ -988,8 +977,7 @@ export const RETRYABLE_ENGINE_ERROR_CODES: ReadonlySet = new Se | `INVALID_STATE` | `APP_NOT_OPEN` | no | | `ENGINE_FAILURE`, any non-`EngineError` | infrastructure `ENGINE_FAILURE` | no | -An `OPERATION_TIMEOUT` on a locate or read that started with under 100 ms -(or the engine's declared `readLatency`, whichever is larger) of +An `OPERATION_TIMEOUT` on a locate or read that started with under 100 ms of a waiting call's deadline left, after one that completed, ends the wait instead: an action's or `waitFor`'s `LOCATOR_NOT_FOUND`, or an assertion's `ASSERTION_FAILED`, with the timeout as its cause. A negated assertion passes @@ -1034,7 +1022,7 @@ Use these helpers to keep behavior consistent across engines: | `matchesText(actual, pattern)`, `toTextPattern(match, { exact? })`, `describePattern(pattern)` | `TextPattern` matching for `locate` and text matchers; `toTextPattern` throws `INVALID_ARGUMENT` for a `match` that is neither a string nor a `RegExp` | | `resolveExpression(expression, nodes, { selector? })` | The reference `locate`. Resolves one `LocatorExpression` over a semantic tree in document order, with `text` and `label` answering the innermost match when a container echoes a descendant's text. An engine whose tree is the whole truth (a device) calls it; a browser engine reproduces it. A `selector` needs the hook, a `frame` is `FRAME_NOT_FOUND` | | `urlMatches(current, expected, baseHref, ignoreCase?)` | `toHaveURL`-style comparison, `ignoreCase` as on the locator matchers; an `expected` that is neither a string nor a `RegExp` throws `INVALID_ARGUMENT`, as in `toTextPattern` | -| `pollCondition({ deadline, signal, negated, evaluate, onTimeout, readLatency? })`, `Deadline` | Assertion polling and the negation grace window. An `OPERATION_TIMEOUT` from an `evaluate` that started with under 100 ms (or `readLatency`, whichever is larger) of the deadline left, after one that completed, ends the poll: `onTimeout(cause)` builds the failure with it, and a negation passes if it had held long enough by the time that read was cut off. Bound each read `evaluate` makes by `deadline` (`context.operation(deadline.remaining())`): one on the action timeout can outlast the assertion's own | +| `pollCondition({ deadline, signal, negated, evaluate, onTimeout })`, `Deadline` | Assertion polling and the negation grace window. An `OPERATION_TIMEOUT` from an `evaluate` that started with under 100 ms of the deadline left, after one that completed, ends the poll: `onTimeout(cause)` builds the failure with it, and a negation passes if it had held long enough by the time that read was cut off. Bound each read `evaluate` makes by `deadline` (`context.operation(deadline.remaining())`): one on the action timeout can outlast the assertion's own | | `validateJsonValue(value, label, rules?)` | Checks a value is plain JSON: a top-level `undefined` passes; a non-finite number, a cycle, an object that is not a plain object or array, or a function, symbol, bigint, or `undefined` inside is `INVALID_ARGUMENT` naming `label`. A `Secret` is `POLICY_DENIED` unless `rules.onSecret` throws its own; `rules.maxDepth` caps the nesting | | `rejectUnknownKeys(label, value, keys, code?)` | Throws `INVALID_CONFIG` for the first own key of `value` outside `keys`, naming the nearest known key when one is a plausible typo (`web() has unknown key "viewprt"; did you mean "viewport"?`) and the known keys otherwise: the check the runner runs on its own config objects, for an engine factory's options. Pass `'INVALID_ARGUMENT'` as `code` for an object a test passes to a fixture method | | `rejectUnknownOptions(api, options, known, code?)` | `INVALID_ARGUMENT` (or `code`, `INVALID_LOCATOR` for a locator option) for an option key a fixture method or matcher does not take, so a JavaScript caller's unsupported option fails instead of running on the default. `options` must be a plain object or `undefined`; a class instance or array is refused, and non-enumerable own keys count as keys | diff --git a/packages/e2e/src/engine/index.ts b/packages/e2e/src/engine/index.ts index 84bb4aa502..9185b5549c 100644 --- a/packages/e2e/src/engine/index.ts +++ b/packages/e2e/src/engine/index.ts @@ -680,13 +680,6 @@ export interface Engine { * targets without being over-subscribed. Omit when there is no such bound. */ readonly workers?: number; - /** - * Expected latency in ms for a single observation or locator snapshot read. - * A slow engine declares this so poll loops (like locator engine or pollCondition) - * classify a read timing out near the deadline as a normal deadline cut-off - * rather than a hung read. - */ - readonly readLatency?: number; /** * The `secrets.get()` handles the engine's options hold, such as a * basic-auth password. The config load checks each names a configured diff --git a/packages/e2e/src/engine/manifest.ts b/packages/e2e/src/engine/manifest.ts index ff80be0b1a..305e007dd6 100644 --- a/packages/e2e/src/engine/manifest.ts +++ b/packages/e2e/src/engine/manifest.ts @@ -20,7 +20,6 @@ const KNOWN_KEYS = [ 'spiVersion', 'platform', 'workers', - 'readLatency', 'observe', 'locate', 'perform', @@ -202,9 +201,6 @@ export function defineEngine(spec: Engine): EngineHandle { if (spec.workers !== undefined && (!Number.isSafeInteger(spec.workers) || spec.workers < 1)) { throw invalid(name, 'workers must be a positive safe integer: the most workers the engine serves per target'); } - if (spec.readLatency !== undefined && (!Number.isSafeInteger(spec.readLatency) || spec.readLatency < 0)) { - throw invalid(name, 'readLatency must be a non-negative safe integer'); - } const capabilities = new Set(); if (spec.observe !== undefined) capabilities.add('observation'); @@ -252,7 +248,6 @@ export function defineEngine(spec: Engine): EngineHandle { spiVersion: spec.spiVersion, platform: spec.platform, workers: spec.workers, - ...(spec.readLatency === undefined ? {} : { readLatency: spec.readLatency }), }); for (const member of FUNCTION_MEMBERS) { const fn = spec[member]; diff --git a/packages/e2e/src/engine/session.ts b/packages/e2e/src/engine/session.ts index 3e2ac6508c..e248a8a4e9 100644 --- a/packages/e2e/src/engine/session.ts +++ b/packages/e2e/src/engine/session.ts @@ -323,7 +323,6 @@ export function createEngineSession(options: EngineSessionOptions): TargetSessio actions, pointerActions, tapModifiers, - ...(engine?.readLatency === undefined ? {} : { readLatency: engine.readLatency }), app, artifacts, appLog: createAppLogRoute(), diff --git a/packages/e2e/src/engine/surface.ts b/packages/e2e/src/engine/surface.ts index 149daf419a..fe5815c9ea 100644 --- a/packages/e2e/src/engine/surface.ts +++ b/packages/e2e/src/engine/surface.ts @@ -165,8 +165,6 @@ export interface TargetSession { readonly pointerActions: ReadonlySet; /** Whether the engine declared `tapModifiers`, so the tap actions may carry `modifiers`. */ readonly tapModifiers: boolean; - /** Expected latency in ms for a single observation or locator snapshot read, if declared by the engine. */ - readonly readLatency?: number; /** Captures one atomic agent observation; the harness redacts it downstream. */ observe(operation: OperationContext, options?: EngineObserveOptions): Promise; /** Resolves immediately; the runner owns query polling and strictness. */ diff --git a/packages/e2e/src/expect/async.ts b/packages/e2e/src/expect/async.ts index 022e9d04f8..90b233066f 100644 --- a/packages/e2e/src/expect/async.ts +++ b/packages/e2e/src/expect/async.ts @@ -177,7 +177,6 @@ class AsyncExpectationImpl implements AsyncExpectation { deadline, signal: engine.signal, negated: this.negated, - ...(engine.session.readLatency === undefined ? {} : { readLatency: engine.session.readLatency }), evaluate: async () => { const sample = await this.sample(spec, deadline); lastSample = sample; diff --git a/packages/e2e/src/internal/time.ts b/packages/e2e/src/internal/time.ts index 04eef2a9e7..27e656cdf7 100644 --- a/packages/e2e/src/internal/time.ts +++ b/packages/e2e/src/internal/time.ts @@ -89,8 +89,6 @@ export interface PollConditionOptions { readonly deadline: Deadline; readonly signal: AbortSignal; readonly negated: boolean; - /** Expected latency in ms for a single read, if declared by the engine. */ - readonly readLatency?: number; /** * Evaluates the positive condition once, every read bounded by * `deadline`: a read on a longer budget outlasts the poll. Returns @@ -116,7 +114,7 @@ export interface PollConditionOptions { * `cutOffAtDeadline`) after an earlier one completed saw nothing: the poll * ends there. A negation then passes if it has held long enough by the * time the read is cut off: the read left less than one poll tick (or the - * declared read latency) unseen, no more than the gap between any two reads, + * measured read latency) unseen, no more than the gap between any two reads, * so a short budget still passes at its deadline on what it saw. */ export async function pollCondition(options: PollConditionOptions): Promise { @@ -127,14 +125,17 @@ export async function pollCondition(options: PollConditionOptions): Promise holdingSince !== undefined && now - holdingSince >= grace; let sampled = false; + let latency = 0; for (;;) { const startedWithMs = deadline.remaining(readAt); let value: boolean | undefined; + const evaluateStart = Date.now(); try { value = await options.evaluate(); sampled = true; + latency = Math.max(latency, Date.now() - evaluateStart); } catch (cause) { - if (!sampled || !cutOffAtDeadline(cause, startedWithMs, options.readLatency)) throw cause; + if (!sampled || !cutOffAtDeadline(cause, startedWithMs, latency)) throw cause; if (negated && holds(Math.min(Date.now(), deadline.endsAt))) return; throw await options.onTimeout(cause); } diff --git a/packages/e2e/src/locator/engine.ts b/packages/e2e/src/locator/engine.ts index 9313e6ddb1..571811d337 100644 --- a/packages/e2e/src/locator/engine.ts +++ b/packages/e2e/src/locator/engine.ts @@ -94,6 +94,10 @@ export class LocatorEngine { return this.options.budget.signal; } + get budget(): AttemptBudget { + return this.options.budget; + } + get assertionTimeout(): number { return this.options.assertionTimeout; } @@ -122,10 +126,10 @@ export class LocatorEngine { } /** An operation context bounded by a deadline a caller shares across steps, such as the swipes of one scroll. */ - operationWithin(deadline: Deadline): OperationContext { + operationWithin(deadline: Deadline, timeoutMs?: number): OperationContext { return { signal: this.signal, - timeoutMs: Math.max(1, deadline.remaining()), + timeoutMs: timeoutMs ?? Math.max(1, deadline.remaining()), runId: this.options.runId, attemptId: this.options.attemptId, origin: 'test', @@ -181,15 +185,16 @@ export class LocatorEngine { expression: LocatorExpression, deadline: Deadline, missingFrame: MissingFrame = 'wait', + timeoutMs?: number, ): Promise { let retried: unknown; for (;;) { const startedWithMs = deadline.remaining(); try { - return await this.session.locate(expression, this.operationWithin(deadline)); + return await this.session.locate(expression, this.operationWithin(deadline, timeoutMs)); } catch (cause) { // A retry the deadline cut off saw nothing new: the wait ends on the failure it was retrying. - if (retried !== undefined && cutOffAtDeadline(cause, startedWithMs, this.session.readLatency)) throw translateLocatorError(retried, expression); + if (retried !== undefined && cutOffAtDeadline(cause, startedWithMs)) throw translateLocatorError(retried, expression); const engineError = asEngineError(cause); if (engineError?.retryable === true) { if (missingFrame === 'empty' && engineError.code === 'FRAME_NOT_FOUND') return []; @@ -223,14 +228,18 @@ export class LocatorEngine { ...(cause === undefined ? {} : { cause }), }); let sampled = false; + let latency = 0; for (let rounds = 1; ; rounds += 1) { const startedWithMs = deadline.remaining(); + const readTimeout = Math.min(this.options.budget.deadline.remaining(), startedWithMs + latency); let refs: readonly NodeRef[]; + const readStart = Date.now(); try { - refs = await this.resolveOnce(expression, deadline); + refs = await this.resolveOnce(expression, deadline, 'wait', readTimeout); + latency = Math.max(latency, Date.now() - readStart); } catch (cause) { // A resolve the deadline cut off, after one that matched nothing, is the wait running out. - if (sampled && cutOffAtDeadline(cause, startedWithMs, this.session.readLatency)) throw notFound(cause); + if (sampled && cutOffAtDeadline(cause, startedWithMs, latency)) throw notFound(cause); throw cause; } sampled = true; @@ -330,8 +339,9 @@ export class LocatorEngine { expression: LocatorExpression, deadline: Deadline, missingFrame: MissingFrame = 'wait', + timeoutMs?: number, ): Promise<{ node: SemanticNode | null; count: number }> { - return this.readOne(expression, deadline, missingFrame); + return this.readOne(expression, deadline, missingFrame, timeoutMs); } /** @@ -348,11 +358,12 @@ export class LocatorEngine { expression: LocatorExpression, deadline: Deadline, missingFrame: MissingFrame, + timeoutMs?: number, ): Promise<{ node: SemanticNode | null; count: number }> { - const ref = assertSingle(await this.resolveOnce(expression, deadline, missingFrame), expression); + const ref = assertSingle(await this.resolveOnce(expression, deadline, missingFrame, timeoutMs), expression); if (ref === null) return { node: null, count: 0 }; try { - const node = await this.session.read(ref, this.operationWithin(deadline)); + const node = await this.session.read(ref, this.operationWithin(deadline, timeoutMs)); return { node, count: 1 }; } catch (cause) { if (asEngineError(cause)?.code === 'NODE_STALE') { @@ -408,13 +419,17 @@ export class LocatorEngine { ...(cause === undefined ? {} : { cause }), }); let sampled = false; + let latency = 0; for (;;) { const startedWithMs = deadline.remaining(); + const readTimeout = Math.min(this.options.budget.deadline.remaining(), startedWithMs + latency); let node: SemanticNode | null; + const readStart = Date.now(); try { - ({ node } = await this.tryRead(expression, deadline)); + ({ node } = await this.tryRead(expression, deadline, 'wait', readTimeout)); + latency = Math.max(latency, Date.now() - readStart); } catch (cause) { - if (sampled && cutOffAtDeadline(cause, startedWithMs, this.session.readLatency)) throw notVisible(cause); + if (sampled && cutOffAtDeadline(cause, startedWithMs, latency)) throw notVisible(cause); throw cause; } sampled = true; diff --git a/packages/e2e/src/locator/screen.ts b/packages/e2e/src/locator/screen.ts index d98f613f6b..6d91fbf168 100644 --- a/packages/e2e/src/locator/screen.ts +++ b/packages/e2e/src/locator/screen.ts @@ -274,13 +274,17 @@ class ScreenImpl implements Screen { ...(cause === undefined ? {} : { cause }), }); let sampled = false; + let latency = 0; for (;;) { const startedWithMs = deadline.remaining(); + const readTimeout = Math.min(engine.budget.deadline.remaining(), startedWithMs + latency); let node: SemanticNode | null; + const readStart = Date.now(); try { - ({ node } = await engine.tryRead(internals.expression, deadline)); + ({ node } = await engine.tryRead(internals.expression, deadline, 'wait', readTimeout)); + latency = Math.max(latency, Date.now() - readStart); } catch (cause) { - if (sampled && cutOffAtDeadline(cause, startedWithMs, engine.session.readLatency)) throw notVisible(cause); + if (sampled && cutOffAtDeadline(cause, startedWithMs, latency)) throw notVisible(cause); throw cause; } sampled = true; @@ -610,7 +614,6 @@ class LocatorImpl extends ScreenImpl implements Locator { deadline, signal: engine.signal, negated: false, - ...(engine.session.readLatency === undefined ? {} : { readLatency: engine.session.readLatency }), evaluate: async () => { const { node } = await engine.tryRead(this.expression, deadline, ABSENCE_STATES.has(state) ? 'empty' : 'wait'); samples.add(node === null ? 'absent' : isNodeVisible(node) ? 'visible' : 'hidden'); diff --git a/packages/e2e/tests/types/sdk-types.ts b/packages/e2e/tests/types/sdk-types.ts index c2cc6d141c..f94cd4c76f 100644 --- a/packages/e2e/tests/types/sdk-types.ts +++ b/packages/e2e/tests/types/sdk-types.ts @@ -145,7 +145,6 @@ engineSnapshot.treeUnavailable satisfies true | undefined; ({ targets: [{ engine, url: 'http://localhost:3000' }] }) satisfies E2EConfig; // @ts-expect-error an engine only drives the app: the target declares it ({ name: 'toy', version: '1.0.0', spiVersion: 1, app: { url: 'http://localhost:3000' } }) satisfies Engine; -({ name: 'toy', version: '1.0.0', spiVersion: 1, readLatency: 2_000 }) satisfies Engine; ({ name: 'toy', version: '1.0.0', spiVersion: 1, validateApp: (app, { targetName }) => void [app.bundleId, targetName] }) satisfies Engine; // @ts-expect-error a target declares its app; there is no services list in this version ({ targets: [{ engine, app: { url: 'http://localhost:3000' }, services: [] }] }) satisfies E2EConfig; diff --git a/packages/e2e/tests/unit/engine.test.ts b/packages/e2e/tests/unit/engine.test.ts index 03fe2376d3..fafdc5eca7 100644 --- a/packages/e2e/tests/unit/engine.test.ts +++ b/packages/e2e/tests/unit/engine.test.ts @@ -211,15 +211,6 @@ describe('defineEngine', () => { } }); - it('copies a declared readLatency bound and rejects one that is not a non-negative safe integer', () => { - expect(defineEngine(observingEngine({ readLatency: 1_500 })).readLatency).toBe(1_500); - expect(defineEngine(observingEngine({ readLatency: 0 })).readLatency).toBe(0); - expect(defineEngine(observingEngine()).readLatency).toBeUndefined(); - for (const readLatency of [-1, 1.5, Number.NaN, '1500' as never]) { - expect(() => defineEngine(observingEngine({ readLatency }))).toThrow(/readLatency must be a non-negative safe integer/); - } - }); - it('requires a version: provenance and the trace cache key depend on it', () => { expect(() => defineEngine({ ...observingEngine(), version: '' })).toThrow(/version/); expect(() => diff --git a/packages/e2e/tests/unit/locator-engine.test.ts b/packages/e2e/tests/unit/locator-engine.test.ts index d91789dceb..59b394e1a6 100644 --- a/packages/e2e/tests/unit/locator-engine.test.ts +++ b/packages/e2e/tests/unit/locator-engine.test.ts @@ -3,7 +3,7 @@ * complete EngineError -> runner taxonomy mapping. */ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { EngineError, type EngineErrorCode, @@ -51,7 +51,7 @@ function foreignStale(retryable: boolean): Error { return error; } -function makeEngine(script: ScreenScript, options: { actionTimeout?: number; readLatency?: number } = {}) { +function makeEngine(script: ScreenScript, options: { actionTimeout?: number } = {}) { const calls = { resolve: 0, read: 0, perform: 0 }; const origins: (string | undefined)[] = []; const resolveTimeouts: number[] = []; @@ -61,7 +61,6 @@ function makeEngine(script: ScreenScript, options: { actionTimeout?: number; rea return step; }; const session = { - ...(options.readLatency === undefined ? {} : { readLatency: options.readLatency }), async locate(_expression: LocatorExpression, operation: OperationContext) { resolveTimeouts.push(operation.timeoutMs); const step = next(script.resolve, 'resolve'); @@ -201,36 +200,83 @@ describe('LocatorEngine resolve retry contract', () => { }); }); - it('keeps a resolve that hung with budget to spare a timeout, never a missing node', async () => { - const deadline = new Deadline(600); - let calls = 0; - const { engine } = makeEngine({ - resolve: Array.from({ length: 50 }, () => async () => { - calls += 1; - if (calls === 1) return [] as readonly NodeRef[]; - await new Promise((resolve) => setTimeout(resolve, deadline.remaining())); - throw new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); - }) as unknown as Array<() => readonly NodeRef[]>, - }); - await expect(engine.resolveExactlyOne(EXPRESSION, deadline)).rejects.toMatchObject({ code: 'ACTION_FAILED' }); - }); - - it('treats a slow last resolve within declared readLatency as missing, not a timeout', async () => { - const deadline = new Deadline(200); - let calls = 0; - const { engine } = makeEngine( - { - resolve: Array.from({ length: 50 }, () => async () => { + it('keeps a resolve that hung with budget to spare a timeout (ACTION_FAILED), never a missing node', async () => { + vi.useFakeTimers(); + try { + const deadline = new Deadline(2_000); + let calls = 0; + const { engine } = makeEngine({ + resolve: Array.from({ length: 10 }, () => async () => { calls += 1; - if (calls === 1) return [] as readonly NodeRef[]; - // Second round starts with 100ms left, taking full 100ms and timing out - await new Promise((resolve) => setTimeout(resolve, deadline.remaining())); + if (calls === 1) { + await new Promise((resolve) => setTimeout(resolve, 100)); + return [] as readonly NodeRef[]; + } + await new Promise((resolve) => setTimeout(resolve, 500)); throw new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); }) as unknown as Array<() => readonly NodeRef[]>, - }, - { readLatency: 1_500 }, - ); - await expect(engine.resolveExactlyOne(EXPRESSION, deadline)).rejects.toMatchObject({ code: 'LOCATOR_NOT_FOUND' }); + }); + const pending = engine.resolveExactlyOne(EXPRESSION, deadline); + const assertion = expect(pending).rejects.toMatchObject({ code: 'ACTION_FAILED' }); + await vi.advanceTimersByTimeAsync(1_000); + await assertion; + expect(calls).toBe(2); + } finally { + vi.useRealTimers(); + } + }); + + it('treats a slow-but-valid last resolve past deadline as LOCATOR_NOT_FOUND', async () => { + vi.useFakeTimers(); + try { + const deadline = new Deadline(300); + let calls = 0; + const { engine } = makeEngine({ + resolve: Array.from({ length: 10 }, () => async () => { + calls += 1; + if (calls === 1) { + await new Promise((resolve) => setTimeout(resolve, 150)); + return [] as readonly NodeRef[]; + } + await new Promise((resolve) => setTimeout(resolve, 150)); + return [] as readonly NodeRef[]; + }) as unknown as Array<() => readonly NodeRef[]>, + }); + const pending = engine.resolveExactlyOne(EXPRESSION, deadline); + const assertion = expect(pending).rejects.toMatchObject({ code: 'LOCATOR_NOT_FOUND' }); + await vi.advanceTimersByTimeAsync(500); + await assertion; + expect(calls).toBe(2); + } finally { + vi.useRealTimers(); + } + }); + + it('grants a slow last resolve extra time past the deadline and resolves a late match', async () => { + vi.useFakeTimers(); + try { + const deadline = new Deadline(300); + let calls = 0; + const { engine, resolveTimeouts } = makeEngine({ + resolve: Array.from({ length: 10 }, () => async () => { + calls += 1; + if (calls === 1) { + await new Promise((resolve) => setTimeout(resolve, 150)); + return [] as readonly NodeRef[]; + } + await new Promise((resolve) => setTimeout(resolve, 150)); + return [REF]; + }) as unknown as Array<() => readonly NodeRef[]>, + }); + const pending = engine.resolveExactlyOne(EXPRESSION, deadline); + await vi.advanceTimersByTimeAsync(500); + const resolved = await pending; + expect(resolved).toBe(REF); + expect(calls).toBe(2); + expect(resolveTimeouts[1]).toBe(200); + } finally { + vi.useRealTimers(); + } }); it('ends a frame wait the deadline cut off on the missing frame, as LOCATOR_NOT_FOUND', async () => { diff --git a/packages/e2e/tests/unit/time.test.ts b/packages/e2e/tests/unit/time.test.ts index 3ec2a71d56..31d139354e 100644 --- a/packages/e2e/tests/unit/time.test.ts +++ b/packages/e2e/tests/unit/time.test.ts @@ -351,7 +351,7 @@ describe('pollCondition', () => { await expect(early).rejects.toBe(cut); }); - it('honors readLatency in pollCondition: treats a last read within readLatency as cut off, not hung', async () => { + it('treats a last read in pollCondition within measured latency as cut off, not hung', async () => { vi.useFakeTimers(); const cut = new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); const deadline = new Deadline(600); @@ -360,11 +360,13 @@ describe('pollCondition', () => { deadline, signal: new AbortController().signal, negated: false, - readLatency: 1_500, evaluate: async () => { calls += 1; - if (calls === 1) return false; - // Started with ~500ms remaining, which is < readLatency (1500ms) but >= POLL_INTERVAL_MS (100ms) + if (calls === 1) { + await sleep(300); + return false; + } + // Started with ~200ms remaining, which is < measured latency (300ms) but >= POLL_INTERVAL_MS (100ms) throw cut; }, onTimeout: () => new Error('poll timed out'), diff --git a/packages/mobile/src/engine.ts b/packages/mobile/src/engine.ts index 3d54ba29a2..8e3e064d59 100644 --- a/packages/mobile/src/engine.ts +++ b/packages/mobile/src/engine.ts @@ -17,9 +17,6 @@ import { AgentDeviceSurface, assertPermissions } from './surface.ts'; const surfaces = new WeakMap(); -/** Expected latency for a device snapshot read, granting slow reads past the deadline. */ -const MOBILE_READ_LATENCY_MS = 2_000; - /** Assembles the manifest for one surface. Exported for tests that script the client. */ export function buildEngine(surface: AgentDeviceSurface): EngineHandle { const handle = defineEngine({ @@ -27,7 +24,6 @@ export function buildEngine(surface: AgentDeviceSurface): EngineHandle { version: ownVersion(), spiVersion: 1, platform: surface.options.platform, - readLatency: MOBILE_READ_LATENCY_MS, ...(surface.pool.size === undefined ? {} : { workers: surface.pool.size }), prepare: (info) => surface.pool.prepare(info), finish: (info) => surface.pool.finish(info), @@ -35,8 +31,8 @@ export function buildEngine(surface: AgentDeviceSurface): EngineHandle { startAttempt: (context) => surface.startAttempt(context), endAttempt: (context) => surface.endAttempt(context), dispose: (context) => surface.dispose(context), - observe: (operation, options) => surface.observe(operation, options), - locate: (expression, operation) => surface.locate(expression, operation), + observe: (operation, options) => surface.observe(operation, options, surface.inputBudget('observe', operation.timeoutMs)), + locate: (expression, operation) => surface.locate(expression, operation, surface.inputBudget('locate', operation.timeoutMs)), // The harness signal does not abort at an operation's budget, so each // input enforces its own; see InputBudget in support.ts. perform: (ref, action, operation) => surface.perform(ref, action, operation, surface.inputBudget(action.kind, operation.timeoutMs)), @@ -58,7 +54,7 @@ export function buildEngine(surface: AgentDeviceSurface): EngineHandle { reset: (operation) => surface.reset(operation), }, artifacts: { - screenshot: (label, operation) => surface.screenshot(label, operation), + screenshot: (label, operation) => surface.screenshot(label, operation, surface.inputBudget('screenshot', operation.timeoutMs)), startVideo: (operation) => surface.startVideo(operation), stopVideo: (operation) => surface.stopVideo(operation), }, diff --git a/packages/mobile/src/support.ts b/packages/mobile/src/support.ts index 3597af0cbf..05e47278ae 100644 --- a/packages/mobile/src/support.ts +++ b/packages/mobile/src/support.ts @@ -76,40 +76,15 @@ export class InputBudget { return withTimeout(pending, Math.max(remaining, MIN_ANSWER_MS), () => this.maybeCommitted()); } - private maybeCommitted(): EngineError { - return new EngineError('ACTION_MAY_HAVE_COMMITTED', `${this.label} timed out before the device answered; its input may have been dispatched`, { - retryable: false, - }); - } -} - -/** - * Bounds a device read (snapshot or screenshot) by the operation's budget. - * When the budget expires before or during the read, it fails with - * `OPERATION_TIMEOUT` instead of holding the test until the test timeout. - */ -export class ReadBudget { - private readonly endsAt: number; - - constructor( - private readonly label: string, - timeoutMs: number, - private readonly track: (pending: Promise) => Promise, - ) { - this.endsAt = Date.now() + timeoutMs; - } - - remainingMs(): number { - return Math.max(0, this.endsAt - Date.now()); - } - check(): void { if (this.endsAt - Date.now() <= 0) { - throw new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }); + throw this.sent + ? this.maybeCommitted() + : new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }); } } - send(dispatch: () => Promise): Promise { + read(dispatch: () => Promise): Promise { const remaining = this.endsAt - Date.now(); if (remaining <= 0) { return Promise.reject( @@ -121,6 +96,12 @@ export class ReadBudget { new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before the device answered`, { retryable: false }), ); } + + private maybeCommitted(): EngineError { + return new EngineError('ACTION_MAY_HAVE_COMMITTED', `${this.label} timed out before the device answered; its input may have been dispatched`, { + retryable: false, + }); + } } export function invalidState(text: string): EngineError { diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index c6b51acdd3..f5e11585e7 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -75,7 +75,6 @@ import { type Rect, type Point, InputBudget, - ReadBudget, } from './support.ts'; /** Hinge poses `device.fold` accepts: `closed` lights the outer display, `half-open` and `open` the inner one. */ @@ -471,11 +470,6 @@ export class AgentDeviceSurface { return new InputBudget(label, timeoutMs, (pending) => this.track(pending)); } - /** The budget one read runs under; its device commands stay in flight until they settle. */ - readBudget(label: string, timeoutMs: number): ReadBudget { - return new ReadBudget(label, timeoutMs, (pending) => this.track(pending)); - } - /** Registers one device command as in flight until it settles. */ private track(pending: Promise): Promise { this.inflight.add(pending); @@ -850,7 +844,7 @@ export class AgentDeviceSurface { return installed; } - private async snapshot(signal: AbortSignal, interactiveOnly: boolean, budget?: ReadBudget): Promise { + private async snapshot(signal: AbortSignal, interactiveOnly: boolean, budget?: InputBudget): Promise { let last: RawSnapshot = {}; for (const backoffMs of SPARSE_RETRY_BACKOFF_MS) { if (backoffMs > 0) { @@ -859,7 +853,7 @@ export class AgentDeviceSurface { } budget?.check(); if (signal.aborted) throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false }); - last = await this.capture(signal, interactiveOnly); + last = await (budget ? budget.read(() => this.capture(signal, interactiveOnly)) : this.capture(signal, interactiveOnly)); if (last.appBundleId !== undefined || last.appName !== undefined) { this.appIdentity = last.appBundleId ?? last.appName; } @@ -890,10 +884,9 @@ export class AgentDeviceSurface { * anything is open is an empty screen rather than a fault: the model's next * move is the open tool, and failing the step would take that move away. */ - private async snapshotOrEmpty(operation: OperationContext, interactiveOnly: boolean): Promise { - const budget = this.readBudget('snapshot', operation.timeoutMs); + private async snapshotOrEmpty(operation: OperationContext, interactiveOnly: boolean, budget?: InputBudget): Promise { try { - return await budget.send(() => this.snapshot(operation.signal, interactiveOnly, budget)); + return await this.snapshot(operation.signal, interactiveOnly, budget); } catch (cause) { if (!this.managesApp && cause instanceof EngineError && cause.code === 'INVALID_STATE') return { nodes: [] }; throw cause; @@ -920,10 +913,10 @@ export class AgentDeviceSurface { * the harness measures every rect and point against the viewport, so an * invented one would misplace every tap. */ - private async viewportFor(projected: ProjectedSnapshot, signal: AbortSignal, timeoutMs?: number): Promise { + private async viewportFor(projected: ProjectedSnapshot, signal: AbortSignal, budget?: InputBudget): Promise { const known = projected.viewport ?? this.knownViewport; if (known !== undefined) return known; - const probed = await this.probeViewport(signal, timeoutMs); + const probed = await this.probeViewport(signal, budget); if (probed === undefined) { throw new EngineError( 'ENGINE_FAILURE', @@ -936,18 +929,18 @@ export class AgentDeviceSurface { } /** The device's logical screen size as its screenshot reports it; undefined when it reports none. */ - private async probeViewport(signal: AbortSignal, timeoutMs?: number): Promise { - const result = await this.captureScreenshot(signal, async (shot) => shot, {}, timeoutMs); + private async probeViewport(signal: AbortSignal, budget?: InputBudget): Promise { + const result = await this.captureScreenshot(signal, async (shot) => shot, {}, budget); return logicalScreenSize(result); } - async observe(operation: OperationContext, options?: EngineObserveOptions): Promise { - const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive'); + async observe(operation: OperationContext, options?: EngineObserveOptions, budget?: InputBudget): Promise { + const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive', budget); const projected = this.project(raw); this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected.index)])); - const viewport = await this.viewportFor(projected, operation.signal, operation.timeoutMs); + const viewport = await this.viewportFor(projected, operation.signal, budget); const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected)); - const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined; + const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }, budget) : undefined; return { root: screenRoot(projected.roots, viewport), viewport, @@ -958,8 +951,8 @@ export class AgentDeviceSurface { } /** Retains only action bindings for matches, independently of the observation generation. */ - async locate(expression: LocatorExpression, operation: OperationContext): Promise { - const raw = await this.snapshotOrEmpty(operation, false); + async locate(expression: LocatorExpression, operation: OperationContext, budget?: InputBudget): Promise { + const raw = await this.snapshotOrEmpty(operation, false, budget); const projected = this.project(raw); this.traceScreen(raw, projected); const matches = resolveExpression(expression, projected.index, this.options.platform); @@ -1076,7 +1069,7 @@ export class AgentDeviceSurface { * mid-flight would send the action where the control was: below the * screen, for a modal rising from the bottom. */ - private async settled(entry: NodeBinding, operation: OperationContext): Promise { + private async settled(entry: NodeBinding, operation: OperationContext, budget?: InputBudget): Promise { const since = Date.now() - this.lastActionAt; const remaining = this.transitionMs - since; if (remaining <= 0) return entry; @@ -1091,12 +1084,12 @@ export class AgentDeviceSurface { if (operation.signal.aborted) { throw new EngineError('CANCELLED', 'transition wait cancelled', { retryable: false }); } - return this.relocated(entry, operation); + return this.relocated(entry, operation, budget); } /** The same control in a fresh snapshot; the binding as it was when the snapshot no longer lists it. */ - private async relocated(entry: NodeBinding, operation: OperationContext): Promise { - const projected = this.project(await this.snapshotOrEmpty(operation, false)); + private async relocated(entry: NodeBinding, operation: OperationContext, budget?: InputBudget): Promise { + const projected = this.project(await this.snapshotOrEmpty(operation, false, budget)); const found = this.refind(entry, projected.index); return found === undefined ? entry : this.bind(found, projected.index); } @@ -1143,7 +1136,7 @@ export class AgentDeviceSurface { } const before = this.latestIndex; const run = async (): Promise => { - const target = deterministic ? await this.settled(entry, operation) : entry; + const target = deterministic ? await this.settled(entry, operation, input) : entry; switch (action.kind) { case 'tap': return input.send(() => client.interactions.press({ ...this.actionTarget(target, true), ...settle })); @@ -1335,10 +1328,10 @@ export class AgentDeviceSurface { * one-node tree has no keys and would pass for a dismissed keyboard. */ private async dismissKeyboardByGesture(signal: AbortSignal, input: InputBudget | undefined): Promise { - let raw = await this.dismissKeyboardTip(await this.snapshot(signal, false), signal, input); + let raw = await this.dismissKeyboardTip(await this.snapshot(signal, false, input), signal, input); if (!keyboardShowing(raw)) return; const projected = projectSnapshot(raw.nodes ?? [], { mintId: () => 'keyboard', viewport: raw.viewport }); - const viewport = await this.viewportFor(projected, signal); + const viewport = await this.viewportFor(projected, signal, input); const centre = { x: viewport.width / 2, y: viewport.height / 2 }; const drags: readonly Point[] = [ { x: centre.x - viewport.width * KEYBOARD_DRAG_RATIO, y: centre.y }, @@ -1346,7 +1339,7 @@ export class AgentDeviceSurface { ]; for (const to of drags) { await this.command('keyboard.dismiss', (client) => sendInput(input, () => client.interactions.swipe({ from: centre, to })), signal); - raw = await this.snapshot(signal, false); + raw = await this.snapshot(signal, false, input); if (!keyboardShowing(raw)) return; } throw new EngineError( @@ -1370,7 +1363,7 @@ export class AgentDeviceSurface { const button = nodes.find((node) => node.label === 'Continue' && node.rect !== undefined && node.rect.y > tipTop); if (button?.rect === undefined) return raw; await this.command('keyboard.dismiss', (client) => sendInput(input, () => client.interactions.press(centreOf(button.rect!))), signal); - return await this.snapshot(signal, false); + return await this.snapshot(signal, false, input); } async back(operation: OperationContext, input: InputBudget): Promise { @@ -1432,21 +1425,21 @@ export class AgentDeviceSurface { * bounds cannot be masked, and an image that cannot be redacted is not * written at all. */ - async screenshot(label: string | undefined, operation: OperationContext): Promise { + async screenshot(label: string | undefined, operation: OperationContext, budget?: InputBudget): Promise { const attempt = this.attempt; if (attempt === undefined) throw invalidState('screenshot outside an attempt'); - const masked = await this.maskedScreenshot(operation.signal, operation.timeoutMs); + const masked = await this.maskedScreenshot(operation.signal, budget); attempt.screenshots += 1; const name = `${String(attempt.screenshots).padStart(3, '0')}-${sanitizeFilename(label ?? 'screenshot')}.png`; - const relative = path.join('screenshots', name); + const relative = path.posix.join('screenshots', name); mkdirSync(path.join(attempt.artifactsDir, 'screenshots'), { recursive: true }); - writeFileSync(path.join(attempt.artifactsDir, relative), masked.data); + writeFileSync(path.join(attempt.artifactsDir, 'screenshots', name), masked.data); return relative; } /** Raw device pixels; cleanup follows the capture even when its caller abandons it. */ - private rawScreenshot(signal?: AbortSignal, capture: ScreenshotCapture = {}, timeoutMs?: number): Promise { - return this.captureScreenshot(signal, async (shot, file) => new Uint8Array(readFileSync(shot.path ?? file)), capture, timeoutMs); + private rawScreenshot(signal?: AbortSignal, capture: ScreenshotCapture = {}, budget?: InputBudget): Promise { + return this.captureScreenshot(signal, async (shot, file) => new Uint8Array(readFileSync(shot.path ?? file)), capture, budget); } /** @@ -1458,7 +1451,7 @@ export class AgentDeviceSurface { signal: AbortSignal | undefined, read: (shot: RawScreenshotResult, file: string) => Promise, capture: ScreenshotCapture = {}, - timeoutMs?: number, + budget?: InputBudget, ): Promise { const run = () => this.command('screenshot', async (client) => { @@ -1470,9 +1463,8 @@ export class AgentDeviceSurface { rmSync(directory, { recursive: true, force: true }); } }, signal); - if (timeoutMs === undefined) return run(); - const budget = this.readBudget('screenshot', timeoutMs); - return budget.send(run); + if (budget === undefined) return run(); + return budget.read(run); } /** @@ -1481,16 +1473,16 @@ export class AgentDeviceSurface { * throws when a secure field cannot be covered, because an image that may * hold a credential must not leave the engine. */ - private async maskedScreenshot(signal?: AbortSignal, timeoutMs = 30_000): Promise<{ data: Uint8Array; masked: number }> { + private async maskedScreenshot(signal?: AbortSignal, budget?: InputBudget): Promise<{ data: Uint8Array; masked: number }> { const operation: OperationContext = { signal: signal ?? new AbortController().signal, - timeoutMs, + timeoutMs: 30_000, runId: '', attemptId: '', origin: 'test', }; - const projected = this.project(await this.snapshotOrEmpty(operation, false)); - const data = await this.rawScreenshot(signal, {}, timeoutMs); + const projected = this.project(await this.snapshotOrEmpty(operation, false, budget)); + const data = await this.rawScreenshot(signal, {}, budget); const masked = redactSecure(data, projected, projected.viewport ?? this.knownViewport); if (masked === undefined) { throw new EngineError('ENGINE_FAILURE', 'a secure field on screen could not be masked; screenshot withheld', { @@ -1510,10 +1502,11 @@ export class AgentDeviceSurface { projected: ProjectedSnapshot, viewport: ViewportSize, capture: ScreenshotCapture, + budget?: InputBudget, ): Promise<{ pixels: ObservationPixels; masked: number } | undefined> { let raw: Uint8Array; try { - raw = await this.rawScreenshot(operation.signal, capture, operation.timeoutMs); + raw = await this.rawScreenshot(operation.signal, capture, budget); } catch (cause) { if (cause instanceof EngineError && cause.code === 'OPERATION_TIMEOUT') throw cause; return undefined; diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index c037904e01..60812d4fa5 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -961,6 +961,43 @@ describe('observation', () => { } }); + it('shares one budget across observe steps so screenshot cannot take a fresh budget', async () => { + const h = harness(); + await openAttempt(h); + vi.useFakeTimers(); + try { + // Snapshot takes 1,500ms + h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve(SETTINGS_SNAPSHOT), 1_500))); + // Screenshot hangs + h.fake.respond('capture.screenshot', () => new Promise(() => undefined)); + const obs = h.engine.observe!({ ...operation(), timeoutMs: 2_000 }, { pixels: true }); + const assertion = expect(obs).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + // Total operation budget is 2,000ms. At 2,000ms (1,500ms snapshot + 500ms screenshot), it must time out. + await vi.advanceTimersByTimeAsync(2_000); + await assertion; + } finally { + vi.useRealTimers(); + } + }); + + it('bounds keyboard dismiss snapshots by the operation budget', async () => { + const h = harness(); + await openAttempt(h); + vi.useFakeTimers(); + try { + h.fake.respond('command.keyboard', () => { + throw new AppError('UNSUPPORTED_OPERATION', 'Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key'); + }); + h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve(SETTINGS_SNAPSHOT), 5_000))); + const dismiss = h.engine.keyboard!.dismiss!({ ...operation(), timeoutMs: 1_000 }); + const assertion = expect(dismiss).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + await vi.advanceTimersByTimeAsync(1_000); + await assertion; + } finally { + vi.useRealTimers(); + } + }); + it('cancels a snapshot when the operation aborts', async () => { const h = harness(); h.fake.respond('capture.snapshot', () => new Promise(() => undefined)); diff --git a/packages/mobile/tests/unit/support.test.ts b/packages/mobile/tests/unit/support.test.ts index 1dbc32679a..460d4af01c 100644 --- a/packages/mobile/tests/unit/support.test.ts +++ b/packages/mobile/tests/unit/support.test.ts @@ -42,6 +42,28 @@ describe('support helpers', () => { message: expect.stringMatching(/^longPress holds for 3000 ms, but only 2000 ms of its timeout are left/), }); expect(held).toBe(false); + + // A read under budget succeeds, and one that hangs fails with OPERATION_TIMEOUT + const readBudget = new InputBudget('snapshot', 300, (pending) => pending); + const readSuccess = readBudget.read(() => Promise.resolve('data')); + await expect(readSuccess).resolves.toBe('data'); + + const hangingRead = new InputBudget('snapshot', 200, (pending) => pending); + const hangingPromise = hangingRead.read(hung); + const hangingOutcome = expect(hangingPromise).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT', message: 'snapshot timed out before the device answered' }); + await vi.advanceTimersByTimeAsync(200); + await hangingOutcome; + + // A read with no budget left fails immediately with OPERATION_TIMEOUT + const spentRead = new InputBudget('snapshot', 100, (pending) => pending); + await vi.advanceTimersByTimeAsync(100); + await expect(spentRead.read(async () => 'late')).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT', message: 'snapshot timed out before it was sent' }); + + // check() throws when budget has expired + const checkBudget = new InputBudget('observe', 100, (pending) => pending); + expect(() => checkBudget.check()).not.toThrow(); + await vi.advanceTimersByTimeAsync(100); + expect(() => checkBudget.check()).toThrowError(/observe timed out before it was sent/); } finally { vi.useRealTimers(); } From 7c9aed8c8246cb5ee0dc50d5a27b4e561b97c344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 16 Oct 2026 15:11:13 +0200 Subject: [PATCH 06/12] fix(mobile): give the last read of a wait the time reads take Address review: one ReadWindow helper for every wait (pollCondition, locator waits, scrollUntilVisible, toHaveScreenshot), InputBudget.read honors sent, no snapshot retry after an abandoned capture, sparse retries stop when the budget cannot fit one, mobile peer e2e >=0.21.2. Co-Authored-By: Claude --- .changeset/last-read-grace.md | 5 + .changeset/mobile-reads-budget.md | 3 +- docs/mobile.mdx | 12 +- docs/reference/engine.mdx | 16 +- packages/e2e/src/engine/contract.ts | 6 +- packages/e2e/src/expect/async.ts | 15 +- packages/e2e/src/expect/screenshot-capture.ts | 20 +-- packages/e2e/src/expect/screenshot.ts | 10 +- packages/e2e/src/internal/time.ts | 81 ++++++--- packages/e2e/src/locator/engine.ts | 102 +++++------ packages/e2e/src/locator/screen.ts | 16 +- packages/e2e/tests/helpers/screen-over.ts | 6 +- .../e2e/tests/unit/locator-engine.test.ts | 167 ++++++++---------- .../e2e/tests/unit/locator-wait-for.test.ts | 31 +++- packages/e2e/tests/unit/time.test.ts | 58 ++++-- packages/mobile/package.json | 2 +- packages/mobile/src/engine.ts | 4 +- packages/mobile/src/support.ts | 53 +++--- packages/mobile/src/surface.ts | 46 +++-- packages/mobile/tests/unit/engine.test.ts | 73 ++++++-- packages/mobile/tests/unit/support.test.ts | 39 ++-- 21 files changed, 457 insertions(+), 308 deletions(-) create mode 100644 .changeset/last-read-grace.md diff --git a/.changeset/last-read-grace.md b/.changeset/last-read-grace.md new file mode 100644 index 0000000000..80b11c58cd --- /dev/null +++ b/.changeset/last-read-grace.md @@ -0,0 +1,5 @@ +--- +'e2e': patch +--- + +`waitFor`, `expect` on a locator, and `toHaveScreenshot` give their last read extra time past the timeout, as long as their slowest read so far, so a screen that shows up during a slow last read still passes. diff --git a/.changeset/mobile-reads-budget.md b/.changeset/mobile-reads-budget.md index 178390828d..19ccb1bd15 100644 --- a/.changeset/mobile-reads-budget.md +++ b/.changeset/mobile-reads-budget.md @@ -1,6 +1,5 @@ --- -'e2e': patch '@e2e-dev/mobile': patch --- -The mobile engine now bounds reads (snapshots and screenshots) by their operation budget so a hung device does not hold the step until the test timeout. The locator poll measures how long reads take and grants the last read extra time past the deadline so a slow read still finishes, avoiding timeouts when elements appear late. +Snapshots and screenshots now get the step's timeout, so a device that stops answering fails the step instead of holding it until the test timeout. Requires `e2e` 0.21.2 or later. diff --git a/docs/mobile.mdx b/docs/mobile.mdx index b58adf09da..660ac6cc37 100644 --- a/docs/mobile.mdx +++ b/docs/mobile.mdx @@ -687,11 +687,13 @@ message and hint, never `LAUNCH_TIMEOUT`, `ACTION_FAILED`, or `back` gets its `timeout` (default `actionTimeout`). If the device doesn't answer in time, the step fails with `ACTION_FAILED` and isn't retried: the input may have landed. A `longPress` that holds at least as long as the -time left is refused before it's sent; raise its `timeout`. Device reads -(snapshots and screenshots) are bounded by their operation timeout, and -locator polls grant slow reads extra time past the deadline so missing -elements end with `LOCATOR_NOT_FOUND` on locator actions and reads, or with -`ASSERTION_FAILED` on assertion polls. +time left is refused before it's sent; raise its `timeout`. Snapshots and +screenshots get the same timeout, so a device that stops answering fails the +step with `ACTION_FAILED` instead of holding it until the test timeout. +`waitFor`, `expect`, and `toHaveScreenshot` give their last snapshot extra +time, as long as their slowest one so far, so a screen that shows up during +it still passes. An element that never shows up fails as +`LOCATOR_NOT_FOUND`, or `ASSERTION_FAILED` for `expect`. ```bash npm diff --git a/docs/reference/engine.mdx b/docs/reference/engine.mdx index 6995679e3e..972145964a 100644 --- a/docs/reference/engine.mdx +++ b/docs/reference/engine.mdx @@ -225,9 +225,16 @@ export interface Engine { ```ts /** What every engine call carries: budget, ids, and who is acting. */ export interface OperationContext { - /** Aborts with the attempt or when the operation budget is spent. */ + /** + * Aborts when the attempt, or the hook running the call, ends. It does not + * abort at `timeoutMs`: bound each call by that budget yourself. + */ readonly signal: AbortSignal; - /** Remaining operation budget when the call starts. */ + /** + * Remaining operation budget when the call starts. A waiting call's last + * locate or read can get more than its deadline leaves: as long as the + * slowest one it has seen. + */ readonly timeoutMs: number; /** The run id. */ readonly runId: string; @@ -986,8 +993,9 @@ export const RETRYABLE_ENGINE_ERROR_CODES: ReadonlySet = new Se | `INVALID_STATE` | `APP_NOT_OPEN` | no | | `ENGINE_FAILURE`, any non-`EngineError` | infrastructure `ENGINE_FAILURE` | no | -An `OPERATION_TIMEOUT` on a locate or read that started with under 100 ms of -a waiting call's deadline left, after one that completed, ends the wait +An `OPERATION_TIMEOUT` on a locate or read that started with under 100 ms, or +under its slowest completed locate or read, of a waiting call's deadline left, +after one that completed, ends the wait instead: an action's or `waitFor`'s `LOCATOR_NOT_FOUND`, or an assertion's `ASSERTION_FAILED`, with the timeout as its cause. A negated assertion passes there if its negation had held long enough by the time the read was cut off, as diff --git a/packages/e2e/src/engine/contract.ts b/packages/e2e/src/engine/contract.ts index 2866ec2750..d60269e407 100644 --- a/packages/e2e/src/engine/contract.ts +++ b/packages/e2e/src/engine/contract.ts @@ -32,7 +32,11 @@ export interface OperationContext { * abort at `timeoutMs`: bound each call by that budget yourself. */ readonly signal: AbortSignal; - /** Remaining operation budget when the call starts. */ + /** + * Remaining operation budget when the call starts. A waiting call's last + * locate or read can get more than its deadline leaves: as long as the + * slowest one it has seen. + */ readonly timeoutMs: number; /** The run id. */ readonly runId: string; diff --git a/packages/e2e/src/expect/async.ts b/packages/e2e/src/expect/async.ts index 90b233066f..4748759da5 100644 --- a/packages/e2e/src/expect/async.ts +++ b/packages/e2e/src/expect/async.ts @@ -14,7 +14,7 @@ import { type TextPattern, } from '../internal/text.ts'; import { isValueControl } from '../internal/roles.ts'; -import { Deadline, pollCondition } from '../internal/time.ts'; +import { pollCondition, type ReadWindow } from '../internal/time.ts'; import { SampleHistory } from './samples.ts'; import { attributeOf, denySecureRead, isNodeVisible } from '../locator/engine.ts'; import { describeExpression } from '../locator/expression.ts'; @@ -175,10 +175,11 @@ class AsyncExpectationImpl implements AsyncExpectation { await samples.record(engine, () => pollCondition({ deadline, + reads: engine.reads(deadline), signal: engine.signal, negated: this.negated, - evaluate: async () => { - const sample = await this.sample(spec, deadline); + evaluate: async (reads) => { + const sample = await this.sample(spec, reads); lastSample = sample; if (spec.readsWithheld === true) { denySecureRead(sample.node === null ? sample.nodes : [sample.node], this.label); @@ -221,18 +222,18 @@ class AsyncExpectationImpl implements AsyncExpectation { return spec.evaluableNode?.(sample.node) ?? true; } - private async sample(spec: MatcherSpec, deadline: Deadline): Promise { + private async sample(spec: MatcherSpec, reads: ReadWindow): Promise { const { engine } = this.internals.context; if (spec.wholeSet === 'read') { - const nodes = await engine.readAll(this.internals.expression, deadline); + const nodes = await engine.readAll(this.internals.expression, reads); return { count: nodes.length, node: null, nodes }; } if (spec.wholeSet === true) { - const refs = await engine.resolveAll(this.internals.expression, deadline); + const refs = await engine.resolveAll(this.internals.expression, reads); return { count: refs.length, node: null, nodes: [] }; } const missingFrame = spec.evaluableWithoutNode === true ? 'empty' : 'wait'; - const { node, count } = await engine.tryRead(this.internals.expression, deadline, missingFrame); + const { node, count } = await engine.tryRead(this.internals.expression, reads, missingFrame); return { count, node, nodes: [] }; } diff --git a/packages/e2e/src/expect/screenshot-capture.ts b/packages/e2e/src/expect/screenshot-capture.ts index 94c74dc58e..08c9f384e7 100644 --- a/packages/e2e/src/expect/screenshot-capture.ts +++ b/packages/e2e/src/expect/screenshot-capture.ts @@ -8,7 +8,7 @@ import type { LocatorExpression } from '../engine/surface.ts'; import { ConfigurationError, isE2EError } from '../internal/errors.ts'; import { clipBox, cropImage, decodePng, fillBoxes, resampleImage, type ImageBox, type RgbaImage } from '../internal/image.ts'; -import { isOperationTimeout, type Deadline } from '../internal/time.ts'; +import { isOperationTimeout, type ReadWindow } from '../internal/time.ts'; import { isNodeVisible } from '../locator/engine.ts'; import { describeExpression } from '../locator/expression.ts'; import type { ScreenContext } from '../locator/screen.ts'; @@ -33,40 +33,40 @@ export type Capture = { readonly image: RgbaImage } | { readonly missing: string * a capture the deadline cut short: the call then fails on what the * captures before it showed. */ -export function screenshotCapturer(context: ScreenContext, request: CaptureRequest, deadline: Deadline): () => Promise { +export function screenshotCapturer(context: ScreenContext, request: CaptureRequest, reads: ReadWindow): () => Promise { const { engine } = context; const state = { scrolled: !engine.session.actions.has('scrollIntoView') }; return async () => { try { - return await captureOnce(context, request, deadline, state); + return await captureOnce(context, request, reads, state); } catch (cause) { - if (deadline.expired() && isOperationTimeout(cause)) return { missing: 'the last screenshot ran past the timeout' }; + if (reads.deadline.expired() && isOperationTimeout(cause)) return { missing: 'the last screenshot ran past the timeout' }; throw cause; } }; } /** One capture; `state.scrolled` records that the subject was scrolled into view. */ -async function captureOnce(context: ScreenContext, request: CaptureRequest, deadline: Deadline, state: { scrolled: boolean }): Promise { +async function captureOnce(context: ScreenContext, request: CaptureRequest, reads: ReadWindow, state: { scrolled: boolean }): Promise { const { engine } = context; const shown = request.subject === undefined ? 'screen' : describeExpression(request.subject); let box: ImageBox | undefined; if (request.subject !== undefined) { - let { node } = await engine.tryRead(request.subject, deadline); + let { node } = await engine.tryRead(request.subject, reads); if (node !== null && !state.scrolled) { try { - await engine.performUntil(request.subject, { kind: 'scrollIntoView' }, deadline); + await engine.performUntil(request.subject, { kind: 'scrollIntoView' }, reads.deadline); } catch (cause) { if (!isE2EError(cause) || cause.code !== 'LOCATOR_NOT_FOUND') throw cause; } state.scrolled = true; - ({ node } = await engine.tryRead(request.subject, deadline)); + ({ node } = await engine.tryRead(request.subject, reads)); } if (node === null) return { missing: `no node matches ${shown}` }; if (!isNodeVisible(node) || node.rect === undefined) return { missing: `${shown} is not visible` }; box = node.rect; } - const observation = await engine.session.observe(engine.operationWithin(deadline), { pixels: true, comparable: true }); + const observation = await reads.read((timeoutMs) => engine.session.observe(engine.readOperation(timeoutMs), { pixels: true, comparable: true })); if (observation.pixels === undefined) return { missing: 'the engine returned no screenshot' }; if (!pixelsProvenMasked(observation.redaction)) { throw new ConfigurationError('POLICY_DENIED', `${request.api} could not prove every secure field on screen was masked, so the screenshot is not kept`); @@ -85,7 +85,7 @@ async function captureOnce(context: ScreenContext, request: CaptureRequest, dead if (request.masks.length === 0) return { image }; const boxes: ImageBox[] = []; for (const mask of request.masks) { - for (const node of await engine.readAllNow(mask, deadline)) { + for (const node of await engine.readAllNow(mask, reads)) { if (isNodeVisible(node) && node.rect !== undefined) boxes.push({ ...node.rect, x: node.rect.x - origin.x, y: node.rect.y - origin.y }); } } diff --git a/packages/e2e/src/expect/screenshot.ts b/packages/e2e/src/expect/screenshot.ts index 8c8b85ef96..be4c7a1bad 100644 --- a/packages/e2e/src/expect/screenshot.ts +++ b/packages/e2e/src/expect/screenshot.ts @@ -11,7 +11,7 @@ import type { LocatorExpression } from '../engine/surface.ts'; import { writeFileAtomic } from '../internal/atomic-write.ts'; import { ConfigurationError, TestError } from '../internal/errors.ts'; import { compareImages, decodePng, encodePng, type ImageComparison, type RgbaImage } from '../internal/image.ts'; -import { pollCondition, type Deadline } from '../internal/time.ts'; +import { pollCondition, type Deadline, type ReadWindow } from '../internal/time.ts'; import { describeExpression } from '../locator/expression.ts'; import type { ScreenContext } from '../locator/screen.ts'; import type { ScreenshotContext } from '../run/screenshots.ts'; @@ -32,6 +32,7 @@ interface Run { readonly api: string; readonly call: ScreenshotCall; readonly stored: Stored; + readonly reads: ReadWindow; readonly capture: () => Promise; } @@ -78,8 +79,9 @@ export async function expectScreenshot( throw new ConfigurationError('POLICY_DENIED', `${api} is denied after a secret fill because the app may display the secret outside a secure field`); } const deadline = context.engine.deadline(call.timeout ?? context.engine.assertionTimeout); - const capture = screenshotCapturer(context, { api, subject, masks: call.masks, maskColor: call.maskColor }, deadline); - const run: Run = { context, store, api, call, stored, capture }; + const reads = context.engine.reads(deadline); + const capture = screenshotCapturer(context, { api, subject, masks: call.masks, maskColor: call.maskColor }, reads); + const run: Run = { context, store, api, call, stored, reads, capture }; const expected = existsSync(file) ? readStored(api, stored) : undefined; if (expected !== undefined && (negated || !store.update) && store.written.has(file)) { throw failure(api, `${stored.shown} was written earlier in this run and is not reviewed yet: look at it, commit it, and run again`, stored, 'unreviewed stored screenshot'); @@ -112,6 +114,7 @@ async function matchOrSettle(run: Run, expected: RgbaImage | undefined, deadline let captures = 0; await pollCondition({ deadline, + reads: run.reads, signal: run.context.engine.signal, negated: false, evaluate: async () => { @@ -186,6 +189,7 @@ async function expectDifferent(run: Run, expected: RgbaImage | undefined, deadli let missing: string | undefined; await pollCondition({ deadline, + reads: run.reads, signal: run.context.engine.signal, negated: true, evaluate: async () => { diff --git a/packages/e2e/src/internal/time.ts b/packages/e2e/src/internal/time.ts index 07cd5d57c9..5f95a96d66 100644 --- a/packages/e2e/src/internal/time.ts +++ b/packages/e2e/src/internal/time.ts @@ -68,15 +68,44 @@ export const MAX_TIMER_MS = 2 ** 31 - 1; export const POLL_INTERVAL_MS = 100; /** - * Whether `cause` is a read a wait's deadline cut off: an operation timeout - * of a read that started with less than one poll tick (or the engine's read - * latency, whichever is larger) of the wait left, so it timed out because the - * wait did, not because the app stopped answering. A read that started with - * more and still timed out hung, and its timeout is the failure. Reads the - * engine error a locator failure wraps as well as a bare one. + * The reads of one wait. Each engine read runs on the time left before the + * wait's deadline plus the longest read the wait has finished, capped by + * `cap` (the running phase's deadline), so a slow last read still finishes. + * Without a cap, reads end at the deadline. */ -export function cutOffAtDeadline(cause: unknown, startedWithMs: number, latencyMs = 0): boolean { - return startedWithMs < Math.max(POLL_INTERVAL_MS, latencyMs) && isOperationTimeout(cause); +export class ReadWindow { + private slowestMs = 0; + + constructor( + readonly deadline: Deadline, + private readonly cap: Deadline = deadline, + ) {} + + /** The budget for an engine read starting now, never under 1 ms. */ + timeoutMs(): number { + const now = Date.now(); + return Math.max(1, Math.min(this.cap.remaining(now), this.deadline.remaining(now) + this.slowestMs)); + } + + /** Runs one engine read on `timeoutMs()`, timing it when it answers. */ + async read(call: (timeoutMs: number) => Promise): Promise { + const startedAt = Date.now(); + const value = await call(this.timeoutMs()); + this.slowestMs = Math.max(this.slowestMs, Date.now() - startedAt); + return value; + } + + /** + * Whether `cause` is a read the deadline cut off: an operation timeout of a + * read that started with less than one poll tick, or the longest read this + * wait finished, of the deadline left, so it timed out because the wait + * did, not because the app stopped answering. A read that started with + * more and still timed out hung, and its timeout is the failure. Reads the + * engine error a locator failure wraps as well as a bare one. + */ + cutOff(cause: unknown, startedWithMs: number): boolean { + return startedWithMs < Math.max(POLL_INTERVAL_MS, this.slowestMs) && isOperationTimeout(cause); + } } /** Whether `cause` is the engine's `OPERATION_TIMEOUT`, bare or wrapped by the locator failure it was translated into. */ @@ -90,15 +119,17 @@ export const NEGATION_GRACE_MS = 1000; export interface PollConditionOptions { readonly deadline: Deadline; + /** The window over `deadline` that `evaluate`'s reads run in; without one, reads end at `deadline`. */ + readonly reads?: ReadWindow; readonly signal: AbortSignal; readonly negated: boolean; /** - * Evaluates the positive condition once, every read bounded by - * `deadline`: a read on a longer budget outlasts the poll. Returns - * undefined when the condition cannot be evaluated yet: the positive poll - * keeps waiting and the negation grace window resets. + * Evaluates the positive condition once, every read run through `reads` + * or bounded by `deadline`: a read on a longer budget outlasts the poll. + * Returns undefined when the condition cannot be evaluated yet: the + * positive poll keeps waiting and the negation grace window resets. */ - evaluate(): Promise; + evaluate(reads: ReadWindow): Promise; /** Builds the poll's failure; `cause` is the read the deadline cut off, when one did. */ onTimeout(cause?: unknown): Error | Promise; } @@ -107,18 +138,18 @@ export interface PollConditionOptions { * Polls a condition until it holds (or, when negated, until its negation has * held continuously for the negation grace window), throwing the caller's * error at the deadline. No sleep runs past the deadline, so the last read - * starts at it and never sees a state that arrived later. + * starts at it; a `reads` window with a cap lets it run past by the longest + * read the poll has finished. * * A negation holds from the moment the read that first saw it was issued, so * a slow read counts toward the window. A budget shorter than the window * still has to be satisfiable: the negation then only needs to hold for the - * whole budget, and passes at the deadline on what it has seen, since a read - * past the deadline has no budget left. A read the deadline cut off (see - * `cutOffAtDeadline`) after an earlier one completed saw nothing: the poll - * ends there. A negation then passes if it has held long enough by the - * time the read is cut off: the read left less than one poll tick (or the - * measured read latency) unseen, no more than the gap between any two reads, - * so a short budget still passes at its deadline on what it saw. + * whole budget, and passes at the deadline on what it has seen. A read the + * deadline cut off (see `ReadWindow.cutOff`) after an earlier one completed + * saw nothing: the poll ends there. A negation then passes if it has held + * long enough by the time the read is cut off: the read left less than one + * poll tick, or one read's time, unseen, so a short budget still passes at + * its deadline on what it saw. */ export async function pollCondition(options: PollConditionOptions): Promise { const { deadline, negated } = options; @@ -127,18 +158,16 @@ export async function pollCondition(options: PollConditionOptions): Promise holdingSince !== undefined && now - holdingSince >= grace; + const reads = options.reads ?? new ReadWindow(deadline); let sampled = false; - let latency = 0; for (;;) { const startedWithMs = deadline.remaining(readAt); let value: boolean | undefined; - const evaluateStart = Date.now(); try { - value = await options.evaluate(); + value = await options.evaluate(reads); sampled = true; - latency = Math.max(latency, Date.now() - evaluateStart); } catch (cause) { - if (!sampled || !cutOffAtDeadline(cause, startedWithMs, latency)) throw cause; + if (!sampled || !reads.cutOff(cause, startedWithMs)) throw cause; if (negated && holds(Math.min(Date.now(), deadline.endsAt))) return; throw await options.onTimeout(cause); } diff --git a/packages/e2e/src/locator/engine.ts b/packages/e2e/src/locator/engine.ts index 571811d337..cfb4f8dbcc 100644 --- a/packages/e2e/src/locator/engine.ts +++ b/packages/e2e/src/locator/engine.ts @@ -20,7 +20,7 @@ import { } from '../internal/errors.ts'; import { requireKey } from '../internal/keys.ts'; import { describeExpression, expressionHints } from './expression.ts'; -import { cutOffAtDeadline, Deadline, POLL_INTERVAL_MS, sleep } from '../internal/time.ts'; +import { Deadline, POLL_INTERVAL_MS, ReadWindow, sleep } from '../internal/time.ts'; import { timestamp } from '../internal/ids.ts'; import { obj } from '../internal/objects.ts'; import { targetLabel } from '../cache/trace.ts'; @@ -94,10 +94,6 @@ export class LocatorEngine { return this.options.budget.signal; } - get budget(): AttemptBudget { - return this.options.budget; - } - get assertionTimeout(): number { return this.options.assertionTimeout; } @@ -117,6 +113,11 @@ export class LocatorEngine { return this.operationWithin(this.deadline(timeoutMs)); } + /** The reads of one wait on `deadline`: the last may run past it, up to the running phase's deadline. */ + reads(deadline: Deadline): ReadWindow { + return new ReadWindow(deadline, this.options.budget.deadline); + } + /** Deadline for one action-family operation, capped by the running phase's deadline. */ deadline(timeoutMs?: number): Deadline { return Deadline.min( @@ -126,10 +127,15 @@ export class LocatorEngine { } /** An operation context bounded by a deadline a caller shares across steps, such as the swipes of one scroll. */ - operationWithin(deadline: Deadline, timeoutMs?: number): OperationContext { + operationWithin(deadline: Deadline): OperationContext { + return this.readOperation(deadline.remaining()); + } + + /** An operation context on `timeoutMs`, never under 1 ms: the budget a `ReadWindow` hands one read. */ + readOperation(timeoutMs: number): OperationContext { return { signal: this.signal, - timeoutMs: timeoutMs ?? Math.max(1, deadline.remaining()), + timeoutMs: Math.max(1, timeoutMs), runId: this.options.runId, attemptId: this.options.attemptId, origin: 'test', @@ -183,22 +189,21 @@ export class LocatorEngine { */ private async resolveOnce( expression: LocatorExpression, - deadline: Deadline, + reads: ReadWindow, missingFrame: MissingFrame = 'wait', - timeoutMs?: number, ): Promise { let retried: unknown; for (;;) { - const startedWithMs = deadline.remaining(); + const startedWithMs = reads.deadline.remaining(); try { - return await this.session.locate(expression, this.operationWithin(deadline, timeoutMs)); + return await reads.read((timeoutMs) => this.session.locate(expression, this.readOperation(timeoutMs))); } catch (cause) { // A retry the deadline cut off saw nothing new: the wait ends on the failure it was retrying. - if (retried !== undefined && cutOffAtDeadline(cause, startedWithMs)) throw translateLocatorError(retried, expression); + if (retried !== undefined && reads.cutOff(cause, startedWithMs)) throw translateLocatorError(retried, expression); const engineError = asEngineError(cause); if (engineError?.retryable === true) { if (missingFrame === 'empty' && engineError.code === 'FRAME_NOT_FOUND') return []; - if (!deadline.expired()) { + if (!reads.deadline.expired()) { retried = cause; await sleep(POLL_INTERVAL_MS, this.signal); continue; @@ -214,11 +219,12 @@ export class LocatorEngine { * multiple matches fail immediately with LOCATOR_AMBIGUOUS. */ async resolveExactlyOne(expression: LocatorExpression, deadline: Deadline): Promise { - return (await this.resolveCounted(expression, deadline)).ref; + return (await this.resolveCounted(expression, this.reads(deadline))).ref; } /** `resolveExactlyOne`, timing how long it polled before the match appeared. */ - private async resolveCounted(expression: LocatorExpression, deadline: Deadline): Promise { + private async resolveCounted(expression: LocatorExpression, reads: ReadWindow): Promise { + const { deadline } = reads; const startedMs = Date.now(); // The wait as it happened: a deadline capped by the test's remaining // budget waited less than the action timeout, and the report says so. @@ -228,18 +234,14 @@ export class LocatorEngine { ...(cause === undefined ? {} : { cause }), }); let sampled = false; - let latency = 0; for (let rounds = 1; ; rounds += 1) { const startedWithMs = deadline.remaining(); - const readTimeout = Math.min(this.options.budget.deadline.remaining(), startedWithMs + latency); let refs: readonly NodeRef[]; - const readStart = Date.now(); try { - refs = await this.resolveOnce(expression, deadline, 'wait', readTimeout); - latency = Math.max(latency, Date.now() - readStart); + refs = await this.resolveOnce(expression, reads); } catch (cause) { // A resolve the deadline cut off, after one that matched nothing, is the wait running out. - if (sampled && cutOffAtDeadline(cause, startedWithMs, latency)) throw notFound(cause); + if (sampled && reads.cutOff(cause, startedWithMs)) throw notFound(cause); throw cause; } sampled = true; @@ -255,8 +257,8 @@ export class LocatorEngine { * stale resolve is retried within the caller's deadline, like a node that * is not there yet. */ - async resolveAll(expression: LocatorExpression, deadline: Deadline): Promise { - return this.resolveOnce(expression, deadline); + async resolveAll(expression: LocatorExpression, reads: ReadWindow): Promise { + return this.resolveOnce(expression, reads); } /** @@ -267,7 +269,7 @@ export class LocatorEngine { * repairs a race with a navigation, it does not wait for a match. */ async resolveNow(expression: LocatorExpression): Promise { - return this.resolveOnce(expression, this.deadline(this.options.actionTimeout), 'empty'); + return this.resolveOnce(expression, new ReadWindow(this.deadline(this.options.actionTimeout)), 'empty'); } /** @@ -278,14 +280,14 @@ export class LocatorEngine { * re-resolves while the deadline remains instead of failing the test. */ async read(expression: LocatorExpression): Promise { - const deadline = this.deadline(this.options.actionTimeout); + const reads = this.reads(this.deadline(this.options.actionTimeout)); for (;;) { - const ref = await this.resolveExactlyOne(expression, deadline); + const { ref } = await this.resolveCounted(expression, reads); try { - return await this.session.read(ref, this.operationWithin(deadline)); + return await reads.read((timeoutMs) => this.session.read(ref, this.readOperation(timeoutMs))); } catch (cause) { const engineError = asEngineError(cause); - if (engineError?.code === 'NODE_STALE' && engineError.retryable && !deadline.expired()) { + if (engineError?.code === 'NODE_STALE' && engineError.retryable && !reads.deadline.expired()) { continue; } throw translateLocatorError(cause, expression); @@ -299,28 +301,31 @@ export class LocatorEngine { * the deadline remains, since one stale member says nothing about the * others; a stale set at the deadline is `LOCATOR_NOT_FOUND`. */ - async readAll(expression: LocatorExpression, deadline: Deadline): Promise { - return this.readEvery(expression, deadline, 'wait'); + async readAll(expression: LocatorExpression, reads: ReadWindow): Promise { + return this.readEvery(expression, reads, 'wait'); } /** Reads every current match at once, for `allTextContents`: a missing frame reads as no matches. */ - async readAllNow(expression: LocatorExpression, deadline: Deadline = this.deadline(this.options.actionTimeout)): Promise { - return this.readEvery(expression, deadline, 'empty'); + async readAllNow( + expression: LocatorExpression, + reads: ReadWindow = new ReadWindow(this.deadline(this.options.actionTimeout)), + ): Promise { + return this.readEvery(expression, reads, 'empty'); } private async readEvery( expression: LocatorExpression, - deadline: Deadline, + reads: ReadWindow, missingFrame: MissingFrame, ): Promise { for (;;) { - const refs = await this.resolveOnce(expression, deadline, missingFrame); + const refs = await this.resolveOnce(expression, reads, missingFrame); try { const nodes: SemanticNode[] = []; - for (const ref of refs) nodes.push(await this.session.read(ref, this.operationWithin(deadline))); + for (const ref of refs) nodes.push(await reads.read((timeoutMs) => this.session.read(ref, this.readOperation(timeoutMs)))); return nodes; } catch (cause) { - if (asEngineError(cause)?.code === 'NODE_STALE' && !deadline.expired()) { + if (asEngineError(cause)?.code === 'NODE_STALE' && !reads.deadline.expired()) { await sleep(POLL_INTERVAL_MS, this.signal); continue; } @@ -337,11 +342,10 @@ export class LocatorEngine { */ async tryRead( expression: LocatorExpression, - deadline: Deadline, + reads: ReadWindow, missingFrame: MissingFrame = 'wait', - timeoutMs?: number, ): Promise<{ node: SemanticNode | null; count: number }> { - return this.readOne(expression, deadline, missingFrame, timeoutMs); + return this.readOne(expression, reads, missingFrame); } /** @@ -350,20 +354,19 @@ export class LocatorEngine { * matches are `LOCATOR_AMBIGUOUS`. */ async readNow(expression: LocatorExpression): Promise { - const { node } = await this.readOne(expression, this.deadline(this.options.actionTimeout), 'empty'); + const { node } = await this.readOne(expression, new ReadWindow(this.deadline(this.options.actionTimeout)), 'empty'); return node; } private async readOne( expression: LocatorExpression, - deadline: Deadline, + reads: ReadWindow, missingFrame: MissingFrame, - timeoutMs?: number, ): Promise<{ node: SemanticNode | null; count: number }> { - const ref = assertSingle(await this.resolveOnce(expression, deadline, missingFrame, timeoutMs), expression); + const ref = assertSingle(await this.resolveOnce(expression, reads, missingFrame), expression); if (ref === null) return { node: null, count: 0 }; try { - const node = await this.session.read(ref, this.operationWithin(deadline, timeoutMs)); + const node = await reads.read((timeoutMs) => this.session.read(ref, this.readOperation(timeoutMs))); return { node, count: 1 }; } catch (cause) { if (asEngineError(cause)?.code === 'NODE_STALE') { @@ -418,18 +421,15 @@ export class LocatorEngine { details: locatorDetails(expression, Date.now() - startedMs), ...(cause === undefined ? {} : { cause }), }); + const reads = this.reads(deadline); let sampled = false; - let latency = 0; for (;;) { const startedWithMs = deadline.remaining(); - const readTimeout = Math.min(this.options.budget.deadline.remaining(), startedWithMs + latency); let node: SemanticNode | null; - const readStart = Date.now(); try { - ({ node } = await this.tryRead(expression, deadline, 'wait', readTimeout)); - latency = Math.max(latency, Date.now() - readStart); + ({ node } = await this.tryRead(expression, reads)); } catch (cause) { - if (sampled && cutOffAtDeadline(cause, startedWithMs, latency)) throw notVisible(cause); + if (sampled && reads.cutOff(cause, startedWithMs)) throw notVisible(cause); throw cause; } sampled = true; @@ -528,7 +528,7 @@ export class LocatorEngine { const startedMs = Date.now(); let stale = 0; for (;;) { - const { ref, waitedMs } = await this.resolveCounted(expression, deadline); + const { ref, waitedMs } = await this.resolveCounted(expression, this.reads(deadline)); // Read now, from the resolve's own cache: a dragTo's target resolve // below supersedes this ref, and the node it named would be gone. const node = this.options.recordEvent === undefined ? undefined : await this.session.read(ref, this.operationWithin(deadline)).catch(() => undefined); diff --git a/packages/e2e/src/locator/screen.ts b/packages/e2e/src/locator/screen.ts index b3e4f0f47d..bc0ed6f18c 100644 --- a/packages/e2e/src/locator/screen.ts +++ b/packages/e2e/src/locator/screen.ts @@ -43,7 +43,7 @@ import { testIdQuery, textQuery, } from './expression.ts'; -import { cutOffAtDeadline, type Deadline, isOperationTimeout, POLL_INTERVAL_MS, pollCondition } from '../internal/time.ts'; +import { type Deadline, isOperationTimeout, POLL_INTERVAL_MS, pollCondition } from '../internal/time.ts'; import { SampleHistory } from '../expect/samples.ts'; import type { ScreenshotContext } from '../run/screenshots.ts'; @@ -273,18 +273,15 @@ class ScreenImpl implements Screen { details: locatorDetails(internals.expression, Date.now() - startedMs), ...(cause === undefined ? {} : { cause }), }); + const reads = engine.reads(deadline); let sampled = false; - let latency = 0; for (;;) { const startedWithMs = deadline.remaining(); - const readTimeout = Math.min(engine.budget.deadline.remaining(), startedWithMs + latency); let node: SemanticNode | null; - const readStart = Date.now(); try { - ({ node } = await engine.tryRead(internals.expression, deadline, 'wait', readTimeout)); - latency = Math.max(latency, Date.now() - readStart); + ({ node } = await engine.tryRead(internals.expression, reads)); } catch (cause) { - if (sampled && cutOffAtDeadline(cause, startedWithMs, latency)) throw notVisible(cause); + if (sampled && reads.cutOff(cause, startedWithMs)) throw notVisible(cause); throw cause; } sampled = true; @@ -611,10 +608,11 @@ class LocatorImpl extends ScreenImpl implements Locator { await samples.record(engine, () => pollCondition({ deadline, + reads: engine.reads(deadline), signal: engine.signal, negated: false, - evaluate: async () => { - const { node } = await engine.tryRead(this.expression, deadline, ABSENCE_STATES.has(state) ? 'empty' : 'wait'); + evaluate: async (reads) => { + const { node } = await engine.tryRead(this.expression, reads, ABSENCE_STATES.has(state) ? 'empty' : 'wait'); samples.add(node === null ? 'absent' : isNodeVisible(node) ? 'visible' : 'hidden'); return inWaitForState(node, state); }, diff --git a/packages/e2e/tests/helpers/screen-over.ts b/packages/e2e/tests/helpers/screen-over.ts index bf37bf0a6b..7f18046055 100644 --- a/packages/e2e/tests/helpers/screen-over.ts +++ b/packages/e2e/tests/helpers/screen-over.ts @@ -27,8 +27,8 @@ import { StepRecorder } from '../../src/run/steps.ts'; import type { Screen } from '../../src/types.ts'; export interface ScreenOverOptions { - /** Answers every `locate`. */ - readonly locate: (expression: LocatorExpression) => readonly SemanticNode[] | Promise; + /** Answers every `locate`, with the operation it runs under. */ + readonly locate: (expression: LocatorExpression, operation: OperationContext) => readonly SemanticNode[] | Promise; /** Answers every `observe`; the root's ref is what a viewport swipe is addressed to. */ readonly observe: () => EngineSnapshot; /** Receives every node action with the operation it runs under; without it the engine declares no actions. */ @@ -51,7 +51,7 @@ export function screenOver(options: ScreenOverOptions): { screen: Screen; steps: version: '1', spiVersion: 1, observe: async () => options.observe(), - locate: async (expression) => options.locate(expression), + locate: async (expression, operation) => options.locate(expression, operation), ...(perform === undefined ? {} : { diff --git a/packages/e2e/tests/unit/locator-engine.test.ts b/packages/e2e/tests/unit/locator-engine.test.ts index 59b394e1a6..10746ebab0 100644 --- a/packages/e2e/tests/unit/locator-engine.test.ts +++ b/packages/e2e/tests/unit/locator-engine.test.ts @@ -3,7 +3,7 @@ * complete EngineError -> runner taxonomy mapping. */ -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it } from 'vitest'; import { EngineError, type EngineErrorCode, @@ -15,7 +15,7 @@ import { } from '../../src/engine/surface.ts'; import { LocatorEngine, isNodeVisible, translateLocatorError } from '../../src/locator/engine.ts'; import { E2EError } from '../../src/internal/errors.ts'; -import { Deadline } from '../../src/internal/time.ts'; +import { Deadline, sleep, withTimeout } from '../../src/internal/time.ts'; import { AttemptBudget } from '../../src/run/budget.ts'; import { defineEngine, LOCATOR_ACTION_KINDS } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; @@ -32,6 +32,14 @@ const EXPRESSION: LocatorExpression = { query: { kind: 'role', value: { kind: 'string', value: 'button', exact: true } }, }; +/** A resolve that answers `refs` after `ms`. */ +function slowResolve(ms: number, refs: readonly NodeRef[]): () => Promise { + return async () => { + await sleep(ms); + return refs; + }; +} + interface ScreenScript { resolve?: Array<(() => readonly NodeRef[]) | 'stale' | 'frame' | 'failure' | 'foreign-stale'>; read?: Array<(() => SemanticNode | Promise) | 'stale' | 'stale-retryable' | 'failure' | 'foreign-stale'>; @@ -51,6 +59,11 @@ function foreignStale(retryable: boolean): Error { return error; } +/** Runs one fake engine call on its operation's budget, as an engine must: past it, the call is `OPERATION_TIMEOUT`. */ +function withinBudget(operation: OperationContext, call: () => T | Promise): Promise { + return withTimeout(Promise.resolve().then(call), operation.timeoutMs, () => new EngineError('OPERATION_TIMEOUT', 'operation timed out', { retryable: false })); +} + function makeEngine(script: ScreenScript, options: { actionTimeout?: number } = {}) { const calls = { resolve: 0, read: 0, perform: 0 }; const origins: (string | undefined)[] = []; @@ -64,16 +77,16 @@ function makeEngine(script: ScreenScript, options: { actionTimeout?: number } = async locate(_expression: LocatorExpression, operation: OperationContext) { resolveTimeouts.push(operation.timeoutMs); const step = next(script.resolve, 'resolve'); - if (step === undefined || typeof step === 'function') return step?.() ?? [REF]; + if (step === undefined || typeof step === 'function') return withinBudget(operation, () => step?.() ?? [REF]); if (step === 'stale') throw new EngineError('NODE_STALE', 'stale', { retryable: true }); if (step === 'foreign-stale') throw foreignStale(true); if (step === 'frame') throw new EngineError('FRAME_NOT_FOUND', 'frame missing', { retryable: true }); throw new EngineError('ENGINE_FAILURE', 'engine died', { retryable: false }); }, - async read() { + async read(_ref: NodeRef, operation: OperationContext) { const step = next(script.read, 'read'); - if (step === undefined || typeof step === 'function') return step?.() ?? NODE; + if (step === undefined || typeof step === 'function') return withinBudget(operation, () => step?.() ?? NODE); if (step === 'stale') throw new EngineError('NODE_STALE', 'stale', { retryable: false }); if (step === 'foreign-stale') throw foreignStale(false); if (step === 'stale-retryable') throw new EngineError('NODE_STALE', 'stale', { retryable: true }); @@ -150,14 +163,14 @@ describe('LocatorEngine read retry contract', () => { describe('LocatorEngine resolve retry contract', () => { it('retries retryable frame misses until the engine recovers', async () => { const { engine, calls } = makeEngine({ resolve: ['frame', 'stale', () => [REF]] }); - const refs = await engine.resolveAll(EXPRESSION, engine.deadline()); + const refs = await engine.resolveAll(EXPRESSION, engine.reads(engine.deadline())); expect(refs).toEqual([REF]); expect(calls.resolve).toBe(3); }); it('translates a non-retryable resolve failure immediately without retrying', async () => { const { engine, calls } = makeEngine({ resolve: ['failure'] }); - await expect(engine.resolveAll(EXPRESSION, engine.deadline())).rejects.toMatchObject({ + await expect(engine.resolveAll(EXPRESSION, engine.reads(engine.deadline()))).rejects.toMatchObject({ category: 'infrastructure', code: 'ENGINE_FAILURE', }); @@ -169,7 +182,7 @@ describe('LocatorEngine resolve retry contract', () => { { resolve: Array.from({ length: 50 }, () => 'stale' as const) }, { actionTimeout: 350 }, ); - await expect(engine.resolveAll(EXPRESSION, engine.deadline())).rejects.toMatchObject({ + await expect(engine.resolveAll(EXPRESSION, engine.reads(engine.deadline()))).rejects.toMatchObject({ category: 'test', code: 'LOCATOR_NOT_FOUND', }); @@ -200,83 +213,45 @@ describe('LocatorEngine resolve retry contract', () => { }); }); - it('keeps a resolve that hung with budget to spare a timeout (ACTION_FAILED), never a missing node', async () => { - vi.useFakeTimers(); - try { - const deadline = new Deadline(2_000); - let calls = 0; - const { engine } = makeEngine({ - resolve: Array.from({ length: 10 }, () => async () => { - calls += 1; - if (calls === 1) { - await new Promise((resolve) => setTimeout(resolve, 100)); - return [] as readonly NodeRef[]; - } - await new Promise((resolve) => setTimeout(resolve, 500)); - throw new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); - }) as unknown as Array<() => readonly NodeRef[]>, - }); - const pending = engine.resolveExactlyOne(EXPRESSION, deadline); - const assertion = expect(pending).rejects.toMatchObject({ code: 'ACTION_FAILED' }); - await vi.advanceTimersByTimeAsync(1_000); - await assertion; - expect(calls).toBe(2); - } finally { - vi.useRealTimers(); - } - }); - - it('treats a slow-but-valid last resolve past deadline as LOCATOR_NOT_FOUND', async () => { - vi.useFakeTimers(); - try { - const deadline = new Deadline(300); - let calls = 0; - const { engine } = makeEngine({ - resolve: Array.from({ length: 10 }, () => async () => { - calls += 1; - if (calls === 1) { - await new Promise((resolve) => setTimeout(resolve, 150)); - return [] as readonly NodeRef[]; - } - await new Promise((resolve) => setTimeout(resolve, 150)); - return [] as readonly NodeRef[]; - }) as unknown as Array<() => readonly NodeRef[]>, - }); - const pending = engine.resolveExactlyOne(EXPRESSION, deadline); - const assertion = expect(pending).rejects.toMatchObject({ code: 'LOCATOR_NOT_FOUND' }); - await vi.advanceTimersByTimeAsync(500); - await assertion; - expect(calls).toBe(2); - } finally { - vi.useRealTimers(); - } - }); - - it('grants a slow last resolve extra time past the deadline and resolves a late match', async () => { - vi.useFakeTimers(); - try { - const deadline = new Deadline(300); - let calls = 0; - const { engine, resolveTimeouts } = makeEngine({ - resolve: Array.from({ length: 10 }, () => async () => { - calls += 1; - if (calls === 1) { - await new Promise((resolve) => setTimeout(resolve, 150)); - return [] as readonly NodeRef[]; - } - await new Promise((resolve) => setTimeout(resolve, 150)); - return [REF]; - }) as unknown as Array<() => readonly NodeRef[]>, - }); - const pending = engine.resolveExactlyOne(EXPRESSION, deadline); - await vi.advanceTimersByTimeAsync(500); - const resolved = await pending; - expect(resolved).toBe(REF); - expect(calls).toBe(2); - expect(resolveTimeouts[1]).toBe(200); - } finally { - vi.useRealTimers(); - } + it('keeps a resolve that hung with budget to spare a timeout, never a missing node', async () => { + const deadline = new Deadline(2_000); + const { engine, calls } = makeEngine({ + resolve: [ + async () => { + await sleep(100); + return []; + }, + () => new Promise(() => undefined), + ] as unknown as Array<() => readonly NodeRef[]>, + }); + await expect(engine.resolveExactlyOne(EXPRESSION, deadline)).rejects.toMatchObject({ code: 'ACTION_FAILED' }); + expect(calls.resolve).toBe(2); + }); + + it('lets a slow last resolve finish past the deadline: a late match resolves', async () => { + const deadline = new Deadline(450); + const { engine, resolveTimeouts } = makeEngine({ + resolve: [slowResolve(200, []), slowResolve(200, [REF])] as unknown as Array<() => readonly NodeRef[]>, + }); + await expect(engine.resolveExactlyOne(EXPRESSION, deadline)).resolves.toBe(REF); + expect(resolveTimeouts).toEqual([450, 350]); + }); + + it('ends a wait whose slow last resolve found nothing as LOCATOR_NOT_FOUND, not a timeout', async () => { + const deadline = new Deadline(450); + const { engine, calls } = makeEngine({ + resolve: [slowResolve(200, []), slowResolve(200, [])] as unknown as Array<() => readonly NodeRef[]>, + }); + const failure = await engine.resolveExactlyOne(EXPRESSION, deadline).catch((error: unknown) => error); + expect(failure).toMatchObject({ code: 'LOCATOR_NOT_FOUND' }); + expect((failure as Error).cause).toBeUndefined(); + expect(calls.resolve).toBe(2); + }); + + it('gives each retry of a resolve what is left of its budget, not the whole budget again', async () => { + const { engine, resolveTimeouts } = makeEngine({ resolve: ['stale', 'stale', () => [REF]] }, { actionTimeout: 1_000 }); + await engine.resolveAll(EXPRESSION, engine.reads(engine.deadline())); + expect(resolveTimeouts).toEqual([1_000, 900, 800]); }); it('ends a frame wait the deadline cut off on the missing frame, as LOCATOR_NOT_FOUND', async () => { @@ -343,13 +318,13 @@ describe('LocatorEngine perform contract', () => { describe('LocatorEngine read contract', () => { it('collapses stale reads to a zero-count sample so assertion polls re-resolve', async () => { const { engine } = makeEngine({ read: ['stale'] }); - const sample = await engine.tryRead(EXPRESSION, new Deadline(5_000)); + const sample = await engine.tryRead(EXPRESSION, engine.reads(new Deadline(5_000))); expect(sample).toEqual({ node: null, count: 0 }); }); it('translates non-stale read failures', async () => { const { engine } = makeEngine({ read: ['failure'] }); - await expect(engine.tryRead(EXPRESSION, new Deadline(5_000))).rejects.toMatchObject({ + await expect(engine.tryRead(EXPRESSION, engine.reads(new Deadline(5_000)))).rejects.toMatchObject({ code: 'ENGINE_FAILURE', category: 'infrastructure', }); @@ -370,13 +345,13 @@ describe('LocatorEngine readAll contract', () => { resolve: [() => [REF, OTHER_REF]], read: [() => NODE, () => OTHER_NODE], }); - expect(await engine.readAll(EXPRESSION, engine.deadline())).toEqual([NODE, OTHER_NODE]); + expect(await engine.readAll(EXPRESSION, engine.reads(engine.deadline()))).toEqual([NODE, OTHER_NODE]); expect(calls).toMatchObject({ resolve: 1, read: 2 }); }); it('answers zero matches with an empty list without waiting', async () => { const { engine, calls } = makeEngine({ resolve: [() => []] }); - expect(await engine.readAll(EXPRESSION, engine.deadline())).toEqual([]); + expect(await engine.readAll(EXPRESSION, engine.reads(engine.deadline()))).toEqual([]); expect(calls).toMatchObject({ resolve: 1, read: 0 }); }); @@ -385,20 +360,20 @@ describe('LocatorEngine readAll contract', () => { resolve: [() => [REF, OTHER_REF], () => [OTHER_REF]], read: ['stale', () => OTHER_NODE], }); - expect(await engine.readAll(EXPRESSION, engine.deadline())).toEqual([OTHER_NODE]); + expect(await engine.readAll(EXPRESSION, engine.reads(engine.deadline()))).toEqual([OTHER_NODE]); expect(calls).toMatchObject({ resolve: 2, read: 2 }); }); it('gives up on a set that stays stale at the deadline as LOCATOR_NOT_FOUND', async () => { const { engine } = makeEngine({ read: ['stale', 'stale', 'stale', 'stale', 'stale'] }); - await expect(engine.readAll(EXPRESSION, new Deadline(150))).rejects.toMatchObject({ + await expect(engine.readAll(EXPRESSION, engine.reads(new Deadline(150)))).rejects.toMatchObject({ code: 'LOCATOR_NOT_FOUND', }); }); it('translates non-stale read failures', async () => { const { engine } = makeEngine({ read: ['failure'] }); - await expect(engine.readAll(EXPRESSION, engine.deadline())).rejects.toMatchObject({ code: 'ENGINE_FAILURE' }); + await expect(engine.readAll(EXPRESSION, engine.reads(engine.deadline()))).rejects.toMatchObject({ code: 'ENGINE_FAILURE' }); }); }); @@ -562,7 +537,7 @@ describe('LocatorEngine visible queries', () => { const { engine } = makeLocatorEngineOverEngine(); const ref = await engine.resolveExactlyOne(build(true), new Deadline(5_000)); expect(ref.id).toBe('shown'); - expect(await engine.resolveAll(build(true), engine.deadline())).toHaveLength(1); + expect(await engine.resolveAll(build(true), engine.reads(engine.deadline()))).toHaveLength(1); }); it.each(kinds)('%s without visible keeps the hidden twin, so the pair is LOCATOR_AMBIGUOUS', async (_kind, build) => { @@ -573,7 +548,7 @@ describe('LocatorEngine visible queries', () => { await expect(engine.resolveExactlyOne(build(false), new Deadline(5_000))).rejects.toMatchObject({ code: 'LOCATOR_AMBIGUOUS', }); - expect(await engine.resolveAll(build(undefined), engine.deadline())).toHaveLength(2); + expect(await engine.resolveAll(build(undefined), engine.reads(engine.deadline()))).toHaveLength(2); }); it('names the predicate in the ambiguity message when two visible nodes remain', async () => { @@ -598,7 +573,7 @@ describe('LocatorEngine visible queries', () => { const indexed: LocatorExpression = { kind: 'index', source: kinds[1]![1](true), index: 'first' }; // The adapter cannot know which of the engine's answers came first, so it // must not second-guess them; the engine applied `visible` before `first`. - expect(await engine.resolveAll(indexed, engine.deadline())).toHaveLength(2); + expect(await engine.resolveAll(indexed, engine.reads(engine.deadline()))).toHaveLength(2); expect(expressions[0]).toEqual(indexed); }); }); @@ -621,7 +596,7 @@ describe('LocatorEngine retries a cross-realm driver failure', () => { it('treats a foreign stale read as zero matches while polling', async () => { const { engine } = makeEngine({ read: ['foreign-stale'] }); - await expect(engine.tryRead(EXPRESSION, new Deadline(0))).resolves.toEqual({ + await expect(engine.tryRead(EXPRESSION, engine.reads(new Deadline(0)))).resolves.toEqual({ node: null, count: 0, }); diff --git a/packages/e2e/tests/unit/locator-wait-for.test.ts b/packages/e2e/tests/unit/locator-wait-for.test.ts index ce9eb6b63b..b75f33fd51 100644 --- a/packages/e2e/tests/unit/locator-wait-for.test.ts +++ b/packages/e2e/tests/unit/locator-wait-for.test.ts @@ -6,7 +6,9 @@ */ import { afterEach, describe, expect, it } from 'vitest'; -import { resolveExpression, type SemanticNode } from '../../src/engine/index.ts'; +import { EngineError, resolveExpression, type SemanticNode } from '../../src/engine/index.ts'; +import { expect as expectFixture } from '../../src/expect/index.ts'; +import { sleep, withTimeout } from '../../src/internal/time.ts'; import type { Locator } from '../../src/types.ts'; import { invalid } from '../helpers/invalid.ts'; import { screenOver } from '../helpers/screen-over.ts'; @@ -117,3 +119,30 @@ describe('locator.waitFor validation', () => { expect(steps.all()).toEqual([]); }); }); + +describe('a slow last read', () => { + /** A screen whose every read takes 300 ms, bounded by its operation's budget, and whose banner shows up at 1050 ms. */ + function slowScreen() { + let nodes: readonly SemanticNode[] = []; + timers.push(setTimeout(() => (nodes = [banner]), 1_050)); + const { screen } = screenOver({ + locate: (expression, operation) => + withTimeout( + sleep(300).then(() => resolveExpression(expression, nodes)), + operation.timeoutMs, + () => new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }), + ), + observe: () => snapshot(nodes), + timeoutMs: 1_000, + }); + return screen.getByTestId('banner'); + } + + it('still finishes past the deadline, so waitFor sees a node that showed up during it', async () => { + await expect(slowScreen().waitFor()).resolves.toBeUndefined(); + }); + + it('still finishes past the deadline, so an assertion sees a node that showed up during it', async () => { + await expect(expectFixture(slowScreen()).toBeVisible()).resolves.toBeUndefined(); + }); +}); diff --git a/packages/e2e/tests/unit/time.test.ts b/packages/e2e/tests/unit/time.test.ts index 31d139354e..d6fe0821e1 100644 --- a/packages/e2e/tests/unit/time.test.ts +++ b/packages/e2e/tests/unit/time.test.ts @@ -5,6 +5,7 @@ import { NEGATION_GRACE_MS, POLL_INTERVAL_MS, pollCondition, + ReadWindow, sleep, withTimeout, } from '../../src/internal/time.ts'; @@ -351,30 +352,57 @@ describe('pollCondition', () => { await expect(early).rejects.toBe(cut); }); - it('treats a last read in pollCondition within measured latency as cut off, not hung', async () => { + it('lets the last read run past the deadline by the longest read the poll finished, up to the window cap', async () => { vi.useFakeTimers(); - const cut = new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); - const deadline = new Deadline(600); + const deadline = new Deadline(450); + const timeouts: number[] = []; let calls = 0; const promise = pollCondition({ deadline, + reads: new ReadWindow(deadline, new Deadline(10_000)), signal: new AbortController().signal, negated: false, - evaluate: async () => { - calls += 1; - if (calls === 1) { - await sleep(300); - return false; - } - // Started with ~200ms remaining, which is < measured latency (300ms) but >= POLL_INTERVAL_MS (100ms) - throw cut; - }, + evaluate: (reads) => + reads.read(async (timeoutMs) => { + timeouts.push(timeoutMs); + calls += 1; + await withTimeout(sleep(200), timeoutMs, () => new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false })); + return calls === 2; + }), onTimeout: () => new Error('poll timed out'), }); - const assertion = expect(promise).rejects.toThrow('poll timed out'); await vi.advanceTimersByTimeAsync(1_000); - await assertion; - expect(calls).toBe(2); + await expect(promise).resolves.toBeUndefined(); + expect(timeouts).toEqual([450, 350]); + }); + + it('ends on a read cut off within the longest read time of the deadline, and fails on one that hung with more left', async () => { + vi.useFakeTimers(); + const cut = new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false }); + const poll = (timeoutMs: number) => { + const deadline = new Deadline(timeoutMs); + let calls = 0; + return pollCondition({ + deadline, + signal: new AbortController().signal, + negated: false, + evaluate: (reads) => + reads.read(async () => { + calls += 1; + await sleep(300); + if (calls === 1) return false; + throw cut; + }), + onTimeout: () => new Error('poll timed out'), + }); + }; + // The second read starts with 200 ms left: under the 300 ms the first took, so the deadline cut it. + const within = expect(poll(600)).rejects.toThrow('poll timed out'); + // With 600 ms left, more than any read took, the read hung. + const hung = expect(poll(1_000)).rejects.toBe(cut); + await vi.advanceTimersByTimeAsync(2_000); + await within; + await hung; }); it('keeps a read that hung most of the wait a failure, and never lets a negation pass on it', async () => { diff --git a/packages/mobile/package.json b/packages/mobile/package.json index 3381510c38..6de01e05f5 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -65,7 +65,7 @@ }, "peerDependencies": { "ai": "^7.0.0", - "e2e": ">=0.15.0 <1" + "e2e": ">=0.21.2 <1" }, "peerDependenciesMeta": { "ai": { diff --git a/packages/mobile/src/engine.ts b/packages/mobile/src/engine.ts index 8e3e064d59..ac4e675737 100644 --- a/packages/mobile/src/engine.ts +++ b/packages/mobile/src/engine.ts @@ -31,10 +31,10 @@ export function buildEngine(surface: AgentDeviceSurface): EngineHandle { startAttempt: (context) => surface.startAttempt(context), endAttempt: (context) => surface.endAttempt(context), dispose: (context) => surface.dispose(context), + // The harness signal does not abort at an operation's budget, so each + // operation enforces its own; see InputBudget in support.ts. observe: (operation, options) => surface.observe(operation, options, surface.inputBudget('observe', operation.timeoutMs)), locate: (expression, operation) => surface.locate(expression, operation, surface.inputBudget('locate', operation.timeoutMs)), - // The harness signal does not abort at an operation's budget, so each - // input enforces its own; see InputBudget in support.ts. perform: (ref, action, operation) => surface.perform(ref, action, operation, surface.inputBudget(action.kind, operation.timeoutMs)), actions: DEVICE_ACTIONS, performAt: (point, action, operation) => surface.performAt(point, action, operation, surface.inputBudget(action.kind, operation.timeoutMs)), diff --git a/packages/mobile/src/support.ts b/packages/mobile/src/support.ts index 05e47278ae..19f75da9c6 100644 --- a/packages/mobile/src/support.ts +++ b/packages/mobile/src/support.ts @@ -26,9 +26,9 @@ export function cancelled(text: string): EngineError { const MIN_ANSWER_MS = 250; /** - * One input operation's budget (`actionTimeout`, or the call's `timeout`), - * which the harness signal does not enforce. Only the device commands that - * send input run under it. + * One operation's budget (its `timeoutMs`), which the harness signal does not + * enforce. Every device command the operation sends runs under it: the + * inputs through `send`, the snapshots and screenshots through `read`. */ export class InputBudget { private readonly endsAt: number; @@ -56,12 +56,8 @@ export class InputBudget { * landed, so it is `ACTION_MAY_HAVE_COMMITTED` and the runner never repeats it. */ send(dispatch: () => Promise, holdMs = 0): Promise { - const remaining = this.endsAt - Date.now(); - if (remaining <= 0) { - return Promise.reject( - this.sent ? this.maybeCommitted() : new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }), - ); - } + const remaining = this.remainingMs(); + if (remaining <= 0) return Promise.reject(this.spent()); if (holdMs >= remaining) { return Promise.reject( new EngineError( @@ -76,25 +72,32 @@ export class InputBudget { return withTimeout(pending, Math.max(remaining, MIN_ANSWER_MS), () => this.maybeCommitted()); } - check(): void { - if (this.endsAt - Date.now() <= 0) { - throw this.sent + /** + * Runs one device read with what is left of the budget. With none left it + * reads nothing. A read that does not answer in time is `OPERATION_TIMEOUT`, + * or `ACTION_MAY_HAVE_COMMITTED` once an input of this operation went out, + * so the runner never repeats that input. + */ + read(dispatch: () => Promise): Promise { + const remaining = this.remainingMs(); + if (remaining <= 0) return Promise.reject(this.spent()); + return withTimeout(this.track(dispatch()), remaining, () => + this.sent ? this.maybeCommitted() - : new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }); - } + : new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before the device answered`, { retryable: false }), + ); } - read(dispatch: () => Promise): Promise { - const remaining = this.endsAt - Date.now(); - if (remaining <= 0) { - return Promise.reject( - new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }), - ); - } - const pending = this.track(dispatch()); - return withTimeout(pending, remaining, () => - new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before the device answered`, { retryable: false }), - ); + /** Milliseconds left of the budget; zero or less once it is spent. */ + remainingMs(): number { + return this.endsAt - Date.now(); + } + + /** The failure of a command the spent budget did not send. */ + private spent(): EngineError { + return this.sent + ? this.maybeCommitted() + : new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false }); } private maybeCommitted(): EngineError { diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 1d30300f13..33ddfeac0c 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -280,6 +280,11 @@ function sendInput(input: InputBudget | undefined, dispatch: () => Promise return input === undefined ? dispatch() : input.send(dispatch); } +/** Runs a device read under its operation's budget, or directly from a call that has none. */ +function readWithin(budget: InputBudget | undefined, dispatch: () => Promise): Promise { + return budget === undefined ? dispatch() : budget.read(dispatch); +} + function sleep(ms: number, signal: AbortSignal): Promise { return new Promise((resolve) => { const timer = setTimeout(done, ms); @@ -474,7 +479,7 @@ export class AgentDeviceSurface { return result; } - /** The budget one input runs under; its device commands stay in flight until they settle. */ + /** The budget one operation runs under; its device commands stay in flight until they settle. */ inputBudget(label: string, timeoutMs: number): InputBudget { return new InputBudget(label, timeoutMs, (pending) => this.track(pending)); } @@ -862,14 +867,17 @@ export class AgentDeviceSurface { private async snapshot(signal: AbortSignal, interactiveOnly: boolean, budget?: InputBudget): Promise { let last: RawSnapshot = {}; + let captureMs = 0; for (const backoffMs of SPARSE_RETRY_BACKOFF_MS) { if (backoffMs > 0) { - budget?.check(); + // A sparse capture is still the screen: a retry the budget cannot fit returns it instead of timing out. + if (budget !== undefined && budget.remainingMs() <= backoffMs + captureMs) return last; await sleep(backoffMs, signal); } - budget?.check(); if (signal.aborted) throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false }); - last = await (budget ? budget.read(() => this.capture(signal, interactiveOnly)) : this.capture(signal, interactiveOnly)); + const startedAt = Date.now(); + last = await this.capture(signal, interactiveOnly, budget); + captureMs = Date.now() - startedAt; if (last.appBundleId !== undefined || last.appName !== undefined) { this.appIdentity = last.appBundleId ?? last.appName; } @@ -882,11 +890,12 @@ export class AgentDeviceSurface { * One capture, taken again once when the iOS runner acquired the tree but * failed its own presentation check on it (a viewport it could not read, a * malformed graph). The check is per capture and a capture is a read, so - * the retry is safe; a second failure is the runner's and propagates. + * the retry is safe; a second failure is the runner's and propagates. Each + * capture runs under `budget`, so one the budget abandoned is not retried. */ - private async capture(signal: AbortSignal, interactiveOnly: boolean): Promise { + private async capture(signal: AbortSignal, interactiveOnly: boolean, budget?: InputBudget): Promise { const take = (): Promise => - this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal); + readWithin(budget, () => this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal)); try { return await take(); } catch (cause) { @@ -913,15 +922,17 @@ export class AgentDeviceSurface { * Snapshot for a locate; undefined for a screen agent-device could not read * yet (an app still on its splash). The harness polls a locate until its * deadline, so that screen matches nothing and the poll waits for it to - * render. A capture that leaves no budget for another poll fails with the - * reason instead, so a wait that ends on an unreadable screen says so. + * render. A capture that leaves less of the budget than it took, too + * little for another, fails with the reason instead, so a wait that ends on + * an unreadable screen says so. */ private async locateSnapshot(operation: OperationContext, budget?: InputBudget): Promise { const startedMs = Date.now(); try { return await this.snapshotOrEmpty(operation, false, budget); } catch (cause) { - if (operation.signal.aborted || !isUnreadableCapture(cause) || Date.now() - startedMs >= operation.timeoutMs) throw cause; + const tookMs = Date.now() - startedMs; + if (operation.signal.aborted || !isUnreadableCapture(cause) || tookMs >= operation.timeoutMs - tookMs) throw cause; return undefined; } } @@ -1467,9 +1478,9 @@ export class AgentDeviceSurface { const masked = await this.maskedScreenshot(operation.signal, budget); attempt.screenshots += 1; const name = `${String(attempt.screenshots).padStart(3, '0')}-${sanitizeFilename(label ?? 'screenshot')}.png`; - const relative = path.posix.join('screenshots', name); + const relative = path.join('screenshots', name); mkdirSync(path.join(attempt.artifactsDir, 'screenshots'), { recursive: true }); - writeFileSync(path.join(attempt.artifactsDir, 'screenshots', name), masked.data); + writeFileSync(path.join(attempt.artifactsDir, relative), masked.data); return relative; } @@ -1489,7 +1500,7 @@ export class AgentDeviceSurface { capture: ScreenshotCapture = {}, budget?: InputBudget, ): Promise { - const run = () => + return readWithin(budget, () => this.command('screenshot', async (client) => { const directory = mkdtempSync(path.join(tmpdir(), 'e2e-agent-device-')); const file = path.join(directory, 'screenshot.png'); @@ -1498,9 +1509,8 @@ export class AgentDeviceSurface { } finally { rmSync(directory, { recursive: true, force: true }); } - }, signal); - if (budget === undefined) return run(); - return budget.read(run); + }, signal), + ); } /** @@ -1530,8 +1540,8 @@ export class AgentDeviceSurface { /** * Viewport pixels for an observation. Best-effort: a screenshot that cannot - * be produced or redacted costs the observation its image, not the step; - * timeouts propagate so an expired read budget fails the observation. + * be produced or redacted costs the observation its image, not the step, + * unless the operation's budget ran out. */ private async capturePixels( operation: OperationContext, diff --git a/packages/mobile/tests/unit/engine.test.ts b/packages/mobile/tests/unit/engine.test.ts index bb98198f01..7d0a3a1cf3 100644 --- a/packages/mobile/tests/unit/engine.test.ts +++ b/packages/mobile/tests/unit/engine.test.ts @@ -929,7 +929,7 @@ describe('observation', () => { } }); - it('stops sparse retries when the read budget expires', async () => { + it('returns a sparse capture when the budget cannot fit a retry, instead of timing out', async () => { const h = harness(); await openAttempt(h); vi.useFakeTimers(); @@ -943,8 +943,7 @@ describe('observation', () => { { kind: 'query', query: { kind: 'text', value: { kind: 'string', value: 'About', exact: true } } }, { ...operation(), timeoutMs: 500 }, ); - const assertion = expect(read).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); - // At 500ms, first backoff (1200ms) has not completed, budget expires + const assertion = expect(read).resolves.toEqual([]); await vi.advanceTimersByTimeAsync(1_500); await assertion; expect(snapshots).toBe(1); @@ -988,21 +987,26 @@ describe('observation', () => { await openAttempt(h); vi.useFakeTimers(); try { - // Snapshot takes 1,500ms - h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve(SETTINGS_SNAPSHOT), 1_500))); - // Screenshot hangs - h.fake.respond('capture.screenshot', () => new Promise(() => undefined)); + // The snapshot carries its geometry, so the only screenshot is the observation's pixels. + h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve({ ...SETTINGS_SNAPSHOT, viewport: { width: 390, height: 844 } }), 1_500))); + let screenshots = 0; + h.fake.respond('capture.screenshot', () => { + screenshots += 1; + return new Promise(() => undefined); + }); + const started = Date.now(); const obs = h.engine.observe!({ ...operation(), timeoutMs: 2_000 }, { pixels: true }); const assertion = expect(obs).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); - // Total operation budget is 2,000ms. At 2,000ms (1,500ms snapshot + 500ms screenshot), it must time out. await vi.advanceTimersByTimeAsync(2_000); await assertion; + expect(screenshots).toBe(1); + expect(Date.now() - started).toBe(2_000); } finally { vi.useRealTimers(); } }); - it('bounds keyboard dismiss snapshots by the operation budget', async () => { + it('bounds the snapshots of a keyboard dismissal by its budget, and reports one cut off after the drag as possibly committed', async () => { const h = harness(); await openAttempt(h); vi.useFakeTimers(); @@ -1010,11 +1014,54 @@ describe('observation', () => { h.fake.respond('command.keyboard', () => { throw new AppError('UNSUPPORTED_OPERATION', 'Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key'); }); - h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve(SETTINGS_SNAPSHOT), 5_000))); + const keyboardUp = { ...SETTINGS_SNAPSHOT, viewport: { width: 390, height: 844 }, keyboard: { kind: 'visible', frame: { x: 0, y: 583, width: 390, height: 261 } } }; + let snapshots = 0; + h.fake.respond('capture.snapshot', () => (snapshots++ === 0 ? keyboardUp : new Promise(() => undefined))); const dismiss = h.engine.keyboard!.dismiss!({ ...operation(), timeoutMs: 1_000 }); - const assertion = expect(dismiss).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + const assertion = expect(dismiss).rejects.toMatchObject({ code: 'ACTION_MAY_HAVE_COMMITTED' }); await vi.advanceTimersByTimeAsync(1_000); await assertion; + expect(h.fake.calls.filter((call) => call.method === 'interactions.swipe')).toHaveLength(1); + } finally { + vi.useRealTimers(); + } + }); + + it('finishes a slow read the budget covers: a poll grants its last read the time reads take', async () => { + const h = harness(); + await openAttempt(h); + vi.useFakeTimers(); + try { + h.fake.respond('capture.snapshot', () => new Promise((resolve) => setTimeout(() => resolve(SETTINGS_SNAPSHOT), 3_000))); + const about = { kind: 'query', query: { kind: 'text', value: { kind: 'string', value: 'About', exact: true } } } as const; + // The last read of a poll that has 500 ms left, after reads that took 3 s. + const read = h.engine.locate!(about, { ...operation(), timeoutMs: 3_500 }); + const assertion = expect(read).resolves.toHaveLength(1); + await vi.advanceTimersByTimeAsync(3_000); + await assertion; + } finally { + vi.useRealTimers(); + } + }); + + it('takes no second snapshot for a capture the budget abandoned', async () => { + const h = harness(); + await openAttempt(h); + vi.useFakeTimers(); + try { + let snapshots = 0; + const presentationFailure = () => + new AppError('COMMAND_FAILED', 'regular iOS snapshot presentation requires a valid viewport', { reason: 'invalid-viewport' }); + h.fake.respond('capture.snapshot', () => { + snapshots += 1; + return new Promise((_, reject) => setTimeout(() => reject(presentationFailure()), 1_500)); + }); + const about = { kind: 'query', query: { kind: 'text', value: { kind: 'string', value: 'About', exact: true } } } as const; + const read = h.engine.locate!(about, { ...operation(), timeoutMs: 1_000 }); + const assertion = expect(read).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT' }); + await vi.advanceTimersByTimeAsync(3_000); + await assertion; + expect(snapshots).toBe(1); } finally { vi.useRealTimers(); } @@ -1096,10 +1143,10 @@ describe('a screen agent-device cannot read yet', () => { await openAttempt(h); vi.useFakeTimers(); try { - h.fake.respond('capture.snapshot', () => new Promise((_, reject) => setTimeout(() => reject(insufficientContent()), 3_000))); + h.fake.respond('capture.snapshot', () => new Promise((_, reject) => setTimeout(() => reject(insufficientContent()), 600))); const located = h.engine.locate!(ABOUT, { ...operation(), timeoutMs: 1_000 }); const settled = expect(located).rejects.toMatchObject({ code: 'ENGINE_FAILURE', message: expect.stringContaining('insufficient foreground app content') }); - await vi.advanceTimersByTimeAsync(3_000); + await vi.advanceTimersByTimeAsync(600); await settled; } finally { vi.useRealTimers(); diff --git a/packages/mobile/tests/unit/support.test.ts b/packages/mobile/tests/unit/support.test.ts index 460d4af01c..683ad42794 100644 --- a/packages/mobile/tests/unit/support.test.ts +++ b/packages/mobile/tests/unit/support.test.ts @@ -43,27 +43,34 @@ describe('support helpers', () => { }); expect(held).toBe(false); - // A read under budget succeeds, and one that hangs fails with OPERATION_TIMEOUT - const readBudget = new InputBudget('snapshot', 300, (pending) => pending); - const readSuccess = readBudget.read(() => Promise.resolve('data')); - await expect(readSuccess).resolves.toBe('data'); + } finally { + vi.useRealTimers(); + } + }); - const hangingRead = new InputBudget('snapshot', 200, (pending) => pending); - const hangingPromise = hangingRead.read(hung); - const hangingOutcome = expect(hangingPromise).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT', message: 'snapshot timed out before the device answered' }); + it('runs a read with what is left of its budget, and reports one cut off after an input as possibly committed', async () => { + const hung = () => new Promise(() => undefined); + vi.useFakeTimers(); + try { + await expect(new InputBudget('snapshot', 300, (pending) => pending).read(async () => 'tree')).resolves.toBe('tree'); + + const cut = new InputBudget('snapshot', 200, (pending) => pending).read(hung); + const timedOut = expect(cut).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT', message: 'snapshot timed out before the device answered' }); await vi.advanceTimersByTimeAsync(200); - await hangingOutcome; + await timedOut; - // A read with no budget left fails immediately with OPERATION_TIMEOUT - const spentRead = new InputBudget('snapshot', 100, (pending) => pending); + const spent = new InputBudget('snapshot', 100, (pending) => pending); await vi.advanceTimersByTimeAsync(100); - await expect(spentRead.read(async () => 'late')).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT', message: 'snapshot timed out before it was sent' }); + await expect(spent.read(async () => 'late')).rejects.toMatchObject({ code: 'OPERATION_TIMEOUT', message: 'snapshot timed out before it was sent' }); - // check() throws when budget has expired - const checkBudget = new InputBudget('observe', 100, (pending) => pending); - expect(() => checkBudget.check()).not.toThrow(); - await vi.advanceTimersByTimeAsync(100); - expect(() => checkBudget.check()).toThrowError(/observe timed out before it was sent/); + // After a swipe went out, a read that does not answer must not let a replay send the swipe again. + const dismissal = new InputBudget('keyboard dismiss', 300, (pending) => pending); + await dismissal.send(async () => undefined); + const after = dismissal.read(hung); + const committed = expect(after).rejects.toMatchObject({ code: 'ACTION_MAY_HAVE_COMMITTED' }); + await vi.advanceTimersByTimeAsync(300); + await committed; + await expect(dismissal.read(async () => 'late')).rejects.toMatchObject({ code: 'ACTION_MAY_HAVE_COMMITTED' }); } finally { vi.useRealTimers(); } From 21aeaa90a3f884f6545a00e88224c78a4fe24e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 16 Oct 2026 15:14:37 +0200 Subject: [PATCH 07/12] fix(engine): keep ReadWindow off the public poll options Co-Authored-By: Claude --- packages/e2e/src/expect/async.ts | 4 +-- packages/e2e/src/expect/screenshot.ts | 6 ++-- packages/e2e/src/internal/time.ts | 50 +++++++++++++++++---------- packages/e2e/src/locator/screen.ts | 4 +-- packages/e2e/tests/unit/time.test.ts | 6 ++-- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/packages/e2e/src/expect/async.ts b/packages/e2e/src/expect/async.ts index 4748759da5..5402c80685 100644 --- a/packages/e2e/src/expect/async.ts +++ b/packages/e2e/src/expect/async.ts @@ -14,7 +14,7 @@ import { type TextPattern, } from '../internal/text.ts'; import { isValueControl } from '../internal/roles.ts'; -import { pollCondition, type ReadWindow } from '../internal/time.ts'; +import { pollReads, type ReadWindow } from '../internal/time.ts'; import { SampleHistory } from './samples.ts'; import { attributeOf, denySecureRead, isNodeVisible } from '../locator/engine.ts'; import { describeExpression } from '../locator/expression.ts'; @@ -173,7 +173,7 @@ class AsyncExpectationImpl implements AsyncExpectation { let lastSample: Sample = { count: 0, node: null, nodes: [] }; const samples = new SampleHistory('expect', (text) => engine.redact(text)); await samples.record(engine, () => - pollCondition({ + pollReads({ deadline, reads: engine.reads(deadline), signal: engine.signal, diff --git a/packages/e2e/src/expect/screenshot.ts b/packages/e2e/src/expect/screenshot.ts index be4c7a1bad..667467c720 100644 --- a/packages/e2e/src/expect/screenshot.ts +++ b/packages/e2e/src/expect/screenshot.ts @@ -11,7 +11,7 @@ import type { LocatorExpression } from '../engine/surface.ts'; import { writeFileAtomic } from '../internal/atomic-write.ts'; import { ConfigurationError, TestError } from '../internal/errors.ts'; import { compareImages, decodePng, encodePng, type ImageComparison, type RgbaImage } from '../internal/image.ts'; -import { pollCondition, type Deadline, type ReadWindow } from '../internal/time.ts'; +import { pollReads, type Deadline, type ReadWindow } from '../internal/time.ts'; import { describeExpression } from '../locator/expression.ts'; import type { ScreenContext } from '../locator/screen.ts'; import type { ScreenshotContext } from '../run/screenshots.ts'; @@ -112,7 +112,7 @@ async function matchOrSettle(run: Run, expected: RgbaImage | undefined, deadline let unsettled: readonly [RgbaImage, RgbaImage] | undefined; let settled: RgbaImage | undefined; let captures = 0; - await pollCondition({ + await pollReads({ deadline, reads: run.reads, signal: run.context.engine.signal, @@ -187,7 +187,7 @@ async function expectDifferent(run: Run, expected: RgbaImage | undefined, deadli } let compared = false; let missing: string | undefined; - await pollCondition({ + await pollReads({ deadline, reads: run.reads, signal: run.context.engine.signal, diff --git a/packages/e2e/src/internal/time.ts b/packages/e2e/src/internal/time.ts index 5f95a96d66..3ee8ec55aa 100644 --- a/packages/e2e/src/internal/time.ts +++ b/packages/e2e/src/internal/time.ts @@ -119,46 +119,60 @@ export const NEGATION_GRACE_MS = 1000; export interface PollConditionOptions { readonly deadline: Deadline; - /** The window over `deadline` that `evaluate`'s reads run in; without one, reads end at `deadline`. */ - readonly reads?: ReadWindow; readonly signal: AbortSignal; readonly negated: boolean; /** - * Evaluates the positive condition once, every read run through `reads` - * or bounded by `deadline`: a read on a longer budget outlasts the poll. - * Returns undefined when the condition cannot be evaluated yet: the - * positive poll keeps waiting and the negation grace window resets. + * Evaluates the positive condition once, every read bounded by + * `deadline`: a read on a longer budget outlasts the poll. Returns + * undefined when the condition cannot be evaluated yet: the positive poll + * keeps waiting and the negation grace window resets. */ - evaluate(reads: ReadWindow): Promise; + evaluate(): Promise; /** Builds the poll's failure; `cause` is the read the deadline cut off, when one did. */ onTimeout(cause?: unknown): Error | Promise; } +/** `pollCondition` options whose reads run in a `ReadWindow` over the deadline. */ +export interface PollReadsOptions extends Omit { + readonly reads: ReadWindow; + /** Evaluates the positive condition once, every read run through `reads`. */ + evaluate(reads: ReadWindow): Promise; +} + /** * Polls a condition until it holds (or, when negated, until its negation has * held continuously for the negation grace window), throwing the caller's * error at the deadline. No sleep runs past the deadline, so the last read - * starts at it; a `reads` window with a cap lets it run past by the longest - * read the poll has finished. + * starts at it and never sees a state that arrived later. * * A negation holds from the moment the read that first saw it was issued, so * a slow read counts toward the window. A budget shorter than the window * still has to be satisfiable: the negation then only needs to hold for the - * whole budget, and passes at the deadline on what it has seen. A read the - * deadline cut off (see `ReadWindow.cutOff`) after an earlier one completed - * saw nothing: the poll ends there. A negation then passes if it has held - * long enough by the time the read is cut off: the read left less than one - * poll tick, or one read's time, unseen, so a short budget still passes at - * its deadline on what it saw. + * whole budget, and passes at the deadline on what it has seen, since a read + * past the deadline has no budget left. A read the deadline cut off (an + * `OPERATION_TIMEOUT` on a read that started with under one poll tick left) + * after an earlier one completed saw nothing: the poll ends there. A negation + * then passes if it has held long enough by the time the read is cut off: the + * read left less than one poll tick unseen, no more than the gap between any + * two reads, so a short budget still passes at its deadline on what it saw. + */ +export function pollCondition(options: PollConditionOptions): Promise { + return pollReads({ ...options, reads: new ReadWindow(options.deadline) }); +} + +/** + * `pollCondition` with its reads run through `options.reads`: a window with a + * cap lets the last read run past the deadline by the longest read the poll + * has finished, and a read cut off (see `ReadWindow.cutOff`) is one that + * started with under one poll tick, or that longest read, left. */ -export async function pollCondition(options: PollConditionOptions): Promise { - const { deadline, negated } = options; +export async function pollReads(options: PollReadsOptions): Promise { + const { deadline, negated, reads } = options; const startedAt = Date.now(); const grace = Math.min(NEGATION_GRACE_MS, deadline.remaining(startedAt)); let readAt = startedAt; let holdingSince: number | undefined; const holds = (now: number): boolean => holdingSince !== undefined && now - holdingSince >= grace; - const reads = options.reads ?? new ReadWindow(deadline); let sampled = false; for (;;) { const startedWithMs = deadline.remaining(readAt); diff --git a/packages/e2e/src/locator/screen.ts b/packages/e2e/src/locator/screen.ts index bc0ed6f18c..68b0e1149b 100644 --- a/packages/e2e/src/locator/screen.ts +++ b/packages/e2e/src/locator/screen.ts @@ -43,7 +43,7 @@ import { testIdQuery, textQuery, } from './expression.ts'; -import { type Deadline, isOperationTimeout, POLL_INTERVAL_MS, pollCondition } from '../internal/time.ts'; +import { type Deadline, isOperationTimeout, POLL_INTERVAL_MS, pollReads } from '../internal/time.ts'; import { SampleHistory } from '../expect/samples.ts'; import type { ScreenshotContext } from '../run/screenshots.ts'; @@ -606,7 +606,7 @@ class LocatorImpl extends ScreenImpl implements Locator { const startedMs = Date.now(); const samples = new SampleHistory('waitFor', (text) => engine.redact(text)); await samples.record(engine, () => - pollCondition({ + pollReads({ deadline, reads: engine.reads(deadline), signal: engine.signal, diff --git a/packages/e2e/tests/unit/time.test.ts b/packages/e2e/tests/unit/time.test.ts index d6fe0821e1..17c5d0a637 100644 --- a/packages/e2e/tests/unit/time.test.ts +++ b/packages/e2e/tests/unit/time.test.ts @@ -5,6 +5,7 @@ import { NEGATION_GRACE_MS, POLL_INTERVAL_MS, pollCondition, + pollReads, ReadWindow, sleep, withTimeout, @@ -357,7 +358,7 @@ describe('pollCondition', () => { const deadline = new Deadline(450); const timeouts: number[] = []; let calls = 0; - const promise = pollCondition({ + const promise = pollReads({ deadline, reads: new ReadWindow(deadline, new Deadline(10_000)), signal: new AbortController().signal, @@ -382,8 +383,9 @@ describe('pollCondition', () => { const poll = (timeoutMs: number) => { const deadline = new Deadline(timeoutMs); let calls = 0; - return pollCondition({ + return pollReads({ deadline, + reads: new ReadWindow(deadline), signal: new AbortController().signal, negated: false, evaluate: (reads) => From 2018295e2352fe21ff05033ac43533dd7595fe12 Mon Sep 17 00:00:00 2001 From: Totopo27 <35828647+Totopo27@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:58:15 -0600 Subject: [PATCH 08/12] fix(mobile): check signal on sparse snapshot budget exhaustion and exercise ReadWindow cap --- packages/e2e/tests/unit/time.test.ts | 13 +++++++++---- packages/mobile/src/surface.ts | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/e2e/tests/unit/time.test.ts b/packages/e2e/tests/unit/time.test.ts index 17c5d0a637..50bb887af5 100644 --- a/packages/e2e/tests/unit/time.test.ts +++ b/packages/e2e/tests/unit/time.test.ts @@ -355,26 +355,31 @@ describe('pollCondition', () => { it('lets the last read run past the deadline by the longest read the poll finished, up to the window cap', async () => { vi.useFakeTimers(); - const deadline = new Deadline(450); + const deadline = new Deadline(350); const timeouts: number[] = []; let calls = 0; const promise = pollReads({ deadline, - reads: new ReadWindow(deadline, new Deadline(10_000)), + reads: new ReadWindow(deadline, new Deadline(400)), signal: new AbortController().signal, negated: false, evaluate: (reads) => reads.read(async (timeoutMs) => { timeouts.push(timeoutMs); calls += 1; - await withTimeout(sleep(200), timeoutMs, () => new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false })); + const readDuration = calls === 1 ? 200 : 100; + await withTimeout(sleep(readDuration), timeoutMs, () => new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false })); return calls === 2; }), onTimeout: () => new Error('poll timed out'), }); await vi.advanceTimersByTimeAsync(1_000); await expect(promise).resolves.toBeUndefined(); - expect(timeouts).toEqual([450, 350]); + // Round 1 started at t=0: deadline remaining 350, cap 400 -> 350. Takes 200 ms. + // Sleep between reads takes min(100, 350 - 200) = 100 ms. So t = 300 ms. + // Round 2 started at t=300: deadline remaining 50, slowestMs is 200 -> allowed 250 ms. + // Cap remaining is 400 - 300 = 100 ms! So capped at 100 ms (< 250 ms). + expect(timeouts).toEqual([350, 100]); }); it('ends on a read cut off within the longest read time of the deadline, and fails on one that hung with more left', async () => { diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 33ddfeac0c..6e208f7aa2 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -871,7 +871,10 @@ export class AgentDeviceSurface { for (const backoffMs of SPARSE_RETRY_BACKOFF_MS) { if (backoffMs > 0) { // A sparse capture is still the screen: a retry the budget cannot fit returns it instead of timing out. - if (budget !== undefined && budget.remainingMs() <= backoffMs + captureMs) return last; + if (budget !== undefined && budget.remainingMs() <= backoffMs + captureMs) { + if (signal.aborted) throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false }); + return last; + } await sleep(backoffMs, signal); } if (signal.aborted) throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false }); From e62ea7c827a306eba121b310c168f51a941a152e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Fri, 16 Oct 2026 19:15:21 +0200 Subject: [PATCH 09/12] fix(mobile): require e2e 0.21.3, revert extra abort check, tighten read window test --- .changeset/mobile-reads-budget.md | 2 +- packages/e2e/tests/unit/time.test.ts | 6 +----- packages/mobile/package.json | 2 +- packages/mobile/src/surface.ts | 5 +---- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.changeset/mobile-reads-budget.md b/.changeset/mobile-reads-budget.md index 19ccb1bd15..8d30d46b28 100644 --- a/.changeset/mobile-reads-budget.md +++ b/.changeset/mobile-reads-budget.md @@ -2,4 +2,4 @@ '@e2e-dev/mobile': patch --- -Snapshots and screenshots now get the step's timeout, so a device that stops answering fails the step instead of holding it until the test timeout. Requires `e2e` 0.21.2 or later. +Snapshots and screenshots now get the step's timeout, so a device that stops answering fails the step instead of holding it until the test timeout. Requires `e2e` 0.21.3 or later. diff --git a/packages/e2e/tests/unit/time.test.ts b/packages/e2e/tests/unit/time.test.ts index 50bb887af5..a5294bd69e 100644 --- a/packages/e2e/tests/unit/time.test.ts +++ b/packages/e2e/tests/unit/time.test.ts @@ -367,7 +367,7 @@ describe('pollCondition', () => { reads.read(async (timeoutMs) => { timeouts.push(timeoutMs); calls += 1; - const readDuration = calls === 1 ? 200 : 100; + const readDuration = calls === 1 ? 200 : 50; await withTimeout(sleep(readDuration), timeoutMs, () => new EngineError('OPERATION_TIMEOUT', 'locate timed out', { retryable: false })); return calls === 2; }), @@ -375,10 +375,6 @@ describe('pollCondition', () => { }); await vi.advanceTimersByTimeAsync(1_000); await expect(promise).resolves.toBeUndefined(); - // Round 1 started at t=0: deadline remaining 350, cap 400 -> 350. Takes 200 ms. - // Sleep between reads takes min(100, 350 - 200) = 100 ms. So t = 300 ms. - // Round 2 started at t=300: deadline remaining 50, slowestMs is 200 -> allowed 250 ms. - // Cap remaining is 400 - 300 = 100 ms! So capped at 100 ms (< 250 ms). expect(timeouts).toEqual([350, 100]); }); diff --git a/packages/mobile/package.json b/packages/mobile/package.json index 7e6e45df24..eff1f5ecbb 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -65,7 +65,7 @@ }, "peerDependencies": { "ai": "^7.0.0", - "e2e": ">=0.21.2 <1" + "e2e": ">=0.21.3 <1" }, "peerDependenciesMeta": { "ai": { diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 6e208f7aa2..33ddfeac0c 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -871,10 +871,7 @@ export class AgentDeviceSurface { for (const backoffMs of SPARSE_RETRY_BACKOFF_MS) { if (backoffMs > 0) { // A sparse capture is still the screen: a retry the budget cannot fit returns it instead of timing out. - if (budget !== undefined && budget.remainingMs() <= backoffMs + captureMs) { - if (signal.aborted) throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false }); - return last; - } + if (budget !== undefined && budget.remainingMs() <= backoffMs + captureMs) return last; await sleep(backoffMs, signal); } if (signal.aborted) throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false }); From f7f9584a1cb4d6fa6dae8e88b4d3cb8ef0aebe1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Sun, 25 Oct 2026 23:24:12 +0100 Subject: [PATCH 10/12] fix(mobile): require e2e 0.21.4, restore locateSnapshot JSDoc --- .changeset/mobile-reads-budget.md | 2 +- packages/mobile/package.json | 2 +- packages/mobile/src/surface.ts | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.changeset/mobile-reads-budget.md b/.changeset/mobile-reads-budget.md index 8d30d46b28..be06c4028f 100644 --- a/.changeset/mobile-reads-budget.md +++ b/.changeset/mobile-reads-budget.md @@ -2,4 +2,4 @@ '@e2e-dev/mobile': patch --- -Snapshots and screenshots now get the step's timeout, so a device that stops answering fails the step instead of holding it until the test timeout. Requires `e2e` 0.21.3 or later. +Snapshots and screenshots now get the step's timeout, so a device that stops answering fails the step instead of holding it until the test timeout. Requires `e2e` 0.21.4 or later. diff --git a/packages/mobile/package.json b/packages/mobile/package.json index 1eda329cc6..a38d6fcdb0 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -65,7 +65,7 @@ }, "peerDependencies": { "ai": "^7.0.0", - "e2e": ">=0.21.3 <1" + "e2e": ">=0.21.4 <1" }, "peerDependenciesMeta": { "ai": { diff --git a/packages/mobile/src/surface.ts b/packages/mobile/src/surface.ts index 93dbaad461..345cb1971e 100644 --- a/packages/mobile/src/surface.ts +++ b/packages/mobile/src/surface.ts @@ -976,8 +976,9 @@ export class AgentDeviceSurface { * Snapshot for a locate; undefined for a screen agent-device could not read * yet (an app still on its splash, or one iOS does not report running yet * after a launch). The harness polls a locate until its deadline, so that - * screen matches nothing and the poll waits for it to render. A capture that leaves no budget for another poll fails with the - * reason instead, so a wait that ends on an unreadable screen says so. + * screen matches nothing and the poll waits for it to render. A capture + * that leaves less of the budget than it took fails with the reason + * instead, so a wait that ends on an unreadable screen says so. */ private async locateSnapshot(operation: OperationContext, budget?: InputBudget): Promise { const startedMs = Date.now(); From 39e3e999441217ed6a035ef0937bb6cb51a79c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Tue, 27 Oct 2026 00:10:37 +0100 Subject: [PATCH 11/12] test(locator): restore readAllNow call-count assertion lost in merge --- packages/e2e/tests/unit/locator-engine.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/e2e/tests/unit/locator-engine.test.ts b/packages/e2e/tests/unit/locator-engine.test.ts index 4de88a4a2b..732b725480 100644 --- a/packages/e2e/tests/unit/locator-engine.test.ts +++ b/packages/e2e/tests/unit/locator-engine.test.ts @@ -345,6 +345,7 @@ describe('LocatorEngine readAllNow contract', () => { read: [() => NODE, () => OTHER_NODE], }); expect(await engine.readAllNow(EXPRESSION, engine.deadline())).toEqual([NODE, OTHER_NODE]); + expect(calls).toMatchObject({ resolve: 1, read: 2 }); }); it('answers zero matches with an empty list without waiting', async () => { From 7f1d05ed8689ef4aec6b2f4f96984832d1cb71e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oskar=20Kwas=CC=81niewski?= Date: Wed, 28 Oct 2026 04:36:29 +0100 Subject: [PATCH 12/12] refactor(locator): take a ReadWindow in resolveNow and readAllNow Only tests passed a Deadline; drop the instanceof branch. --- packages/e2e/src/locator/engine.ts | 6 ++---- .../e2e/tests/unit/locator-engine.test.ts | 20 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/e2e/src/locator/engine.ts b/packages/e2e/src/locator/engine.ts index 3f43347273..4cf31e3aba 100644 --- a/packages/e2e/src/locator/engine.ts +++ b/packages/e2e/src/locator/engine.ts @@ -268,9 +268,8 @@ export class LocatorEngine { */ async resolveNow( expression: LocatorExpression, - deadlineOrReads: Deadline | ReadWindow = this.deadline(this.options.actionTimeout), + reads: ReadWindow = new ReadWindow(this.deadline(this.options.actionTimeout)), ): Promise { - const reads = deadlineOrReads instanceof ReadWindow ? deadlineOrReads : new ReadWindow(deadlineOrReads); return this.resolveOnce(expression, reads, 'empty'); } @@ -306,9 +305,8 @@ export class LocatorEngine { */ async readAllNow( expression: LocatorExpression, - deadlineOrReads: Deadline | ReadWindow = this.deadline(this.options.actionTimeout), + reads: ReadWindow = new ReadWindow(this.deadline(this.options.actionTimeout)), ): Promise { - const reads = deadlineOrReads instanceof ReadWindow ? deadlineOrReads : new ReadWindow(deadlineOrReads); for (;;) { const refs = await this.resolveOnce(expression, reads, 'empty'); try { diff --git a/packages/e2e/tests/unit/locator-engine.test.ts b/packages/e2e/tests/unit/locator-engine.test.ts index 233674dc56..39b9afbd7b 100644 --- a/packages/e2e/tests/unit/locator-engine.test.ts +++ b/packages/e2e/tests/unit/locator-engine.test.ts @@ -171,7 +171,7 @@ describe('LocatorEngine resolve retry contract', () => { it('translates a non-retryable resolve failure immediately without retrying', async () => { const { engine, calls } = makeEngine({ resolve: ['failure'] }); - await expect(engine.resolveNow(EXPRESSION, engine.deadline())).rejects.toMatchObject({ + await expect(engine.resolveNow(EXPRESSION, engine.reads(engine.deadline()))).rejects.toMatchObject({ category: 'infrastructure', code: 'ENGINE_FAILURE', }); @@ -183,7 +183,7 @@ describe('LocatorEngine resolve retry contract', () => { { resolve: Array.from({ length: 50 }, () => 'stale' as const) }, { actionTimeout: 350 }, ); - await expect(engine.resolveNow(EXPRESSION, engine.deadline())).rejects.toMatchObject({ + await expect(engine.resolveNow(EXPRESSION, engine.reads(engine.deadline()))).rejects.toMatchObject({ category: 'test', code: 'LOCATOR_NOT_FOUND', }); @@ -387,13 +387,13 @@ describe('LocatorEngine readAllNow contract', () => { resolve: [() => [REF, OTHER_REF]], read: [() => NODE, () => OTHER_NODE], }); - expect(await engine.readAllNow(EXPRESSION, engine.deadline())).toEqual([NODE, OTHER_NODE]); + expect(await engine.readAllNow(EXPRESSION, engine.reads(engine.deadline()))).toEqual([NODE, OTHER_NODE]); expect(calls).toMatchObject({ resolve: 1, read: 2 }); }); it('answers zero matches with an empty list without waiting', async () => { const { engine, calls } = makeEngine({ resolve: [() => []] }); - expect(await engine.readAllNow(EXPRESSION, engine.deadline())).toEqual([]); + expect(await engine.readAllNow(EXPRESSION, engine.reads(engine.deadline()))).toEqual([]); expect(calls).toMatchObject({ resolve: 1, read: 0 }); }); @@ -402,20 +402,20 @@ describe('LocatorEngine readAllNow contract', () => { resolve: [() => [REF, OTHER_REF], () => [OTHER_REF]], read: ['stale', () => OTHER_NODE], }); - expect(await engine.readAllNow(EXPRESSION, engine.deadline())).toEqual([OTHER_NODE]); + expect(await engine.readAllNow(EXPRESSION, engine.reads(engine.deadline()))).toEqual([OTHER_NODE]); expect(calls).toMatchObject({ resolve: 2, read: 2 }); }); it('gives up on a set that stays stale at the deadline as LOCATOR_NOT_FOUND', async () => { const { engine } = makeEngine({ read: ['stale', 'stale', 'stale', 'stale', 'stale'] }); - await expect(engine.readAllNow(EXPRESSION, new Deadline(150))).rejects.toMatchObject({ + await expect(engine.readAllNow(EXPRESSION, engine.reads(new Deadline(150)))).rejects.toMatchObject({ code: 'LOCATOR_NOT_FOUND', }); }); it('translates non-stale read failures', async () => { const { engine } = makeEngine({ read: ['failure'] }); - await expect(engine.readAllNow(EXPRESSION, engine.deadline())).rejects.toMatchObject({ code: 'ENGINE_FAILURE' }); + await expect(engine.readAllNow(EXPRESSION, engine.reads(engine.deadline()))).rejects.toMatchObject({ code: 'ENGINE_FAILURE' }); }); }); @@ -579,7 +579,7 @@ describe('LocatorEngine visible queries', () => { const { engine } = makeLocatorEngineOverEngine(); const ref = await engine.resolveExactlyOne(build(true), new Deadline(5_000)); expect(ref.id).toBe('shown'); - expect(await engine.resolveNow(build(true), engine.deadline())).toHaveLength(1); + expect(await engine.resolveNow(build(true), engine.reads(engine.deadline()))).toHaveLength(1); }); it.each(kinds)('%s without visible keeps the hidden twin, so the pair is LOCATOR_AMBIGUOUS', async (_kind, build) => { @@ -590,7 +590,7 @@ describe('LocatorEngine visible queries', () => { await expect(engine.resolveExactlyOne(build(false), new Deadline(5_000))).rejects.toMatchObject({ code: 'LOCATOR_AMBIGUOUS', }); - expect(await engine.resolveNow(build(undefined), engine.deadline())).toHaveLength(2); + expect(await engine.resolveNow(build(undefined), engine.reads(engine.deadline()))).toHaveLength(2); }); it('names the predicate in the ambiguity message when two visible nodes remain', async () => { @@ -615,7 +615,7 @@ describe('LocatorEngine visible queries', () => { const indexed: LocatorExpression = { kind: 'index', source: kinds[1]![1](true), index: 'first' }; // The adapter cannot know which of the engine's answers came first, so it // must not second-guess them; the engine applied `visible` before `first`. - expect(await engine.resolveNow(indexed, engine.deadline())).toHaveLength(2); + expect(await engine.resolveNow(indexed, engine.reads(engine.deadline()))).toHaveLength(2); expect(expressions[0]).toEqual(indexed); }); });