diff --git a/.changeset/locator-filter-step-counts.md b/.changeset/locator-filter-step-counts.md new file mode 100644 index 0000000000..e47a5963f4 --- /dev/null +++ b/.changeset/locator-filter-step-counts.md @@ -0,0 +1,5 @@ +--- +'e2e': patch +--- + +Filtered-locator failures can list how many candidates each filter step left, when the post-failure diagnostic pass can read the chain, so a step that narrowed too much or too little names itself. The message says the counts were read then. diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index 7cc5b9cd27..fb710a9750 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -109,8 +109,8 @@ category a blocked verdict reports as the obstacle. | `CANCELLED` | The runner cancelled the operation without a process signal: an `e2e mcp` `open_session` cut short by a client cancel, a closed connection, or the server's shutdown, after it stopped what it had started | infrastructure | 3 | | `MODEL_OUTPUT_INVALID` | Structured output still failed validation after the repair budget was spent. Under a [decision model](/decision-models): an answer without a valid choice distribution, a choice the request never offered, or a question the provider refused. Decisions are not repaired | test | 1 | | `APP_NOT_OPEN` | An operation needed an open app: `app.open()` had not run, or the engine reported invalid state | test | 1 | -| `LOCATOR_NOT_FOUND` | No matching node within the deadline, or an agent action refers to a node that is no longer available | test | 1 | -| `LOCATOR_AMBIGUOUS` | More than one match for a single-node operation. Fails at once, with no polling. | test | 1 | +| `LOCATOR_NOT_FOUND` | No matching node within the deadline, or an agent action refers to a node that is no longer available. For a filtered locator the message can also list how many candidates each filter step left, counted after the failure, when that diagnostic pass can read the chain | test | 1 | +| `LOCATOR_AMBIGUOUS` | More than one match for a single-node operation. Fails at once, with no polling, and for a filtered locator can list how many candidates each filter step left, counted after the failure, when that diagnostic pass can read the chain | test | 1 | | `ACTION_FAILED` | Actionability failure, operation timeout, or an action that may already have committed | test | 1 | | `STEP_BUDGET_EXHAUSTED` | The model-call or action-step budget ran out before a conclusion | test | 1 | | `STEP_TIMEOUT` | The invocation deadline expired, including an operation that timed out within a poll tick of it. Carries the last judgment where one exists, with the [vision hint](/reference/agent#agent-assert) when that judgment was inconclusive | test | 1 | diff --git a/packages/e2e/src/locator/engine.ts b/packages/e2e/src/locator/engine.ts index 3d653494ef..42397157c1 100644 --- a/packages/e2e/src/locator/engine.ts +++ b/packages/e2e/src/locator/engine.ts @@ -19,8 +19,8 @@ import { type ErrorDetails, } from '../internal/errors.ts'; import { requireKey } from '../internal/keys.ts'; -import { describeExpression, expressionHints } from './expression.ts'; -import { Deadline, POLL_INTERVAL_MS, ReadWindow, sleep } from '../internal/time.ts'; +import { describeExpression, expressionHints, filterSpine } from './expression.ts'; +import { abortError, cancelledError, Deadline, POLL_INTERVAL_MS, ReadWindow, sleep } from '../internal/time.ts'; import { timestamp } from '../internal/ids.ts'; import { obj } from '../internal/objects.ts'; import { targetLabel } from '../cache/trace.ts'; @@ -61,6 +61,14 @@ export function denySecureRead(nodes: readonly SemanticNode[], label: string): v */ type MissingFrame = 'wait' | 'empty'; +/** The budget the whole diagnostic failure walk shares: best effort, never a wait. + * One deadline bounds every prefix resolve, so a multi-step chain cannot cost + * more than one short budget even on mobile, where `session.locate()` ignores + * the operation timeout and a snapshot that starts just before the deadline + * still runs to completion — the real bound is the budget plus one snapshot. + */ +const STEP_COUNT_BUDGET_MS = 500; + interface LocatorEngineOptions { readonly session: TargetSession; /** The running phase's signal and deadline, read per operation. */ @@ -234,11 +242,16 @@ export class LocatorEngine { const startedMs = Date.now(); // The wait as it happened: a deadline capped by the test's remaining // budget waited less than the action timeout, and the report says so. - const notFound = (cause?: unknown) => - new TestError('LOCATOR_NOT_FOUND', `locator matched no nodes within ${describeExpression(expression)}`, { - details: locatorDetails(expression, Date.now() - startedMs), + const notFound = async (cause?: unknown) => { + // The wait as it happened, read before the diagnostic pass runs: the + // prefix resolves below are not part of the locator's wait. + const waitedMs = Date.now() - startedMs; + const lines = await this.candidateCounts(expression); + return new TestError('LOCATOR_NOT_FOUND', `locator matched no nodes within ${describeExpression(expression)}${lines}`, { + details: locatorDetails(expression, waitedMs), ...(cause === undefined ? {} : { cause }), }); + }; let sampled = false; for (let rounds = 1; ; rounds += 1) { let refs: readonly NodeRef[]; @@ -246,17 +259,58 @@ export class LocatorEngine { refs = await this.resolveOnce(expression, reads); } catch (cause) { // A resolve the deadline cut off, after one that matched nothing, is the wait running out. - if (sampled && reads.cutOff(cause)) throw notFound(cause); +if (sampled && reads.cutOff(cause)) throw await notFound(cause); throw cause; } sampled = true; - const ref = assertSingle(refs, expression); + const ref = await this.singleOf(refs, expression); if (ref !== null) return { ref, waitedMs: rounds > 1 ? Date.now() - startedMs : 0 }; - if (deadline.expired()) throw notFound(); + if (deadline.expired()) throw await notFound(); await sleep(POLL_INTERVAL_MS, this.signal); } } + /** + * The diagnostic on a filtered locator's failure: one immediate resolve per + * prefix of its chain, innermost first, and how many candidates each left, + * as the message's per-step lines. One deadline bounds the whole walk, so + * even on mobile, where `session.locate()` ignores the operation timeout, + * a chain whose walk cannot finish lists only the prefixes it read and says + * so, or lists nothing when a read failed outright. The counts are read + * after the failure, so a page that changed since can disagree with the + * failing attempt, and the message says so. + */ + private async candidateCounts(expression: LocatorExpression): Promise { + const spine = filterSpine(expression); + if (spine.length === 0) return ''; + const deadline = Deadline.min(new Deadline(STEP_COUNT_BUDGET_MS), this.options.budget.deadline); + const lines: string[] = []; + try { + for (const prefix of spine) { + if (deadline.expired()) break; + const refs = await this.resolveNow(prefix, new ReadWindow(deadline)); + lines.push(`${describeExpression(prefix)}: ${refs.length}`); + } + } catch { + if (this.signal.aborted) throw abortError(this.signal, () => cancelledError()); + // A diagnostic read failure must not fail the failure: partial counts + // must not read as a completed walk, so return nothing. + return ''; + } + if (lines.length === 0) return ''; + const stopped = lines.length < spine.length ? `, walk stopped after ${lines.length} of ${spine.length} steps` : ''; + return `; candidates left after each filter step${stopped}, counted after the failure:\n${lines.join('\n')}`; + } + + /** Zero matches -> null; one -> the ref; many -> LOCATOR_AMBIGUOUS with the per-step counts. */ + private async singleOf(refs: readonly NodeRef[], expression: LocatorExpression): Promise { + if (refs.length <= 1) return refs[0] ?? null; + const lines = await this.candidateCounts(expression); + throw new TestError('LOCATOR_AMBIGUOUS', `locator matched ${refs.length} nodes, expected exactly one: ${describeExpression(expression)}${lines}`, { + details: { ...locatorDetails(expression), matches: refs.length }, + }); + } + /** * Resolves every current match at once, without waiting for one. A frame * the engine reports missing is zero matches, since nothing under it is in @@ -350,7 +404,7 @@ export class LocatorEngine { reads: ReadWindow, missingFrame: MissingFrame, ): Promise<{ node: SemanticNode | null; count: number }> { - const ref = assertSingle(await this.resolveOnce(expression, reads, missingFrame), expression); +const ref = await this.singleOf(await this.resolveOnce(expression, reads, missingFrame), expression); if (ref === null) return { node: null, count: 0 }; try { const node = await reads.read((timeoutMs) => this.session.read(ref, this.readOperation(timeoutMs))); @@ -630,18 +684,6 @@ function describeNode(node: SemanticNode, redact: (text: string) => string): str })); } -/** Zero matches -> null; one -> the ref; many -> LOCATOR_AMBIGUOUS. */ -function assertSingle(refs: readonly NodeRef[], expression: LocatorExpression): NodeRef | null { - if (refs.length > 1) { - throw new TestError( - 'LOCATOR_AMBIGUOUS', - `locator matched ${refs.length} nodes, expected exactly one: ${describeExpression(expression)}`, - { details: { ...locatorDetails(expression), matches: refs.length } }, - ); - } - return refs[0] ?? null; -} - /** * The facts of a locator failure the report keeps beside the message: the * locator as written, what it asked for, and how long it actually waited, in ms. diff --git a/packages/e2e/src/locator/expression.ts b/packages/e2e/src/locator/expression.ts index 9e669adee2..01324cb1ff 100644 --- a/packages/e2e/src/locator/expression.ts +++ b/packages/e2e/src/locator/expression.ts @@ -123,6 +123,26 @@ export function indexExpression( return { kind: 'index', source, index }; } +/** + * The prefixes of a filtered locator's chain, innermost first: the base and + * every `filter` or `index` step applied over it. Empty when the chain has + * no filter step, since a plain query, selector, or frame has no step a + * candidate count can explain. A union at the base counts once like any other + * base; the walk stops there and never descends into its two sides. + */ +export function filterSpine(expression: LocatorExpression): LocatorExpression[] { + const steps: LocatorExpression[] = []; + let current = expression; + let filtered = false; + while (current.kind === 'filter' || current.kind === 'index') { + if (current.kind === 'filter') filtered = true; + steps.push(current); + current = current.source; + } + if (!filtered) return []; + return [current, ...steps.toReversed()]; +} + /** * What an expression asks for, as the facts a failure report keeps beside * the rendered locator: the role and name of a role query, the text a label, diff --git a/packages/e2e/tests/unit/expression.test.ts b/packages/e2e/tests/unit/expression.test.ts index b7da1c3f66..7e54fc8473 100644 --- a/packages/e2e/tests/unit/expression.test.ts +++ b/packages/e2e/tests/unit/expression.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from 'vitest'; +import type { LocatorExpression } from '../../src/engine/surface.ts'; import { describeExpression, expressionHints, filterExpression, + filterSpine, indexExpression, roleQuery, testIdQuery, @@ -60,6 +62,43 @@ describe('locator expressions', () => { expect(() => filterExpression(source, {})).toThrow(/hasText, hasNotText, has, and\/or hasNot/); }); + it('lists every prefix of a filtered chain, innermost first', () => { + const base = roleQuery('listitem', undefined, undefined); + const first = filterExpression(base, { hasText: 'Order' }); + const second = filterExpression(first, { has: roleQuery('button', undefined, undefined) }); + expect(filterSpine(second)).toEqual([base, first, second]); + }); + + it('follows an index step between filters', () => { + const base = roleQuery('listitem', undefined, undefined); + const filtered = filterExpression(base, { hasText: 'Order' }); + const indexed = indexExpression(filtered, 'first'); + const chained = filterExpression(indexed, { hasText: '1042' }); + expect(filterSpine(chained)).toEqual([base, filtered, indexed, chained]); + }); + + it('is empty without a filter step', () => { + const base = roleQuery('listitem', undefined, undefined); + expect(filterSpine(base)).toEqual([]); + expect(filterSpine(indexExpression(base, 0))).toEqual([]); + expect(filterSpine(textQuery('text', 'Pro', undefined, undefined))).toEqual([]); + }); + + it('is empty for a union of chains', () => { + const base = roleQuery('listitem', undefined, undefined); + const filtered = filterExpression(base, { hasText: 'Order' }); + const union: LocatorExpression = { kind: 'union', left: filtered, right: base }; + expect(filterSpine(union)).toEqual([]); + }); + + it('treats a union as a single base to count once, never descending into its sides', () => { + const base = roleQuery('listitem', undefined, undefined); + const other = roleQuery('button', undefined, undefined); + const union: LocatorExpression = { kind: 'union', left: base, right: other }; + const filtered = filterExpression(union, { hasText: 'Order' }); + expect(filterSpine(filtered)).toEqual([union, filtered]); + }); + it('rejects negative and fractional nth indices', () => { const source = testIdQuery('card', undefined, undefined); expect(() => indexExpression(source, -1)).toThrow(/nonnegative/); diff --git a/packages/e2e/tests/unit/locator-engine.test.ts b/packages/e2e/tests/unit/locator-engine.test.ts index a0b96c6e0e..61a5ea41f9 100644 --- a/packages/e2e/tests/unit/locator-engine.test.ts +++ b/packages/e2e/tests/unit/locator-engine.test.ts @@ -19,7 +19,7 @@ import { Deadline, sleep, withTimeout } from '../../src/internal/time.ts'; import { AttemptBudget } from '../../src/run/budget.ts'; import { defineEngine, LOCATOR_ACTION_KINDS } from '../../src/engine/index.ts'; import { createEngineSession } from '../../src/engine/session.ts'; -import { roleQuery, testIdQuery, textQuery } from '../../src/locator/expression.ts'; +import { describeExpression, filterExpression, indexExpression, roleQuery, testIdQuery, textQuery } from '../../src/locator/expression.ts'; import { snapshot } from '../helpers/snapshot.ts'; import { useFakeTime } from '../helpers/fake-time.ts'; @@ -682,3 +682,247 @@ describe('LocatorEngine retries a cross-realm driver failure', () => { expect(calls.perform).toBe(2); }); }); + +describe('locator failure candidate counts', () => { + const base = roleQuery('listitem', undefined, undefined); + const texted = filterExpression(base, { hasText: 'Order 1042' }); + const chain = filterExpression(texted, { has: roleQuery('button', { name: 'Cancel order' }, undefined) }); + + /** + * A session that answers per expression, keyed by the rendered locator, so a + * test pins exactly which expressions a failure path resolves and how much + * deadline each resolve was given. + */ + function makeCountingEngine(answers: ReadonlyMap, options: { actionTimeout?: number } = {}) { + const resolved: string[] = []; + const resolveTimeouts: number[] = []; + const session = { + async locate(expression: LocatorExpression, operation: OperationContext) { + const key = describeExpression(expression); + resolved.push(key); + resolveTimeouts.push(operation.timeoutMs); + const answer = answers.get(key); + if (answer === undefined) throw new Error(`unexpected locate: ${key}`); + if (answer instanceof Error) throw answer; + return answer; + }, + async read() { + throw new Error('unexpected read'); + }, + async perform() { + throw new Error('unexpected perform'); + }, + actions: new Set(LOCATOR_ACTION_KINDS), + } as unknown as TargetSession; + const engine = new LocatorEngine({ + session, + budget: new AttemptBudget(new AbortController().signal, new Deadline(30_000)), + runId: 'run-1', + attemptId: 'attempt-1', + actionTimeout: options.actionTimeout ?? 20, + assertionTimeout: 1_000, + }); + return { engine, resolved, resolveTimeouts }; + } + + function answersFor(...entries: Array<[LocatorExpression, readonly NodeRef[] | Error]>): ReadonlyMap { + return new Map(entries.map(([expression, answer]) => [describeExpression(expression), answer])); + } + + it('lists how many candidates each filter step left, counted after the failure', async () => { + const { engine, resolved, resolveTimeouts } = makeCountingEngine( + answersFor( + [chain, []], + [base, Array.from({ length: 6 }, () => REF)], + [texted, [REF]], + ), + ); + const failure = await engine.read(chain).then( + () => { + throw new Error('expected a rejection'); + }, + (cause: E2EError) => cause, + ); + expect(failure.code).toBe('LOCATOR_NOT_FOUND'); + expect(failure.message).toContain('candidates left after each filter step, counted after the failure'); + expect(failure.message).toContain(`${describeExpression(base)}: 6`); + expect(failure.message).toContain(`${describeExpression(texted)}: 1`); + expect(failure.message).toContain(`${describeExpression(chain)}: 0`); + // The walk re-resolves the prefixes innermost first, after the failing resolve's polls. + expect(resolved[0]).toBe(describeExpression(chain)); + expect(resolved.slice(-3)).toEqual([describeExpression(base), describeExpression(texted), describeExpression(chain)]); + // The failing resolve polled on the action deadline; the diagnostic resolves + // arrived after it, each bounded by the 500 ms diagnostic budget. + expect(resolveTimeouts[0]).toBe(20); + const diagnostic = resolveTimeouts.slice(-3); + expect(diagnostic.length).toBe(3); + for (const timeoutMs of diagnostic) { + expect(timeoutMs).toBeGreaterThan(0); + expect(timeoutMs).toBeLessThanOrEqual(500); + } + }); + + it('keeps a locator with no filter step unlisted', async () => { + const { engine, resolved } = makeCountingEngine(answersFor([base, []])); + await expect(engine.read(base)).rejects.toMatchObject({ + code: 'LOCATOR_NOT_FOUND', + message: expect.not.stringContaining('counted after the failure'), + }); + // Only the failing resolve polled; no walk ran. + expect(resolved.every((key) => key === describeExpression(base))).toBe(true); + }); + + it('keeps the plain failure when the diagnostic pass cannot read', async () => { + const { engine, resolved } = makeCountingEngine( + answersFor( + [chain, []], + [base, new EngineError('ENGINE_FAILURE', 'engine died', { retryable: false })], + ), + ); + await expect(engine.read(chain)).rejects.toMatchObject({ + code: 'LOCATOR_NOT_FOUND', + message: expect.not.stringContaining('counted after the failure'), + }); + // The walk stopped at the prefix that refused to read; only the full chain polled. + expect(resolved.filter((key) => key !== describeExpression(chain))).toEqual([describeExpression(base)]); + }); + + it('lists the step counts on an ambiguous filtered locator', async () => { + const { engine, resolved } = makeCountingEngine( + answersFor( + [chain, [REF, REF]], + [base, Array.from({ length: 6 }, () => REF)], + [texted, [REF, REF, REF]], + ), + ); + await expect(engine.read(chain)).rejects.toMatchObject({ + code: 'LOCATOR_AMBIGUOUS', + details: { matches: 2 }, + message: expect.stringContaining(`${describeExpression(base)}: 6`), + }); + expect(resolved).toEqual([describeExpression(chain), describeExpression(base), describeExpression(texted), describeExpression(chain)]); + }); + + it('counts an index step between filters under its own label', async () => { + const indexed = indexExpression(texted, 'first'); + const chained = filterExpression(indexed, { hasText: '1042' }); + const { engine } = makeCountingEngine( + answersFor( + [chained, []], + [base, [REF, REF]], + [texted, [REF]], + [indexed, [REF]], + ), + ); + const message = await engine.read(chained).catch((cause: E2EError) => cause.message); + expect(message).toContain(`${describeExpression(base)}: 2`); + expect(message).toContain(`${describeExpression(texted)}: 1`); + expect(message).toContain(`${describeExpression(indexed)}: 1`); + expect(message).toContain(`${describeExpression(chained)}: 0`); + }); + + /** + * A session the way mobile's `locate()` behaves: it ignores the operation + * timeout and every call runs to completion. The walk shares one deadline, + * so a three-step chain reads two prefixes and then says it stopped, rather + * than starting a third full snapshot the budget cannot cover. + */ + it('bounds the whole walk by one shared deadline when locate ignores the operation timeout', async () => { + const resolved: string[] = []; + const resolveTimeouts: number[] = []; + const session = { + async locate(expression: LocatorExpression, operation: OperationContext) { + resolved.push(describeExpression(expression)); + resolveTimeouts.push(operation.timeoutMs); + await new Promise((resolve) => setTimeout(resolve, 300)); + return []; + }, + async read() { + throw new Error('unexpected read'); + }, + async perform() { + throw new Error('unexpected perform'); + }, + actions: new Set(LOCATOR_ACTION_KINDS), + } as unknown as TargetSession; + const engine = new LocatorEngine({ + session, + budget: new AttemptBudget(new AbortController().signal, new Deadline(30_000)), + runId: 'run-1', + attemptId: 'attempt-1', + actionTimeout: 20, + assertionTimeout: 1_000, + }); + + const failure = await engine.read(chain).then( + () => { + throw new Error('expected a rejection'); + }, + (cause: E2EError) => cause, + ); + expect(failure.code).toBe('LOCATOR_NOT_FOUND'); + // The failing resolve polls once, then the walk resolves two of the three + // prefixes inside the shared budget and refuses the third. + expect(resolved).toEqual([describeExpression(chain), describeExpression(base), describeExpression(texted)]); + expect(failure.message).toContain('walk stopped after 2 of 3 steps'); + for (const timeoutMs of resolveTimeouts.slice(1)) { + expect(timeoutMs).toBeGreaterThan(0); + expect(timeoutMs).toBeLessThanOrEqual(500); + } + }); + + it('surfaces cancellation when the signal aborts mid-walk', async () => { + const controller = new AbortController(); + const resolved: string[] = []; + const session = { + async locate(expression: LocatorExpression) { + const key = describeExpression(expression); + resolved.push(key); + // Abort during the walk's second prefix, as a mobile session would + // when the attempt's signal fires between two snapshot reads. + if (key === describeExpression(texted)) { + controller.abort(); + throw new Error('boom'); + } + return []; + }, + async read() { + throw new Error('unexpected read'); + }, + async perform() { + throw new Error('unexpected perform'); + }, + actions: new Set(LOCATOR_ACTION_KINDS), + } as unknown as TargetSession; + const engine = new LocatorEngine({ + session, + budget: new AttemptBudget(controller.signal, new Deadline(30_000)), + runId: 'run-1', + attemptId: 'attempt-1', + actionTimeout: 20, + assertionTimeout: 1_000, + }); + + await expect(engine.read(chain)).rejects.toMatchObject({ code: 'CANCELLED' }); + }); + + it('drops the counts when a later prefix cannot be read', async () => { + const { engine, resolved } = makeCountingEngine( + answersFor( + [chain, []], + [base, [REF]], + [texted, new EngineError('ENGINE_FAILURE', 'engine died', { retryable: false })], + ), + ); + await expect(engine.read(chain)).rejects.toMatchObject({ + code: 'LOCATOR_NOT_FOUND', + message: expect.not.stringContaining('counted after the failure'), + }); + // The walk read the base, then the next prefix refused: a partial list + // must not present itself as the per-step counts of a completed walk. + expect(resolved.filter((key) => key !== describeExpression(chain))).toEqual([ + describeExpression(base), + describeExpression(texted), + ]); + }); +});