Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/locator-filter-step-counts.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'e2e': patch
---

Filtered-locator failures can list how many candidates each filter step left, when the post-failure diagnostic pass can read the chain, so a step that narrowed too much or too little names itself. The message says the counts were read then.
4 changes: 2 additions & 2 deletions docs/reference/errors.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ category a blocked verdict reports as the obstacle.
| `CANCELLED` | The runner cancelled the operation without a process signal: an `e2e mcp` `open_session` cut short by a client cancel, a closed connection, or the server's shutdown, after it stopped what it had started | infrastructure | 3 |
| `MODEL_OUTPUT_INVALID` | Structured output still failed validation after the repair budget was spent. Under a [decision model](/decision-models): an answer without a valid choice distribution, a choice the request never offered, or a question the provider refused. Decisions are not repaired | test | 1 |
| `APP_NOT_OPEN` | An operation needed an open app: `app.open()` had not run, or the engine reported invalid state | test | 1 |
| `LOCATOR_NOT_FOUND` | No matching node within the deadline, or an agent action refers to a node that is no longer available | test | 1 |
| `LOCATOR_AMBIGUOUS` | More than one match for a single-node operation. Fails at once, with no polling. | test | 1 |
| `LOCATOR_NOT_FOUND` | No matching node within the deadline, or an agent action refers to a node that is no longer available. For a filtered locator the message can also list how many candidates each filter step left, counted after the failure, when that diagnostic pass can read the chain | test | 1 |
| `LOCATOR_AMBIGUOUS` | More than one match for a single-node operation. Fails at once, with no polling, and for a filtered locator can list how many candidates each filter step left, counted after the failure, when that diagnostic pass can read the chain | test | 1 |
| `ACTION_FAILED` | Actionability failure, operation timeout, or an action that may already have committed | test | 1 |
| `STEP_BUDGET_EXHAUSTED` | The model-call or action-step budget ran out before a conclusion | test | 1 |
| `STEP_TIMEOUT` | The invocation deadline expired, including an operation that timed out within a poll tick of it. Carries the last judgment where one exists, with the [vision hint](/reference/agent#agent-assert) when that judgment was inconclusive | test | 1 |
Expand Down
84 changes: 63 additions & 21 deletions packages/e2e/src/locator/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ import {
type ErrorDetails,
} from '../internal/errors.ts';
import { requireKey } from '../internal/keys.ts';
import { describeExpression, expressionHints } from './expression.ts';
import { Deadline, POLL_INTERVAL_MS, ReadWindow, sleep } from '../internal/time.ts';
import { describeExpression, expressionHints, filterSpine } from './expression.ts';
import { abortError, cancelledError, Deadline, POLL_INTERVAL_MS, ReadWindow, sleep } from '../internal/time.ts';
import { timestamp } from '../internal/ids.ts';
import { obj } from '../internal/objects.ts';
import { targetLabel } from '../cache/trace.ts';
Expand Down Expand Up @@ -61,6 +61,14 @@ export function denySecureRead(nodes: readonly SemanticNode[], label: string): v
*/
type MissingFrame = 'wait' | 'empty';

/** The budget the whole diagnostic failure walk shares: best effort, never a wait.
* One deadline bounds every prefix resolve, so a multi-step chain cannot cost
* more than one short budget even on mobile, where `session.locate()` ignores
* the operation timeout and a snapshot that starts just before the deadline
* still runs to completion — the real bound is the budget plus one snapshot.
*/
const STEP_COUNT_BUDGET_MS = 500;

interface LocatorEngineOptions {
readonly session: TargetSession;
/** The running phase's signal and deadline, read per operation. */
Expand Down Expand Up @@ -234,29 +242,75 @@ export class LocatorEngine {
const startedMs = Date.now();
// The wait as it happened: a deadline capped by the test's remaining
// budget waited less than the action timeout, and the report says so.
const notFound = (cause?: unknown) =>
new TestError('LOCATOR_NOT_FOUND', `locator matched no nodes within ${describeExpression(expression)}`, {
details: locatorDetails(expression, Date.now() - startedMs),
const notFound = async (cause?: unknown) => {
// The wait as it happened, read before the diagnostic pass runs: the
// prefix resolves below are not part of the locator's wait.
const waitedMs = Date.now() - startedMs;
const lines = await this.candidateCounts(expression);
return new TestError('LOCATOR_NOT_FOUND', `locator matched no nodes within ${describeExpression(expression)}${lines}`, {
details: locatorDetails(expression, waitedMs),
...(cause === undefined ? {} : { cause }),
});
};
let sampled = false;
for (let rounds = 1; ; rounds += 1) {
let refs: readonly NodeRef[];
try {
refs = await this.resolveOnce(expression, reads);
} catch (cause) {
// A resolve the deadline cut off, after one that matched nothing, is the wait running out.
if (sampled && reads.cutOff(cause)) throw notFound(cause);
if (sampled && reads.cutOff(cause)) throw await notFound(cause);
throw cause;
}
sampled = true;
const ref = assertSingle(refs, expression);
const ref = await this.singleOf(refs, expression);
if (ref !== null) return { ref, waitedMs: rounds > 1 ? Date.now() - startedMs : 0 };
if (deadline.expired()) throw notFound();
if (deadline.expired()) throw await notFound();
await sleep(POLL_INTERVAL_MS, this.signal);
}
}

/**
* The diagnostic on a filtered locator's failure: one immediate resolve per
* prefix of its chain, innermost first, and how many candidates each left,
* as the message's per-step lines. One deadline bounds the whole walk, so
* even on mobile, where `session.locate()` ignores the operation timeout,
* a chain whose walk cannot finish lists only the prefixes it read and says
* so, or lists nothing when a read failed outright. The counts are read
* after the failure, so a page that changed since can disagree with the
* failing attempt, and the message says so.
*/
private async candidateCounts(expression: LocatorExpression): Promise<string> {
const spine = filterSpine(expression);
if (spine.length === 0) return '';
const deadline = Deadline.min(new Deadline(STEP_COUNT_BUDGET_MS), this.options.budget.deadline);
const lines: string[] = [];
try {
for (const prefix of spine) {
if (deadline.expired()) break;
const refs = await this.resolveNow(prefix, new ReadWindow(deadline));
lines.push(`${describeExpression(prefix)}: ${refs.length}`);
}
} catch {
if (this.signal.aborted) throw abortError(this.signal, () => cancelledError());
// A diagnostic read failure must not fail the failure: partial counts
// must not read as a completed walk, so return nothing.
return '';
}
if (lines.length === 0) return '';
const stopped = lines.length < spine.length ? `, walk stopped after ${lines.length} of ${spine.length} steps` : '';
return `; candidates left after each filter step${stopped}, counted after the failure:\n${lines.join('\n')}`;
}

/** Zero matches -> null; one -> the ref; many -> LOCATOR_AMBIGUOUS with the per-step counts. */
private async singleOf(refs: readonly NodeRef[], expression: LocatorExpression): Promise<NodeRef | null> {
if (refs.length <= 1) return refs[0] ?? null;
const lines = await this.candidateCounts(expression);
throw new TestError('LOCATOR_AMBIGUOUS', `locator matched ${refs.length} nodes, expected exactly one: ${describeExpression(expression)}${lines}`, {
details: { ...locatorDetails(expression), matches: refs.length },
});
}

/**
* Resolves every current match at once, without waiting for one. A frame
* the engine reports missing is zero matches, since nothing under it is in
Expand Down Expand Up @@ -350,7 +404,7 @@ export class LocatorEngine {
reads: ReadWindow,
missingFrame: MissingFrame,
): Promise<{ node: SemanticNode | null; count: number }> {
const ref = assertSingle(await this.resolveOnce(expression, reads, missingFrame), expression);
const ref = await this.singleOf(await this.resolveOnce(expression, reads, missingFrame), expression);
if (ref === null) return { node: null, count: 0 };
try {
const node = await reads.read((timeoutMs) => this.session.read(ref, this.readOperation(timeoutMs)));
Expand Down Expand Up @@ -630,18 +684,6 @@ function describeNode(node: SemanticNode, redact: (text: string) => string): str
}));
}

/** Zero matches -> null; one -> the ref; many -> LOCATOR_AMBIGUOUS. */
function assertSingle(refs: readonly NodeRef[], expression: LocatorExpression): NodeRef | null {
if (refs.length > 1) {
throw new TestError(
'LOCATOR_AMBIGUOUS',
`locator matched ${refs.length} nodes, expected exactly one: ${describeExpression(expression)}`,
{ details: { ...locatorDetails(expression), matches: refs.length } },
);
}
return refs[0] ?? null;
}

/**
* The facts of a locator failure the report keeps beside the message: the
* locator as written, what it asked for, and how long it actually waited, in ms.
Expand Down
20 changes: 20 additions & 0 deletions packages/e2e/src/locator/expression.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,26 @@ export function indexExpression(
return { kind: 'index', source, index };
}

/**
* The prefixes of a filtered locator's chain, innermost first: the base and
* every `filter` or `index` step applied over it. Empty when the chain has
* no filter step, since a plain query, selector, or frame has no step a
* candidate count can explain. A union at the base counts once like any other
* base; the walk stops there and never descends into its two sides.
*/
export function filterSpine(expression: LocatorExpression): LocatorExpression[] {
const steps: LocatorExpression[] = [];
let current = expression;
let filtered = false;
while (current.kind === 'filter' || current.kind === 'index') {
if (current.kind === 'filter') filtered = true;
steps.push(current);
current = current.source;
}
if (!filtered) return [];
return [current, ...steps.toReversed()];

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: A filter applied to locator.or(...) still produces candidate-count diagnostics because the walk accepts the union as its base. Preserve the existing union-root message by returning undefined when current.kind === 'union' as well as when no filter was found.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At packages/e2e/src/locator/expression.ts, line 136:

<comment>A filter applied to `locator.or(...)` still produces candidate-count diagnostics because the walk accepts the union as its base. Preserve the existing union-root message by returning `undefined` when `current.kind === 'union'` as well as when no filter was found.</comment>

<file context>
@@ -116,6 +116,26 @@ export function indexExpression(
+    current = current.source;
+  }
+  if (!filtered) return undefined;
+  return [current, ...steps.toReversed()];
+}
+
</file context>
Suggested change
return [current, ...steps.toReversed()];
if (!filtered || current.kind === 'union') return undefined;
return [current, ...steps.toReversed()];

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in a11af50. filterSpine now returns undefined when the chain bottoms out in a union as well, so a filter over or() keeps today's message, and a test covers a filter sitting on a union.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 8c5b55b (amended): filterSpine now returns the union root as a single base entry, so a filter over or() counts the union once and never descends into its sides, with a unit test.

}

/**
* What an expression asks for, as the facts a failure report keeps beside
* the rendered locator: the role and name of a role query, the text a label,
Expand Down
39 changes: 39 additions & 0 deletions packages/e2e/tests/unit/expression.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { describe, expect, it } from 'vitest';
import type { LocatorExpression } from '../../src/engine/surface.ts';
import {
describeExpression,
expressionHints,
filterExpression,
filterSpine,
indexExpression,
roleQuery,
testIdQuery,
Expand Down Expand Up @@ -60,6 +62,43 @@ describe('locator expressions', () => {
expect(() => filterExpression(source, {})).toThrow(/hasText, hasNotText, has, and\/or hasNot/);
});

it('lists every prefix of a filtered chain, innermost first', () => {
const base = roleQuery('listitem', undefined, undefined);
const first = filterExpression(base, { hasText: 'Order' });
const second = filterExpression(first, { has: roleQuery('button', undefined, undefined) });
expect(filterSpine(second)).toEqual([base, first, second]);
});

it('follows an index step between filters', () => {
const base = roleQuery('listitem', undefined, undefined);
const filtered = filterExpression(base, { hasText: 'Order' });
const indexed = indexExpression(filtered, 'first');
const chained = filterExpression(indexed, { hasText: '1042' });
expect(filterSpine(chained)).toEqual([base, filtered, indexed, chained]);
});

it('is empty without a filter step', () => {
const base = roleQuery('listitem', undefined, undefined);
expect(filterSpine(base)).toEqual([]);
expect(filterSpine(indexExpression(base, 0))).toEqual([]);
expect(filterSpine(textQuery('text', 'Pro', undefined, undefined))).toEqual([]);
});

it('is empty for a union of chains', () => {
const base = roleQuery('listitem', undefined, undefined);
const filtered = filterExpression(base, { hasText: 'Order' });
const union: LocatorExpression = { kind: 'union', left: filtered, right: base };
expect(filterSpine(union)).toEqual([]);
});

it('treats a union as a single base to count once, never descending into its sides', () => {
const base = roleQuery('listitem', undefined, undefined);
const other = roleQuery('button', undefined, undefined);
const union: LocatorExpression = { kind: 'union', left: base, right: other };
const filtered = filterExpression(union, { hasText: 'Order' });
expect(filterSpine(filtered)).toEqual([union, filtered]);
});

it('rejects negative and fractional nth indices', () => {
const source = testIdQuery('card', undefined, undefined);
expect(() => indexExpression(source, -1)).toThrow(/nonnegative/);
Expand Down
Loading
Loading