Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/max-failures-worker-wait.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"e2e": patch
---

With several workers, `--max-failures` no longer lets the worker whose failure reaches the limit start its next test. That test was cancelled a moment later and reported `interrupted`; it is now skipped with cause `failure-limit`, as the docs say.
3 changes: 3 additions & 0 deletions packages/e2e/src/run/execute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ export interface TargetExecutorOptions extends EngineSessionsOptions {
readonly holdStep?: ((step: HeldStep, signal: AbortSignal) => Promise<void>) | undefined;
/** Whether each hold reads the screen it starts on into `HeldStep.screen`. */
readonly heldScreens?: boolean | undefined;
/** Awaited before each pair starts, before the interrupt check. */
readonly beforePair?: (() => Promise<void>) | undefined;
readonly events?: ExecutionEvents;
/** The virtual file of tests registered in memory, and the agents they run as; see `InMemoryAttempts`. */
readonly inMemory?: InMemoryAttempts | undefined;
Expand Down Expand Up @@ -271,6 +273,7 @@ export class TargetExecutor extends EngineSessions implements SerialHost {
const serialUnitKey =
pair.test.serialId === undefined ? undefined : pairKey(pair.test.serialId, pair.agent, pair.repeat);
if (serialUnitKey !== undefined && executedSerialUnits.has(serialUnitKey)) continue;
await this.options.beforePair?.();
if (this.interruptSignal.aborted) {
this.emit(unstartedResult(pair, interruptedSkip(this.interruptSignal)));
continue;
Expand Down
35 changes: 18 additions & 17 deletions packages/e2e/src/run/scheduler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ import {
} from './units.ts';
import {
type AttemptDeadlineMessage,
type FailureLimit,
type OutputMessage,
type PairStart,
type WirePair,
Expand Down Expand Up @@ -234,8 +233,8 @@ class SchedulerWorker {
});
}

/** Sends a unit, with the run's failure limit when it has one, and starts tracking it. */
dispatch(unit: WorkUnit, failureLimit: FailureLimit | undefined): void {
/** Sends a unit, saying whether the run has a failure limit, and starts tracking it. */
dispatch(unit: WorkUnit, answersFailures: boolean): void {
this.state = 'busy';
this.unit = unit;
this.reported.clear();
Expand All @@ -256,7 +255,7 @@ class SchedulerWorker {
file: unit.file,
absolutePath: unit.absolutePath,
pairs,
...(failureLimit === undefined ? {} : { failureLimit }),
...(answersFailures ? { answersFailures: true as const } : {}),
});
}

Expand Down Expand Up @@ -404,16 +403,19 @@ class Scheduler {
/**
* Every result leaves through here, so the failure count is one number
* however the result came about: a worker's verdict, a synthesized crash,
* or a skip. Reaching the limit stops the run at once.
* or a skip. Reaching the limit stops the run at once. Returns whether the
* result was a counted failure the run goes on after.
*/
private report(result: ResultRecord): void {
private report(result: ResultRecord): boolean {
this.options.events.onResult(result);
if (!countsTowardFailureLimit(result.status)) return;
if (!countsTowardFailureLimit(result.status)) return false;
this.failures += 1;
const limit = this.options.maxFailures;
if (limit === undefined || this.stopSkip !== undefined || this.failures < limit) return;
this.stopEarly(failureLimitSkip(this.failures, limit));
this.options.events.onFailureLimit?.(this.failures, limit);
if (limit !== undefined && this.stopSkip === undefined && this.failures >= limit) {
this.stopEarly(failureLimitSkip(this.failures, limit));
this.options.events.onFailureLimit?.(this.failures, limit);
}
return !this.interrupting;
}

/**
Expand All @@ -440,10 +442,9 @@ class Scheduler {
this.wakeUp();
}

/** The failure limit a unit dispatched now carries, when the run has one. */
private get failureLimit(): FailureLimit | undefined {
const limit = this.options.maxFailures;
return limit === undefined ? undefined : { limit, failures: this.failures };
/** Whether the run has a failure limit, so its workers wait for an answer to each failure they report. */
private get answersFailures(): boolean {
return this.options.maxFailures !== undefined;
}

/** Empties a target's queues, reporting every pair they held as skipped for `skip`. */
Expand Down Expand Up @@ -674,7 +675,7 @@ class Scheduler {
this.returnUnit(state, unit);
return;
}
if (worker.state === 'idle') worker.dispatch(unit, this.failureLimit);
if (worker.state === 'idle') worker.dispatch(unit, this.answersFailures);
else worker.queued = unit;
}
}
Expand Down Expand Up @@ -870,7 +871,7 @@ class Scheduler {
worker.state = 'idle';
const queued = worker.queued;
worker.queued = undefined;
if (queued !== undefined) worker.dispatch(queued, this.failureLimit);
if (queued !== undefined) worker.dispatch(queued, this.answersFailures);
this.wakeUp();
break;
}
Expand Down Expand Up @@ -933,7 +934,7 @@ class Scheduler {
worker.sawFailure = true;
}
if (result.test.kind === 'setup') this.recordSetupOutcome(state, result);
this.report(result);
if (this.report(result) && this.answersFailures) worker.runner.send({ type: 'continue' });
break;
}
case 'serial-group': {
Expand Down
36 changes: 24 additions & 12 deletions packages/e2e/src/run/worker/protocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,18 +85,14 @@ export interface RunUnitMessage {
readonly absolutePath: string;
readonly pairs: readonly WirePair[];
/**
* The run's `--max-failures` and the failures it had counted when the unit
* was dispatched. The worker stops starting the unit's tests once its own
* failures reach the limit, without waiting for the runner's interrupt to
* arrive: by then it could have started the next test in the file.
* Set when the run has a `--max-failures` limit. Only the runner counts
* failures toward it, so after each result that counts the worker starts
* no further test until the runner answers: `continue` while the run is
* below the limit, `interrupt` once a failure reached it. One round-trip
* per failure: with several workers, two failures can land faster than any
* count the runner could send ahead.
*/
readonly failureLimit?: FailureLimit;
}

/** A run's failure limit and the failures counted toward it so far. */
export interface FailureLimit {
readonly limit: number;
readonly failures: number;
readonly answersFailures?: true;
}

export interface InterruptMessage {
Expand Down Expand Up @@ -133,13 +129,29 @@ export interface PingMessage {
readonly type: 'ping';
}

/**
* The runner counted a failure the worker reported, and the run is still
* below its failure limit. Handled at once, like `ping`; see
* `RunUnitMessage.answersFailures`.
*/
export interface ContinueMessage {
readonly type: 'continue';
}

/** The held step `holdId` names may run. Handled at once, like `ping`; an id the worker no longer holds is ignored. */
export interface StepReleaseMessage {
readonly type: 'step-release';
readonly holdId: string;
}

export type MainToWorker = RunUnitMessage | InterruptMessage | ShutdownMessage | TerminateMessage | PingMessage | StepReleaseMessage;
export type MainToWorker =
| RunUnitMessage
| InterruptMessage
| ShutdownMessage
| TerminateMessage
| PingMessage
| StepReleaseMessage
| ContinueMessage;

export interface ReadyMessage {
readonly type: 'ready';
Expand Down
51 changes: 34 additions & 17 deletions packages/e2e/src/run/worker/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import { TargetExecutor, type InMemoryAttempts } from '../execute.ts';
import { encodeResult, type RunError } from '../records.ts';
import type { SessionStore } from '../sessions.ts';
import type { HeldStep, StepHolding } from '../step-hold.ts';
import { countsTowardFailureLimit, disappearedResult, failureLimitSkip } from '../units.ts';
import type { FailureLimit, MainToWorker, RunUnitMessage, WirePair, WorkerToMain } from './protocol.ts';
import { countsTowardFailureLimit, disappearedResult } from '../units.ts';
import type { MainToWorker, RunUnitMessage, WirePair, WorkerToMain } from './protocol.ts';

/** Pairs resolved locally for one unit, plus identities that vanished. */
export interface ResolvedUnitPairs {
Expand Down Expand Up @@ -89,8 +89,12 @@ export class TargetWorker {
get pairInFlight(): { readonly testId: string; readonly agent: string; readonly repeat: number } | undefined {
return this.inFlight;
}
/** The failure limit of the unit running now, counting this worker's own failures since its dispatch. */
private failureLimit: FailureLimit | undefined;
/** Whether the run has a failure limit, so each failure this worker reports waits for the runner's answer. */
private answersFailures = false;
/** Failures this worker reported that the runner has not answered yet. */
private unanswered = 0;
/** Wakes the wait in `failuresAnswered` once the last of them is answered. */
private answered: (() => void) | undefined;
/** Serializes message handling so units never overlap on one worker. */
private queue: Promise<void> = Promise.resolve();
/** Disposal happens once, whichever of shutdown or terminate asks first. */
Expand Down Expand Up @@ -125,14 +129,16 @@ export class TargetWorker {
: { holdStep: (step: HeldStep, signal: AbortSignal) => this.holdStep(step, signal), heldScreens: deps.step === 'screen' }),
...(deps.debug !== undefined ? { debug: deps.debug } : {}),
inMemory: deps.inMemory,
beforePair: () => this.failuresAnswered(),
events: {
onResult: (result) => {
// Teardown after the result (an afterAll) belongs to no pair.
if (this.inFlight?.testId === result.test.id && this.inFlight.agent === result.agent && this.inFlight.repeat === result.repeat) {
this.inFlight = undefined;
}
// Counted before it is sent: an in-process runner answers inside `emit`.
if (this.answersFailures && countsTowardFailureLimit(result.status)) this.unanswered += 1;
this.host.emit({ type: 'result', result: encodeResult(result) });
if (countsTowardFailureLimit(result.status)) this.countFailure();
},
onSerialGroup: (group) => this.host.emit({ type: 'serial-group', group }),
onAttemptStart: (pair, index) =>
Expand Down Expand Up @@ -217,6 +223,11 @@ export class TargetWorker {
// Now too: the unit it releases is the one running.
this.holds.get(message.holdId)?.();
return;
case 'continue':
// Now too: the running unit is what waits for it.
this.unanswered -= 1;
if (this.unanswered === 0) this.answered?.();
return;
default:
message satisfies never;
}
Expand Down Expand Up @@ -249,18 +260,24 @@ export class TargetWorker {
}

/**
* Counts a failure toward the unit's failure limit. Reaching it stops the
* unit the way the runner's interrupt would, at once: the tests the worker
* has not started are skipped with the limit's reason, however long the
* runner's interrupt takes to arrive. Other workers' failures only add to
* the run's count, so a limit reached here is reached for the run too.
* Resolves once the runner has answered every failure this worker reported,
* or the interrupt arrived. The runner alone counts failures toward the
* run's limit, so a worker whose failure reached it never starts another
* test: the interrupt answering that failure skips the rest with the
* limit's reason.
*/
private countFailure(): void {
const current = this.failureLimit;
if (current === undefined) return;
const counted = { limit: current.limit, failures: current.failures + 1 };
this.failureLimit = counted;
if (counted.failures >= counted.limit) this.interruptController.abort(failureLimitSkip(counted.failures, counted.limit));
private failuresAnswered(): Promise<void> {
const signal = this.interruptController.signal;
if (this.unanswered === 0 || signal.aborted) return Promise.resolve();
return new Promise<void>((resolve) => {
const done = (): void => {
this.answered = undefined;
signal.removeEventListener('abort', done);
resolve();
};
this.answered = done;
signal.addEventListener('abort', done, { once: true });
});
}

/**
Expand Down Expand Up @@ -294,7 +311,7 @@ export class TargetWorker {
// fatal for the worker; the scheduler still holds the unit and synthesizes
// failed results for every pair it never heard about. Reporting the unit
// done first would clear that bookkeeping and drop those tests silently.
this.failureLimit = message.failureLimit;
this.answersFailures = message.answersFailures === true;
const { pairs, missing, registration } = await deps.resolvePairs(message);
for (const wire of missing) {
executor.recordDisappeared(
Expand Down
77 changes: 77 additions & 0 deletions packages/e2e/tests/integration/parallel-workers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,83 @@ test('would run last', async () => {});
120_000,
);

it(
'stops at --max-failures across workers: the worker whose failure reaches the limit starts no further test',
async () => {
// Each file fails first, then holds its worker in a long test. The file
// that fails later reaches the limit; its worker must wait for the
// runner's answer instead of starting its next test on a stale count.
// The 500 ms gap orders the two failures, so the tripping file is the
// one whose failure ended second.
const file = (name: string, failAfterMs: number) => `import { test } from 'e2e';

test('${name} fails', async ({ app }) => {
await app.open();
await new Promise((resolve) => setTimeout(resolve, ${failAfterMs}));
throw new Error('${name}');
});
test('${name} next', async ({ app }) => {
await app.open();
await new Promise((resolve) => setTimeout(resolve, 2_000));
});
test('${name} last', async () => {});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/a.e2e.ts': file('a', 0), 'tests/b.e2e.ts': file('b', 500) },
{ configSource: workerFakeConfigSource(2), runOptions: { maxFailures: 2 } },
);
expect(outcome.exitCode).toBe(1);
const failed = outcome.results.filter((result) => result.status === 'failed');
expect(failed.map((result) => result.test.title).toSorted()).toEqual(['a fails', 'b fails']);
const endOf = (result: (typeof failed)[number]): number => {
const last = result.attempts.at(-1)!;
return Date.parse(last.startedAt) + last.durationMs;
};
const tripping = failed.toSorted((a, b) => endOf(a) - endOf(b))[1]!.test.file;
const trippingRest = outcome.results.filter((result) => result.test.file === tripping && result.status !== 'failed');
expect(trippingRest.map((result) => [result.status, result.skip?.cause, result.attempts.length])).toEqual([
['skipped', 'failure-limit', 0],
['skipped', 'failure-limit', 0],
]);
expect(outcome.results.filter((result) => result.status === 'interrupted').length).toBeLessThanOrEqual(1);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);

it(
'below --max-failures, each worker gets the runner\'s answer to its failure and runs its next test',
async () => {
// Both workers fail once and wait for the runner's `continue`; a lost
// answer would hold them forever, so both `next` tests must run.
const file = (name: string) => `import { test } from 'e2e';

test('${name} fails', async ({ app }) => {
await app.open();
throw new Error('${name}');
});
test('${name} next', async ({ app }) => {
await app.open();
});
`;
const { outcome, project } = await runProjectWithConfigFile(
{ 'tests/a.e2e.ts': file('a'), 'tests/b.e2e.ts': file('b') },
{ configSource: workerFakeConfigSource(2), runOptions: { maxFailures: 3 } },
);
expect(outcome.exitCode).toBe(1);
expect(outcome.results.map((result) => [result.test.title, result.status]).toSorted()).toEqual([
['a fails', 'failed'],
['a next', 'passed'],
['b fails', 'failed'],
['b next', 'passed'],
]);
assertValidReport(outcome.report);
project.cleanup();
},
120_000,
);

it(
'reports a target whose workers cannot boot once, whatever the worker count',
async () => {
Expand Down
17 changes: 17 additions & 0 deletions packages/e2e/tests/unit/scheduler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -761,6 +761,23 @@ describe('failure limit', () => {
expect(collected.failureLimits).toEqual([]);
expect(fleet.interruptSkips).toEqual([]);
});

it('answers each counted failure with continue below the limit, and with nothing at the limit or without one', async () => {
const target = makeTarget('web', 0);
const pairs = ['a', 'b', 'c'].map((name) => makePair(makeTest(`tests/${name}.e2e.ts`, name), target));
const status = { [pairs[0]!.test.id]: 'timed-out' as const, [pairs[1]!.test.id]: 'failed' as const };
const controlsWith = async (maxFailures: number | undefined): Promise<MainToWorker['type'][]> => {
const fleet = new FakeFleet({ status });
await run(makeSelection([{ target, pairs }]), makeCollection(pairs.map((pair) => pair.test.file), pairs), fleet, {
workers: 1,
...(maxFailures === undefined ? {} : { maxFailures }),
});
return fleet.controlMessages.filter((type) => type === 'continue' || type === 'interrupt');
};
expect(await controlsWith(3)).toEqual(['continue', 'continue']);
expect(await controlsWith(2)).toEqual(['continue', 'interrupt']);
expect(await controlsWith(undefined)).toEqual([]);
});
});

describe('time limit', () => {
Expand Down
Loading