diff --git a/.changeset/max-failures-worker-wait.md b/.changeset/max-failures-worker-wait.md new file mode 100644 index 000000000..f6c310f6a --- /dev/null +++ b/.changeset/max-failures-worker-wait.md @@ -0,0 +1,5 @@ +--- +"e2e": patch +--- + +With several workers, `--max-failures` no longer lets the worker whose failure reaches the limit start its next test. That test was cancelled a moment later and reported `interrupted`; it is now skipped with cause `failure-limit`, as the docs say. diff --git a/packages/e2e/src/run/execute.ts b/packages/e2e/src/run/execute.ts index 06f525b15..e3ae3cd7d 100644 --- a/packages/e2e/src/run/execute.ts +++ b/packages/e2e/src/run/execute.ts @@ -108,6 +108,8 @@ export interface TargetExecutorOptions extends EngineSessionsOptions { readonly holdStep?: ((step: HeldStep, signal: AbortSignal) => Promise) | undefined; /** Whether each hold reads the screen it starts on into `HeldStep.screen`. */ readonly heldScreens?: boolean | undefined; + /** Awaited before each pair starts, before the interrupt check. */ + readonly beforePair?: (() => Promise) | undefined; readonly events?: ExecutionEvents; /** The virtual file of tests registered in memory, and the agents they run as; see `InMemoryAttempts`. */ readonly inMemory?: InMemoryAttempts | undefined; @@ -271,6 +273,7 @@ export class TargetExecutor extends EngineSessions implements SerialHost { const serialUnitKey = pair.test.serialId === undefined ? undefined : pairKey(pair.test.serialId, pair.agent, pair.repeat); if (serialUnitKey !== undefined && executedSerialUnits.has(serialUnitKey)) continue; + await this.options.beforePair?.(); if (this.interruptSignal.aborted) { this.emit(unstartedResult(pair, interruptedSkip(this.interruptSignal))); continue; diff --git a/packages/e2e/src/run/scheduler.ts b/packages/e2e/src/run/scheduler.ts index 89ea63063..0a87a575e 100644 --- a/packages/e2e/src/run/scheduler.ts +++ b/packages/e2e/src/run/scheduler.ts @@ -38,7 +38,6 @@ import { } from './units.ts'; import { type AttemptDeadlineMessage, - type FailureLimit, type OutputMessage, type PairStart, type WirePair, @@ -234,8 +233,8 @@ class SchedulerWorker { }); } - /** Sends a unit, with the run's failure limit when it has one, and starts tracking it. */ - dispatch(unit: WorkUnit, failureLimit: FailureLimit | undefined): void { + /** Sends a unit, saying whether the run has a failure limit, and starts tracking it. */ + dispatch(unit: WorkUnit, answersFailures: boolean): void { this.state = 'busy'; this.unit = unit; this.reported.clear(); @@ -256,7 +255,7 @@ class SchedulerWorker { file: unit.file, absolutePath: unit.absolutePath, pairs, - ...(failureLimit === undefined ? {} : { failureLimit }), + ...(answersFailures ? { answersFailures: true as const } : {}), }); } @@ -404,16 +403,19 @@ class Scheduler { /** * Every result leaves through here, so the failure count is one number * however the result came about: a worker's verdict, a synthesized crash, - * or a skip. Reaching the limit stops the run at once. + * or a skip. Reaching the limit stops the run at once. Returns whether the + * result was a counted failure the run goes on after. */ - private report(result: ResultRecord): void { + private report(result: ResultRecord): boolean { this.options.events.onResult(result); - if (!countsTowardFailureLimit(result.status)) return; + if (!countsTowardFailureLimit(result.status)) return false; this.failures += 1; const limit = this.options.maxFailures; - if (limit === undefined || this.stopSkip !== undefined || this.failures < limit) return; - this.stopEarly(failureLimitSkip(this.failures, limit)); - this.options.events.onFailureLimit?.(this.failures, limit); + if (limit !== undefined && this.stopSkip === undefined && this.failures >= limit) { + this.stopEarly(failureLimitSkip(this.failures, limit)); + this.options.events.onFailureLimit?.(this.failures, limit); + } + return !this.interrupting; } /** @@ -440,10 +442,9 @@ class Scheduler { this.wakeUp(); } - /** The failure limit a unit dispatched now carries, when the run has one. */ - private get failureLimit(): FailureLimit | undefined { - const limit = this.options.maxFailures; - return limit === undefined ? undefined : { limit, failures: this.failures }; + /** Whether the run has a failure limit, so its workers wait for an answer to each failure they report. */ + private get answersFailures(): boolean { + return this.options.maxFailures !== undefined; } /** Empties a target's queues, reporting every pair they held as skipped for `skip`. */ @@ -674,7 +675,7 @@ class Scheduler { this.returnUnit(state, unit); return; } - if (worker.state === 'idle') worker.dispatch(unit, this.failureLimit); + if (worker.state === 'idle') worker.dispatch(unit, this.answersFailures); else worker.queued = unit; } } @@ -870,7 +871,7 @@ class Scheduler { worker.state = 'idle'; const queued = worker.queued; worker.queued = undefined; - if (queued !== undefined) worker.dispatch(queued, this.failureLimit); + if (queued !== undefined) worker.dispatch(queued, this.answersFailures); this.wakeUp(); break; } @@ -933,7 +934,7 @@ class Scheduler { worker.sawFailure = true; } if (result.test.kind === 'setup') this.recordSetupOutcome(state, result); - this.report(result); + if (this.report(result) && this.answersFailures) worker.runner.send({ type: 'continue' }); break; } case 'serial-group': { diff --git a/packages/e2e/src/run/worker/protocol.ts b/packages/e2e/src/run/worker/protocol.ts index dfa5478d4..32841039e 100644 --- a/packages/e2e/src/run/worker/protocol.ts +++ b/packages/e2e/src/run/worker/protocol.ts @@ -85,18 +85,14 @@ export interface RunUnitMessage { readonly absolutePath: string; readonly pairs: readonly WirePair[]; /** - * The run's `--max-failures` and the failures it had counted when the unit - * was dispatched. The worker stops starting the unit's tests once its own - * failures reach the limit, without waiting for the runner's interrupt to - * arrive: by then it could have started the next test in the file. + * Set when the run has a `--max-failures` limit. Only the runner counts + * failures toward it, so after each result that counts the worker starts + * no further test until the runner answers: `continue` while the run is + * below the limit, `interrupt` once a failure reached it. One round-trip + * per failure: with several workers, two failures can land faster than any + * count the runner could send ahead. */ - readonly failureLimit?: FailureLimit; -} - -/** A run's failure limit and the failures counted toward it so far. */ -export interface FailureLimit { - readonly limit: number; - readonly failures: number; + readonly answersFailures?: true; } export interface InterruptMessage { @@ -133,13 +129,29 @@ export interface PingMessage { readonly type: 'ping'; } +/** + * The runner counted a failure the worker reported, and the run is still + * below its failure limit. Handled at once, like `ping`; see + * `RunUnitMessage.answersFailures`. + */ +export interface ContinueMessage { + readonly type: 'continue'; +} + /** The held step `holdId` names may run. Handled at once, like `ping`; an id the worker no longer holds is ignored. */ export interface StepReleaseMessage { readonly type: 'step-release'; readonly holdId: string; } -export type MainToWorker = RunUnitMessage | InterruptMessage | ShutdownMessage | TerminateMessage | PingMessage | StepReleaseMessage; +export type MainToWorker = + | RunUnitMessage + | InterruptMessage + | ShutdownMessage + | TerminateMessage + | PingMessage + | StepReleaseMessage + | ContinueMessage; export interface ReadyMessage { readonly type: 'ready'; diff --git a/packages/e2e/src/run/worker/session.ts b/packages/e2e/src/run/worker/session.ts index 9910851aa..632032a4e 100644 --- a/packages/e2e/src/run/worker/session.ts +++ b/packages/e2e/src/run/worker/session.ts @@ -16,8 +16,8 @@ import { TargetExecutor, type InMemoryAttempts } from '../execute.ts'; import { encodeResult, type RunError } from '../records.ts'; import type { SessionStore } from '../sessions.ts'; import type { HeldStep, StepHolding } from '../step-hold.ts'; -import { countsTowardFailureLimit, disappearedResult, failureLimitSkip } from '../units.ts'; -import type { FailureLimit, MainToWorker, RunUnitMessage, WirePair, WorkerToMain } from './protocol.ts'; +import { countsTowardFailureLimit, disappearedResult } from '../units.ts'; +import type { MainToWorker, RunUnitMessage, WirePair, WorkerToMain } from './protocol.ts'; /** Pairs resolved locally for one unit, plus identities that vanished. */ export interface ResolvedUnitPairs { @@ -89,8 +89,12 @@ export class TargetWorker { get pairInFlight(): { readonly testId: string; readonly agent: string; readonly repeat: number } | undefined { return this.inFlight; } - /** The failure limit of the unit running now, counting this worker's own failures since its dispatch. */ - private failureLimit: FailureLimit | undefined; + /** Whether the run has a failure limit, so each failure this worker reports waits for the runner's answer. */ + private answersFailures = false; + /** Failures this worker reported that the runner has not answered yet. */ + private unanswered = 0; + /** Wakes the wait in `failuresAnswered` once the last of them is answered. */ + private answered: (() => void) | undefined; /** Serializes message handling so units never overlap on one worker. */ private queue: Promise = Promise.resolve(); /** Disposal happens once, whichever of shutdown or terminate asks first. */ @@ -125,14 +129,16 @@ export class TargetWorker { : { holdStep: (step: HeldStep, signal: AbortSignal) => this.holdStep(step, signal), heldScreens: deps.step === 'screen' }), ...(deps.debug !== undefined ? { debug: deps.debug } : {}), inMemory: deps.inMemory, + beforePair: () => this.failuresAnswered(), events: { onResult: (result) => { // Teardown after the result (an afterAll) belongs to no pair. if (this.inFlight?.testId === result.test.id && this.inFlight.agent === result.agent && this.inFlight.repeat === result.repeat) { this.inFlight = undefined; } + // Counted before it is sent: an in-process runner answers inside `emit`. + if (this.answersFailures && countsTowardFailureLimit(result.status)) this.unanswered += 1; this.host.emit({ type: 'result', result: encodeResult(result) }); - if (countsTowardFailureLimit(result.status)) this.countFailure(); }, onSerialGroup: (group) => this.host.emit({ type: 'serial-group', group }), onAttemptStart: (pair, index) => @@ -217,6 +223,11 @@ export class TargetWorker { // Now too: the unit it releases is the one running. this.holds.get(message.holdId)?.(); return; + case 'continue': + // Now too: the running unit is what waits for it. + this.unanswered -= 1; + if (this.unanswered === 0) this.answered?.(); + return; default: message satisfies never; } @@ -249,18 +260,24 @@ export class TargetWorker { } /** - * Counts a failure toward the unit's failure limit. Reaching it stops the - * unit the way the runner's interrupt would, at once: the tests the worker - * has not started are skipped with the limit's reason, however long the - * runner's interrupt takes to arrive. Other workers' failures only add to - * the run's count, so a limit reached here is reached for the run too. + * Resolves once the runner has answered every failure this worker reported, + * or the interrupt arrived. The runner alone counts failures toward the + * run's limit, so a worker whose failure reached it never starts another + * test: the interrupt answering that failure skips the rest with the + * limit's reason. */ - private countFailure(): void { - const current = this.failureLimit; - if (current === undefined) return; - const counted = { limit: current.limit, failures: current.failures + 1 }; - this.failureLimit = counted; - if (counted.failures >= counted.limit) this.interruptController.abort(failureLimitSkip(counted.failures, counted.limit)); + private failuresAnswered(): Promise { + const signal = this.interruptController.signal; + if (this.unanswered === 0 || signal.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const done = (): void => { + this.answered = undefined; + signal.removeEventListener('abort', done); + resolve(); + }; + this.answered = done; + signal.addEventListener('abort', done, { once: true }); + }); } /** @@ -294,7 +311,7 @@ export class TargetWorker { // fatal for the worker; the scheduler still holds the unit and synthesizes // failed results for every pair it never heard about. Reporting the unit // done first would clear that bookkeeping and drop those tests silently. - this.failureLimit = message.failureLimit; + this.answersFailures = message.answersFailures === true; const { pairs, missing, registration } = await deps.resolvePairs(message); for (const wire of missing) { executor.recordDisappeared( diff --git a/packages/e2e/tests/integration/parallel-workers.test.ts b/packages/e2e/tests/integration/parallel-workers.test.ts index 53f909e26..18b8ef188 100644 --- a/packages/e2e/tests/integration/parallel-workers.test.ts +++ b/packages/e2e/tests/integration/parallel-workers.test.ts @@ -867,6 +867,83 @@ test('would run last', async () => {}); 120_000, ); + it( + 'stops at --max-failures across workers: the worker whose failure reaches the limit starts no further test', + async () => { + // Each file fails first, then holds its worker in a long test. The file + // that fails later reaches the limit; its worker must wait for the + // runner's answer instead of starting its next test on a stale count. + // The 500 ms gap orders the two failures, so the tripping file is the + // one whose failure ended second. + const file = (name: string, failAfterMs: number) => `import { test } from 'e2e'; + +test('${name} fails', async ({ app }) => { + await app.open(); + await new Promise((resolve) => setTimeout(resolve, ${failAfterMs})); + throw new Error('${name}'); +}); +test('${name} next', async ({ app }) => { + await app.open(); + await new Promise((resolve) => setTimeout(resolve, 2_000)); +}); +test('${name} last', async () => {}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/a.e2e.ts': file('a', 0), 'tests/b.e2e.ts': file('b', 500) }, + { configSource: workerFakeConfigSource(2), runOptions: { maxFailures: 2 } }, + ); + expect(outcome.exitCode).toBe(1); + const failed = outcome.results.filter((result) => result.status === 'failed'); + expect(failed.map((result) => result.test.title).toSorted()).toEqual(['a fails', 'b fails']); + const endOf = (result: (typeof failed)[number]): number => { + const last = result.attempts.at(-1)!; + return Date.parse(last.startedAt) + last.durationMs; + }; + const tripping = failed.toSorted((a, b) => endOf(a) - endOf(b))[1]!.test.file; + const trippingRest = outcome.results.filter((result) => result.test.file === tripping && result.status !== 'failed'); + expect(trippingRest.map((result) => [result.status, result.skip?.cause, result.attempts.length])).toEqual([ + ['skipped', 'failure-limit', 0], + ['skipped', 'failure-limit', 0], + ]); + expect(outcome.results.filter((result) => result.status === 'interrupted').length).toBeLessThanOrEqual(1); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + + it( + 'below --max-failures, each worker gets the runner\'s answer to its failure and runs its next test', + async () => { + // Both workers fail once and wait for the runner's `continue`; a lost + // answer would hold them forever, so both `next` tests must run. + const file = (name: string) => `import { test } from 'e2e'; + +test('${name} fails', async ({ app }) => { + await app.open(); + throw new Error('${name}'); +}); +test('${name} next', async ({ app }) => { + await app.open(); +}); +`; + const { outcome, project } = await runProjectWithConfigFile( + { 'tests/a.e2e.ts': file('a'), 'tests/b.e2e.ts': file('b') }, + { configSource: workerFakeConfigSource(2), runOptions: { maxFailures: 3 } }, + ); + expect(outcome.exitCode).toBe(1); + expect(outcome.results.map((result) => [result.test.title, result.status]).toSorted()).toEqual([ + ['a fails', 'failed'], + ['a next', 'passed'], + ['b fails', 'failed'], + ['b next', 'passed'], + ]); + assertValidReport(outcome.report); + project.cleanup(); + }, + 120_000, + ); + it( 'reports a target whose workers cannot boot once, whatever the worker count', async () => { diff --git a/packages/e2e/tests/unit/scheduler.test.ts b/packages/e2e/tests/unit/scheduler.test.ts index ce69a2329..0beb13c16 100644 --- a/packages/e2e/tests/unit/scheduler.test.ts +++ b/packages/e2e/tests/unit/scheduler.test.ts @@ -761,6 +761,23 @@ describe('failure limit', () => { expect(collected.failureLimits).toEqual([]); expect(fleet.interruptSkips).toEqual([]); }); + + it('answers each counted failure with continue below the limit, and with nothing at the limit or without one', async () => { + const target = makeTarget('web', 0); + const pairs = ['a', 'b', 'c'].map((name) => makePair(makeTest(`tests/${name}.e2e.ts`, name), target)); + const status = { [pairs[0]!.test.id]: 'timed-out' as const, [pairs[1]!.test.id]: 'failed' as const }; + const controlsWith = async (maxFailures: number | undefined): Promise => { + const fleet = new FakeFleet({ status }); + await run(makeSelection([{ target, pairs }]), makeCollection(pairs.map((pair) => pair.test.file), pairs), fleet, { + workers: 1, + ...(maxFailures === undefined ? {} : { maxFailures }), + }); + return fleet.controlMessages.filter((type) => type === 'continue' || type === 'interrupt'); + }; + expect(await controlsWith(3)).toEqual(['continue', 'continue']); + expect(await controlsWith(2)).toEqual(['continue', 'interrupt']); + expect(await controlsWith(undefined)).toEqual([]); + }); }); describe('time limit', () => {