diff --git a/scripts/ci-failure-shapes.d.mts b/scripts/ci-failure-shapes.d.mts index b03640c021..2acd955a16 100644 --- a/scripts/ci-failure-shapes.d.mts +++ b/scripts/ci-failure-shapes.d.mts @@ -15,3 +15,12 @@ export function extractTapCounts(output: string): TapCounts | null; export function countNotOkEntries(output: string): { lock: number; other: number }; export function classifyPhaseFailure(output: string, baseToFile: Map): PhaseFailureVerdict; export function describeRetryOutcome(verdict: PhaseFailureVerdict): string; +export function isMergeGateEvent(eventName: string): boolean; +export function shouldDrainPass(first: PhaseFailureVerdict, retry: PhaseFailureVerdict, eventName: string): boolean; +export function extractDrainCancelledTests(output: string): string[]; +export function describeDrainPass( + first: { kind: "cancelled-drain"; counts: TapCounts }, + retry: { kind: "cancelled-drain"; counts: TapCounts }, + cancelledTests: string[], + eventName: string, +): string; diff --git a/scripts/ci-failure-shapes.mjs b/scripts/ci-failure-shapes.mjs index c61c1f846c..443f4ab235 100644 --- a/scripts/ci-failure-shapes.mjs +++ b/scripts/ci-failure-shapes.mjs @@ -121,6 +121,64 @@ export function classifyPhaseFailure(output, baseToFile) { // concrete/swallowed failures on the retry are genuine regressions. The caller // must never print a fixed "genuine failure" line without reclassifying — that // mislabels a reproduced drain as a code regression (#3665 round-2 log). +// +// #4212 (#3665 round 3): "a fresh runner is the only escape" — but nothing +// ever drew one. A twice-drained job went red with a message asking a human to +// rerun, and merge gates stalled on a zero-assertion-failure infrastructure +// flake (PR #4206 sat red ~40 min; owner had to merge manually). The runner +// therefore downgrades a TWICE-drained phase to a loud warning + exit 0, but +// ONLY on merge-gating events (pull_request/merge_group): those checks exist +// to gate merges, and fail=0 on both attempts is by definition not an +// assertion regression. push/schedule/workflow_dispatch stay strict — each +// draws a fresh runner, so they double as the cross-runner probe that catches +// a genuinely deterministic orphaned-promise regression within one push run / +// the 30-min scheduled cadence. Local runs (no GITHUB_EVENT_NAME) stay strict. +export function isMergeGateEvent(eventName) { + return eventName === "pull_request" || eventName === "merge_group"; +} + +// Strict by construction: BOTH phases must be the pure drain shape. A +// swallowed-then-drained or drained-then-concrete sequence is NOT the +// twice-drained signature and keeps failing, whatever the event. +export function shouldDrainPass(first, retry, eventName) { + return ( + first.kind === "cancelled-drain" && + retry.kind === "cancelled-drain" && + isMergeGateEvent(eventName) + ); +} + +// Top-level `not ok N - ` entries whose YAML block carries the drain +// cancellation marker — the drain-pass warning NAMES the tests it lost, not +// just counts them. Nested subtests are indented, so they never match +// column-0; concrete failures carry a different failureType. +export function extractDrainCancelledTests(output) { + const lines = output.split(/\r?\n/); + const found = new Set(); + for (let i = 0; i < lines.length; i++) { + const m = lines[i].match(/^not ok \d+ - (.+?)\s*$/); + if (!m) continue; + const window = lines.slice(i, i + 20).join("\n"); + if (!/failureType:\s*'cancelledByParent'/.test(window)) continue; + found.add(m[1].trim()); + } + return [...found]; +} + +export function describeDrainPass(first, retry, cancelledTests, eventName) { + return ( + `DRAIN-PASS (#3665 round 3, #4212): first run drained ` + + `(pass=${first.counts.pass} fail=0 cancelled=${first.counts.cancelled}) and the same-runner retry ` + + `drained identically (pass=${retry.counts.pass} fail=0 cancelled=${retry.counts.cancelled}) — zero ` + + `assertion failures on BOTH attempts, so this is the runner-pool drain condition, not an assertion ` + + `regression. This phase runs on a merge-gating event (${eventName}) and passes WITH A WARNING instead ` + + `of blocking the merge on infrastructure. Cancelled tests (not executed to completion): ` + + `${cancelledTests.length > 0 ? cancelledTests.join("; ") : "(names not recoverable from the TAP above)"}. ` + + `Backstop: push/schedule/dispatch lanes draw fresh runners and stay strict — if this drain is ` + + `deterministic rather than environmental, the next push or scheduled run goes red there.` + ); +} + export function describeRetryOutcome(verdict) { if (verdict.kind === "cancelled-drain") { return ( diff --git a/scripts/run-ci-tests.mjs b/scripts/run-ci-tests.mjs index 846f3b1d9a..6942bae4c3 100644 --- a/scripts/run-ci-tests.mjs +++ b/scripts/run-ci-tests.mjs @@ -55,9 +55,19 @@ // DETERMINISTIC lock (a genuine handle leak) reproduces on the retry and still // fails, so no regression can be masked. Raw user-code unlinkSync EBUSY (no // rimraf frames) stays concrete and unretried — see ci-failure-shapes.mjs. +// +// #4212 (#3665 round 3): the same-runner retry provably cannot recover a +// persistent drain (it reproduced 8/8 across jobs in a 30-min window on +// 2026-10-21, then a fresh-runner full matrix went green). So a TWICE-drained +// phase on a merge-gating event (pull_request/merge_group) now passes WITH a +// loud ::warning naming the cancelled tests instead of going red and waiting +// for a human to draw a fresh runner (PR #4206 sat red ~40 min with zero +// assertion failures). push/schedule/dispatch — the fresh-runner probes — and +// every non-drain shape fail exactly as before, so a genuinely deterministic +// drain is still caught within one push run / the 30-min scheduled cadence. import { execFileSync, spawn } from "node:child_process"; import { LIVE_FILES } from "./live-files.mjs"; -import { classifyPhaseFailure, describeRetryOutcome } from "./ci-failure-shapes.mjs"; +import { classifyPhaseFailure, describeDrainPass, describeRetryOutcome, extractDrainCancelledTests, shouldDrainPass } from "./ci-failure-shapes.mjs"; import { readdirSync, readFileSync } from "node:fs"; import { freemem, loadavg, totalmem } from "node:os"; import { join } from "node:path"; @@ -320,6 +330,22 @@ function runIsolated(fileRel) { // a code regression — the old fixed "genuine failure" line mislabelled it // (#3665 round-2 log). Reclassify before printing the conclusion. const retryVerdict = classifyPhaseFailure(retry.out, baseToFile); + // #4212 (#3665 round 3): twice-drained + merge-gating event = zero + // assertion failures on both attempts on this runner — pass with a loud + // warning instead of blocking merges on a runner-pool condition no repo + // change can fix. push/schedule/dispatch (fresh-runner probes), local runs + // (no GITHUB_EVENT_NAME), and every non-drain shape fail exactly as before. + const eventName = process.env.GITHUB_EVENT_NAME ?? ""; + if (shouldDrainPass(verdict, retryVerdict, eventName)) { + const cancelledTests = extractDrainCancelledTests(retry.out); + console.error( + `::warning::DRAIN-PASS (#3665/#4212): both attempts on this runner lost tests to event-loop ` + + `drain with ZERO assertion failures — merge gate not blocked. Cancelled: ` + + `${cancelledTests.join("; ") || "see TAP above"}`, + ); + console.error(`[ci-tests] ${describeDrainPass(verdict, retryVerdict, cancelledTests, eventName)}`); + process.exit(0); + } console.error( `[ci-tests] full retry also FAILED (exit=${retry.code} signal=${retry.signal}) -> ` + describeRetryOutcome(retryVerdict), diff --git a/tests/ci-runner-failure-shapes.test.ts b/tests/ci-runner-failure-shapes.test.ts index 7739242195..ee12f1b2bf 100644 --- a/tests/ci-runner-failure-shapes.test.ts +++ b/tests/ci-runner-failure-shapes.test.ts @@ -10,9 +10,12 @@ import assert from "node:assert/strict"; import { classifyPhaseFailure, countNotOkEntries, + describeDrainPass, describeRetryOutcome, + extractDrainCancelledTests, extractSwallowedFiles, extractTapCounts, + shouldDrainPass, } from "../scripts/ci-failure-shapes.mjs"; const baseToFile = new Map([ @@ -307,3 +310,55 @@ test("extractTapCounts: missing '# cancelled' line defaults to 0; indented looka assert.ok(c !== null && c.fail === 0 && c.pass === 4352); assert.equal(extractTapCounts("no summary here"), null); }); + +// #4212 (#3665 round 3): the twice-drained merge-gate downgrade. Gate +// discipline (#3617): both directions asserted — downgraded when it should be, +// strict everywhere else. +test("shouldDrainPass: twice-drained + merge-gating event -> downgraded to warning-pass", () => { + const v = classifyPhaseFailure(DRAIN_TAP, baseToFile); + assert.equal(shouldDrainPass(v, v, "pull_request"), true); + assert.equal(shouldDrainPass(v, v, "merge_group"), true); +}); + +test("shouldDrainPass: fresh-runner probe events and local runs stay strict", () => { + const v = classifyPhaseFailure(DRAIN_TAP, baseToFile); + for (const ev of ["push", "schedule", "workflow_dispatch", ""]) { + assert.equal(shouldDrainPass(v, v, ev), false, `event=${JSON.stringify(ev)}`); + } +}); + +test("shouldDrainPass: only the TWICE-drained signature downgrades — any other shape pair fails (any event)", () => { + const drain = classifyPhaseFailure(DRAIN_TAP, baseToFile); + const swallowed = classifyPhaseFailure(SWALLOWED_TAP, baseToFile); + const concrete = classifyPhaseFailure(CONCRETE_TAP, baseToFile); + assert.equal(shouldDrainPass(swallowed, drain, "pull_request"), false); + assert.equal(shouldDrainPass(drain, swallowed, "pull_request"), false); + assert.equal(shouldDrainPass(drain, concrete, "pull_request"), false); + assert.equal(shouldDrainPass(concrete, drain, "pull_request"), false); +}); + +test("extractDrainCancelledTests: names the drain-cancelled top-level tests", () => { + assert.deepEqual(extractDrainCancelledTests(DRAIN_TAP), [ + "preflight rejects malformed proxy args", + ]); +}); + +test("extractDrainCancelledTests: concrete failures and swallowed file deaths are not drain names", () => { + assert.deepEqual(extractDrainCancelledTests(CONCRETE_TAP), []); + assert.deepEqual(extractDrainCancelledTests(SWALLOWED_TAP), []); +}); + +test("describeDrainPass: counts both attempts, names the lost tests, never claims a genuine failure", () => { + const v = classifyPhaseFailure(DRAIN_TAP, baseToFile); + if (v.kind !== "cancelled-drain") { + assert.fail("fixture must classify as cancelled-drain"); + return; + } + const msg = describeDrainPass(v, v, ["preflight rejects malformed proxy args"], "pull_request"); + assert.match(msg, /DRAIN-PASS/); + assert.match(msg, /pass=4352 fail=0 cancelled=5/); + assert.match(msg, /preflight rejects malformed proxy args/); + assert.match(msg, /pull_request/); + assert.match(msg, /fresh runners and stay strict/); + assert.doesNotMatch(msg, /genuine failure/); +});