Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions scripts/ci-failure-shapes.d.mts
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,12 @@ export function extractTapCounts(output: string): TapCounts | null;
export function countNotOkEntries(output: string): { lock: number; other: number };
export function classifyPhaseFailure(output: string, baseToFile: Map<string, string>): PhaseFailureVerdict;
export function describeRetryOutcome(verdict: PhaseFailureVerdict): string;
export function isMergeGateEvent(eventName: string): boolean;
export function shouldDrainPass(first: PhaseFailureVerdict, retry: PhaseFailureVerdict, eventName: string): boolean;
export function extractDrainCancelledTests(output: string): string[];
export function describeDrainPass(
first: { kind: "cancelled-drain"; counts: TapCounts },
retry: { kind: "cancelled-drain"; counts: TapCounts },
cancelledTests: string[],
eventName: string,
): string;
58 changes: 58 additions & 0 deletions scripts/ci-failure-shapes.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,64 @@ export function classifyPhaseFailure(output, baseToFile) {
// concrete/swallowed failures on the retry are genuine regressions. The caller
// must never print a fixed "genuine failure" line without reclassifying — that
// mislabels a reproduced drain as a code regression (#3665 round-2 log).
//
// #4212 (#3665 round 3): "a fresh runner is the only escape" — but nothing
// ever drew one. A twice-drained job went red with a message asking a human to
// rerun, and merge gates stalled on a zero-assertion-failure infrastructure
// flake (PR #4206 sat red ~40 min; owner had to merge manually). The runner
// therefore downgrades a TWICE-drained phase to a loud warning + exit 0, but
// ONLY on merge-gating events (pull_request/merge_group): those checks exist
// to gate merges, and fail=0 on both attempts is by definition not an
// assertion regression. push/schedule/workflow_dispatch stay strict — each
// draws a fresh runner, so they double as the cross-runner probe that catches
// a genuinely deterministic orphaned-promise regression within one push run /
// the 30-min scheduled cadence. Local runs (no GITHUB_EVENT_NAME) stay strict.
export function isMergeGateEvent(eventName) {
return eventName === "pull_request" || eventName === "merge_group";
}

// Strict by construction: BOTH phases must be the pure drain shape. A
// swallowed-then-drained or drained-then-concrete sequence is NOT the
// twice-drained signature and keeps failing, whatever the event.
export function shouldDrainPass(first, retry, eventName) {
return (
first.kind === "cancelled-drain" &&
retry.kind === "cancelled-drain" &&
isMergeGateEvent(eventName)
);
}

// Top-level `not ok N - <name>` entries whose YAML block carries the drain
// cancellation marker — the drain-pass warning NAMES the tests it lost, not
// just counts them. Nested subtests are indented, so they never match
// column-0; concrete failures carry a different failureType.
export function extractDrainCancelledTests(output) {
const lines = output.split(/\r?\n/);
const found = new Set();
for (let i = 0; i < lines.length; i++) {
const m = lines[i].match(/^not ok \d+ - (.+?)\s*$/);
if (!m) continue;
const window = lines.slice(i, i + 20).join("\n");
if (!/failureType:\s*'cancelledByParent'/.test(window)) continue;
found.add(m[1].trim());
}
return [...found];
}

export function describeDrainPass(first, retry, cancelledTests, eventName) {
return (
`DRAIN-PASS (#3665 round 3, #4212): first run drained ` +
`(pass=${first.counts.pass} fail=0 cancelled=${first.counts.cancelled}) and the same-runner retry ` +
`drained identically (pass=${retry.counts.pass} fail=0 cancelled=${retry.counts.cancelled}) — zero ` +
`assertion failures on BOTH attempts, so this is the runner-pool drain condition, not an assertion ` +
`regression. This phase runs on a merge-gating event (${eventName}) and passes WITH A WARNING instead ` +
`of blocking the merge on infrastructure. Cancelled tests (not executed to completion): ` +
`${cancelledTests.length > 0 ? cancelledTests.join("; ") : "(names not recoverable from the TAP above)"}. ` +
`Backstop: push/schedule/dispatch lanes draw fresh runners and stay strict — if this drain is ` +
`deterministic rather than environmental, the next push or scheduled run goes red there.`
);
}

export function describeRetryOutcome(verdict) {
if (verdict.kind === "cancelled-drain") {
return (
Expand Down
28 changes: 27 additions & 1 deletion scripts/run-ci-tests.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,19 @@
// DETERMINISTIC lock (a genuine handle leak) reproduces on the retry and still
// fails, so no regression can be masked. Raw user-code unlinkSync EBUSY (no
// rimraf frames) stays concrete and unretried — see ci-failure-shapes.mjs.
//
// #4212 (#3665 round 3): the same-runner retry provably cannot recover a
// persistent drain (it reproduced 8/8 across jobs in a 30-min window on
// 2026-10-21, then a fresh-runner full matrix went green). So a TWICE-drained
// phase on a merge-gating event (pull_request/merge_group) now passes WITH a
// loud ::warning naming the cancelled tests instead of going red and waiting
// for a human to draw a fresh runner (PR #4206 sat red ~40 min with zero
// assertion failures). push/schedule/dispatch — the fresh-runner probes — and
// every non-drain shape fail exactly as before, so a genuinely deterministic
// drain is still caught within one push run / the 30-min scheduled cadence.
import { execFileSync, spawn } from "node:child_process";
import { LIVE_FILES } from "./live-files.mjs";
import { classifyPhaseFailure, describeRetryOutcome } from "./ci-failure-shapes.mjs";
import { classifyPhaseFailure, describeDrainPass, describeRetryOutcome, extractDrainCancelledTests, shouldDrainPass } from "./ci-failure-shapes.mjs";
import { readdirSync, readFileSync } from "node:fs";
import { freemem, loadavg, totalmem } from "node:os";
import { join } from "node:path";
Expand Down Expand Up @@ -320,6 +330,22 @@ function runIsolated(fileRel) {
// a code regression — the old fixed "genuine failure" line mislabelled it
// (#3665 round-2 log). Reclassify before printing the conclusion.
const retryVerdict = classifyPhaseFailure(retry.out, baseToFile);
// #4212 (#3665 round 3): twice-drained + merge-gating event = zero
// assertion failures on both attempts on this runner — pass with a loud
// warning instead of blocking merges on a runner-pool condition no repo
// change can fix. push/schedule/dispatch (fresh-runner probes), local runs
// (no GITHUB_EVENT_NAME), and every non-drain shape fail exactly as before.
const eventName = process.env.GITHUB_EVENT_NAME ?? "";
if (shouldDrainPass(verdict, retryVerdict, eventName)) {
const cancelledTests = extractDrainCancelledTests(retry.out);
console.error(
`::warning::DRAIN-PASS (#3665/#4212): both attempts on this runner lost tests to event-loop ` +
`drain with ZERO assertion failures — merge gate not blocked. Cancelled: ` +
`${cancelledTests.join("; ") || "see TAP above"}`,
);
console.error(`[ci-tests] ${describeDrainPass(verdict, retryVerdict, cancelledTests, eventName)}`);
process.exit(0);
}
console.error(
`[ci-tests] full retry also FAILED (exit=${retry.code} signal=${retry.signal}) -> ` +
describeRetryOutcome(retryVerdict),
Expand Down
55 changes: 55 additions & 0 deletions tests/ci-runner-failure-shapes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,12 @@ import assert from "node:assert/strict";
import {
classifyPhaseFailure,
countNotOkEntries,
describeDrainPass,
describeRetryOutcome,
extractDrainCancelledTests,
extractSwallowedFiles,
extractTapCounts,
shouldDrainPass,
} from "../scripts/ci-failure-shapes.mjs";

const baseToFile = new Map([
Expand Down Expand Up @@ -307,3 +310,55 @@ test("extractTapCounts: missing '# cancelled' line defaults to 0; indented looka
assert.ok(c !== null && c.fail === 0 && c.pass === 4352);
assert.equal(extractTapCounts("no summary here"), null);
});

// #4212 (#3665 round 3): the twice-drained merge-gate downgrade. Gate
// discipline (#3617): both directions asserted — downgraded when it should be,
// strict everywhere else.
test("shouldDrainPass: twice-drained + merge-gating event -> downgraded to warning-pass", () => {
const v = classifyPhaseFailure(DRAIN_TAP, baseToFile);
assert.equal(shouldDrainPass(v, v, "pull_request"), true);
assert.equal(shouldDrainPass(v, v, "merge_group"), true);
});

test("shouldDrainPass: fresh-runner probe events and local runs stay strict", () => {
const v = classifyPhaseFailure(DRAIN_TAP, baseToFile);
for (const ev of ["push", "schedule", "workflow_dispatch", ""]) {
assert.equal(shouldDrainPass(v, v, ev), false, `event=${JSON.stringify(ev)}`);
}
});

test("shouldDrainPass: only the TWICE-drained signature downgrades — any other shape pair fails (any event)", () => {
const drain = classifyPhaseFailure(DRAIN_TAP, baseToFile);
const swallowed = classifyPhaseFailure(SWALLOWED_TAP, baseToFile);
const concrete = classifyPhaseFailure(CONCRETE_TAP, baseToFile);
assert.equal(shouldDrainPass(swallowed, drain, "pull_request"), false);
assert.equal(shouldDrainPass(drain, swallowed, "pull_request"), false);
assert.equal(shouldDrainPass(drain, concrete, "pull_request"), false);
assert.equal(shouldDrainPass(concrete, drain, "pull_request"), false);
});

test("extractDrainCancelledTests: names the drain-cancelled top-level tests", () => {
assert.deepEqual(extractDrainCancelledTests(DRAIN_TAP), [
"preflight rejects malformed proxy args",
]);
});

test("extractDrainCancelledTests: concrete failures and swallowed file deaths are not drain names", () => {
assert.deepEqual(extractDrainCancelledTests(CONCRETE_TAP), []);
assert.deepEqual(extractDrainCancelledTests(SWALLOWED_TAP), []);
});

test("describeDrainPass: counts both attempts, names the lost tests, never claims a genuine failure", () => {
const v = classifyPhaseFailure(DRAIN_TAP, baseToFile);
if (v.kind !== "cancelled-drain") {
assert.fail("fixture must classify as cancelled-drain");
return;
}
const msg = describeDrainPass(v, v, ["preflight rejects malformed proxy args"], "pull_request");
assert.match(msg, /DRAIN-PASS/);
assert.match(msg, /pass=4352 fail=0 cancelled=5/);
assert.match(msg, /preflight rejects malformed proxy args/);
assert.match(msg, /pull_request/);
assert.match(msg, /fresh runners and stay strict/);
assert.doesNotMatch(msg, /genuine failure/);
});
Loading