From 9e9100c449528c6b3121714ecdf391e949014a6b Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 17:23:01 -0500 Subject: [PATCH 1/6] ci: fail a test phase that prints nothing for 10 minutes and name what is stuck (#69) A hung test file made a whole shard silent until the 90-minute job timeout, and the log never named it (#61: twice on windows-latest / Node 24). Every runner phase now runs under scripts/ci-stall-watch.mjs. After 10 minutes without output it lists the processes still alive under the phase, kills the process tree and fails the job. A stalled phase skips the failure-shape classifier and the retry, so it can neither be retried nor pass. Assisted-by: Cindy (https://cindy.app) --- reference/testing.md | 18 +++ scripts/ci-stall-watch.d.mts | 37 ++++++ scripts/ci-stall-watch.mjs | 217 ++++++++++++++++++++++++++++++ scripts/live-files.mjs | 2 + scripts/run-ci-tests.mjs | 116 +++++++++------- tests/ci-stall-watch.test.ts | 249 +++++++++++++++++++++++++++++++++++ 6 files changed, 590 insertions(+), 49 deletions(-) create mode 100644 scripts/ci-stall-watch.d.mts create mode 100644 scripts/ci-stall-watch.mjs create mode 100644 tests/ci-stall-watch.test.ts diff --git a/reference/testing.md b/reference/testing.md index 580ef7a06e..27d0e60472 100644 --- a/reference/testing.md +++ b/reference/testing.md @@ -46,6 +46,24 @@ A bare `node --import tsx --test ...` still works (the storage family pins its own dirs), but any suite that relies on DEFAULT path resolution will write into your real machine state — prefer the preload chain. +## CI: stall watch (#69) + +Every phase `scripts/run-ci-tests.mjs` starts (the main run, the full retry and +the isolated re-runs) runs under `scripts/ci-stall-watch.mjs`. A phase that +prints nothing on stdout and stderr for 10 minutes is treated as stalled: the +runner lists the processes still alive under it (a finished test file's process +is gone, so the list names the stuck file and anything it left running), kills +the whole process tree, and fails the job. A stall is never retried and never +reaches the failure-shape classifier — it stays a red job until its cause is +known. `node --test` holds back later files' output until the file at the head +of its order finishes, which is why one hung file used to silence a whole shard +until the 90-minute job timeout (#61). + +The limit is a constant, not a setting. It was chosen against the longest quiet +gap seen in 20 passing windows-latest / Node 24 shard jobs on 2026-11-14 +(238.5 s). A hang that keeps printing is not detected. When a job fails with +`STALL:` lines, copy the process list to the issue tracking that stall. + ## CI: test schedule (#2816) The main CI `test` job does not run the flat `npm test` glob — it runs diff --git a/scripts/ci-stall-watch.d.mts b/scripts/ci-stall-watch.d.mts new file mode 100644 index 0000000000..48fcc2af05 --- /dev/null +++ b/scripts/ci-stall-watch.d.mts @@ -0,0 +1,37 @@ +export interface ProcRow { + pid: number; + ppid: number; + cmd: string; +} + +export interface ProcNode extends ProcRow { + depth: number; +} + +export interface WatchedResult { + code: number | null; + signal: NodeJS.Signals | null; + stalled: boolean; + ms: number; + spawnError: Error | undefined; +} + +export interface WatchOptions { + cwd?: string; + env?: NodeJS.ProcessEnv; + label?: string; + limitMs?: number; + closeGraceMs?: number; + onStdout?: (chunk: Buffer) => void; + onStderr?: (chunk: Buffer) => void; + log?: (line: string) => void; + list?: () => ProcRow[]; + killTree?: (rootPid: number, known: ProcNode[]) => void; +} + +export const STALL_LIMIT_MS: number; +export function listProcesses(): ProcRow[]; +export function descendantsOf(rootPid: number, rows: ProcRow[]): ProcNode[]; +export function testFilesIn(procs: ProcNode[]): string[]; +export function killProcessTree(rootPid: number, known?: ProcNode[]): void; +export function runWatched(command: string, args: string[], opts?: WatchOptions): Promise; diff --git a/scripts/ci-stall-watch.mjs b/scripts/ci-stall-watch.mjs new file mode 100644 index 0000000000..9859f89173 --- /dev/null +++ b/scripts/ci-stall-watch.mjs @@ -0,0 +1,217 @@ +// #69: stall watch for the CI test phases. `node --test` holds back the output +// of later files until the file at the head of its order finishes, so one hung +// test file makes the whole phase silent and the log never names it (#61: 40 +// and 22 minutes of silence on windows-latest / Node 24, cancelled by hand). +// +// runWatched() spawns a phase and, when it prints nothing on stdout AND stderr +// for `limitMs`, (1) reports the processes still alive under it — a finished +// test file's process is gone, so what remains is what is stuck — (2) ends the +// whole process tree, and (3) resolves with `stalled: true`. The caller fails +// the job; a stall is never retried here (owner decision on #69: the stall +// stays a red job while its cause is unknown). +// +// Lives in its own module because run-ci-tests.mjs has top-level side effects +// and cannot be imported by a test. The limit is a parameter, not a setting: +// no environment variable reads it. +import { execFileSync, spawn } from "node:child_process"; + +// 10 minutes (owner decision on #69). Longest gap between two output lines in +// 20 passing windows-latest / Node 24 shard jobs on 2026-11-14: 238.5 s. +export const STALL_LIMIT_MS = 10 * 60 * 1000; + +const QUERY_TIMEOUT_MS = 30_000; +const TAIL_BYTES = 2048; +const TEST_FILE_RE = /[^\s"']+\.test\.[cm]?[jt]sx?/g; + +/** Every process on the machine as { pid, ppid, cmd }. Throws when the query + * fails or exceeds its own deadline — the caller must treat that as "no + * snapshot", never as "nothing is running". */ +export function listProcesses() { + if (process.platform === "win32") { + const json = execFileSync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + "Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CommandLine | ConvertTo-Json -Compress", + ], + { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: QUERY_TIMEOUT_MS, windowsHide: true }, + ); + const rows = JSON.parse(json); + return (Array.isArray(rows) ? rows : [rows]).map((r) => ({ + pid: Number(r.ProcessId), + ppid: Number(r.ParentProcessId), + cmd: String(r.CommandLine ?? ""), + })); + } + const text = execFileSync("ps", ["-eo", "pid=,ppid=,args="], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + timeout: QUERY_TIMEOUT_MS, + }); + const rows = []; + for (const line of text.split("\n")) { + const m = line.match(/^\s*(\d+)\s+(\d+)\s+(.*)$/); + if (m) rows.push({ pid: Number(m[1]), ppid: Number(m[2]), cmd: m[3] }); + } + return rows; +} + +/** The root and its descendants, parents before children. Pure. */ +export function descendantsOf(rootPid, rows) { + const byParent = new Map(); + for (const r of rows) { + if (!byParent.has(r.ppid)) byParent.set(r.ppid, []); + byParent.get(r.ppid).push(r); + } + const out = []; + const seen = new Set(); + const walk = (pid, depth) => { + for (const r of byParent.get(pid) ?? []) { + if (seen.has(r.pid) || r.pid === rootPid) continue; // pid reuse can fake a cycle + seen.add(r.pid); + out.push({ ...r, depth }); + walk(r.pid, depth + 1); + } + }; + const root = rows.find((r) => r.pid === rootPid); + if (root) out.push({ ...root, depth: 0 }); + walk(rootPid, 1); + return out; +} + +/** Test files named on the command lines of the given processes. Pure. */ +export function testFilesIn(procs) { + const files = new Set(); + for (const p of procs) { + // A file-level child carries exactly one test file. A command line + // with several (the phase root's shard list) names nothing. + const hits = p.cmd.match(TEST_FILE_RE) ?? []; + if (hits.length === 1) files.add(hits[0].replace(/\\/g, "/")); + } + return [...files]; +} + +/** Ends the root and everything under it. `known` is the pre-kill snapshot + * (may be empty when the snapshot failed). Never throws. */ +export function killProcessTree(rootPid, known = []) { + if (process.platform === "win32") { + try { + execFileSync("taskkill", ["/PID", String(rootPid), "/T", "/F"], { + stdio: "ignore", + timeout: QUERY_TIMEOUT_MS, + windowsHide: true, + }); + } catch { + // Already gone, or taskkill itself failed: the close-grace in + // runWatched still resolves the phase. + } + return; + } + const pids = known.length > 0 ? known.map((p) => p.pid).reverse() : [rootPid]; + if (!pids.includes(rootPid)) pids.push(rootPid); + for (const pid of pids) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone + } + } +} + +function tailOf(cmd, n = 240) { + const flat = cmd.replace(/\s+/g, " ").trim(); + return flat.length <= n ? flat : `…${flat.slice(-n)}`; +} + +/** + * Spawn `command args` and watch it for silence. + * + * Resolves with { code, signal, stalled, ms, spawnError }. `stalled: true` + * means the phase was ended by this watch; `code`/`signal` then describe the + * kill and MUST NOT be interpreted as a test result. + */ +export function runWatched(command, args, opts = {}) { + const { + cwd, + env, + label = "phase", + limitMs = STALL_LIMIT_MS, + closeGraceMs = 15_000, + onStdout = () => {}, + onStderr = () => {}, + log = (line) => console.error(line), + list = listProcesses, + killTree = killProcessTree, + } = opts; + const t0 = Date.now(); + return new Promise((resolve) => { + const child = spawn(command, args, { cwd, env }); + let stalled = false; + let settled = false; + let graceTimer; + let tail = Buffer.alloc(0); + const settle = (r) => { + if (settled) return; + settled = true; + clearTimeout(stallTimer); + clearTimeout(graceTimer); + resolve({ spawnError: undefined, ...r, stalled, ms: Date.now() - t0 }); + }; + const onStall = () => { + if (settled) return; + stalled = true; + log( + `::error::[ci-tests] STALL: ${label} printed nothing for ${Math.round(limitMs / 1000)}s — ` + + `ending the phase and failing the job (#69)`, + ); + let procs = []; + try { + procs = descendantsOf(child.pid, list()); + log(`[ci-tests] STALL: ${procs.length} process(es) still alive under ${label} (pid ${child.pid}), taken before the kill:`); + for (const p of procs) log(`[ci-tests] STALL: ${" ".repeat(p.depth)}pid=${p.pid} ppid=${p.ppid} ${tailOf(p.cmd)}`); + const files = testFilesIn(procs); + log( + files.length > 0 + ? `[ci-tests] STALL: test file(s) still running: ${files.join(", ")}` + : `[ci-tests] STALL: no test file could be read from the command lines above`, + ); + } catch (e) { + procs = []; + log(`[ci-tests] STALL: process snapshot unavailable (${e && e.message ? e.message.split("\n")[0] : e}) — killing anyway`); + } + const tailText = tail.toString("utf8").trimEnd(); + log(`[ci-tests] STALL: last output before the silence:\n${tailText === "" ? "(none)" : tailText}`); + killTree(child.pid, procs); + // A descendant that survived the kill keeps the pipes open and + // 'close' never fires: do not wait for it forever. + graceTimer = setTimeout(() => { + log(`[ci-tests] STALL: ${label} did not close within ${Math.round(closeGraceMs / 1000)}s of the kill — abandoning its streams`); + // Drop every handle on it so the abandoned phase cannot keep this + // process alive either. + child.stdin?.destroy(); + child.stdout?.destroy(); + child.stderr?.destroy(); + child.unref(); + settle({ code: null, signal: null }); + }, closeGraceMs); + }; + const stallTimer = setTimeout(onStall, limitMs); + const seen = (d) => { + if (!stalled) stallTimer.refresh(); + tail = Buffer.concat([tail, d]); + if (tail.length > TAIL_BYTES) tail = tail.subarray(tail.length - TAIL_BYTES); + }; + child.stdout.on("data", (d) => { + seen(d); + onStdout(d); + }); + child.stderr.on("data", (d) => { + seen(d); + onStderr(d); + }); + child.on("error", (e) => settle({ code: -1, signal: null, spawnError: e })); + child.on("close", (code, signal) => settle({ code, signal })); + }); +} diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index da1abd7521..e2cd7a7a94 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -118,6 +118,8 @@ export const LIVE_FILES = [ // the tripwire alone, like #3890/#5114 "tests/cache-keepalive.test.ts", "tests/mcp-eof.test.ts", + // #69: spawns real process trees to exercise the CI stall watch; no ports. + "tests/ci-stall-watch.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/scripts/run-ci-tests.mjs b/scripts/run-ci-tests.mjs index ec18706b71..3509ff936f 100644 --- a/scripts/run-ci-tests.mjs +++ b/scripts/run-ci-tests.mjs @@ -80,10 +80,18 @@ // own files (tests/issue760-percall-conversation.test.ts, // tests/storage-full-flip.test.ts) — classifier and widening are belt and // suspenders, not alternatives. -import { execFileSync, spawn } from "node:child_process"; +// +// #69: every phase (main, full retry, isolated re-runs) runs under the stall +// watch in ci-stall-watch.mjs. A phase that prints nothing for 10 minutes is +// reported (processes still alive under it), killed as a tree, and FAILS the +// job right there: it never reaches the classifier or the retry below, because +// a TAP stream cut off by the kill has no tail and must not be read as one of +// the recoverable shapes. A stall is not a flake until its cause is known (#61). +import { execFileSync } from "node:child_process"; import { LIVE_FILES } from "./live-files.mjs"; import { classifyPhaseFailure, describeDrainPass, describeRetryOutcome, extractDrainCancelledTests, shouldDrainPass } from "./ci-failure-shapes.mjs"; import { partitionAll as partitionFiles } from "./ci-shard-partition.mjs"; +import { runWatched } from "./ci-stall-watch.mjs"; import { readdirSync, readFileSync } from "node:fs"; import { freemem, loadavg, totalmem } from "node:os"; import { join } from "node:path"; @@ -187,48 +195,57 @@ const PRELOAD_ARGS = ["--import", "./tests/preload-storage-baseline.mjs", "--imp const MAX_CAPTURE = 32 * 1024 * 1024; // bound in-memory capture; real runs are MB-scale function runNodeTest(files, concurrency, label) { - const t0 = Date.now(); console.log(`[ci-tests] ${label}: ${files.length} files, concurrency=${concurrency}`); - return new Promise((resolve) => { - const child = spawn( - process.execPath, - [...PRELOAD_ARGS, "--test", "--test-concurrency", String(concurrency), "--test-reporter=tap", ...files], - { cwd: root }, - ); - const out = []; - const err = []; - let outLen = 0; - let errLen = 0; - child.stdout.on("data", (d) => { - if (outLen < MAX_CAPTURE) { - out.push(d); - outLen += d.length; - } - process.stdout.write(d); - }); - child.stderr.on("data", (d) => { - if (errLen < MAX_CAPTURE) { - err.push(d); - errLen += d.length; - } - process.stderr.write(d); - }); - child.on("error", (e) => { - console.error(`[ci-tests] ${label}: spawn error: ${e.message}`); - resolve({ code: -1, signal: null, out: "", err: "", ms: Date.now() - t0 }); - }); - child.on("close", (code, signal) => { - resolve({ - code, - signal, - out: Buffer.concat(out).toString("utf8"), - err: Buffer.concat(err).toString("utf8"), - ms: Date.now() - t0, - }); - }); + const out = []; + const err = []; + let outLen = 0; + let errLen = 0; + return runWatched( + process.execPath, + [...PRELOAD_ARGS, "--test", "--test-concurrency", String(concurrency), "--test-reporter=tap", ...files], + { + cwd: root, + label, + onStdout: (d) => { + if (outLen < MAX_CAPTURE) { + out.push(d); + outLen += d.length; + } + process.stdout.write(d); + }, + onStderr: (d) => { + if (errLen < MAX_CAPTURE) { + err.push(d); + errLen += d.length; + } + process.stderr.write(d); + }, + }, + ).then((r) => { + if (r.spawnError) console.error(`[ci-tests] ${label}: spawn error: ${r.spawnError.message}`); + return { + code: r.code, + signal: r.signal, + stalled: r.stalled, + out: r.spawnError ? "" : Buffer.concat(out).toString("utf8"), + err: r.spawnError ? "" : Buffer.concat(err).toString("utf8"), + ms: r.ms, + }; }); } +// #69: a stalled phase is terminal. Called before anything reads the phase's +// exit code or TAP, so a stall can neither be retried nor pass. +function failIfStalled(r, label) { + if (!r.stalled) return; + console.error( + `[ci-tests] ${label} STALLED after ${(r.ms / 1000).toFixed(1)}s and was killed -> failing WITHOUT retry (#69). ` + + `The process list above names what was still running; record it on the stall's issue (#61 for tests/dsh-native.test.ts).`, + ); + console.error(`[ci-tests] resource snapshot: ${resourceSnapshot()}`); + process.exit(1); +} + function resourceSnapshot() { const mi = (n) => `${Math.round(n / 1048576)}MiB`; return ( @@ -239,22 +256,21 @@ function resourceSnapshot() { } function runIsolated(fileRel) { - const t0 = Date.now(); console.log(`[ci-tests] DIAGNOSTIC isolated re-run (serial): ${fileRel} — full output below`); - return new Promise((resolve) => { - const child = spawn(process.execPath, [...PRELOAD_ARGS, "--test", "--test-reporter=tap", fileRel], { cwd: root }); - child.stdout.on("data", (d) => process.stdout.write(d)); - child.stderr.on("data", (d) => process.stderr.write(d)); - child.on("error", (e) => { - console.error(`[ci-tests] isolated re-run ${fileRel}: spawn error: ${e.message}`); - resolve({ code: -1, signal: null, ms: Date.now() - t0 }); - }); - child.on("close", (code, signal) => resolve({ code, signal, ms: Date.now() - t0 })); + return runWatched(process.execPath, [...PRELOAD_ARGS, "--test", "--test-reporter=tap", fileRel], { + cwd: root, + label: `isolated re-run ${fileRel}`, + onStdout: (d) => process.stdout.write(d), + onStderr: (d) => process.stderr.write(d), + }).then((r) => { + if (r.spawnError) console.error(`[ci-tests] isolated re-run ${fileRel}: spawn error: ${r.spawnError.message}`); + return { code: r.code, signal: r.signal, stalled: r.stalled, ms: r.ms }; }); } (async () => { const first = await runNodeTest(selectedFiles, 4, "all"); + failIfStalled(first, "all"); if (first.code === 0) { console.log(`[ci-tests] all ok in ${(first.ms / 1000).toFixed(1)}s`); return; @@ -289,6 +305,7 @@ function runIsolated(fileRel) { ); for (const rel of verdict.files) { const r = await runIsolated(rel); + failIfStalled(r, `isolated re-run ${rel}`); console.error( `[ci-tests] isolated re-run ${rel}: exit=${r.code} signal=${r.signal} (${(r.ms / 1000).toFixed(1)}s) -> ` + (r.code === 0 @@ -322,6 +339,7 @@ function runIsolated(fileRel) { const retryIssue = { swallowed: "#3029", "cancelled-drain": "#3665", "cleanup-lock": "#4347", "timing-window": "#4161" }[verdict.kind]; console.error(`[ci-tests] retrying full phase once to confirm flake (${retryIssue})`); const retry = await runNodeTest(selectedFiles, 4, "all (retry)"); + failIfStalled(retry, "all (retry)"); if (retry.code === 0) { const firstSummary = verdict.kind === "swallowed" diff --git a/tests/ci-stall-watch.test.ts b/tests/ci-stall-watch.test.ts new file mode 100644 index 0000000000..6ee4fea5b5 --- /dev/null +++ b/tests/ci-stall-watch.test.ts @@ -0,0 +1,249 @@ +// #69: the CI runner's stall watch (scripts/ci-stall-watch.mjs). A phase that +// prints nothing for the limit is reported, killed as a tree and flagged +// `stalled`; a phase that keeps printing, or finishes inside the limit, is +// left alone. Fixtures are real child processes: the point of the watch is +// what happens to a process tree, which a stub cannot show. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { descendantsOf, runWatched, STALL_LIMIT_MS, testFilesIn } from "../scripts/ci-stall-watch.mjs"; + +// Short stand-in for the 10-minute limit. It also has to cover a fixture's +// start-up (spawn + node boot) on a loaded runner, since nothing is printed +// before that: 3 s leaves that room without making the file slow. +const STALL_AFTER_MS = 3000; + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function waitDead(pids: number[], ms = 10_000): Promise { + const end = Date.now() + ms; + for (;;) { + const left = pids.filter(alive); + if (left.length === 0 || Date.now() > end) return left; + await new Promise((r) => setTimeout(r, 100)); + } +} + +/** A parent that starts `stuck-fixture.test.mjs` sharing its stdout, prints + * both pids, and then both stay silent forever — the #61 shape: a file-level + * process that never finishes under a phase root that waits for it. */ +function stuckTree(dir: string): string { + writeFileSync(join(dir, "stuck-fixture.test.mjs"), "setInterval(() => {}, 1000);\n"); + const parent = join(dir, "phase-root.mjs"); + writeFileSync( + parent, + [ + 'import { spawn } from "node:child_process";', + 'const kid = spawn(process.execPath, ["stuck-fixture.test.mjs"], { stdio: "inherit" });', + "console.log(`pids ${process.pid} ${kid.pid}`);", + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + ); + return parent; +} + +function pidsFrom(out: string): number[] { + const m = out.match(/pids (\d+) (\d+)/); + assert.ok(m, `fixture printed its pids (got: ${JSON.stringify(out)})`); + return [Number(m[1]), Number(m[2])]; +} + +test("the limit is the owner-approved 10 minutes (#69)", () => { + assert.equal(STALL_LIMIT_MS, 600_000); +}); + +test("a silent phase is reported, killed as a tree and flagged stalled", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-stall-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const logs: string[] = []; + let out = ""; + const t0 = Date.now(); + const r = await runWatched(process.execPath, [stuckTree(dir)], { + cwd: dir, + label: "fixture", + limitMs: STALL_AFTER_MS, + onStdout: (d) => (out += d.toString()), + log: (l) => logs.push(l), + }); + const [rootPid, kidPid] = pidsFrom(out); + t.after(() => { + for (const pid of [kidPid, rootPid]) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone — the expected case + } + } + }); + const report = logs.join("\n"); + + assert.equal(r.stalled, true, "flagged stalled"); + assert.ok(Date.now() - t0 >= STALL_AFTER_MS, "not before the limit"); + assert.deepEqual(await waitDead([rootPid, kidPid]), [], "the root and the process it started are both gone"); + assert.match(report, /::error::\[ci-tests\] STALL: fixture printed nothing for 3s/, "names the phase and the silence"); + assert.match(report, /test file\(s\) still running: stuck-fixture\.test\.mjs/, "names the stuck test file"); + assert.match(report, new RegExp(`pid=${kidPid} ppid=${rootPid} `), "lists the stuck process under its parent"); + assert.match(report, new RegExp(`last output before the silence:\\npids ${rootPid} ${kidPid}`), "shows the last output"); + assert.doesNotMatch(report, /snapshot unavailable|did not close within/, "took the snapshot and closed normally"); +}); + +for (const stream of ["stdout", "stderr"] as const) { + test(`a phase that keeps printing on ${stream} past the limit is not stalled`, async () => { + const logs: string[] = []; + let seen = 0; + // Prints every 100 ms for 4.5 s against a 3 s limit: longer than the + // limit in total, never more than a fraction of it between two lines. + const script = + `let n = 0; const i = setInterval(() => { process.${stream}.write("tick\\n"); ` + + `if (++n === 45) { clearInterval(i); process.exitCode = 7; } }, 100);`; + const r = await runWatched(process.execPath, ["-e", script], { + limitMs: STALL_AFTER_MS, + onStdout: () => seen++, + onStderr: () => seen++, + log: (l) => logs.push(l), + }); + assert.equal(r.stalled, false); + assert.equal(r.code, 7, "the phase's own exit code comes through"); + assert.ok(r.ms >= STALL_AFTER_MS, `ran past the limit (${r.ms} ms)`); + assert.ok(seen > 0, "output was forwarded"); + assert.deepEqual(logs, [], "no stall report"); + }); +} + +test("a phase silent for less than the limit that then exits is not stalled, and the watch is disarmed", async () => { + const logs: string[] = []; + const r = await runWatched(process.execPath, ["-e", "setTimeout(() => process.exit(3), 400)"], { + limitMs: STALL_AFTER_MS, + log: (l) => logs.push(l), + }); + assert.equal(r.stalled, false); + assert.equal(r.code, 3); + // Outlive the limit: a timer left armed would report now. + await new Promise((res) => setTimeout(res, STALL_AFTER_MS + 300)); + assert.deepEqual(logs, [], "nothing reported after the phase closed"); +}); + +test("a finished phase leaves nothing that holds the calling process open", async () => { + // The real limit is 10 minutes: a timer left armed after the phase closed + // would keep the runner alive that long after its last phase. Run a + // caller with the real default limit and a phase that ends at once; the + // caller must end on its own well inside it. + const mod = pathToFileURL(join(process.cwd(), "scripts", "ci-stall-watch.mjs")).href; + const caller = + `const m = await import(${JSON.stringify(mod)}); ` + + `const r = await m.runWatched(process.execPath, ["-e", ""]); console.log("stalled=" + r.stalled + " code=" + r.code);`; + let out = ""; + const logs: string[] = []; + const r = await runWatched(process.execPath, ["--input-type=module", "-e", caller], { + limitMs: 60_000, + onStdout: (d) => (out += d.toString()), + log: (l) => logs.push(l), + }); + assert.equal(r.stalled, false, `the caller ended on its own (${logs.join(" | ")})`); + assert.equal(r.code, 0); + assert.match(out, /stalled=false code=0/); + assert.ok(r.ms < 30_000, `promptly (${r.ms} ms)`); +}); + +test("a failed process snapshot is reported and the phase is still killed and flagged", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-stall-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const logs: string[] = []; + let out = ""; + const r = await runWatched(process.execPath, [stuckTree(dir)], { + cwd: dir, + limitMs: STALL_AFTER_MS, + onStdout: (d) => (out += d.toString()), + log: (l) => logs.push(l), + list: () => { + throw new Error("query refused"); + }, + }); + const pids = pidsFrom(out); + t.after(() => { + for (const pid of pids) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone + } + } + }); + const report = logs.join("\n"); + assert.equal(r.stalled, true); + assert.match(report, /process snapshot unavailable \(query refused\) — killing anyway/); + assert.doesNotMatch(report, /still running:/, "claims nothing about what was running"); + if (process.platform === "win32") { + // taskkill /T does not need the snapshot. + assert.deepEqual(await waitDead(pids), [], "whole tree ended without a snapshot"); + } else { + assert.equal(alive(pids[0]), false, "the phase root ended without a snapshot"); + } +}); + +test("a phase whose streams stay open after the kill is abandoned after the grace, still flagged stalled", async (t) => { + const logs: string[] = []; + let pid = 0; + t.after(() => { + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone + } + }); + const r = await runWatched(process.execPath, ["-e", "console.log(process.pid); setInterval(() => {}, 1000)"], { + limitMs: STALL_AFTER_MS, + closeGraceMs: 500, + onStdout: (d) => (pid = Number(d.toString())), + log: (l) => logs.push(l), + list: () => [], + killTree: () => {}, // the kill "did not work": nothing closes the pipes + }); + assert.equal(r.stalled, true); + assert.equal(r.code, null, "no exit code is invented"); + assert.match(logs.join("\n"), /did not close within 1s of the kill — abandoning its streams/); + assert.ok(pid > 0 && alive(pid), "the fixture really was still running when the watch gave up"); +}); + +test("descendantsOf walks only the phase's own tree, parents first", () => { + const rows = [ + { pid: 10, ppid: 1, cmd: "node --test tests/a.test.ts tests/b.test.ts" }, + { pid: 11, ppid: 10, cmd: "node --import tsx tests/a.test.ts" }, + { pid: 12, ppid: 11, cmd: "node dist/index.js start" }, + { pid: 20, ppid: 1, cmd: "node --import tsx tests/unrelated.test.ts" }, + { pid: 10, ppid: 12, cmd: "reused pid" }, + ]; + const tree = descendantsOf(10, rows); + assert.deepEqual( + tree.map((p) => [p.pid, p.depth]), + [ + [10, 0], + [11, 1], + [12, 2], + ], + ); + assert.deepEqual(testFilesIn(tree), ["tests/a.test.ts"], "the root's file list and other trees name nothing"); + assert.deepEqual(descendantsOf(999, rows), [], "an unknown root has no tree"); +}); + +test("testFilesIn normalises Windows separators and skips ambiguous command lines", () => { + assert.deepEqual( + testFilesIn([ + { pid: 2, ppid: 1, depth: 1, cmd: '"C:\\node.exe" --import tsx tests\\dsh-native.test.ts' }, + { pid: 3, ppid: 1, depth: 1, cmd: "node tests/x.test.ts tests/y.test.ts" }, + { pid: 4, ppid: 2, depth: 2, cmd: "node dist/index.js" }, + ]), + ["tests/dsh-native.test.ts"], + ); +}); From ddbac087a103b0fd6a787be687328fc5fd702dff Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 17:32:49 -0500 Subject: [PATCH 2/6] ci: list parentless node processes in the stall report; keep helper types internal (#69) A test file that exits while something it started still holds its output leaves the phase silent with nothing under its process tree. The report now lists node processes with no living parent as a hint (never killed), and the wording no longer claims the stuck file is always named. Also drops the exports on the .d.mts helper types (unused-exports check) and moves the reference section after the one that introduces the runner. Assisted-by: Cindy (https://cindy.app) --- reference/testing.md | 39 +++++++++++++++++++----------------- scripts/ci-stall-watch.d.mts | 9 +++++---- scripts/ci-stall-watch.mjs | 29 ++++++++++++++++++++++++--- tests/ci-stall-watch.test.ts | 18 ++++++++++++++++- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/reference/testing.md b/reference/testing.md index 27d0e60472..8bd619552b 100644 --- a/reference/testing.md +++ b/reference/testing.md @@ -46,24 +46,6 @@ A bare `node --import tsx --test ...` still works (the storage family pins its own dirs), but any suite that relies on DEFAULT path resolution will write into your real machine state — prefer the preload chain. -## CI: stall watch (#69) - -Every phase `scripts/run-ci-tests.mjs` starts (the main run, the full retry and -the isolated re-runs) runs under `scripts/ci-stall-watch.mjs`. A phase that -prints nothing on stdout and stderr for 10 minutes is treated as stalled: the -runner lists the processes still alive under it (a finished test file's process -is gone, so the list names the stuck file and anything it left running), kills -the whole process tree, and fails the job. A stall is never retried and never -reaches the failure-shape classifier — it stays a red job until its cause is -known. `node --test` holds back later files' output until the file at the head -of its order finishes, which is why one hung file used to silence a whole shard -until the 90-minute job timeout (#61). - -The limit is a constant, not a setting. It was chosen against the longest quiet -gap seen in 20 passing windows-latest / Node 24 shard jobs on 2026-11-14 -(238.5 s). A hang that keeps printing is not detected. When a job fails with -`STALL:` lines, copy the process list to the issue tracking that stall. - ## CI: test schedule (#2816) The main CI `test` job does not run the flat `npm test` glob — it runs @@ -102,6 +84,27 @@ The `static-gates` job (same os×node matrix) runs typecheck + kernel suite + pi-subagents suite + build IN PARALLEL with the `test` job; they used to be sequential steps ahead of the tests (~90s of critical path per leg). +## CI: stall watch (#69) + +Every phase `scripts/run-ci-tests.mjs` starts (the main run, the full retry and +the isolated re-runs) runs under `scripts/ci-stall-watch.mjs`. A phase that +prints nothing on stdout and stderr for 10 minutes is treated as stalled: the +runner lists the processes still alive under it (a finished test file's process +is gone, so the list usually names the stuck file and what it started), kills +that process tree, and fails the job. A stall is never retried and never +reaches the failure-shape classifier — it stays a red job until its cause is +known. `node --test` holds back later files' output until the file at the head +of its order finishes, which is why one hung file used to silence a whole shard +until the 90-minute job timeout (#61). + +The limit is a constant, not a setting. It was chosen against the longest quiet +gap seen in 20 passing windows-latest / Node 24 shard jobs on 2026-11-14 +(238.5 s). A hang that keeps printing is not detected. A process left behind by +a test file that has already exited is outside the phase's tree: the report +lists node processes with no living parent as a hint, names no file in that +case, and does not kill them. When a job fails with +`STALL:` lines, copy the process list to the issue tracking that stall. + ## Hermetic data roots (#5107) Running the suite on a machine that also has bili installed must never touch diff --git a/scripts/ci-stall-watch.d.mts b/scripts/ci-stall-watch.d.mts index 48fcc2af05..730eac708a 100644 --- a/scripts/ci-stall-watch.d.mts +++ b/scripts/ci-stall-watch.d.mts @@ -1,14 +1,14 @@ -export interface ProcRow { +interface ProcRow { pid: number; ppid: number; cmd: string; } -export interface ProcNode extends ProcRow { +interface ProcNode extends ProcRow { depth: number; } -export interface WatchedResult { +interface WatchedResult { code: number | null; signal: NodeJS.Signals | null; stalled: boolean; @@ -16,7 +16,7 @@ export interface WatchedResult { spawnError: Error | undefined; } -export interface WatchOptions { +interface WatchOptions { cwd?: string; env?: NodeJS.ProcessEnv; label?: string; @@ -32,6 +32,7 @@ export interface WatchOptions { export const STALL_LIMIT_MS: number; export function listProcesses(): ProcRow[]; export function descendantsOf(rootPid: number, rows: ProcRow[]): ProcNode[]; +export function orphansRunning(execPath: string, rows: ProcRow[]): ProcRow[]; export function testFilesIn(procs: ProcNode[]): string[]; export function killProcessTree(rootPid: number, known?: ProcNode[]): void; export function runWatched(command: string, args: string[], opts?: WatchOptions): Promise; diff --git a/scripts/ci-stall-watch.mjs b/scripts/ci-stall-watch.mjs index 9859f89173..eaaa6f5932 100644 --- a/scripts/ci-stall-watch.mjs +++ b/scripts/ci-stall-watch.mjs @@ -5,8 +5,10 @@ // // runWatched() spawns a phase and, when it prints nothing on stdout AND stderr // for `limitMs`, (1) reports the processes still alive under it — a finished -// test file's process is gone, so what remains is what is stuck — (2) ends the -// whole process tree, and (3) resolves with `stalled: true`. The caller fails +// test file's process is gone, so what remains is usually what is stuck — plus +// node processes whose parent has exited, (2) ends the phase's process tree, +// and (3) resolves with `stalled: true`. Limit: a process orphaned by a test +// file that already exited is outside the tree; it is listed, not killed. The caller fails // the job; a stall is never retried here (owner decision on #69: the stall // stays a red job while its cause is unknown). // @@ -81,6 +83,18 @@ export function descendantsOf(rootPid, rows) { return out; } +/** Processes whose parent is gone and whose command line runs `execPath`. + * A test file that exits while something it started still holds its output + * leaves exactly this: the phase stays silent, but the holder is no longer + * under the phase's tree (Windows keeps the dead parent's pid on it), so + * descendantsOf() cannot see it. Listed as a hint only — nothing proves these + * belong to this phase, so they are never killed. Pure. */ +export function orphansRunning(execPath, rows) { + const live = new Set(rows.map((r) => r.pid)); + const needle = execPath.toLowerCase(); + return rows.filter((r) => r.ppid > 0 && !live.has(r.ppid) && r.cmd.toLowerCase().includes(needle)); +} + /** Test files named on the command lines of the given processes. Pure. */ export function testFilesIn(procs) { const files = new Set(); @@ -168,7 +182,8 @@ export function runWatched(command, args, opts = {}) { ); let procs = []; try { - procs = descendantsOf(child.pid, list()); + const rows = list(); + procs = descendantsOf(child.pid, rows); log(`[ci-tests] STALL: ${procs.length} process(es) still alive under ${label} (pid ${child.pid}), taken before the kill:`); for (const p of procs) log(`[ci-tests] STALL: ${" ".repeat(p.depth)}pid=${p.pid} ppid=${p.ppid} ${tailOf(p.cmd)}`); const files = testFilesIn(procs); @@ -177,6 +192,14 @@ export function runWatched(command, args, opts = {}) { ? `[ci-tests] STALL: test file(s) still running: ${files.join(", ")}` : `[ci-tests] STALL: no test file could be read from the command lines above`, ); + const orphans = orphansRunning(process.execPath, rows); + if (orphans.length > 0) { + log( + `[ci-tests] STALL: ${orphans.length} node process(es) on this machine have no living parent — possibly left by a ` + + `test file that already exited and still holding its output; NOT killed, not proven to be this phase's:`, + ); + for (const p of orphans) log(`[ci-tests] STALL: pid=${p.pid} ppid=${p.ppid} (gone) ${tailOf(p.cmd)}`); + } } catch (e) { procs = []; log(`[ci-tests] STALL: process snapshot unavailable (${e && e.message ? e.message.split("\n")[0] : e}) — killing anyway`); diff --git a/tests/ci-stall-watch.test.ts b/tests/ci-stall-watch.test.ts index 6ee4fea5b5..4a51b90bfd 100644 --- a/tests/ci-stall-watch.test.ts +++ b/tests/ci-stall-watch.test.ts @@ -9,7 +9,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; -import { descendantsOf, runWatched, STALL_LIMIT_MS, testFilesIn } from "../scripts/ci-stall-watch.mjs"; +import { descendantsOf, orphansRunning, runWatched, STALL_LIMIT_MS, testFilesIn } from "../scripts/ci-stall-watch.mjs"; // Short stand-in for the 10-minute limit. It also has to cover a fixture's // start-up (spawn + node boot) on a loaded runner, since nothing is printed @@ -237,6 +237,22 @@ test("descendantsOf walks only the phase's own tree, parents first", () => { assert.deepEqual(descendantsOf(999, rows), [], "an unknown root has no tree"); }); +test("orphansRunning lists node processes whose parent is gone, and only those", () => { + const rows = [ + { pid: 10, ppid: 1, cmd: '"C:\\Node\\node.exe" --test tests/a.test.ts' }, + { pid: 11, ppid: 10, cmd: '"C:\\Node\\node.exe" tests/a.test.ts' }, + { pid: 30, ppid: 777, cmd: '"c:\\node\\NODE.EXE" dist/index.js start' }, + { pid: 31, ppid: 778, cmd: "C:\\Windows\\explorer.exe" }, + { pid: 1, ppid: 0, cmd: "System" }, + ]; + assert.deepEqual( + orphansRunning("C:\\Node\\node.exe", rows).map((r) => r.pid), + [30], + "living parent, other executables and the system root are left out", + ); + assert.deepEqual(orphansRunning("C:\\Node\\node.exe", rows.filter((r) => r.pid !== 30)), []); +}); + test("testFilesIn normalises Windows separators and skips ambiguous command lines", () => { assert.deepEqual( testFilesIn([ From 583d757b19f6a6dbc1cec1d50af0f1ffabd9cc38 Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 17:58:53 -0500 Subject: [PATCH 3/6] ci: test the stall gate through the runner; kill by process group on POSIX (#69) Review follow-up. - tests/ci-runner-stall-gate.test.ts runs a copy of run-ci-tests.mjs (limit shortened in the copy only) against a fixture suite: a stall in the main phase, an isolated re-run and the full retry each fail the job with no further re-run or retry; a clean run and a recoverable failure behave as before. - POSIX: the phase is started as its own process group and killed by group, so a failed `ps` no longer leaves descendants running. SIGINT/SIGTERM to the runner are passed on to that group. - Tests: teardown never signals a pid <= 0; a zombie is not counted as running on Linux. Assisted-by: Cindy (https://cindy.app) --- scripts/ci-stall-watch.mjs | 24 ++++- scripts/live-files.mjs | 2 + tests/ci-runner-stall-gate.test.ts | 157 +++++++++++++++++++++++++++++ tests/ci-stall-watch.test.ts | 62 ++++++------ 4 files changed, 209 insertions(+), 36 deletions(-) create mode 100644 tests/ci-runner-stall-gate.test.ts diff --git a/scripts/ci-stall-watch.mjs b/scripts/ci-stall-watch.mjs index eaaa6f5932..a2303ebad6 100644 --- a/scripts/ci-stall-watch.mjs +++ b/scripts/ci-stall-watch.mjs @@ -123,8 +123,11 @@ export function killProcessTree(rootPid, known = []) { } return; } - const pids = known.length > 0 ? known.map((p) => p.pid).reverse() : [rootPid]; - if (!pids.includes(rootPid)) pids.push(rootPid); + // runWatched starts the phase as the leader of its own process group, so + // the group kill reaches every descendant without needing the snapshot + // (`ps` can be missing or time out). Snapshot pids follow for anything + // that moved itself into another group. + const pids = [-rootPid, ...known.map((p) => p.pid).reverse(), rootPid]; for (const pid of pids) { try { process.kill(pid, "SIGKILL"); @@ -161,7 +164,20 @@ export function runWatched(command, args, opts = {}) { } = opts; const t0 = Date.now(); return new Promise((resolve) => { - const child = spawn(command, args, { cwd, env }); + // POSIX: own process group, so killProcessTree can end the whole tree + // by group. Windows needs nothing here (taskkill /T walks the tree). + const posix = process.platform !== "win32"; + const child = spawn(command, args, { cwd, env, detached: posix }); + // A group leader no longer receives the terminal's or the CI agent's + // signal with us: pass it on, then end the way the signal would have. + const onSignal = (sig) => { + killTree(child.pid, []); + process.exit(sig === "SIGINT" ? 130 : 143); + }; + if (posix) { + process.once("SIGINT", onSignal); + process.once("SIGTERM", onSignal); + } let stalled = false; let settled = false; let graceTimer; @@ -171,6 +187,8 @@ export function runWatched(command, args, opts = {}) { settled = true; clearTimeout(stallTimer); clearTimeout(graceTimer); + process.removeListener("SIGINT", onSignal); + process.removeListener("SIGTERM", onSignal); resolve({ spawnError: undefined, ...r, stalled, ms: Date.now() - t0 }); }; const onStall = () => { diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index e2cd7a7a94..764f687f16 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -120,6 +120,8 @@ export const LIVE_FILES = [ "tests/mcp-eof.test.ts", // #69: spawns real process trees to exercise the CI stall watch; no ports. "tests/ci-stall-watch.test.ts", + // #69: runs copies of run-ci-tests.mjs as subprocesses; no ports. + "tests/ci-runner-stall-gate.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/tests/ci-runner-stall-gate.test.ts b/tests/ci-runner-stall-gate.test.ts new file mode 100644 index 0000000000..a047c043a6 --- /dev/null +++ b/tests/ci-runner-stall-gate.test.ts @@ -0,0 +1,157 @@ +// #69: the stall gate in scripts/run-ci-tests.mjs itself. The runner has +// top-level side effects and cannot be imported, so each case runs a COPY of +// it (and of the modules it imports) as a subprocess against a two-file +// fixture suite. The only edit to the copy is the stall limit, shortened so a +// stall takes seconds; the gate, the classifier and the retry are the real +// code. A stall in ANY phase must fail the job at once — no isolated re-run, +// no retry, no pass — while a clean run and a recoverable failure behave as +// before. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const REPO = process.cwd(); +const LIMIT_MS = 4000; + +/** What tests/flaky.test.ts does on each successive run of the file. */ +type Step = "pass" | "hang" | "die"; + +function makeSuite(steps: Step[]): string { + // Under the repo so `--import tsx` resolves from the copy's cwd. + mkdirSync(join(REPO, ".tmp"), { recursive: true }); + const dir = mkdtempSync(join(REPO, ".tmp", "stall-gate-")); + mkdirSync(join(dir, "scripts")); + mkdirSync(join(dir, "tests")); + for (const f of ["run-ci-tests.mjs", "ci-failure-shapes.mjs", "ci-shard-partition.mjs", "ci-shard-weights.json"]) { + copyFileSync(join(REPO, "scripts", f), join(dir, "scripts", f)); + } + const watch = readFileSync(join(REPO, "scripts", "ci-stall-watch.mjs"), "utf8"); + const shortened = watch.replace("export const STALL_LIMIT_MS = 10 * 60 * 1000;", `export const STALL_LIMIT_MS = ${LIMIT_MS};`); + assert.notEqual(shortened, watch, "the limit constant was found and shortened in the copy"); + writeFileSync(join(dir, "scripts", "ci-stall-watch.mjs"), shortened); + writeFileSync(join(dir, "scripts", "live-files.mjs"), "export const LIVE_FILES = [];\n"); + for (const f of ["preload-storage-baseline.mjs", "preload-test-env.mjs", "preload-zone-slot.mjs"]) { + writeFileSync(join(dir, "tests", f), "\n"); + } + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name: "stall-gate-fixture", private: true, type: "module", scripts: { pretest: 'node -e ""' } }), + ); + writeFileSync(join(dir, "plan.json"), JSON.stringify({ steps, runs: 0 })); + writeFileSync(join(dir, "tests", "always.test.ts"), 'import { test } from "node:test";\ntest("always passes", () => {});\n'); + writeFileSync( + join(dir, "tests", "flaky.test.ts"), + [ + 'import { test } from "node:test";', + 'import { readFileSync, writeFileSync } from "node:fs";', + 'const planFile = new URL("../plan.json", import.meta.url);', + 'const plan = JSON.parse(readFileSync(planFile, "utf8"));', + 'const step: string = plan.steps[plan.runs] ?? "pass";', + "plan.runs += 1;", + "writeFileSync(planFile, JSON.stringify(plan));", + "// A silent death before any test reports: the #3029 swallowed shape,", + "// which the runner re-runs in isolation and then retries.", + 'if (step === "die") process.exit(1);', + 'test("flaky step", async () => {', + ' if (step === "hang") await new Promise(() => void setInterval(() => {}, 1000));', + "});", + "", + ].join("\n"), + ); + return dir; +} + +function runRunner(dir: string): Promise<{ code: number | null; out: string; runs: number }> { + const env = { ...process.env }; + // The copy must run its own two files as a plain top-level `node --test`: + // not this job's shard slice, and not as a nested child of this test run. + delete env.CI_TEST_SHARD; + delete env.NODE_TEST_CONTEXT; + delete env.GITHUB_EVENT_NAME; + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["scripts/run-ci-tests.mjs"], { cwd: dir, env }); + let out = ""; + child.stdout.on("data", (d) => (out += d.toString())); + child.stderr.on("data", (d) => (out += d.toString())); + const deadline = setTimeout(() => { + child.kill(); + reject(new Error(`runner copy did not finish in 120s. Output so far:\n${out}`)); + }, 120_000); + child.on("error", reject); + child.on("close", (code) => { + clearTimeout(deadline); + resolve({ code, out, runs: JSON.parse(readFileSync(join(dir, "plan.json"), "utf8")).runs }); + }); + }); +} + +const RETRY = /retrying full phase once/; +const ISOLATED = /DIAGNOSTIC isolated re-run/; +const STALL = /STALL/; + +interface Case { + name: string; + steps: Step[]; + code: number; + runs: number; + has: RegExp[]; + lacks: RegExp[]; +} + +const CASES: Case[] = [ + { + name: "a stall in the main phase fails the job with no isolated re-run and no retry", + steps: ["hang"], + code: 1, + runs: 1, + has: [/::error::\[ci-tests\] STALL: all printed nothing for 4s/, /test file\(s\) still running: tests\/flaky\.test\.ts/, /\[ci-tests\] all STALLED .* failing WITHOUT retry/], + lacks: [ISOLATED, RETRY, /FLAKE CONFIRMED/, /all ok/], + }, + { + name: "a stall in an isolated re-run fails the job with no retry", + steps: ["die", "hang"], + code: 1, + runs: 2, + has: [/swallowed file-level failure/, ISOLATED, /\[ci-tests\] isolated re-run tests\/flaky\.test\.ts STALLED .* failing WITHOUT retry/], + lacks: [RETRY, /FLAKE CONFIRMED/, /PASSED in isolation|REPRODUCED in isolation/], + }, + { + name: "a stall in the full retry fails the job and is not read as a retry result", + steps: ["die", "die", "hang"], + code: 1, + runs: 3, + has: [ISOLATED, RETRY, /\[ci-tests\] all \(retry\) STALLED .* failing WITHOUT retry/], + lacks: [/FLAKE CONFIRMED/, /full retry also FAILED/, /DRAIN-PASS/], + }, + { + name: "control: a clean run passes and reports no stall", + steps: ["pass"], + code: 0, + runs: 1, + has: [/\[ci-tests\] all ok in/], + lacks: [STALL, ISOLATED, RETRY], + }, + { + name: "control: a recoverable failure is still re-run, retried and passed, with no stall", + steps: ["die", "pass", "pass"], + code: 0, + runs: 3, + has: [/swallowed file-level failure/, /PASSED in isolation/, RETRY, /FLAKE CONFIRMED/], + lacks: [STALL], + }, +]; + +for (const c of CASES) { + test(c.name, async (t) => { + const dir = makeSuite(c.steps); + t.after(() => rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 })); + const r = await runRunner(dir); + const ctx = `\n--- runner output ---\n${r.out}`; + assert.equal(r.code, c.code, `exit code${ctx}`); + assert.equal(r.runs, c.runs, `how many times the flaky file was started${ctx}`); + for (const re of c.has) assert.match(r.out, re, `expected ${re}${ctx}`); + for (const re of c.lacks) assert.doesNotMatch(r.out, re, `unexpected ${re}${ctx}`); + }); +} diff --git a/tests/ci-stall-watch.test.ts b/tests/ci-stall-watch.test.ts index 4a51b90bfd..da522cce2d 100644 --- a/tests/ci-stall-watch.test.ts +++ b/tests/ci-stall-watch.test.ts @@ -5,7 +5,7 @@ // what happens to a process tree, which a stub cannot show. import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; @@ -19,10 +19,31 @@ const STALL_AFTER_MS = 3000; function alive(pid: number): boolean { try { process.kill(pid, 0); - return true; } catch { return false; } + if (process.platform === "linux") { + // kill(pid, 0) also succeeds for a zombie: a killed process nobody has + // reaped yet (a container whose PID 1 does not reap). It is not running. + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + if (stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3) === "Z") return false; + } catch { + return false; + } + } + return true; +} + +/** Teardown kill. Never signals pid 0 or below: on POSIX that addresses a + * whole process group, which would take the test runner with it. */ +function killQuietly(pid: number): void { + if (!Number.isInteger(pid) || pid <= 0) return; + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone — the expected case + } } async function waitDead(pids: number[], ms = 10_000): Promise { @@ -77,15 +98,7 @@ test("a silent phase is reported, killed as a tree and flagged stalled", async ( log: (l) => logs.push(l), }); const [rootPid, kidPid] = pidsFrom(out); - t.after(() => { - for (const pid of [kidPid, rootPid]) { - try { - process.kill(pid, "SIGKILL"); - } catch { - // already gone — the expected case - } - } - }); + t.after(() => [kidPid, rootPid].forEach(killQuietly)); const report = logs.join("\n"); assert.equal(r.stalled, true, "flagged stalled"); @@ -171,37 +184,20 @@ test("a failed process snapshot is reported and the phase is still killed and fl }, }); const pids = pidsFrom(out); - t.after(() => { - for (const pid of pids) { - try { - process.kill(pid, "SIGKILL"); - } catch { - // already gone - } - } - }); + t.after(() => pids.forEach(killQuietly)); const report = logs.join("\n"); assert.equal(r.stalled, true); assert.match(report, /process snapshot unavailable \(query refused\) — killing anyway/); assert.doesNotMatch(report, /still running:/, "claims nothing about what was running"); - if (process.platform === "win32") { - // taskkill /T does not need the snapshot. - assert.deepEqual(await waitDead(pids), [], "whole tree ended without a snapshot"); - } else { - assert.equal(alive(pids[0]), false, "the phase root ended without a snapshot"); - } + // taskkill /T on Windows and the process-group kill elsewhere: neither + // needs the snapshot. + assert.deepEqual(await waitDead(pids), [], "whole tree ended without a snapshot"); }); test("a phase whose streams stay open after the kill is abandoned after the grace, still flagged stalled", async (t) => { const logs: string[] = []; let pid = 0; - t.after(() => { - try { - process.kill(pid, "SIGKILL"); - } catch { - // already gone - } - }); + t.after(() => killQuietly(pid)); const r = await runWatched(process.execPath, ["-e", "console.log(process.pid); setInterval(() => {}, 1000)"], { limitMs: STALL_AFTER_MS, closeGraceMs: 500, From e39d7108467440c9ab0d980626ee336b2aad9780 Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 18:10:14 -0500 Subject: [PATCH 4/6] ci: reach nested process groups on cancellation; settle the runner fixture once (#69) Review follow-up. - POSIX: on SIGINT/SIGTERM the watch takes a best-effort snapshot before the group kill, so a descendant that started its own group is ended by pid. - tests/ci-runner-stall-gate.test.ts: the runner-copy promise settles only after the child closed and never reads the fixture after a timeout. Assisted-by: Cindy (https://cindy.app) --- scripts/ci-stall-watch.mjs | 11 ++++++++++- tests/ci-runner-stall-gate.test.ts | 24 +++++++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/scripts/ci-stall-watch.mjs b/scripts/ci-stall-watch.mjs index a2303ebad6..5946a821ae 100644 --- a/scripts/ci-stall-watch.mjs +++ b/scripts/ci-stall-watch.mjs @@ -170,8 +170,17 @@ export function runWatched(command, args, opts = {}) { const child = spawn(command, args, { cwd, env, detached: posix }); // A group leader no longer receives the terminal's or the CI agent's // signal with us: pass it on, then end the way the signal would have. + // Best-effort snapshot first: a descendant that started its own group + // (a nested watch) is outside ours and is only reachable by pid. If the + // snapshot fails, the group kill still runs. const onSignal = (sig) => { - killTree(child.pid, []); + let known = []; + try { + known = descendantsOf(child.pid, list()); + } catch { + known = []; + } + killTree(child.pid, known); process.exit(sig === "SIGINT" ? 130 : 143); }; if (posix) { diff --git a/tests/ci-runner-stall-gate.test.ts b/tests/ci-runner-stall-gate.test.ts index a047c043a6..f46bfd7243 100644 --- a/tests/ci-runner-stall-gate.test.ts +++ b/tests/ci-runner-stall-gate.test.ts @@ -75,14 +75,32 @@ function runRunner(dir: string): Promise<{ code: number | null; out: string; run let out = ""; child.stdout.on("data", (d) => (out += d.toString())); child.stderr.on("data", (d) => (out += d.toString())); + // Settles once, and only after the child has closed: the caller's + // teardown deletes the fixture directory, so nothing may read it + // after this promise settles. + let timedOut = false; + let spawnError: Error | undefined; const deadline = setTimeout(() => { + timedOut = true; child.kill(); - reject(new Error(`runner copy did not finish in 120s. Output so far:\n${out}`)); }, 120_000); - child.on("error", reject); + child.on("error", (e) => { + spawnError = e; + // A failed spawn emits no 'close'. + if (child.pid === undefined) { + clearTimeout(deadline); + reject(e); + } + }); child.on("close", (code) => { clearTimeout(deadline); - resolve({ code, out, runs: JSON.parse(readFileSync(join(dir, "plan.json"), "utf8")).runs }); + if (spawnError) return reject(spawnError); + if (timedOut) return reject(new Error(`runner copy did not finish in 120s. Output so far:\n${out}`)); + try { + resolve({ code, out, runs: JSON.parse(readFileSync(join(dir, "plan.json"), "utf8")).runs }); + } catch (e) { + reject(e); + } }); }); } From b13a359a33454ae6629ff2f7f900c6cf112de22f Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 18:25:35 -0500 Subject: [PATCH 5/6] ci: bound the runner fixture's deadline when output pipes stay open (#69) At the deadline the fixture helper now kills the whole process tree (its own process group on POSIX) instead of only the runner copy, and if 'close' still has not arrived after a grace period it abandons the streams and rejects anyway. Settlement stays single: nothing reads the fixture directory after the promise settles. Tests: a pipe-holding child is killed with its parent; a holder that survives the kill cannot keep the wait open; a command that finishes in time resolves normally. Assisted-by: Cindy (https://cindy.app) --- tests/ci-runner-stall-gate.test.ts | 196 ++++++++++++++++++++++++----- 1 file changed, 166 insertions(+), 30 deletions(-) diff --git a/tests/ci-runner-stall-gate.test.ts b/tests/ci-runner-stall-gate.test.ts index f46bfd7243..2436815225 100644 --- a/tests/ci-runner-stall-gate.test.ts +++ b/tests/ci-runner-stall-gate.test.ts @@ -11,6 +11,7 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { descendantsOf, killProcessTree, listProcesses } from "../scripts/ci-stall-watch.mjs"; const REPO = process.cwd(); const LIMIT_MS = 4000; @@ -63,48 +64,87 @@ function makeSuite(steps: Step[]): string { return dir; } -function runRunner(dir: string): Promise<{ code: number | null; out: string; runs: number }> { - const env = { ...process.env }; - // The copy must run its own two files as a plain top-level `node --test`: - // not this job's shard slice, and not as a nested child of this test run. - delete env.CI_TEST_SHARD; - delete env.NODE_TEST_CONTEXT; - delete env.GITHUB_EVENT_NAME; +interface Finished { + code: number | null; + out: string; +} + +interface RunOptions { + cwd: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + graceMs?: number; + /** Ends the process tree at the deadline. Injectable so a test can leave + * the tree alive and prove the fallback settlement. */ + kill?: (pid: number) => void; +} + +function killTree(pid: number): void { + let known: ReturnType = []; + try { + known = descendantsOf(pid, listProcesses()); + } catch { + // no snapshot: the tree kill below still reaches the group / the tree + } + killProcessTree(pid, known); +} + +/** Runs a command to its end and settles exactly once. Normally that is the + * child's 'close'. At the deadline the whole tree is killed; if 'close' still + * has not come graceMs later (something the kill did not reach holds the + * output pipes), the streams are abandoned and the promise rejects anyway, so + * the wait is bounded. Nothing runs after settlement: the caller's teardown + * deletes the fixture directory. */ +function runToClose(command: string, args: string[], opts: RunOptions): Promise { + const { cwd, env = process.env, timeoutMs = 120_000, graceMs = 15_000, kill = killTree } = opts; return new Promise((resolve, reject) => { - const child = spawn(process.execPath, ["scripts/run-ci-tests.mjs"], { cwd: dir, env }); + // Its own process group on POSIX, so the deadline kill reaches every + // descendant, including ones started before the runner's own handlers. + const child = spawn(command, args, { cwd, env, detached: process.platform !== "win32" }); let out = ""; child.stdout.on("data", (d) => (out += d.toString())); child.stderr.on("data", (d) => (out += d.toString())); - // Settles once, and only after the child has closed: the caller's - // teardown deletes the fixture directory, so nothing may read it - // after this promise settles. + let settled = false; let timedOut = false; - let spawnError: Error | undefined; + let fallback: NodeJS.Timeout | undefined; + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(deadline); + clearTimeout(fallback); + fn(); + }; + const timeoutError = (how: string) => new Error(`command did not finish in ${timeoutMs}ms (${how}). Output so far:\n${out}`); const deadline = setTimeout(() => { timedOut = true; - child.kill(); - }, 120_000); - child.on("error", (e) => { - spawnError = e; - // A failed spawn emits no 'close'. - if (child.pid === undefined) { - clearTimeout(deadline); - reject(e); - } - }); + if (child.pid !== undefined) kill(child.pid); + fallback = setTimeout(() => { + child.stdin?.destroy(); + child.stdout.destroy(); + child.stderr.destroy(); + child.unref(); + settle(() => reject(timeoutError(`output pipes still open ${graceMs}ms after the kill; abandoned`))); + }, graceMs); + }, timeoutMs); + child.on("error", (e) => settle(() => reject(e))); child.on("close", (code) => { - clearTimeout(deadline); - if (spawnError) return reject(spawnError); - if (timedOut) return reject(new Error(`runner copy did not finish in 120s. Output so far:\n${out}`)); - try { - resolve({ code, out, runs: JSON.parse(readFileSync(join(dir, "plan.json"), "utf8")).runs }); - } catch (e) { - reject(e); - } + if (timedOut) return settle(() => reject(timeoutError("killed"))); + settle(() => resolve({ code, out })); }); }); } +async function runRunner(dir: string): Promise { + const env = { ...process.env }; + // The copy must run its own two files as a plain top-level node --test: + // not this job's shard slice, and not as a nested child of this test run. + delete env.CI_TEST_SHARD; + delete env.NODE_TEST_CONTEXT; + delete env.GITHUB_EVENT_NAME; + const r = await runToClose(process.execPath, ["scripts/run-ci-tests.mjs"], { cwd: dir, env }); + return { ...r, runs: JSON.parse(readFileSync(join(dir, "plan.json"), "utf8")).runs }; +} + const RETRY = /retrying full phase once/; const ISOLATED = /DIAGNOSTIC isolated re-run/; const STALL = /STALL/; @@ -173,3 +213,99 @@ for (const c of CASES) { for (const re of c.lacks) assert.doesNotMatch(r.out, re, `unexpected ${re}${ctx}`); }); } + +// The deadline in runToClose, which every case above relies on to stay bounded. +// The fixture is the shape that used to hang it: a parent whose child inherits +// the output pipes and outlives it. + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + } catch { + return false; + } + if (process.platform === "linux") { + // kill(pid, 0) also succeeds for a zombie, which is not running. + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + if (stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3) === "Z") return false; + } catch { + return false; + } + } + return true; +} + +async function waitDead(pids: number[], ms = 10_000): Promise { + const end = Date.now() + ms; + for (;;) { + const left = pids.filter(alive); + if (left.length === 0 || Date.now() > end) return left; + await new Promise((r) => setTimeout(r, 100)); + } +} + +function pipeHolder(): { dir: string; script: string } { + mkdirSync(join(REPO, ".tmp"), { recursive: true }); + const dir = mkdtempSync(join(REPO, ".tmp", "stall-gate-pipes-")); + writeFileSync(join(dir, "holder.mjs"), "setInterval(() => {}, 1000);\n"); + const script = join(dir, "parent.mjs"); + writeFileSync( + script, + [ + 'import { spawn } from "node:child_process";', + 'const kid = spawn(process.execPath, ["holder.mjs"], { stdio: "inherit" });', + "console.log(`pids ${process.pid} ${kid.pid}`);", + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + ); + return { dir, script }; +} + +function pidsIn(message: string): number[] { + const m = message.match(/pids (\d+) (\d+)/); + assert.ok(m, `the fixture printed its pids (got: ${JSON.stringify(message)})`); + return [Number(m[1]), Number(m[2])]; +} + +test("the deadline kills the whole tree, including a child holding the output pipes", async (t) => { + const { dir, script } = pipeHolder(); + t.after(() => rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 })); + const t0 = Date.now(); + const err = await runToClose(process.execPath, [script], { cwd: dir, timeoutMs: 2000, graceMs: 30_000 }).then( + () => undefined, + (e: Error) => e, + ); + assert.ok(err, "a run past its deadline rejects"); + assert.match(err.message, /did not finish in 2000ms \(killed\)/); + assert.ok(Date.now() - t0 < 20_000, "settled on the kill, well inside the grace"); + assert.deepEqual(await waitDead(pidsIn(err.message)), [], "parent and pipe-holding child are both gone"); +}); + +test("a pipe holder that survives the kill cannot keep the wait open", async (t) => { + const { dir, script } = pipeHolder(); + let pids: number[] = []; + t.after(async () => { + for (const pid of pids) if (pid > 0) killProcessTree(pid); + await waitDead(pids); + rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 }); + }); + const t0 = Date.now(); + // A kill that reaches nothing: both processes keep the pipes open. + const err = await runToClose(process.execPath, [script], { cwd: dir, timeoutMs: 2000, graceMs: 1500, kill: () => {} }).then( + () => undefined, + (e: Error) => e, + ); + assert.ok(err, "the wait ends even though 'close' never came"); + pids = pidsIn(err.message); + assert.match(err.message, /output pipes still open 1500ms after the kill; abandoned/); + assert.ok(Date.now() - t0 < 20_000, "bounded by deadline plus grace"); + assert.ok(pids.every(alive), "the fixture really was still running when the wait ended"); +}); + +test("control: a command that finishes in time resolves with its exit code and output", async () => { + const r = await runToClose(process.execPath, ["-e", "console.log('done'); process.exit(3)"], { cwd: REPO, timeoutMs: 30_000 }); + assert.equal(r.code, 3); + assert.match(r.out, /done/); +}); From 5cbc93affa58b00e739154de0981b5a53358dab9 Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 18:38:25 -0500 Subject: [PATCH 6/6] ci: runner fixture helper cleans up on cancellation and without pid output (#69) On POSIX the helper's child runs in its own process group, so SIGINT/SIGTERM to the test process now kills that tree before exiting; the listeners are removed on settlement. The no-op-kill test records the root pid from the kill callback, so teardown reaches the fixture even if its pids were never printed. Assisted-by: Cindy (https://cindy.app) --- tests/ci-runner-stall-gate.test.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/ci-runner-stall-gate.test.ts b/tests/ci-runner-stall-gate.test.ts index 2436815225..2f63d0c793 100644 --- a/tests/ci-runner-stall-gate.test.ts +++ b/tests/ci-runner-stall-gate.test.ts @@ -112,8 +112,22 @@ function runToClose(command: string, args: string[], opts: RunOptions): Promise< settled = true; clearTimeout(deadline); clearTimeout(fallback); + for (const [sig, h] of onSignal) process.removeListener(sig, h); fn(); }; + // The child is outside the caller's process group on POSIX, so a + // cancelled run (Ctrl+C on a targeted run) must take it down itself. + const onSignal = new Map void>(); + if (process.platform !== "win32") { + for (const [sig, exitCode] of [["SIGINT", 130], ["SIGTERM", 143]] as const) { + const h = () => { + if (child.pid !== undefined) killTree(child.pid); + process.exit(exitCode); + }; + onSignal.set(sig, h); + process.once(sig, h); + } + } const timeoutError = (how: string) => new Error(`command did not finish in ${timeoutMs}ms (${how}). Output so far:\n${out}`); const deadline = setTimeout(() => { timedOut = true; @@ -286,18 +300,23 @@ test("the deadline kills the whole tree, including a child holding the output pi test("a pipe holder that survives the kill cannot keep the wait open", async (t) => { const { dir, script } = pipeHolder(); let pids: number[] = []; + // Known from the kill callback, not from the fixture's output: teardown + // must reach the tree even if the pids were never printed. + let root = 0; t.after(async () => { + if (root > 0) killTree(root); for (const pid of pids) if (pid > 0) killProcessTree(pid); await waitDead(pids); rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 }); }); const t0 = Date.now(); // A kill that reaches nothing: both processes keep the pipes open. - const err = await runToClose(process.execPath, [script], { cwd: dir, timeoutMs: 2000, graceMs: 1500, kill: () => {} }).then( + const err = await runToClose(process.execPath, [script], { cwd: dir, timeoutMs: 2000, graceMs: 1500, kill: (pid) => void (root = pid) }).then( () => undefined, (e: Error) => e, ); assert.ok(err, "the wait ends even though 'close' never came"); + assert.ok(root > 0, "the deadline handed the root pid to the kill"); pids = pidsIn(err.message); assert.match(err.message, /output pipes still open 1500ms after the kill; abandoned/); assert.ok(Date.now() - t0 < 20_000, "bounded by deadline plus grace");