diff --git a/reference/testing.md b/reference/testing.md index 580ef7a06e..8bd619552b 100644 --- a/reference/testing.md +++ b/reference/testing.md @@ -84,6 +84,27 @@ The `static-gates` job (same os×node matrix) runs typecheck + kernel suite + pi-subagents suite + build IN PARALLEL with the `test` job; they used to be sequential steps ahead of the tests (~90s of critical path per leg). +## CI: stall watch (#69) + +Every phase `scripts/run-ci-tests.mjs` starts (the main run, the full retry and +the isolated re-runs) runs under `scripts/ci-stall-watch.mjs`. A phase that +prints nothing on stdout and stderr for 10 minutes is treated as stalled: the +runner lists the processes still alive under it (a finished test file's process +is gone, so the list usually names the stuck file and what it started), kills +that process tree, and fails the job. A stall is never retried and never +reaches the failure-shape classifier — it stays a red job until its cause is +known. `node --test` holds back later files' output until the file at the head +of its order finishes, which is why one hung file used to silence a whole shard +until the 90-minute job timeout (#61). + +The limit is a constant, not a setting. It was chosen against the longest quiet +gap seen in 20 passing windows-latest / Node 24 shard jobs on 2026-11-14 +(238.5 s). A hang that keeps printing is not detected. A process left behind by +a test file that has already exited is outside the phase's tree: the report +lists node processes with no living parent as a hint, names no file in that +case, and does not kill them. When a job fails with +`STALL:` lines, copy the process list to the issue tracking that stall. + ## Hermetic data roots (#5107) Running the suite on a machine that also has bili installed must never touch diff --git a/scripts/ci-stall-watch.d.mts b/scripts/ci-stall-watch.d.mts new file mode 100644 index 0000000000..730eac708a --- /dev/null +++ b/scripts/ci-stall-watch.d.mts @@ -0,0 +1,38 @@ +interface ProcRow { + pid: number; + ppid: number; + cmd: string; +} + +interface ProcNode extends ProcRow { + depth: number; +} + +interface WatchedResult { + code: number | null; + signal: NodeJS.Signals | null; + stalled: boolean; + ms: number; + spawnError: Error | undefined; +} + +interface WatchOptions { + cwd?: string; + env?: NodeJS.ProcessEnv; + label?: string; + limitMs?: number; + closeGraceMs?: number; + onStdout?: (chunk: Buffer) => void; + onStderr?: (chunk: Buffer) => void; + log?: (line: string) => void; + list?: () => ProcRow[]; + killTree?: (rootPid: number, known: ProcNode[]) => void; +} + +export const STALL_LIMIT_MS: number; +export function listProcesses(): ProcRow[]; +export function descendantsOf(rootPid: number, rows: ProcRow[]): ProcNode[]; +export function orphansRunning(execPath: string, rows: ProcRow[]): ProcRow[]; +export function testFilesIn(procs: ProcNode[]): string[]; +export function killProcessTree(rootPid: number, known?: ProcNode[]): void; +export function runWatched(command: string, args: string[], opts?: WatchOptions): Promise; diff --git a/scripts/ci-stall-watch.mjs b/scripts/ci-stall-watch.mjs new file mode 100644 index 0000000000..5946a821ae --- /dev/null +++ b/scripts/ci-stall-watch.mjs @@ -0,0 +1,267 @@ +// #69: stall watch for the CI test phases. `node --test` holds back the output +// of later files until the file at the head of its order finishes, so one hung +// test file makes the whole phase silent and the log never names it (#61: 40 +// and 22 minutes of silence on windows-latest / Node 24, cancelled by hand). +// +// runWatched() spawns a phase and, when it prints nothing on stdout AND stderr +// for `limitMs`, (1) reports the processes still alive under it — a finished +// test file's process is gone, so what remains is usually what is stuck — plus +// node processes whose parent has exited, (2) ends the phase's process tree, +// and (3) resolves with `stalled: true`. Limit: a process orphaned by a test +// file that already exited is outside the tree; it is listed, not killed. The caller fails +// the job; a stall is never retried here (owner decision on #69: the stall +// stays a red job while its cause is unknown). +// +// Lives in its own module because run-ci-tests.mjs has top-level side effects +// and cannot be imported by a test. The limit is a parameter, not a setting: +// no environment variable reads it. +import { execFileSync, spawn } from "node:child_process"; + +// 10 minutes (owner decision on #69). Longest gap between two output lines in +// 20 passing windows-latest / Node 24 shard jobs on 2026-11-14: 238.5 s. +export const STALL_LIMIT_MS = 10 * 60 * 1000; + +const QUERY_TIMEOUT_MS = 30_000; +const TAIL_BYTES = 2048; +const TEST_FILE_RE = /[^\s"']+\.test\.[cm]?[jt]sx?/g; + +/** Every process on the machine as { pid, ppid, cmd }. Throws when the query + * fails or exceeds its own deadline — the caller must treat that as "no + * snapshot", never as "nothing is running". */ +export function listProcesses() { + if (process.platform === "win32") { + const json = execFileSync( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + "Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CommandLine | ConvertTo-Json -Compress", + ], + { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: QUERY_TIMEOUT_MS, windowsHide: true }, + ); + const rows = JSON.parse(json); + return (Array.isArray(rows) ? rows : [rows]).map((r) => ({ + pid: Number(r.ProcessId), + ppid: Number(r.ParentProcessId), + cmd: String(r.CommandLine ?? ""), + })); + } + const text = execFileSync("ps", ["-eo", "pid=,ppid=,args="], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + timeout: QUERY_TIMEOUT_MS, + }); + const rows = []; + for (const line of text.split("\n")) { + const m = line.match(/^\s*(\d+)\s+(\d+)\s+(.*)$/); + if (m) rows.push({ pid: Number(m[1]), ppid: Number(m[2]), cmd: m[3] }); + } + return rows; +} + +/** The root and its descendants, parents before children. Pure. */ +export function descendantsOf(rootPid, rows) { + const byParent = new Map(); + for (const r of rows) { + if (!byParent.has(r.ppid)) byParent.set(r.ppid, []); + byParent.get(r.ppid).push(r); + } + const out = []; + const seen = new Set(); + const walk = (pid, depth) => { + for (const r of byParent.get(pid) ?? []) { + if (seen.has(r.pid) || r.pid === rootPid) continue; // pid reuse can fake a cycle + seen.add(r.pid); + out.push({ ...r, depth }); + walk(r.pid, depth + 1); + } + }; + const root = rows.find((r) => r.pid === rootPid); + if (root) out.push({ ...root, depth: 0 }); + walk(rootPid, 1); + return out; +} + +/** Processes whose parent is gone and whose command line runs `execPath`. + * A test file that exits while something it started still holds its output + * leaves exactly this: the phase stays silent, but the holder is no longer + * under the phase's tree (Windows keeps the dead parent's pid on it), so + * descendantsOf() cannot see it. Listed as a hint only — nothing proves these + * belong to this phase, so they are never killed. Pure. */ +export function orphansRunning(execPath, rows) { + const live = new Set(rows.map((r) => r.pid)); + const needle = execPath.toLowerCase(); + return rows.filter((r) => r.ppid > 0 && !live.has(r.ppid) && r.cmd.toLowerCase().includes(needle)); +} + +/** Test files named on the command lines of the given processes. Pure. */ +export function testFilesIn(procs) { + const files = new Set(); + for (const p of procs) { + // A file-level child carries exactly one test file. A command line + // with several (the phase root's shard list) names nothing. + const hits = p.cmd.match(TEST_FILE_RE) ?? []; + if (hits.length === 1) files.add(hits[0].replace(/\\/g, "/")); + } + return [...files]; +} + +/** Ends the root and everything under it. `known` is the pre-kill snapshot + * (may be empty when the snapshot failed). Never throws. */ +export function killProcessTree(rootPid, known = []) { + if (process.platform === "win32") { + try { + execFileSync("taskkill", ["/PID", String(rootPid), "/T", "/F"], { + stdio: "ignore", + timeout: QUERY_TIMEOUT_MS, + windowsHide: true, + }); + } catch { + // Already gone, or taskkill itself failed: the close-grace in + // runWatched still resolves the phase. + } + return; + } + // runWatched starts the phase as the leader of its own process group, so + // the group kill reaches every descendant without needing the snapshot + // (`ps` can be missing or time out). Snapshot pids follow for anything + // that moved itself into another group. + const pids = [-rootPid, ...known.map((p) => p.pid).reverse(), rootPid]; + for (const pid of pids) { + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone + } + } +} + +function tailOf(cmd, n = 240) { + const flat = cmd.replace(/\s+/g, " ").trim(); + return flat.length <= n ? flat : `…${flat.slice(-n)}`; +} + +/** + * Spawn `command args` and watch it for silence. + * + * Resolves with { code, signal, stalled, ms, spawnError }. `stalled: true` + * means the phase was ended by this watch; `code`/`signal` then describe the + * kill and MUST NOT be interpreted as a test result. + */ +export function runWatched(command, args, opts = {}) { + const { + cwd, + env, + label = "phase", + limitMs = STALL_LIMIT_MS, + closeGraceMs = 15_000, + onStdout = () => {}, + onStderr = () => {}, + log = (line) => console.error(line), + list = listProcesses, + killTree = killProcessTree, + } = opts; + const t0 = Date.now(); + return new Promise((resolve) => { + // POSIX: own process group, so killProcessTree can end the whole tree + // by group. Windows needs nothing here (taskkill /T walks the tree). + const posix = process.platform !== "win32"; + const child = spawn(command, args, { cwd, env, detached: posix }); + // A group leader no longer receives the terminal's or the CI agent's + // signal with us: pass it on, then end the way the signal would have. + // Best-effort snapshot first: a descendant that started its own group + // (a nested watch) is outside ours and is only reachable by pid. If the + // snapshot fails, the group kill still runs. + const onSignal = (sig) => { + let known = []; + try { + known = descendantsOf(child.pid, list()); + } catch { + known = []; + } + killTree(child.pid, known); + process.exit(sig === "SIGINT" ? 130 : 143); + }; + if (posix) { + process.once("SIGINT", onSignal); + process.once("SIGTERM", onSignal); + } + let stalled = false; + let settled = false; + let graceTimer; + let tail = Buffer.alloc(0); + const settle = (r) => { + if (settled) return; + settled = true; + clearTimeout(stallTimer); + clearTimeout(graceTimer); + process.removeListener("SIGINT", onSignal); + process.removeListener("SIGTERM", onSignal); + resolve({ spawnError: undefined, ...r, stalled, ms: Date.now() - t0 }); + }; + const onStall = () => { + if (settled) return; + stalled = true; + log( + `::error::[ci-tests] STALL: ${label} printed nothing for ${Math.round(limitMs / 1000)}s — ` + + `ending the phase and failing the job (#69)`, + ); + let procs = []; + try { + const rows = list(); + procs = descendantsOf(child.pid, rows); + log(`[ci-tests] STALL: ${procs.length} process(es) still alive under ${label} (pid ${child.pid}), taken before the kill:`); + for (const p of procs) log(`[ci-tests] STALL: ${" ".repeat(p.depth)}pid=${p.pid} ppid=${p.ppid} ${tailOf(p.cmd)}`); + const files = testFilesIn(procs); + log( + files.length > 0 + ? `[ci-tests] STALL: test file(s) still running: ${files.join(", ")}` + : `[ci-tests] STALL: no test file could be read from the command lines above`, + ); + const orphans = orphansRunning(process.execPath, rows); + if (orphans.length > 0) { + log( + `[ci-tests] STALL: ${orphans.length} node process(es) on this machine have no living parent — possibly left by a ` + + `test file that already exited and still holding its output; NOT killed, not proven to be this phase's:`, + ); + for (const p of orphans) log(`[ci-tests] STALL: pid=${p.pid} ppid=${p.ppid} (gone) ${tailOf(p.cmd)}`); + } + } catch (e) { + procs = []; + log(`[ci-tests] STALL: process snapshot unavailable (${e && e.message ? e.message.split("\n")[0] : e}) — killing anyway`); + } + const tailText = tail.toString("utf8").trimEnd(); + log(`[ci-tests] STALL: last output before the silence:\n${tailText === "" ? "(none)" : tailText}`); + killTree(child.pid, procs); + // A descendant that survived the kill keeps the pipes open and + // 'close' never fires: do not wait for it forever. + graceTimer = setTimeout(() => { + log(`[ci-tests] STALL: ${label} did not close within ${Math.round(closeGraceMs / 1000)}s of the kill — abandoning its streams`); + // Drop every handle on it so the abandoned phase cannot keep this + // process alive either. + child.stdin?.destroy(); + child.stdout?.destroy(); + child.stderr?.destroy(); + child.unref(); + settle({ code: null, signal: null }); + }, closeGraceMs); + }; + const stallTimer = setTimeout(onStall, limitMs); + const seen = (d) => { + if (!stalled) stallTimer.refresh(); + tail = Buffer.concat([tail, d]); + if (tail.length > TAIL_BYTES) tail = tail.subarray(tail.length - TAIL_BYTES); + }; + child.stdout.on("data", (d) => { + seen(d); + onStdout(d); + }); + child.stderr.on("data", (d) => { + seen(d); + onStderr(d); + }); + child.on("error", (e) => settle({ code: -1, signal: null, spawnError: e })); + child.on("close", (code, signal) => settle({ code, signal })); + }); +} diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index da1abd7521..764f687f16 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -118,6 +118,10 @@ export const LIVE_FILES = [ // the tripwire alone, like #3890/#5114 "tests/cache-keepalive.test.ts", "tests/mcp-eof.test.ts", + // #69: spawns real process trees to exercise the CI stall watch; no ports. + "tests/ci-stall-watch.test.ts", + // #69: runs copies of run-ci-tests.mjs as subprocesses; no ports. + "tests/ci-runner-stall-gate.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/scripts/run-ci-tests.mjs b/scripts/run-ci-tests.mjs index ec18706b71..3509ff936f 100644 --- a/scripts/run-ci-tests.mjs +++ b/scripts/run-ci-tests.mjs @@ -80,10 +80,18 @@ // own files (tests/issue760-percall-conversation.test.ts, // tests/storage-full-flip.test.ts) — classifier and widening are belt and // suspenders, not alternatives. -import { execFileSync, spawn } from "node:child_process"; +// +// #69: every phase (main, full retry, isolated re-runs) runs under the stall +// watch in ci-stall-watch.mjs. A phase that prints nothing for 10 minutes is +// reported (processes still alive under it), killed as a tree, and FAILS the +// job right there: it never reaches the classifier or the retry below, because +// a TAP stream cut off by the kill has no tail and must not be read as one of +// the recoverable shapes. A stall is not a flake until its cause is known (#61). +import { execFileSync } from "node:child_process"; import { LIVE_FILES } from "./live-files.mjs"; import { classifyPhaseFailure, describeDrainPass, describeRetryOutcome, extractDrainCancelledTests, shouldDrainPass } from "./ci-failure-shapes.mjs"; import { partitionAll as partitionFiles } from "./ci-shard-partition.mjs"; +import { runWatched } from "./ci-stall-watch.mjs"; import { readdirSync, readFileSync } from "node:fs"; import { freemem, loadavg, totalmem } from "node:os"; import { join } from "node:path"; @@ -187,48 +195,57 @@ const PRELOAD_ARGS = ["--import", "./tests/preload-storage-baseline.mjs", "--imp const MAX_CAPTURE = 32 * 1024 * 1024; // bound in-memory capture; real runs are MB-scale function runNodeTest(files, concurrency, label) { - const t0 = Date.now(); console.log(`[ci-tests] ${label}: ${files.length} files, concurrency=${concurrency}`); - return new Promise((resolve) => { - const child = spawn( - process.execPath, - [...PRELOAD_ARGS, "--test", "--test-concurrency", String(concurrency), "--test-reporter=tap", ...files], - { cwd: root }, - ); - const out = []; - const err = []; - let outLen = 0; - let errLen = 0; - child.stdout.on("data", (d) => { - if (outLen < MAX_CAPTURE) { - out.push(d); - outLen += d.length; - } - process.stdout.write(d); - }); - child.stderr.on("data", (d) => { - if (errLen < MAX_CAPTURE) { - err.push(d); - errLen += d.length; - } - process.stderr.write(d); - }); - child.on("error", (e) => { - console.error(`[ci-tests] ${label}: spawn error: ${e.message}`); - resolve({ code: -1, signal: null, out: "", err: "", ms: Date.now() - t0 }); - }); - child.on("close", (code, signal) => { - resolve({ - code, - signal, - out: Buffer.concat(out).toString("utf8"), - err: Buffer.concat(err).toString("utf8"), - ms: Date.now() - t0, - }); - }); + const out = []; + const err = []; + let outLen = 0; + let errLen = 0; + return runWatched( + process.execPath, + [...PRELOAD_ARGS, "--test", "--test-concurrency", String(concurrency), "--test-reporter=tap", ...files], + { + cwd: root, + label, + onStdout: (d) => { + if (outLen < MAX_CAPTURE) { + out.push(d); + outLen += d.length; + } + process.stdout.write(d); + }, + onStderr: (d) => { + if (errLen < MAX_CAPTURE) { + err.push(d); + errLen += d.length; + } + process.stderr.write(d); + }, + }, + ).then((r) => { + if (r.spawnError) console.error(`[ci-tests] ${label}: spawn error: ${r.spawnError.message}`); + return { + code: r.code, + signal: r.signal, + stalled: r.stalled, + out: r.spawnError ? "" : Buffer.concat(out).toString("utf8"), + err: r.spawnError ? "" : Buffer.concat(err).toString("utf8"), + ms: r.ms, + }; }); } +// #69: a stalled phase is terminal. Called before anything reads the phase's +// exit code or TAP, so a stall can neither be retried nor pass. +function failIfStalled(r, label) { + if (!r.stalled) return; + console.error( + `[ci-tests] ${label} STALLED after ${(r.ms / 1000).toFixed(1)}s and was killed -> failing WITHOUT retry (#69). ` + + `The process list above names what was still running; record it on the stall's issue (#61 for tests/dsh-native.test.ts).`, + ); + console.error(`[ci-tests] resource snapshot: ${resourceSnapshot()}`); + process.exit(1); +} + function resourceSnapshot() { const mi = (n) => `${Math.round(n / 1048576)}MiB`; return ( @@ -239,22 +256,21 @@ function resourceSnapshot() { } function runIsolated(fileRel) { - const t0 = Date.now(); console.log(`[ci-tests] DIAGNOSTIC isolated re-run (serial): ${fileRel} — full output below`); - return new Promise((resolve) => { - const child = spawn(process.execPath, [...PRELOAD_ARGS, "--test", "--test-reporter=tap", fileRel], { cwd: root }); - child.stdout.on("data", (d) => process.stdout.write(d)); - child.stderr.on("data", (d) => process.stderr.write(d)); - child.on("error", (e) => { - console.error(`[ci-tests] isolated re-run ${fileRel}: spawn error: ${e.message}`); - resolve({ code: -1, signal: null, ms: Date.now() - t0 }); - }); - child.on("close", (code, signal) => resolve({ code, signal, ms: Date.now() - t0 })); + return runWatched(process.execPath, [...PRELOAD_ARGS, "--test", "--test-reporter=tap", fileRel], { + cwd: root, + label: `isolated re-run ${fileRel}`, + onStdout: (d) => process.stdout.write(d), + onStderr: (d) => process.stderr.write(d), + }).then((r) => { + if (r.spawnError) console.error(`[ci-tests] isolated re-run ${fileRel}: spawn error: ${r.spawnError.message}`); + return { code: r.code, signal: r.signal, stalled: r.stalled, ms: r.ms }; }); } (async () => { const first = await runNodeTest(selectedFiles, 4, "all"); + failIfStalled(first, "all"); if (first.code === 0) { console.log(`[ci-tests] all ok in ${(first.ms / 1000).toFixed(1)}s`); return; @@ -289,6 +305,7 @@ function runIsolated(fileRel) { ); for (const rel of verdict.files) { const r = await runIsolated(rel); + failIfStalled(r, `isolated re-run ${rel}`); console.error( `[ci-tests] isolated re-run ${rel}: exit=${r.code} signal=${r.signal} (${(r.ms / 1000).toFixed(1)}s) -> ` + (r.code === 0 @@ -322,6 +339,7 @@ function runIsolated(fileRel) { const retryIssue = { swallowed: "#3029", "cancelled-drain": "#3665", "cleanup-lock": "#4347", "timing-window": "#4161" }[verdict.kind]; console.error(`[ci-tests] retrying full phase once to confirm flake (${retryIssue})`); const retry = await runNodeTest(selectedFiles, 4, "all (retry)"); + failIfStalled(retry, "all (retry)"); if (retry.code === 0) { const firstSummary = verdict.kind === "swallowed" diff --git a/tests/ci-runner-stall-gate.test.ts b/tests/ci-runner-stall-gate.test.ts new file mode 100644 index 0000000000..2f63d0c793 --- /dev/null +++ b/tests/ci-runner-stall-gate.test.ts @@ -0,0 +1,330 @@ +// #69: the stall gate in scripts/run-ci-tests.mjs itself. The runner has +// top-level side effects and cannot be imported, so each case runs a COPY of +// it (and of the modules it imports) as a subprocess against a two-file +// fixture suite. The only edit to the copy is the stall limit, shortened so a +// stall takes seconds; the gate, the classifier and the retry are the real +// code. A stall in ANY phase must fail the job at once — no isolated re-run, +// no retry, no pass — while a clean run and a recoverable failure behave as +// before. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { descendantsOf, killProcessTree, listProcesses } from "../scripts/ci-stall-watch.mjs"; + +const REPO = process.cwd(); +const LIMIT_MS = 4000; + +/** What tests/flaky.test.ts does on each successive run of the file. */ +type Step = "pass" | "hang" | "die"; + +function makeSuite(steps: Step[]): string { + // Under the repo so `--import tsx` resolves from the copy's cwd. + mkdirSync(join(REPO, ".tmp"), { recursive: true }); + const dir = mkdtempSync(join(REPO, ".tmp", "stall-gate-")); + mkdirSync(join(dir, "scripts")); + mkdirSync(join(dir, "tests")); + for (const f of ["run-ci-tests.mjs", "ci-failure-shapes.mjs", "ci-shard-partition.mjs", "ci-shard-weights.json"]) { + copyFileSync(join(REPO, "scripts", f), join(dir, "scripts", f)); + } + const watch = readFileSync(join(REPO, "scripts", "ci-stall-watch.mjs"), "utf8"); + const shortened = watch.replace("export const STALL_LIMIT_MS = 10 * 60 * 1000;", `export const STALL_LIMIT_MS = ${LIMIT_MS};`); + assert.notEqual(shortened, watch, "the limit constant was found and shortened in the copy"); + writeFileSync(join(dir, "scripts", "ci-stall-watch.mjs"), shortened); + writeFileSync(join(dir, "scripts", "live-files.mjs"), "export const LIVE_FILES = [];\n"); + for (const f of ["preload-storage-baseline.mjs", "preload-test-env.mjs", "preload-zone-slot.mjs"]) { + writeFileSync(join(dir, "tests", f), "\n"); + } + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name: "stall-gate-fixture", private: true, type: "module", scripts: { pretest: 'node -e ""' } }), + ); + writeFileSync(join(dir, "plan.json"), JSON.stringify({ steps, runs: 0 })); + writeFileSync(join(dir, "tests", "always.test.ts"), 'import { test } from "node:test";\ntest("always passes", () => {});\n'); + writeFileSync( + join(dir, "tests", "flaky.test.ts"), + [ + 'import { test } from "node:test";', + 'import { readFileSync, writeFileSync } from "node:fs";', + 'const planFile = new URL("../plan.json", import.meta.url);', + 'const plan = JSON.parse(readFileSync(planFile, "utf8"));', + 'const step: string = plan.steps[plan.runs] ?? "pass";', + "plan.runs += 1;", + "writeFileSync(planFile, JSON.stringify(plan));", + "// A silent death before any test reports: the #3029 swallowed shape,", + "// which the runner re-runs in isolation and then retries.", + 'if (step === "die") process.exit(1);', + 'test("flaky step", async () => {', + ' if (step === "hang") await new Promise(() => void setInterval(() => {}, 1000));', + "});", + "", + ].join("\n"), + ); + return dir; +} + +interface Finished { + code: number | null; + out: string; +} + +interface RunOptions { + cwd: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; + graceMs?: number; + /** Ends the process tree at the deadline. Injectable so a test can leave + * the tree alive and prove the fallback settlement. */ + kill?: (pid: number) => void; +} + +function killTree(pid: number): void { + let known: ReturnType = []; + try { + known = descendantsOf(pid, listProcesses()); + } catch { + // no snapshot: the tree kill below still reaches the group / the tree + } + killProcessTree(pid, known); +} + +/** Runs a command to its end and settles exactly once. Normally that is the + * child's 'close'. At the deadline the whole tree is killed; if 'close' still + * has not come graceMs later (something the kill did not reach holds the + * output pipes), the streams are abandoned and the promise rejects anyway, so + * the wait is bounded. Nothing runs after settlement: the caller's teardown + * deletes the fixture directory. */ +function runToClose(command: string, args: string[], opts: RunOptions): Promise { + const { cwd, env = process.env, timeoutMs = 120_000, graceMs = 15_000, kill = killTree } = opts; + return new Promise((resolve, reject) => { + // Its own process group on POSIX, so the deadline kill reaches every + // descendant, including ones started before the runner's own handlers. + const child = spawn(command, args, { cwd, env, detached: process.platform !== "win32" }); + let out = ""; + child.stdout.on("data", (d) => (out += d.toString())); + child.stderr.on("data", (d) => (out += d.toString())); + let settled = false; + let timedOut = false; + let fallback: NodeJS.Timeout | undefined; + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(deadline); + clearTimeout(fallback); + for (const [sig, h] of onSignal) process.removeListener(sig, h); + fn(); + }; + // The child is outside the caller's process group on POSIX, so a + // cancelled run (Ctrl+C on a targeted run) must take it down itself. + const onSignal = new Map void>(); + if (process.platform !== "win32") { + for (const [sig, exitCode] of [["SIGINT", 130], ["SIGTERM", 143]] as const) { + const h = () => { + if (child.pid !== undefined) killTree(child.pid); + process.exit(exitCode); + }; + onSignal.set(sig, h); + process.once(sig, h); + } + } + const timeoutError = (how: string) => new Error(`command did not finish in ${timeoutMs}ms (${how}). Output so far:\n${out}`); + const deadline = setTimeout(() => { + timedOut = true; + if (child.pid !== undefined) kill(child.pid); + fallback = setTimeout(() => { + child.stdin?.destroy(); + child.stdout.destroy(); + child.stderr.destroy(); + child.unref(); + settle(() => reject(timeoutError(`output pipes still open ${graceMs}ms after the kill; abandoned`))); + }, graceMs); + }, timeoutMs); + child.on("error", (e) => settle(() => reject(e))); + child.on("close", (code) => { + if (timedOut) return settle(() => reject(timeoutError("killed"))); + settle(() => resolve({ code, out })); + }); + }); +} + +async function runRunner(dir: string): Promise { + const env = { ...process.env }; + // The copy must run its own two files as a plain top-level node --test: + // not this job's shard slice, and not as a nested child of this test run. + delete env.CI_TEST_SHARD; + delete env.NODE_TEST_CONTEXT; + delete env.GITHUB_EVENT_NAME; + const r = await runToClose(process.execPath, ["scripts/run-ci-tests.mjs"], { cwd: dir, env }); + return { ...r, runs: JSON.parse(readFileSync(join(dir, "plan.json"), "utf8")).runs }; +} + +const RETRY = /retrying full phase once/; +const ISOLATED = /DIAGNOSTIC isolated re-run/; +const STALL = /STALL/; + +interface Case { + name: string; + steps: Step[]; + code: number; + runs: number; + has: RegExp[]; + lacks: RegExp[]; +} + +const CASES: Case[] = [ + { + name: "a stall in the main phase fails the job with no isolated re-run and no retry", + steps: ["hang"], + code: 1, + runs: 1, + has: [/::error::\[ci-tests\] STALL: all printed nothing for 4s/, /test file\(s\) still running: tests\/flaky\.test\.ts/, /\[ci-tests\] all STALLED .* failing WITHOUT retry/], + lacks: [ISOLATED, RETRY, /FLAKE CONFIRMED/, /all ok/], + }, + { + name: "a stall in an isolated re-run fails the job with no retry", + steps: ["die", "hang"], + code: 1, + runs: 2, + has: [/swallowed file-level failure/, ISOLATED, /\[ci-tests\] isolated re-run tests\/flaky\.test\.ts STALLED .* failing WITHOUT retry/], + lacks: [RETRY, /FLAKE CONFIRMED/, /PASSED in isolation|REPRODUCED in isolation/], + }, + { + name: "a stall in the full retry fails the job and is not read as a retry result", + steps: ["die", "die", "hang"], + code: 1, + runs: 3, + has: [ISOLATED, RETRY, /\[ci-tests\] all \(retry\) STALLED .* failing WITHOUT retry/], + lacks: [/FLAKE CONFIRMED/, /full retry also FAILED/, /DRAIN-PASS/], + }, + { + name: "control: a clean run passes and reports no stall", + steps: ["pass"], + code: 0, + runs: 1, + has: [/\[ci-tests\] all ok in/], + lacks: [STALL, ISOLATED, RETRY], + }, + { + name: "control: a recoverable failure is still re-run, retried and passed, with no stall", + steps: ["die", "pass", "pass"], + code: 0, + runs: 3, + has: [/swallowed file-level failure/, /PASSED in isolation/, RETRY, /FLAKE CONFIRMED/], + lacks: [STALL], + }, +]; + +for (const c of CASES) { + test(c.name, async (t) => { + const dir = makeSuite(c.steps); + t.after(() => rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 })); + const r = await runRunner(dir); + const ctx = `\n--- runner output ---\n${r.out}`; + assert.equal(r.code, c.code, `exit code${ctx}`); + assert.equal(r.runs, c.runs, `how many times the flaky file was started${ctx}`); + for (const re of c.has) assert.match(r.out, re, `expected ${re}${ctx}`); + for (const re of c.lacks) assert.doesNotMatch(r.out, re, `unexpected ${re}${ctx}`); + }); +} + +// The deadline in runToClose, which every case above relies on to stay bounded. +// The fixture is the shape that used to hang it: a parent whose child inherits +// the output pipes and outlives it. + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + } catch { + return false; + } + if (process.platform === "linux") { + // kill(pid, 0) also succeeds for a zombie, which is not running. + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + if (stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3) === "Z") return false; + } catch { + return false; + } + } + return true; +} + +async function waitDead(pids: number[], ms = 10_000): Promise { + const end = Date.now() + ms; + for (;;) { + const left = pids.filter(alive); + if (left.length === 0 || Date.now() > end) return left; + await new Promise((r) => setTimeout(r, 100)); + } +} + +function pipeHolder(): { dir: string; script: string } { + mkdirSync(join(REPO, ".tmp"), { recursive: true }); + const dir = mkdtempSync(join(REPO, ".tmp", "stall-gate-pipes-")); + writeFileSync(join(dir, "holder.mjs"), "setInterval(() => {}, 1000);\n"); + const script = join(dir, "parent.mjs"); + writeFileSync( + script, + [ + 'import { spawn } from "node:child_process";', + 'const kid = spawn(process.execPath, ["holder.mjs"], { stdio: "inherit" });', + "console.log(`pids ${process.pid} ${kid.pid}`);", + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + ); + return { dir, script }; +} + +function pidsIn(message: string): number[] { + const m = message.match(/pids (\d+) (\d+)/); + assert.ok(m, `the fixture printed its pids (got: ${JSON.stringify(message)})`); + return [Number(m[1]), Number(m[2])]; +} + +test("the deadline kills the whole tree, including a child holding the output pipes", async (t) => { + const { dir, script } = pipeHolder(); + t.after(() => rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 })); + const t0 = Date.now(); + const err = await runToClose(process.execPath, [script], { cwd: dir, timeoutMs: 2000, graceMs: 30_000 }).then( + () => undefined, + (e: Error) => e, + ); + assert.ok(err, "a run past its deadline rejects"); + assert.match(err.message, /did not finish in 2000ms \(killed\)/); + assert.ok(Date.now() - t0 < 20_000, "settled on the kill, well inside the grace"); + assert.deepEqual(await waitDead(pidsIn(err.message)), [], "parent and pipe-holding child are both gone"); +}); + +test("a pipe holder that survives the kill cannot keep the wait open", async (t) => { + const { dir, script } = pipeHolder(); + let pids: number[] = []; + // Known from the kill callback, not from the fixture's output: teardown + // must reach the tree even if the pids were never printed. + let root = 0; + t.after(async () => { + if (root > 0) killTree(root); + for (const pid of pids) if (pid > 0) killProcessTree(pid); + await waitDead(pids); + rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 }); + }); + const t0 = Date.now(); + // A kill that reaches nothing: both processes keep the pipes open. + const err = await runToClose(process.execPath, [script], { cwd: dir, timeoutMs: 2000, graceMs: 1500, kill: (pid) => void (root = pid) }).then( + () => undefined, + (e: Error) => e, + ); + assert.ok(err, "the wait ends even though 'close' never came"); + assert.ok(root > 0, "the deadline handed the root pid to the kill"); + pids = pidsIn(err.message); + assert.match(err.message, /output pipes still open 1500ms after the kill; abandoned/); + assert.ok(Date.now() - t0 < 20_000, "bounded by deadline plus grace"); + assert.ok(pids.every(alive), "the fixture really was still running when the wait ended"); +}); + +test("control: a command that finishes in time resolves with its exit code and output", async () => { + const r = await runToClose(process.execPath, ["-e", "console.log('done'); process.exit(3)"], { cwd: REPO, timeoutMs: 30_000 }); + assert.equal(r.code, 3); + assert.match(r.out, /done/); +}); diff --git a/tests/ci-stall-watch.test.ts b/tests/ci-stall-watch.test.ts new file mode 100644 index 0000000000..da522cce2d --- /dev/null +++ b/tests/ci-stall-watch.test.ts @@ -0,0 +1,261 @@ +// #69: the CI runner's stall watch (scripts/ci-stall-watch.mjs). A phase that +// prints nothing for the limit is reported, killed as a tree and flagged +// `stalled`; a phase that keeps printing, or finishes inside the limit, is +// left alone. Fixtures are real child processes: the point of the watch is +// what happens to a process tree, which a stub cannot show. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { descendantsOf, orphansRunning, runWatched, STALL_LIMIT_MS, testFilesIn } from "../scripts/ci-stall-watch.mjs"; + +// Short stand-in for the 10-minute limit. It also has to cover a fixture's +// start-up (spawn + node boot) on a loaded runner, since nothing is printed +// before that: 3 s leaves that room without making the file slow. +const STALL_AFTER_MS = 3000; + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + } catch { + return false; + } + if (process.platform === "linux") { + // kill(pid, 0) also succeeds for a zombie: a killed process nobody has + // reaped yet (a container whose PID 1 does not reap). It is not running. + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + if (stat.slice(stat.lastIndexOf(")") + 2, stat.lastIndexOf(")") + 3) === "Z") return false; + } catch { + return false; + } + } + return true; +} + +/** Teardown kill. Never signals pid 0 or below: on POSIX that addresses a + * whole process group, which would take the test runner with it. */ +function killQuietly(pid: number): void { + if (!Number.isInteger(pid) || pid <= 0) return; + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone — the expected case + } +} + +async function waitDead(pids: number[], ms = 10_000): Promise { + const end = Date.now() + ms; + for (;;) { + const left = pids.filter(alive); + if (left.length === 0 || Date.now() > end) return left; + await new Promise((r) => setTimeout(r, 100)); + } +} + +/** A parent that starts `stuck-fixture.test.mjs` sharing its stdout, prints + * both pids, and then both stay silent forever — the #61 shape: a file-level + * process that never finishes under a phase root that waits for it. */ +function stuckTree(dir: string): string { + writeFileSync(join(dir, "stuck-fixture.test.mjs"), "setInterval(() => {}, 1000);\n"); + const parent = join(dir, "phase-root.mjs"); + writeFileSync( + parent, + [ + 'import { spawn } from "node:child_process";', + 'const kid = spawn(process.execPath, ["stuck-fixture.test.mjs"], { stdio: "inherit" });', + "console.log(`pids ${process.pid} ${kid.pid}`);", + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + ); + return parent; +} + +function pidsFrom(out: string): number[] { + const m = out.match(/pids (\d+) (\d+)/); + assert.ok(m, `fixture printed its pids (got: ${JSON.stringify(out)})`); + return [Number(m[1]), Number(m[2])]; +} + +test("the limit is the owner-approved 10 minutes (#69)", () => { + assert.equal(STALL_LIMIT_MS, 600_000); +}); + +test("a silent phase is reported, killed as a tree and flagged stalled", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-stall-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const logs: string[] = []; + let out = ""; + const t0 = Date.now(); + const r = await runWatched(process.execPath, [stuckTree(dir)], { + cwd: dir, + label: "fixture", + limitMs: STALL_AFTER_MS, + onStdout: (d) => (out += d.toString()), + log: (l) => logs.push(l), + }); + const [rootPid, kidPid] = pidsFrom(out); + t.after(() => [kidPid, rootPid].forEach(killQuietly)); + const report = logs.join("\n"); + + assert.equal(r.stalled, true, "flagged stalled"); + assert.ok(Date.now() - t0 >= STALL_AFTER_MS, "not before the limit"); + assert.deepEqual(await waitDead([rootPid, kidPid]), [], "the root and the process it started are both gone"); + assert.match(report, /::error::\[ci-tests\] STALL: fixture printed nothing for 3s/, "names the phase and the silence"); + assert.match(report, /test file\(s\) still running: stuck-fixture\.test\.mjs/, "names the stuck test file"); + assert.match(report, new RegExp(`pid=${kidPid} ppid=${rootPid} `), "lists the stuck process under its parent"); + assert.match(report, new RegExp(`last output before the silence:\\npids ${rootPid} ${kidPid}`), "shows the last output"); + assert.doesNotMatch(report, /snapshot unavailable|did not close within/, "took the snapshot and closed normally"); +}); + +for (const stream of ["stdout", "stderr"] as const) { + test(`a phase that keeps printing on ${stream} past the limit is not stalled`, async () => { + const logs: string[] = []; + let seen = 0; + // Prints every 100 ms for 4.5 s against a 3 s limit: longer than the + // limit in total, never more than a fraction of it between two lines. + const script = + `let n = 0; const i = setInterval(() => { process.${stream}.write("tick\\n"); ` + + `if (++n === 45) { clearInterval(i); process.exitCode = 7; } }, 100);`; + const r = await runWatched(process.execPath, ["-e", script], { + limitMs: STALL_AFTER_MS, + onStdout: () => seen++, + onStderr: () => seen++, + log: (l) => logs.push(l), + }); + assert.equal(r.stalled, false); + assert.equal(r.code, 7, "the phase's own exit code comes through"); + assert.ok(r.ms >= STALL_AFTER_MS, `ran past the limit (${r.ms} ms)`); + assert.ok(seen > 0, "output was forwarded"); + assert.deepEqual(logs, [], "no stall report"); + }); +} + +test("a phase silent for less than the limit that then exits is not stalled, and the watch is disarmed", async () => { + const logs: string[] = []; + const r = await runWatched(process.execPath, ["-e", "setTimeout(() => process.exit(3), 400)"], { + limitMs: STALL_AFTER_MS, + log: (l) => logs.push(l), + }); + assert.equal(r.stalled, false); + assert.equal(r.code, 3); + // Outlive the limit: a timer left armed would report now. + await new Promise((res) => setTimeout(res, STALL_AFTER_MS + 300)); + assert.deepEqual(logs, [], "nothing reported after the phase closed"); +}); + +test("a finished phase leaves nothing that holds the calling process open", async () => { + // The real limit is 10 minutes: a timer left armed after the phase closed + // would keep the runner alive that long after its last phase. Run a + // caller with the real default limit and a phase that ends at once; the + // caller must end on its own well inside it. + const mod = pathToFileURL(join(process.cwd(), "scripts", "ci-stall-watch.mjs")).href; + const caller = + `const m = await import(${JSON.stringify(mod)}); ` + + `const r = await m.runWatched(process.execPath, ["-e", ""]); console.log("stalled=" + r.stalled + " code=" + r.code);`; + let out = ""; + const logs: string[] = []; + const r = await runWatched(process.execPath, ["--input-type=module", "-e", caller], { + limitMs: 60_000, + onStdout: (d) => (out += d.toString()), + log: (l) => logs.push(l), + }); + assert.equal(r.stalled, false, `the caller ended on its own (${logs.join(" | ")})`); + assert.equal(r.code, 0); + assert.match(out, /stalled=false code=0/); + assert.ok(r.ms < 30_000, `promptly (${r.ms} ms)`); +}); + +test("a failed process snapshot is reported and the phase is still killed and flagged", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-stall-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const logs: string[] = []; + let out = ""; + const r = await runWatched(process.execPath, [stuckTree(dir)], { + cwd: dir, + limitMs: STALL_AFTER_MS, + onStdout: (d) => (out += d.toString()), + log: (l) => logs.push(l), + list: () => { + throw new Error("query refused"); + }, + }); + const pids = pidsFrom(out); + t.after(() => pids.forEach(killQuietly)); + const report = logs.join("\n"); + assert.equal(r.stalled, true); + assert.match(report, /process snapshot unavailable \(query refused\) — killing anyway/); + assert.doesNotMatch(report, /still running:/, "claims nothing about what was running"); + // taskkill /T on Windows and the process-group kill elsewhere: neither + // needs the snapshot. + assert.deepEqual(await waitDead(pids), [], "whole tree ended without a snapshot"); +}); + +test("a phase whose streams stay open after the kill is abandoned after the grace, still flagged stalled", async (t) => { + const logs: string[] = []; + let pid = 0; + t.after(() => killQuietly(pid)); + const r = await runWatched(process.execPath, ["-e", "console.log(process.pid); setInterval(() => {}, 1000)"], { + limitMs: STALL_AFTER_MS, + closeGraceMs: 500, + onStdout: (d) => (pid = Number(d.toString())), + log: (l) => logs.push(l), + list: () => [], + killTree: () => {}, // the kill "did not work": nothing closes the pipes + }); + assert.equal(r.stalled, true); + assert.equal(r.code, null, "no exit code is invented"); + assert.match(logs.join("\n"), /did not close within 1s of the kill — abandoning its streams/); + assert.ok(pid > 0 && alive(pid), "the fixture really was still running when the watch gave up"); +}); + +test("descendantsOf walks only the phase's own tree, parents first", () => { + const rows = [ + { pid: 10, ppid: 1, cmd: "node --test tests/a.test.ts tests/b.test.ts" }, + { pid: 11, ppid: 10, cmd: "node --import tsx tests/a.test.ts" }, + { pid: 12, ppid: 11, cmd: "node dist/index.js start" }, + { pid: 20, ppid: 1, cmd: "node --import tsx tests/unrelated.test.ts" }, + { pid: 10, ppid: 12, cmd: "reused pid" }, + ]; + const tree = descendantsOf(10, rows); + assert.deepEqual( + tree.map((p) => [p.pid, p.depth]), + [ + [10, 0], + [11, 1], + [12, 2], + ], + ); + assert.deepEqual(testFilesIn(tree), ["tests/a.test.ts"], "the root's file list and other trees name nothing"); + assert.deepEqual(descendantsOf(999, rows), [], "an unknown root has no tree"); +}); + +test("orphansRunning lists node processes whose parent is gone, and only those", () => { + const rows = [ + { pid: 10, ppid: 1, cmd: '"C:\\Node\\node.exe" --test tests/a.test.ts' }, + { pid: 11, ppid: 10, cmd: '"C:\\Node\\node.exe" tests/a.test.ts' }, + { pid: 30, ppid: 777, cmd: '"c:\\node\\NODE.EXE" dist/index.js start' }, + { pid: 31, ppid: 778, cmd: "C:\\Windows\\explorer.exe" }, + { pid: 1, ppid: 0, cmd: "System" }, + ]; + assert.deepEqual( + orphansRunning("C:\\Node\\node.exe", rows).map((r) => r.pid), + [30], + "living parent, other executables and the system root are left out", + ); + assert.deepEqual(orphansRunning("C:\\Node\\node.exe", rows.filter((r) => r.pid !== 30)), []); +}); + +test("testFilesIn normalises Windows separators and skips ambiguous command lines", () => { + assert.deepEqual( + testFilesIn([ + { pid: 2, ppid: 1, depth: 1, cmd: '"C:\\node.exe" --import tsx tests\\dsh-native.test.ts' }, + { pid: 3, ppid: 1, depth: 1, cmd: "node tests/x.test.ts tests/y.test.ts" }, + { pid: 4, ppid: 2, depth: 2, cmd: "node dist/index.js" }, + ]), + ["tests/dsh-native.test.ts"], + ); +});