diff --git a/scripts/run-ci-tests.mjs b/scripts/run-ci-tests.mjs index e1301371ae..95f7ae0d7f 100644 --- a/scripts/run-ci-tests.mjs +++ b/scripts/run-ci-tests.mjs @@ -15,8 +15,25 @@ // LIVE_FILES is a tripwire, not a gate: every process-spawning suite must be // registered there so adding a new one fails the job until someone reviews // its concurrency assumptions (#1248) instead of silently riding c=4. -import { execFileSync } from "node:child_process"; +// +// #3029: attribution + bounded flake recovery for "swallowed" file-level +// failures. When a test child dies SILENTLY (non-zero exit, no stderr — OOM +// kill, tsx/esbuild worker death, race exit under concurrency), node --test +// collapses it to one file-level `not ok ` whose only payload is +// `error: 'test failed'` and zero subtests; NO reporter can surface an error +// the child never emitted. On failure this runner therefore: captures exit +// code + signal (previously discarded by an empty catch) plus a resource +// snapshot (free/total mem, loadavg, node version, platform); detects the +// swallowed shape (a top-level `not ok … *.test.ts` — successful files promote +// their inner tests to top level by test name and never emit a file-level +// entry); re-runs each such file in ISOLATION for attribution; then retries +// the whole phase ONCE and exits 0 only if that retry is green (a confirmed +// transient). Concrete assertion failures (subtests ran and reported a real +// message) are NOT the swallowed shape → they fail hard immediately, no retry, +// so a real regression is never masked. +import { execFileSync, spawn } from "node:child_process"; import { readdirSync, readFileSync } from "node:fs"; +import { freemem, loadavg, totalmem } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -75,6 +92,8 @@ const allFiles = readdirSync(testsDir) .sort() .map((f) => `tests/${f}`); +const baseToFile = new Map(allFiles.map((f) => [f.split("/").pop(), f])); + function die(msg) { console.error(`::error::${msg}`); console.error(`[ci-tests] ${msg}`); @@ -104,20 +123,153 @@ const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm"; console.log("[ci-tests] pretest (kernel + pi-subagents build)"); execFileSync(npmCmd, ["run", "pretest"], { cwd: root, stdio: "inherit", shell: process.platform === "win32" }); -function runPhase(label, files, concurrency) { +// Loader chain the suites depend on (storage-baseline preload, #2671/#2803, +// then tsx). Shared by the main phase and the isolated diagnostic re-run so +// the latter sees the SAME environment as the in-phase run. +const PRELOAD_ARGS = ["--import", "./tests/preload-storage-baseline.mjs", "--import", "tsx"]; + +const MAX_CAPTURE = 32 * 1024 * 1024; // bound in-memory capture; real runs are MB-scale + +function runNodeTest(files, concurrency, label) { const t0 = Date.now(); console.log(`[ci-tests] ${label}: ${files.length} files, concurrency=${concurrency}`); - try { - execFileSync( + return new Promise((resolve) => { + const child = spawn( process.execPath, - ["--import", "./tests/preload-storage-baseline.mjs", "--import", "tsx", "--test", "--test-concurrency", String(concurrency), ...files], - { cwd: root, stdio: "inherit" }, + [...PRELOAD_ARGS, "--test", "--test-concurrency", String(concurrency), "--test-reporter=tap", ...files], + { cwd: root }, ); - } catch { - console.error(`[ci-tests] ${label} FAILED after ${((Date.now() - t0) / 1000).toFixed(1)}s`); - process.exit(1); + const out = []; + const err = []; + let outLen = 0; + let errLen = 0; + child.stdout.on("data", (d) => { + if (outLen < MAX_CAPTURE) { + out.push(d); + outLen += d.length; + } + process.stdout.write(d); + }); + child.stderr.on("data", (d) => { + if (errLen < MAX_CAPTURE) { + err.push(d); + errLen += d.length; + } + process.stderr.write(d); + }); + child.on("error", (e) => { + console.error(`[ci-tests] ${label}: spawn error: ${e.message}`); + resolve({ code: -1, signal: null, out: "", err: "", ms: Date.now() - t0 }); + }); + child.on("close", (code, signal) => { + resolve({ + code, + signal, + out: Buffer.concat(out).toString("utf8"), + err: Buffer.concat(err).toString("utf8"), + ms: Date.now() - t0, + }); + }); + }); +} + +function resourceSnapshot() { + const mi = (n) => `${Math.round(n / 1048576)}MiB`; + return ( + `freemem=${mi(freemem())} totalmem=${mi(totalmem())} ` + + `loadavg=${loadavg().map((x) => x.toFixed(2)).join("/")} ` + + `node=${process.version} platform=${process.platform}/${process.arch}` + ); +} + +// A top-level `not ok N - ` can only come from a file +// whose child DIED: successful files promote their inner tests to top level by +// test name and never emit a file-level entry. Pairing that with the YAML +// `error: 'test failed'` marker isolates the swallowed/silent-death shape from +// real assertion failures (named by test name, carrying a real message). +function extractSwallowedFiles(output) { + const lines = output.split(/\r?\n/); + const found = new Set(); + for (let i = 0; i < lines.length; i++) { + const m = lines[i].match(/^not ok \d+ - (.+?)\s*$/); + if (!m) continue; + const window = lines.slice(i, i + 20).join("\n"); + if (!/error:\s*'test failed'/.test(window)) continue; + const base = m[1].trim().split(/[\\/]/).pop(); + const rel = baseToFile.get(base); + if (rel) found.add(rel); } - console.log(`[ci-tests] ${label} ok in ${((Date.now() - t0) / 1000).toFixed(1)}s`); + return [...found]; +} + +function runIsolated(fileRel) { + const t0 = Date.now(); + console.log(`[ci-tests] DIAGNOSTIC isolated re-run (serial): ${fileRel} — full output below`); + return new Promise((resolve) => { + const child = spawn(process.execPath, [...PRELOAD_ARGS, "--test", "--test-reporter=tap", fileRel], { cwd: root }); + child.stdout.on("data", (d) => process.stdout.write(d)); + child.stderr.on("data", (d) => process.stderr.write(d)); + child.on("error", (e) => { + console.error(`[ci-tests] isolated re-run ${fileRel}: spawn error: ${e.message}`); + resolve({ code: -1, signal: null, ms: Date.now() - t0 }); + }); + child.on("close", (code, signal) => resolve({ code, signal, ms: Date.now() - t0 })); + }); } -runPhase("all", allFiles, 4); +(async () => { + const first = await runNodeTest(allFiles, 4, "all"); + if (first.code === 0) { + console.log(`[ci-tests] all ok in ${(first.ms / 1000).toFixed(1)}s`); + return; + } + console.error(`[ci-tests] all FAILED after ${(first.ms / 1000).toFixed(1)}s (exit=${first.code} signal=${first.signal})`); + console.error(`[ci-tests] resource snapshot: ${resourceSnapshot()}`); + const errTrim = first.err.replace(/\s+/g, " ").trim(); + if (errTrim) { + console.error( + `[ci-tests] child stderr captured (${first.err.length} chars) — may carry a V8/native fatal (e.g. OOM 'Reached heap limit'): ` + + `${errTrim.slice(0, 2000)}`, + ); + } else { + console.error( + "[ci-tests] child wrote nothing to stderr (node --test reports test failures on the TAP stdout above; nothing extra to surface here)", + ); + } + const swallowed = extractSwallowedFiles(first.out); + if (swallowed.length === 0) { + console.error( + "[ci-tests] failure carries concrete sub-test assertions (not the swallowed 'test failed' shape) " + + "-> failing WITHOUT auto-retry so a real regression is never masked.", + ); + process.exit(first.code ?? 1); + } + console.error( + `[ci-tests] swallowed file-level failure(s) — zero subtests, payload only 'test failed': ` + + `${swallowed.join(", ")} (#3029)`, + ); + for (const rel of swallowed) { + const r = await runIsolated(rel); + console.error( + `[ci-tests] isolated re-run ${rel}: exit=${r.code} signal=${r.signal} (${(r.ms / 1000).toFixed(1)}s) -> ` + + (r.code === 0 + ? "PASSED in isolation (transient / concurrency flake likely)" + : "REPRODUCED in isolation (deterministic break)"), + ); + } + console.error("[ci-tests] retrying full phase once to confirm flake (#3029)"); + const retry = await runNodeTest(allFiles, 4, "all (retry)"); + if (retry.code === 0) { + console.log( + `[ci-tests] FLAKE CONFIRMED: first run failed on ${swallowed.join(", ")}, full retry green in ` + + `${(retry.ms / 1000).toFixed(1)}s -> treating as transient (#3029). If it recurs, check ` + + `node${process.version.replace(/^v/, "").split(".")[0]}-${process.platform} + tsx load + concurrency=4 (see snapshot above).`, + ); + process.exit(0); + } + console.error(`[ci-tests] full retry also FAILED (exit=${retry.code} signal=${retry.signal}) -> genuine failure, not a flake`); + process.exit(retry.code ?? 1); +})().catch((e) => { + console.error(`[ci-tests] runner crashed: ${e && e.stack ? e.stack : e}`); + process.exit(1); +});