Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 163 additions & 11 deletions scripts/run-ci-tests.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,25 @@
// LIVE_FILES is a tripwire, not a gate: every process-spawning suite must be
// registered there so adding a new one fails the job until someone reviews
// its concurrency assumptions (#1248) instead of silently riding c=4.
import { execFileSync } from "node:child_process";
//
// #3029: attribution + bounded flake recovery for "swallowed" file-level
// failures. When a test child dies SILENTLY (non-zero exit, no stderr — OOM
// kill, tsx/esbuild worker death, race exit under concurrency), node --test
// collapses it to one file-level `not ok <file>` whose only payload is
// `error: 'test failed'` and zero subtests; NO reporter can surface an error
// the child never emitted. On failure this runner therefore: captures exit
// code + signal (previously discarded by an empty catch) plus a resource
// snapshot (free/total mem, loadavg, node version, platform); detects the
// swallowed shape (a top-level `not ok … *.test.ts` — successful files promote
// their inner tests to top level by test name and never emit a file-level
// entry); re-runs each such file in ISOLATION for attribution; then retries
// the whole phase ONCE and exits 0 only if that retry is green (a confirmed
// transient). Concrete assertion failures (subtests ran and reported a real
// message) are NOT the swallowed shape → they fail hard immediately, no retry,
// so a real regression is never masked.
import { execFileSync, spawn } from "node:child_process";
import { readdirSync, readFileSync } from "node:fs";
import { freemem, loadavg, totalmem } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";

Expand Down Expand Up @@ -75,6 +92,8 @@ const allFiles = readdirSync(testsDir)
.sort()
.map((f) => `tests/${f}`);

const baseToFile = new Map(allFiles.map((f) => [f.split("/").pop(), f]));

function die(msg) {
console.error(`::error::${msg}`);
console.error(`[ci-tests] ${msg}`);
Expand Down Expand Up @@ -104,20 +123,153 @@ const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm";
console.log("[ci-tests] pretest (kernel + pi-subagents build)");
execFileSync(npmCmd, ["run", "pretest"], { cwd: root, stdio: "inherit", shell: process.platform === "win32" });

function runPhase(label, files, concurrency) {
// Loader chain the suites depend on (storage-baseline preload, #2671/#2803,
// then tsx). Shared by the main phase and the isolated diagnostic re-run so
// the latter sees the SAME environment as the in-phase run.
const PRELOAD_ARGS = ["--import", "./tests/preload-storage-baseline.mjs", "--import", "tsx"];

const MAX_CAPTURE = 32 * 1024 * 1024; // bound in-memory capture; real runs are MB-scale

function runNodeTest(files, concurrency, label) {
const t0 = Date.now();
console.log(`[ci-tests] ${label}: ${files.length} files, concurrency=${concurrency}`);
try {
execFileSync(
return new Promise((resolve) => {
const child = spawn(
process.execPath,
["--import", "./tests/preload-storage-baseline.mjs", "--import", "tsx", "--test", "--test-concurrency", String(concurrency), ...files],
{ cwd: root, stdio: "inherit" },
[...PRELOAD_ARGS, "--test", "--test-concurrency", String(concurrency), "--test-reporter=tap", ...files],
{ cwd: root },
);
} catch {
console.error(`[ci-tests] ${label} FAILED after ${((Date.now() - t0) / 1000).toFixed(1)}s`);
process.exit(1);
const out = [];
const err = [];
let outLen = 0;
let errLen = 0;
child.stdout.on("data", (d) => {
if (outLen < MAX_CAPTURE) {
out.push(d);
outLen += d.length;
}
process.stdout.write(d);
});
child.stderr.on("data", (d) => {
if (errLen < MAX_CAPTURE) {
err.push(d);
errLen += d.length;
}
process.stderr.write(d);
});
child.on("error", (e) => {
console.error(`[ci-tests] ${label}: spawn error: ${e.message}`);
resolve({ code: -1, signal: null, out: "", err: "", ms: Date.now() - t0 });
});
child.on("close", (code, signal) => {
resolve({
code,
signal,
out: Buffer.concat(out).toString("utf8"),
err: Buffer.concat(err).toString("utf8"),
ms: Date.now() - t0,
});
});
});
}

function resourceSnapshot() {
const mi = (n) => `${Math.round(n / 1048576)}MiB`;
return (
`freemem=${mi(freemem())} totalmem=${mi(totalmem())} ` +
`loadavg=${loadavg().map((x) => x.toFixed(2)).join("/")} ` +
`node=${process.version} platform=${process.platform}/${process.arch}`
);
}

// A top-level `not ok N - <path ending .test.ts>` can only come from a file
// whose child DIED: successful files promote their inner tests to top level by
// test name and never emit a file-level entry. Pairing that with the YAML
// `error: 'test failed'` marker isolates the swallowed/silent-death shape from
// real assertion failures (named by test name, carrying a real message).
function extractSwallowedFiles(output) {
const lines = output.split(/\r?\n/);
const found = new Set();
for (let i = 0; i < lines.length; i++) {
const m = lines[i].match(/^not ok \d+ - (.+?)\s*$/);
if (!m) continue;
const window = lines.slice(i, i + 20).join("\n");
if (!/error:\s*'test failed'/.test(window)) continue;
const base = m[1].trim().split(/[\\/]/).pop();
const rel = baseToFile.get(base);
if (rel) found.add(rel);
}
console.log(`[ci-tests] ${label} ok in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
return [...found];
}

function runIsolated(fileRel) {
const t0 = Date.now();
console.log(`[ci-tests] DIAGNOSTIC isolated re-run (serial): ${fileRel} — full output below`);
return new Promise((resolve) => {
const child = spawn(process.execPath, [...PRELOAD_ARGS, "--test", "--test-reporter=tap", fileRel], { cwd: root });
child.stdout.on("data", (d) => process.stdout.write(d));
child.stderr.on("data", (d) => process.stderr.write(d));
child.on("error", (e) => {
console.error(`[ci-tests] isolated re-run ${fileRel}: spawn error: ${e.message}`);
resolve({ code: -1, signal: null, ms: Date.now() - t0 });
});
child.on("close", (code, signal) => resolve({ code, signal, ms: Date.now() - t0 }));
});
}

runPhase("all", allFiles, 4);
(async () => {
const first = await runNodeTest(allFiles, 4, "all");
if (first.code === 0) {
console.log(`[ci-tests] all ok in ${(first.ms / 1000).toFixed(1)}s`);
return;
}
console.error(`[ci-tests] all FAILED after ${(first.ms / 1000).toFixed(1)}s (exit=${first.code} signal=${first.signal})`);
console.error(`[ci-tests] resource snapshot: ${resourceSnapshot()}`);
const errTrim = first.err.replace(/\s+/g, " ").trim();
if (errTrim) {
console.error(
`[ci-tests] child stderr captured (${first.err.length} chars) — may carry a V8/native fatal (e.g. OOM 'Reached heap limit'): ` +
`${errTrim.slice(0, 2000)}`,
);
} else {
console.error(
"[ci-tests] child wrote nothing to stderr (node --test reports test failures on the TAP stdout above; nothing extra to surface here)",
);
}
const swallowed = extractSwallowedFiles(first.out);
if (swallowed.length === 0) {
console.error(
"[ci-tests] failure carries concrete sub-test assertions (not the swallowed 'test failed' shape) " +
"-> failing WITHOUT auto-retry so a real regression is never masked.",
);
process.exit(first.code ?? 1);
}
console.error(
`[ci-tests] swallowed file-level failure(s) — zero subtests, payload only 'test failed': ` +
`${swallowed.join(", ")} (#3029)`,
);
for (const rel of swallowed) {
const r = await runIsolated(rel);
console.error(
`[ci-tests] isolated re-run ${rel}: exit=${r.code} signal=${r.signal} (${(r.ms / 1000).toFixed(1)}s) -> ` +
(r.code === 0
? "PASSED in isolation (transient / concurrency flake likely)"
: "REPRODUCED in isolation (deterministic break)"),
);
}
console.error("[ci-tests] retrying full phase once to confirm flake (#3029)");
const retry = await runNodeTest(allFiles, 4, "all (retry)");
if (retry.code === 0) {
console.log(
`[ci-tests] FLAKE CONFIRMED: first run failed on ${swallowed.join(", ")}, full retry green in ` +
`${(retry.ms / 1000).toFixed(1)}s -> treating as transient (#3029). If it recurs, check ` +
`node${process.version.replace(/^v/, "").split(".")[0]}-${process.platform} + tsx load + concurrency=4 (see snapshot above).`,
);
process.exit(0);
}
console.error(`[ci-tests] full retry also FAILED (exit=${retry.code} signal=${retry.signal}) -> genuine failure, not a flake`);
process.exit(retry.code ?? 1);
})().catch((e) => {
console.error(`[ci-tests] runner crashed: ${e && e.stack ? e.stack : e}`);
process.exit(1);
});
Loading