From 158f34e1881ec6a6b2ea2a0df84f19a525215ad3 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 13 Oct 2026 19:37:41 +0800 Subject: [PATCH] fix: attribute + one-shot flake recovery for swallowed file-level CI test failures (#3029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a test child dies silently (non-zero exit, no stderr — e.g. OOM kill, tsx/esbuild worker death, race exit under concurrency), node --test collapses it to one file-level 'not ok ' whose only payload is error:'test failed' and zero subtests; no reporter can surface an error the child never emitted. CI was left unable to attribute the failure (#3029). run-ci-tests.mjs now, on failure: - captures exit code + signal (previously discarded by an empty catch); - logs a resource snapshot (free/total mem, loadavg, node version, platform); - surfaces child stderr when present (may carry a V8/native fatal such as OOM 'Reached heap limit'); - detects the swallowed shape (top-level 'not ok … *.test.ts' + error:'test failed') and re-runs each such file in isolation for attribution; - retries the whole phase ONCE and exits 0 only if that retry is green (a confirmed transient). Concrete assertion failures are NOT the swallowed shape and still fail hard immediately with no retry, so real regressions are never masked. Behavior changes (flagged): - The test step now forces --test-reporter=tap (Windows log format spec->tap; Linux was already tap). No test-semantics change. - Swallowed-shape failures get one bounded recovery (isolated re-run + full retry -> exit 0 if green) instead of failing immediately. Concrete assertion failures are unchanged (fail immediately). - No new config/env added; concurrency stays 4. --- scripts/run-ci-tests.mjs | 174 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 163 insertions(+), 11 deletions(-) diff --git a/scripts/run-ci-tests.mjs b/scripts/run-ci-tests.mjs index e1301371ae..95f7ae0d7f 100644 --- a/scripts/run-ci-tests.mjs +++ b/scripts/run-ci-tests.mjs @@ -15,8 +15,25 @@ // LIVE_FILES is a tripwire, not a gate: every process-spawning suite must be // registered there so adding a new one fails the job until someone reviews // its concurrency assumptions (#1248) instead of silently riding c=4. -import { execFileSync } from "node:child_process"; +// +// #3029: attribution + bounded flake recovery for "swallowed" file-level +// failures. When a test child dies SILENTLY (non-zero exit, no stderr — OOM +// kill, tsx/esbuild worker death, race exit under concurrency), node --test +// collapses it to one file-level `not ok ` whose only payload is +// `error: 'test failed'` and zero subtests; NO reporter can surface an error +// the child never emitted. On failure this runner therefore: captures exit +// code + signal (previously discarded by an empty catch) plus a resource +// snapshot (free/total mem, loadavg, node version, platform); detects the +// swallowed shape (a top-level `not ok … *.test.ts` — successful files promote +// their inner tests to top level by test name and never emit a file-level +// entry); re-runs each such file in ISOLATION for attribution; then retries +// the whole phase ONCE and exits 0 only if that retry is green (a confirmed +// transient). Concrete assertion failures (subtests ran and reported a real +// message) are NOT the swallowed shape → they fail hard immediately, no retry, +// so a real regression is never masked. +import { execFileSync, spawn } from "node:child_process"; import { readdirSync, readFileSync } from "node:fs"; +import { freemem, loadavg, totalmem } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -75,6 +92,8 @@ const allFiles = readdirSync(testsDir) .sort() .map((f) => `tests/${f}`); +const baseToFile = new Map(allFiles.map((f) => [f.split("/").pop(), f])); + function die(msg) { console.error(`::error::${msg}`); console.error(`[ci-tests] ${msg}`); @@ -104,20 +123,153 @@ const npmCmd = process.platform === "win32" ? "npm.cmd" : "npm"; console.log("[ci-tests] pretest (kernel + pi-subagents build)"); execFileSync(npmCmd, ["run", "pretest"], { cwd: root, stdio: "inherit", shell: process.platform === "win32" }); -function runPhase(label, files, concurrency) { +// Loader chain the suites depend on (storage-baseline preload, #2671/#2803, +// then tsx). Shared by the main phase and the isolated diagnostic re-run so +// the latter sees the SAME environment as the in-phase run. +const PRELOAD_ARGS = ["--import", "./tests/preload-storage-baseline.mjs", "--import", "tsx"]; + +const MAX_CAPTURE = 32 * 1024 * 1024; // bound in-memory capture; real runs are MB-scale + +function runNodeTest(files, concurrency, label) { const t0 = Date.now(); console.log(`[ci-tests] ${label}: ${files.length} files, concurrency=${concurrency}`); - try { - execFileSync( + return new Promise((resolve) => { + const child = spawn( process.execPath, - ["--import", "./tests/preload-storage-baseline.mjs", "--import", "tsx", "--test", "--test-concurrency", String(concurrency), ...files], - { cwd: root, stdio: "inherit" }, + [...PRELOAD_ARGS, "--test", "--test-concurrency", String(concurrency), "--test-reporter=tap", ...files], + { cwd: root }, ); - } catch { - console.error(`[ci-tests] ${label} FAILED after ${((Date.now() - t0) / 1000).toFixed(1)}s`); - process.exit(1); + const out = []; + const err = []; + let outLen = 0; + let errLen = 0; + child.stdout.on("data", (d) => { + if (outLen < MAX_CAPTURE) { + out.push(d); + outLen += d.length; + } + process.stdout.write(d); + }); + child.stderr.on("data", (d) => { + if (errLen < MAX_CAPTURE) { + err.push(d); + errLen += d.length; + } + process.stderr.write(d); + }); + child.on("error", (e) => { + console.error(`[ci-tests] ${label}: spawn error: ${e.message}`); + resolve({ code: -1, signal: null, out: "", err: "", ms: Date.now() - t0 }); + }); + child.on("close", (code, signal) => { + resolve({ + code, + signal, + out: Buffer.concat(out).toString("utf8"), + err: Buffer.concat(err).toString("utf8"), + ms: Date.now() - t0, + }); + }); + }); +} + +function resourceSnapshot() { + const mi = (n) => `${Math.round(n / 1048576)}MiB`; + return ( + `freemem=${mi(freemem())} totalmem=${mi(totalmem())} ` + + `loadavg=${loadavg().map((x) => x.toFixed(2)).join("/")} ` + + `node=${process.version} platform=${process.platform}/${process.arch}` + ); +} + +// A top-level `not ok N - ` can only come from a file +// whose child DIED: successful files promote their inner tests to top level by +// test name and never emit a file-level entry. Pairing that with the YAML +// `error: 'test failed'` marker isolates the swallowed/silent-death shape from +// real assertion failures (named by test name, carrying a real message). +function extractSwallowedFiles(output) { + const lines = output.split(/\r?\n/); + const found = new Set(); + for (let i = 0; i < lines.length; i++) { + const m = lines[i].match(/^not ok \d+ - (.+?)\s*$/); + if (!m) continue; + const window = lines.slice(i, i + 20).join("\n"); + if (!/error:\s*'test failed'/.test(window)) continue; + const base = m[1].trim().split(/[\\/]/).pop(); + const rel = baseToFile.get(base); + if (rel) found.add(rel); } - console.log(`[ci-tests] ${label} ok in ${((Date.now() - t0) / 1000).toFixed(1)}s`); + return [...found]; +} + +function runIsolated(fileRel) { + const t0 = Date.now(); + console.log(`[ci-tests] DIAGNOSTIC isolated re-run (serial): ${fileRel} — full output below`); + return new Promise((resolve) => { + const child = spawn(process.execPath, [...PRELOAD_ARGS, "--test", "--test-reporter=tap", fileRel], { cwd: root }); + child.stdout.on("data", (d) => process.stdout.write(d)); + child.stderr.on("data", (d) => process.stderr.write(d)); + child.on("error", (e) => { + console.error(`[ci-tests] isolated re-run ${fileRel}: spawn error: ${e.message}`); + resolve({ code: -1, signal: null, ms: Date.now() - t0 }); + }); + child.on("close", (code, signal) => resolve({ code, signal, ms: Date.now() - t0 })); + }); } -runPhase("all", allFiles, 4); +(async () => { + const first = await runNodeTest(allFiles, 4, "all"); + if (first.code === 0) { + console.log(`[ci-tests] all ok in ${(first.ms / 1000).toFixed(1)}s`); + return; + } + console.error(`[ci-tests] all FAILED after ${(first.ms / 1000).toFixed(1)}s (exit=${first.code} signal=${first.signal})`); + console.error(`[ci-tests] resource snapshot: ${resourceSnapshot()}`); + const errTrim = first.err.replace(/\s+/g, " ").trim(); + if (errTrim) { + console.error( + `[ci-tests] child stderr captured (${first.err.length} chars) — may carry a V8/native fatal (e.g. OOM 'Reached heap limit'): ` + + `${errTrim.slice(0, 2000)}`, + ); + } else { + console.error( + "[ci-tests] child wrote nothing to stderr (node --test reports test failures on the TAP stdout above; nothing extra to surface here)", + ); + } + const swallowed = extractSwallowedFiles(first.out); + if (swallowed.length === 0) { + console.error( + "[ci-tests] failure carries concrete sub-test assertions (not the swallowed 'test failed' shape) " + + "-> failing WITHOUT auto-retry so a real regression is never masked.", + ); + process.exit(first.code ?? 1); + } + console.error( + `[ci-tests] swallowed file-level failure(s) — zero subtests, payload only 'test failed': ` + + `${swallowed.join(", ")} (#3029)`, + ); + for (const rel of swallowed) { + const r = await runIsolated(rel); + console.error( + `[ci-tests] isolated re-run ${rel}: exit=${r.code} signal=${r.signal} (${(r.ms / 1000).toFixed(1)}s) -> ` + + (r.code === 0 + ? "PASSED in isolation (transient / concurrency flake likely)" + : "REPRODUCED in isolation (deterministic break)"), + ); + } + console.error("[ci-tests] retrying full phase once to confirm flake (#3029)"); + const retry = await runNodeTest(allFiles, 4, "all (retry)"); + if (retry.code === 0) { + console.log( + `[ci-tests] FLAKE CONFIRMED: first run failed on ${swallowed.join(", ")}, full retry green in ` + + `${(retry.ms / 1000).toFixed(1)}s -> treating as transient (#3029). If it recurs, check ` + + `node${process.version.replace(/^v/, "").split(".")[0]}-${process.platform} + tsx load + concurrency=4 (see snapshot above).`, + ); + process.exit(0); + } + console.error(`[ci-tests] full retry also FAILED (exit=${retry.code} signal=${retry.signal}) -> genuine failure, not a flake`); + process.exit(retry.code ?? 1); +})().catch((e) => { + console.error(`[ci-tests] runner crashed: ${e && e.stack ? e.stack : e}`); + process.exit(1); +});