From 096931d9c36df0cd3716d51eb6b5a63f4e308590 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 14 Oct 2026 15:59:56 +0800 Subject: [PATCH] test: harden 3 load-sensitive suites against runner jitter (#3147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A: fork-nudge-deadzone lane proof read bili.log once via readFileSync right after the response returned, racing the logger's async append stream (src/logger.ts createWriteStream) — under a loaded Windows runner the nudge line had not flushed yet and the assertion false-red'd. Bounded poll (5s, 50ms ticks) until the session's nudge line lands; write-order keeps the negative assertion sound. Teeth unchanged (timeout fails the same way). B: golden-storage driveWire scratch moved out of os.tmpdir() into the gitignored workspace-local tmp/golden-drive/ — on Windows hosted runners %TEMP% is an AV/indexer hot path shared by all c=4 legs, and external locks there repeatedly outlasted the ~9.8s rmrf ladder (#2908 family). Teardown cleanup is now best-effort (warn + inert leftover, swept >6h old on next import) since every assertion precedes it. Root-cause part: the tee logger (configured unconditionally by startServer) held a file stream into the deleted tree — any later log line made getStream() reopen the orphaned path and resurrect the tree mid-rmrf (#1646/#1910 race class); closeLogger()+configureLogger("off")+store.cancelAll() now run before rmrf so no in-process writer can recreate it. C: preflight-hold asserted an absolute wall-clock bound (tHeaderMs < SLOW_MS = 1500ms) — a loaded runner stalls the event loop and eats the fixed margin while the behavior stays correct (observed 1724ms). Both stream and non-stream legs now assert the relative invariant the message always claimed: headers arrived BEFORE the mock's actual summarization completion (timestamped in-mock, same clock). A #568 regression (no early commit) lands headers at stream end, i.e. after summarization finished — still caught. Lower bound tEndMs >= SLOW_MS untouched. Targeted local: fork-nudge-deadzone 3/3, preflight-hold 4/4, storage-engine-proofs 12/12, golden-storage-render 5/5, tmp-rm 6/6, typecheck clean. Full-suite verdict: remote CI. --- tests/fork-nudge-deadzone.test.ts | 23 ++++++++++++--- tests/golden-storage-drive.ts | 48 +++++++++++++++++++++++++++++-- tests/preflight-hold.test.ts | 23 +++++++++++++-- 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/tests/fork-nudge-deadzone.test.ts b/tests/fork-nudge-deadzone.test.ts index 8abad9ece9..2f202f6f74 100644 --- a/tests/fork-nudge-deadzone.test.ts +++ b/tests/fork-nudge-deadzone.test.ts @@ -203,12 +203,27 @@ test("the child's first wire turn sizes on the seeded baseline, not 0 (lane proo const r = await fetch(`${h.origin}/bili/${h.upstreamUrl}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", "x-bili-plugin": "test", "x-bili-plugin-conversation": "child" }, body: JSON.stringify({ model: "small-test", max_tokens: 1024, stream: false, messages }) }); assert.equal(r.status, 200); await r.text(); - const log = readFileSync(join(h.dir, "billion-context", "bili.log"), "utf8"); - const childLines = log.split("\n").filter((l) => l.includes(child.id)); - const nudgeLine = childLines.find((l) => l.includes("nudge")); - assert.ok(nudgeLine, "child turn must log its nudge decision"); + // The logger's file leg is an async append stream (src/logger.ts) — + // under a loaded runner the HTTP response can return before the nudge + // line has flushed to disk, so a one-shot readFileSync raced the flush + // and false-red'd on Windows CI (#3147-A). Poll with a bounded + // deadline instead; the stream preserves write order, so once the + // nudge line is visible every earlier line of the same request (incl. + // any preflight warn) is visible too — the negative assertion below + // stays sound without a separate quiesce wait. + const logPath = join(h.dir, "billion-context", "bili.log"); + const readLog = (): string => { try { return readFileSync(logPath, "utf8"); } catch { return ""; } }; + const deadline = Date.now() + 5_000; + let nudgeLine: string | undefined; + for (;;) { + nudgeLine = readLog().split("\n").filter((l) => l.includes(child.id)).find((l) => l.includes("nudge")); + if (nudgeLine || Date.now() >= deadline) break; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + assert.ok(nudgeLine, "child turn must log its nudge decision (waited up to 5s for the async log stream)"); assert.doesNotMatch(nudgeLine, /usage=0% \(0\//, `dead-zone signature (usage 0% on 0 tokens) must be gone: ${nudgeLine}`); assert.match(nudgeLine, /(?:idle|ACTIVE)[^\n]*usage=[1-9]\d*%/); + const childLines = readLog().split("\n").filter((l) => l.includes(child.id)); assert.doesNotMatch(childLines.join("\n"), /preflight compressing before forward/, "the in-band payload must not fall into the preflight path"); t.diagnostic(`lane nudge line: ${nudgeLine?.trim()}`); } finally { await h.close(); } diff --git a/tests/golden-storage-drive.ts b/tests/golden-storage-drive.ts index 18c3102b59..a4cc697a63 100644 --- a/tests/golden-storage-drive.ts +++ b/tests/golden-storage-drive.ts @@ -23,6 +23,7 @@ import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { flushAllSessions, _resetSessionsForTest } from "../src/session.ts"; import type { ProxyOptions } from "../src/config.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { configureLogger, closeLogger } from "../src/logger.ts"; import { rmrf } from "./tmp-rm.ts"; export type Wire = "responses" | "chat" | "anthropic" | "google"; @@ -379,12 +380,35 @@ function goldenProxyOptions(upstreamPort: number, ctx: number): ProxyOptions { * store, reset the session pool, optionally drop the legacy file * ("dropFile" — the unified index must be self-sufficient), then continue * the conversation through a fresh SessionStore. */ +// Scratch root for drive runs (#3147-B): workspace-local + gitignored, NOT +// os.tmpdir(). On Windows hosted runners %TEMP% is an AV/indexer hot path +// AND is shared by all c=4 test legs, so external locks on the golden tree +// have repeatedly outlasted the rmrf retry budget (~9.8s, #2908 family) and +// false-red'd the leg in cleanup. A cwd-relative dir sees none of that. +const SCRATCH_ROOT = path.resolve(process.cwd(), "tmp", "golden-drive"); + +/** Best-effort sweep of scratch dirs left by an aborted earlier run (> 6h + * old). Never touches live dirs, never throws. */ +function sweepStaleScratch(): void { + try { + const cutoff = Date.now() - 6 * 60 * 60 * 1000; + for (const entry of fs.readdirSync(SCRATCH_ROOT)) { + const p = path.join(SCRATCH_ROOT, entry); + try { + if (fs.statSync(p).mtimeMs < cutoff) fs.rmSync(p, { recursive: true, force: true }); + } catch { /* locked or already gone — leave it */ } + } + } catch { /* root absent — nothing to sweep */ } +} +sweepStaleScratch(); + export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: { dumpStateTo?: string; storageUnified?: "shadow" | "full"; reloadAfterEdit?: "keepFile" | "dropFile"; } = {}): Promise { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), `golden-${wire}-`)); + fs.mkdirSync(SCRATCH_ROOT, { recursive: true }); + const tmp = fs.mkdtempSync(path.join(SCRATCH_ROOT, `${wire}-`)); const sessionsDir = path.join(tmp, "sessions"); const prevXdg = process.env.XDG_STATE_HOME; const prevUnified = process.env.BILI_STORAGE_UNIFIED; @@ -572,6 +596,26 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: else process.env.XDG_STATE_HOME = prevXdg; if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prevUnified; - rmrf(tmp); + // Kill the in-process re-creators BEFORE rmrf (#3147-B root cause): + // startServer configures the tee logger unconditionally and its file + // stream targets tmp/billion-context/bili.log for the process life — + // after deletion any later log line makes getStream() detect the + // orphaned inode and REOPEN the deleted path, resurrecting the tree + // mid-rmrf (the #1646/#1910 ENOTEMPTY/EBUSY race class). The detach + // + target-clear below are synchronous, so no interleaving window. + // The 1ms-debounced store writer races the same way (recreating + // sessions// after the final flush). + try { store?.cancelAll(); } catch { /* no store on early-exit paths */ } + void closeLogger(); + configureLogger("off"); + // Cleanup is best-effort (#3147-B): every assertion has already run by + // now, so an external lock outlasting the rmrf ladder must not turn a + // green run red. Leftovers are inert gitignored scratch — the next + // process importing this module sweeps ones > 6h old. + try { + rmrf(tmp); + } catch (err) { + console.warn(`[golden-storage-drive] leftover scratch ${tmp} (cleanup failed: ${(err as Error).message})`); + } } } diff --git a/tests/preflight-hold.test.ts b/tests/preflight-hold.test.ts index 198fe20899..a54bf895bb 100644 --- a/tests/preflight-hold.test.ts +++ b/tests/preflight-hold.test.ts @@ -60,6 +60,8 @@ type Call = { raw: string }; type TimedResponse = { status: number; headers: http.IncomingHttpHeaders; + /** epoch ms when the request was created */ + t0: number; /** ms from request start to response headers */ tHeaderMs: number; /** ms from request start to response end */ @@ -78,6 +80,7 @@ function timedPost(url: string, headers: Record, body: string): resolve({ status: res.statusCode ?? 0, headers: res.headers, + t0, tHeaderMs, tEndMs: Date.now() - t0, body: Buffer.concat(chunks).toString("utf8"), @@ -113,6 +116,7 @@ function startProxy(upstreamPort: number, models: Record { const calls: Call[] = []; + let summaryDoneMs = 0; const upstream = http.createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c: Buffer) => chunks.push(c)); @@ -127,6 +131,7 @@ test("#568: stream + slow preflight → early 200 + SSE keep-alive, then the rea } else { // Slow summarization call — outlives the 300ms hold grace. setTimeout(() => { + summaryDoneMs = Date.now(); res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ id: "msg_summary", @@ -160,7 +165,13 @@ test("#568: stream + slow preflight → early 200 + SSE keep-alive, then the rea assert.equal(r.status, 200); assert.equal(String(r.headers["content-type"]), "text/event-stream"); assert.equal(r.headers["x-bili-preflight"], "compressing", "early commit is marked for observability"); - assert.ok(r.tHeaderMs < SLOW_MS, `headers arrived before summarization finished (${r.tHeaderMs}ms)`); + // #3147-C: assert the RELATIVE invariant (headers before the mock's + // actual summarization completion), not an absolute wall-clock bound — + // a loaded runner stalls both timers equally and used to eat the fixed + // margin while the behavior stayed correct. If early commit regresses, + // headers land at stream end, i.e. AFTER summarization finished. + assert.ok(summaryDoneMs > 0, "the slow summarization call completed"); + assert.ok(r.tHeaderMs < summaryDoneMs - r.t0, `headers arrived before summarization finished (${r.tHeaderMs}ms vs ${summaryDoneMs - r.t0}ms)`); assert.ok(r.tEndMs >= SLOW_MS, `the response waited for the slow preflight (${r.tEndMs}ms) — the hold did real work`); assert.ok(r.body.includes(": bili-preflight"), "an SSE comment keep-alive was sent while compressing"); assert.ok(r.body.includes('"text_delta"'), "the real stream content follows the keep-alive"); @@ -283,6 +294,7 @@ test("#568: Responses protocol + slow preflight 429 → early-committed 200, in- test("#568: non-stream + slow preflight → early 200 + whitespace keep-alive, then valid JSON", async () => { const calls: Call[] = []; + let summaryDoneMs = 0; const upstream = http.createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c: Buffer) => chunks.push(c)); @@ -298,6 +310,7 @@ test("#568: non-stream + slow preflight → early 200 + whitespace keep-alive, t const msgs = parsed.messages; if (Array.isArray(msgs) && msgs.length === 2 && msgs[0]?.role === "system") { setTimeout(() => { + summaryDoneMs = Date.now(); res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ choices: [{ message: { role: "assistant", content: SUMMARY_TEXT } }] })); }, SLOW_MS); @@ -329,7 +342,13 @@ test("#568: non-stream + slow preflight → early 200 + whitespace keep-alive, t assert.equal(r.status, 200); assert.equal(String(r.headers["content-type"]), "application/json"); assert.equal(r.headers["x-bili-preflight"], "compressing"); - assert.ok(r.tHeaderMs < SLOW_MS, `headers arrived before summarization finished (${r.tHeaderMs}ms)`); + // #3147-C: assert the RELATIVE invariant (headers before the mock's + // actual summarization completion), not an absolute wall-clock bound — + // a loaded runner stalls both timers equally and used to eat the fixed + // margin while the behavior stayed correct. If early commit regresses, + // headers land at stream end, i.e. AFTER summarization finished. + assert.ok(summaryDoneMs > 0, "the slow summarization call completed"); + assert.ok(r.tHeaderMs < summaryDoneMs - r.t0, `headers arrived before summarization finished (${r.tHeaderMs}ms vs ${summaryDoneMs - r.t0}ms)`); assert.ok(r.tEndMs >= SLOW_MS, `the response waited for the slow preflight (${r.tEndMs}ms)`); assert.ok(r.body.startsWith(" "), "whitespace keep-alive precedes the JSON body"); const json = JSON.parse(r.body) as { choices?: Array<{ message?: { content?: string } }> };