Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions tests/fork-nudge-deadzone.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,12 +203,27 @@ test("the child's first wire turn sizes on the seeded baseline, not 0 (lane proo
const r = await fetch(`${h.origin}/bili/${h.upstreamUrl}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", "x-bili-plugin": "test", "x-bili-plugin-conversation": "child" }, body: JSON.stringify({ model: "small-test", max_tokens: 1024, stream: false, messages }) });
assert.equal(r.status, 200);
await r.text();
const log = readFileSync(join(h.dir, "billion-context", "bili.log"), "utf8");
const childLines = log.split("\n").filter((l) => l.includes(child.id));
const nudgeLine = childLines.find((l) => l.includes("nudge"));
assert.ok(nudgeLine, "child turn must log its nudge decision");
// The logger's file leg is an async append stream (src/logger.ts) —
// under a loaded runner the HTTP response can return before the nudge
// line has flushed to disk, so a one-shot readFileSync raced the flush
// and false-red'd on Windows CI (#3147-A). Poll with a bounded
// deadline instead; the stream preserves write order, so once the
// nudge line is visible every earlier line of the same request (incl.
// any preflight warn) is visible too — the negative assertion below
// stays sound without a separate quiesce wait.
const logPath = join(h.dir, "billion-context", "bili.log");
const readLog = (): string => { try { return readFileSync(logPath, "utf8"); } catch { return ""; } };
const deadline = Date.now() + 5_000;
let nudgeLine: string | undefined;
for (;;) {
nudgeLine = readLog().split("\n").filter((l) => l.includes(child.id)).find((l) => l.includes("nudge"));
if (nudgeLine || Date.now() >= deadline) break;
await new Promise((resolve) => setTimeout(resolve, 50));
}
assert.ok(nudgeLine, "child turn must log its nudge decision (waited up to 5s for the async log stream)");
assert.doesNotMatch(nudgeLine, /usage=0% \(0\//, `dead-zone signature (usage 0% on 0 tokens) must be gone: ${nudgeLine}`);
assert.match(nudgeLine, /(?:idle|ACTIVE)[^\n]*usage=[1-9]\d*%/);
const childLines = readLog().split("\n").filter((l) => l.includes(child.id));
assert.doesNotMatch(childLines.join("\n"), /preflight compressing before forward/, "the in-band payload must not fall into the preflight path");
t.diagnostic(`lane nudge line: ${nudgeLine?.trim()}`);
} finally { await h.close(); }
Expand Down
48 changes: 46 additions & 2 deletions tests/golden-storage-drive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { flushAllSessions, _resetSessionsForTest } from "../src/session.ts";
import type { ProxyOptions } from "../src/config.ts";
import { _setForTest as setRegistryForTest } from "../src/registry.ts";
import { configureLogger, closeLogger } from "../src/logger.ts";
import { rmrf } from "./tmp-rm.ts";

export type Wire = "responses" | "chat" | "anthropic" | "google";
Expand Down Expand Up @@ -379,12 +380,35 @@ function goldenProxyOptions(upstreamPort: number, ctx: number): ProxyOptions {
* store, reset the session pool, optionally drop the legacy file
* ("dropFile" — the unified index must be self-sufficient), then continue
* the conversation through a fresh SessionStore. */
// Scratch root for drive runs (#3147-B): workspace-local + gitignored, NOT
// os.tmpdir(). On Windows hosted runners %TEMP% is an AV/indexer hot path
// AND is shared by all c=4 test legs, so external locks on the golden tree
// have repeatedly outlasted the rmrf retry budget (~9.8s, #2908 family) and
// false-red'd the leg in cleanup. A cwd-relative dir sees none of that.
const SCRATCH_ROOT = path.resolve(process.cwd(), "tmp", "golden-drive");

/** Best-effort sweep of scratch dirs left by an aborted earlier run (> 6h
* old). Never touches live dirs, never throws. */
function sweepStaleScratch(): void {
try {
const cutoff = Date.now() - 6 * 60 * 60 * 1000;
for (const entry of fs.readdirSync(SCRATCH_ROOT)) {
const p = path.join(SCRATCH_ROOT, entry);
try {
if (fs.statSync(p).mtimeMs < cutoff) fs.rmSync(p, { recursive: true, force: true });
} catch { /* locked or already gone — leave it */ }
}
} catch { /* root absent — nothing to sweep */ }
}
sweepStaleScratch();

export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: {
dumpStateTo?: string;
storageUnified?: "shadow" | "full";
reloadAfterEdit?: "keepFile" | "dropFile";
} = {}): Promise<GoldenRecord> {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), `golden-${wire}-`));
fs.mkdirSync(SCRATCH_ROOT, { recursive: true });
const tmp = fs.mkdtempSync(path.join(SCRATCH_ROOT, `${wire}-`));
const sessionsDir = path.join(tmp, "sessions");
const prevXdg = process.env.XDG_STATE_HOME;
const prevUnified = process.env.BILI_STORAGE_UNIFIED;
Expand Down Expand Up @@ -572,6 +596,26 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts:
else process.env.XDG_STATE_HOME = prevXdg;
if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED;
else process.env.BILI_STORAGE_UNIFIED = prevUnified;
rmrf(tmp);
// Kill the in-process re-creators BEFORE rmrf (#3147-B root cause):
// startServer configures the tee logger unconditionally and its file
// stream targets tmp/billion-context/bili.log for the process life —
// after deletion any later log line makes getStream() detect the
// orphaned inode and REOPEN the deleted path, resurrecting the tree
// mid-rmrf (the #1646/#1910 ENOTEMPTY/EBUSY race class). The detach
// + target-clear below are synchronous, so no interleaving window.
// The 1ms-debounced store writer races the same way (recreating
// sessions/<provider>/ after the final flush).
try { store?.cancelAll(); } catch { /* no store on early-exit paths */ }
void closeLogger();
configureLogger("off");
// Cleanup is best-effort (#3147-B): every assertion has already run by
// now, so an external lock outlasting the rmrf ladder must not turn a
// green run red. Leftovers are inert gitignored scratch — the next
// process importing this module sweeps ones > 6h old.
try {
rmrf(tmp);
} catch (err) {
console.warn(`[golden-storage-drive] leftover scratch ${tmp} (cleanup failed: ${(err as Error).message})`);
}
}
}
23 changes: 21 additions & 2 deletions tests/preflight-hold.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ type Call = { raw: string };
type TimedResponse = {
status: number;
headers: http.IncomingHttpHeaders;
/** epoch ms when the request was created */
t0: number;
/** ms from request start to response headers */
tHeaderMs: number;
/** ms from request start to response end */
Expand All @@ -78,6 +80,7 @@ function timedPost(url: string, headers: Record<string, string>, body: string):
resolve({
status: res.statusCode ?? 0,
headers: res.headers,
t0,
tHeaderMs,
tEndMs: Date.now() - t0,
body: Buffer.concat(chunks).toString("utf8"),
Expand Down Expand Up @@ -113,6 +116,7 @@ function startProxy(upstreamPort: number, models: Record<string, { context: numb

test("#568: stream + slow preflight → early 200 + SSE keep-alive, then the real stream", async () => {
const calls: Call[] = [];
let summaryDoneMs = 0;
const upstream = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
Expand All @@ -127,6 +131,7 @@ test("#568: stream + slow preflight → early 200 + SSE keep-alive, then the rea
} else {
// Slow summarization call — outlives the 300ms hold grace.
setTimeout(() => {
summaryDoneMs = Date.now();
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({
id: "msg_summary",
Expand Down Expand Up @@ -160,7 +165,13 @@ test("#568: stream + slow preflight → early 200 + SSE keep-alive, then the rea
assert.equal(r.status, 200);
assert.equal(String(r.headers["content-type"]), "text/event-stream");
assert.equal(r.headers["x-bili-preflight"], "compressing", "early commit is marked for observability");
assert.ok(r.tHeaderMs < SLOW_MS, `headers arrived before summarization finished (${r.tHeaderMs}ms)`);
// #3147-C: assert the RELATIVE invariant (headers before the mock's
// actual summarization completion), not an absolute wall-clock bound —
// a loaded runner stalls both timers equally and used to eat the fixed
// margin while the behavior stayed correct. If early commit regresses,
// headers land at stream end, i.e. AFTER summarization finished.
assert.ok(summaryDoneMs > 0, "the slow summarization call completed");
assert.ok(r.tHeaderMs < summaryDoneMs - r.t0, `headers arrived before summarization finished (${r.tHeaderMs}ms vs ${summaryDoneMs - r.t0}ms)`);
assert.ok(r.tEndMs >= SLOW_MS, `the response waited for the slow preflight (${r.tEndMs}ms) — the hold did real work`);
assert.ok(r.body.includes(": bili-preflight"), "an SSE comment keep-alive was sent while compressing");
assert.ok(r.body.includes('"text_delta"'), "the real stream content follows the keep-alive");
Expand Down Expand Up @@ -283,6 +294,7 @@ test("#568: Responses protocol + slow preflight 429 → early-committed 200, in-

test("#568: non-stream + slow preflight → early 200 + whitespace keep-alive, then valid JSON", async () => {
const calls: Call[] = [];
let summaryDoneMs = 0;
const upstream = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
Expand All @@ -298,6 +310,7 @@ test("#568: non-stream + slow preflight → early 200 + whitespace keep-alive, t
const msgs = parsed.messages;
if (Array.isArray(msgs) && msgs.length === 2 && msgs[0]?.role === "system") {
setTimeout(() => {
summaryDoneMs = Date.now();
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ choices: [{ message: { role: "assistant", content: SUMMARY_TEXT } }] }));
}, SLOW_MS);
Expand Down Expand Up @@ -329,7 +342,13 @@ test("#568: non-stream + slow preflight → early 200 + whitespace keep-alive, t
assert.equal(r.status, 200);
assert.equal(String(r.headers["content-type"]), "application/json");
assert.equal(r.headers["x-bili-preflight"], "compressing");
assert.ok(r.tHeaderMs < SLOW_MS, `headers arrived before summarization finished (${r.tHeaderMs}ms)`);
// #3147-C: assert the RELATIVE invariant (headers before the mock's
// actual summarization completion), not an absolute wall-clock bound —
// a loaded runner stalls both timers equally and used to eat the fixed
// margin while the behavior stayed correct. If early commit regresses,
// headers land at stream end, i.e. AFTER summarization finished.
assert.ok(summaryDoneMs > 0, "the slow summarization call completed");
assert.ok(r.tHeaderMs < summaryDoneMs - r.t0, `headers arrived before summarization finished (${r.tHeaderMs}ms vs ${summaryDoneMs - r.t0}ms)`);
assert.ok(r.tEndMs >= SLOW_MS, `the response waited for the slow preflight (${r.tEndMs}ms)`);
assert.ok(r.body.startsWith(" "), "whitespace keep-alive precedes the JSON body");
const json = JSON.parse(r.body) as { choices?: Array<{ message?: { content?: string } }> };
Expand Down
Loading