diff --git a/tests/fixtures/multi-instance-child.ts b/tests/fixtures/multi-instance-child.ts new file mode 100644 index 0000000000..55b5c302e8 --- /dev/null +++ b/tests/fixtures/multi-instance-child.ts @@ -0,0 +1,64 @@ +// Child process for tests/multi-instance-live.test.ts — one bili proxy +// instance. The parent spawns TWO of these over the same BILI_SESSIONS_DIR to +// reproduce the #394 two-live-instances shape hermetically (each child has its +// own module-level session pool and its own SessionStore over the shared dir — +// the part one-process tests cannot simulate, because the pool is a +// module singleton). +// +// argv: +// env: BILI_SESSIONS_DIR (shared sessions dir), XDG_STATE_HOME (shared state +// dir — registry lane + tee log), BILI_PERSIST_DEBOUNCE_MS=30 (fast +// settles). +// stdout: "MI_READY " once listening. Exits on SIGTERM (graceful close). + +import { startServer } from "../../src/server.ts"; +import type { ProxyOptions } from "../../src/config.ts"; +import { defaultConfig } from "acp-kernel"; + +const upstreamPort = process.argv[2] ?? ""; +if (upstreamPort.length === 0) throw new Error("usage: multi-instance-child.ts "); +const upstream = `http://127.0.0.1:${upstreamPort}`; + +const proxy = await startServer({ + port: 0, + proxy: "", + proxyFallback: { explicitDirect: true, globalSource: "direct" }, + auxProxyFallback: { explicitDirect: true, globalSource: "direct" }, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [upstream]: {} }, + modelContextLimit: 200_000, + kernelConfig: defaultConfig(200_000), + compress: { injectTool: true, injectNudge: true }, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, +} as ProxyOptions); + +const deadline = Date.now() + 10_000; +for (;;) { + const a = proxy.address(); + if (a && typeof (a as { port?: number }).port === "number") break; + if (Date.now() > deadline) throw new Error("child server never bound a port"); + await new Promise((r) => setTimeout(r, 5)); +} +const addr = proxy.address() as { port: number }; +process.stdout.write(`MI_READY ${addr.port}\n`); + +const die = (): void => { + proxy.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1500).unref(); +}; +process.on("SIGTERM", die); +process.stdin.on("close", die); diff --git a/tests/multi-instance-live.test.ts b/tests/multi-instance-live.test.ts new file mode 100644 index 0000000000..ce48cd40de --- /dev/null +++ b/tests/multi-instance-live.test.ts @@ -0,0 +1,261 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { spawn } from "node:child_process"; +import { tmpdir } from "node:os"; +import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { rmrf } from "./tmp-rm.ts"; + +// Two LIVE proxy instances over one sessions dir (#394 shape, end-to-end). +// +// What is proven here — the documented contract only, not multi-writer +// coherence (src/persist.ts deliberately rejects an election/lock): +// 1. both instances keep serving through interleaved turns on the SAME +// session id (no crash, no head-of-line poisoning across processes); +// 2. the on-disk record stays a valid envelope at EVERY observed moment +// (atomic tmp+rename) while two processes write it; +// 3. stats.requests on disk NEVER decreases (the #405 monotonic-counter +// guard holds under real traffic, not just unit-shaped writes); +// 4. a stale resident's write is rejected AND the resident converges to the +// newer disk lineage (#2401) — observable as: A's post-convergence write +// is accepted and continues the DISK counter, not A's stale one; +// 5. the late-starting instance warns about the overlap (#394) at +// registration. +// +// NOT proven (documented residuals, pinned separately in +// tests/multi-instance-persist.test.ts): tie passthrough, counter catch-up +// re-clobber, content-store last-writer-wins. + +const SESSION = "mi-live-session"; +const MODEL = "claude-sonnet-4-5"; +const TURN_TIMEOUT_MS = 15_000; +const SETTLE_TIMEOUT_MS = 10_000; + +function sseBlock(type: string, data: Record): string { + return `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`; +} + +function textScript(msgId: string, text: string): string { + return [ + sseBlock("message_start", { message: { id: msgId, role: "assistant", usage: { input_tokens: 40 } } }), + sseBlock("content_block_start", { index: 0, content_block: { type: "text", text: "" } }), + sseBlock("content_block_delta", { index: 0, delta: { type: "text_delta", text } }), + sseBlock("content_block_stop", { index: 0 }), + sseBlock("message_delta", { delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 4 } }), + sseBlock("message_stop", {}), + ].join(""); +} + +interface Child { + proc: ReturnType; + port: number; + stderr: string[]; +} + +async function spawnChild(upstreamPort: number | string, env: NodeJS.ProcessEnv): Promise { + const proc = spawn(process.execPath, ["--import", "tsx", join(import.meta.dirname, "fixtures/multi-instance-child.ts"), String(upstreamPort)], { + env, + stdio: ["pipe", "pipe", "pipe"], + }); + const stderr: string[] = []; + proc.stderr!.setEncoding("utf8"); + proc.stderr!.on("data", (c: string) => { + stderr.push(c); + process.stderr.write(`[child] ${c}`); + }); + const port = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("child did not become ready in 30s")), 30_000); + let buf = ""; + proc.stdout!.setEncoding("utf8"); + proc.stdout!.on("data", (c: string) => { + buf += c; + const m = buf.match(/MI_READY (\d+)/); + if (m) { + clearTimeout(timer); + resolve(Number(m[1])); + } + }); + proc.on("exit", (code) => reject(new Error(`child exited early (code ${code})`))); + }); + return { proc, port, stderr }; +} + +interface DiskView { + file: string; + requests: number; +} + +function readDisk(sessionsDir: string): DiskView | null { + let dirs: string[]; + try { + dirs = readdirSync(sessionsDir, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name); + } catch { + return null; + } + for (const d of dirs) { + for (const f of readdirSync(join(sessionsDir, d))) { + if (!f.endsWith(".json") || f.endsWith(".content-store.json") || f.includes(".tmp-")) continue; + const p = join(sessionsDir, d, f); + let parsed: { id?: string; payload?: { stats?: { requests?: number } } }; + try { + parsed = JSON.parse(readFileSync(p, "utf8")); + } catch { + // torn read of an in-flight rename — the reader loop treats any + // parse failure as a violation; here just skip (the loop catches it) + throw new Error(`on-disk session record failed to parse: ${p}`); + } + if (parsed.id === SESSION) { + return { file: p, requests: parsed.payload?.stats?.requests ?? -1 }; + } + } + } + return null; +} + +async function waitFor(sessionsDir: string, pred: (v: DiskView) => boolean, what: string): Promise { + const deadline = Date.now() + SETTLE_TIMEOUT_MS; + for (;;) { + const v = (() => { + try { + return readDisk(sessionsDir); + } catch { + return null; + } + })(); + if (v && pred(v)) return v; + if (Date.now() > deadline) throw new Error(`timeout waiting for: ${what} (last=${JSON.stringify(v)})`); + await new Promise((r) => setTimeout(r, 25)); + } +} + +test("two live instances over one sessions dir: serve, guard, converge (#394/#405/#2401)", { timeout: 120_000 }, async () => { + const sessionsDir = mkdtempSync(join(tmpdir(), "bili-mi-sessions-")); + const stateDir = mkdtempSync(join(tmpdir(), "bili-mi-state-")); + const children: Child[] = []; + + // Reader loop: the shared record must parse at every observed moment and + // its request counter must never decrease (atomicity + monotonic guard). + let reads = 0; + let maxRequests = -1; + let readerBroken: string | null = null; + const reader = setInterval(() => { + try { + const v = readDisk(sessionsDir); + reads++; + if (v) { + if (v.requests < maxRequests) readerBroken = `disk requests rolled back: ${v.requests} < ${maxRequests}`; + maxRequests = Math.max(maxRequests, v.requests); + } + } catch (e) { + readerBroken = readerBroken ?? String(e); + } + }, 5); + + const upstream = http.createServer((req, res) => { + req.on("data", () => {}); + req.on("end", () => { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.write(textScript(`msg_${Date.now()}`, `reply ok`)); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + + const childEnv = (p: number): NodeJS.ProcessEnv => ({ + ...process.env, + BILI_SESSIONS_DIR: sessionsDir, + XDG_STATE_HOME: stateDir, + BILI_PERSIST_DEBOUNCE_MS: "30", + BILI_UPDATE_CHECK_INTERVAL_MS: String(60 * 60 * 1000), + MI_NOP: String(p), + }); + + const turn = async (child: Child, n: number): Promise => { + const url = `http://127.0.0.1:${child.port}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`; + const res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": SESSION }, + body: JSON.stringify({ model: MODEL, max_tokens: 1024, stream: true, messages: [{ role: "user", content: `turn ${n}: hello` }] }), + signal: AbortSignal.timeout(TURN_TIMEOUT_MS), + }); + const body = await res.text(); + assert.equal(res.status, 200, `turn ${n} via :${child.port}: ${body.slice(0, 300)}`); + assert.ok(body.includes("message_stop"), `turn ${n}: stream incomplete`); + }; + + try { + // Phase 1 — instance A alone warms the session (3 turns, persisted). + children.push(await spawnChild(upstreamPort, childEnv(1))); + const A = children[0]!; + await turn(A, 1); + await turn(A, 2); + await turn(A, 3); + const warmed = await waitFor(sessionsDir, (v) => v.requests >= 3, "A's 3 turns persisted"); + assert.equal(warmed.requests, 3, "each turn must count exactly once"); + + // Phase 2 — instance B boots over the SAME dir (boot-hydrates the + // session), serves 2 turns. B's writes must be fresh (accepted). + children.push(await spawnChild(upstreamPort, childEnv(2))); + const B = children[1]!; + await turn(B, 4); + await turn(B, 5); + await waitFor(sessionsDir, (v) => v.requests >= 5, "B's turns persisted (fresh lineage accepted)"); + + // B registered last: it must have seen A and warned (#394). + const bErr = B.stderr.join(""); + assert.ok(bErr.includes("#394"), "late starter must warn about the overlapping instance"); + assert.ok(bErr.includes("another bili instance"), `unexpected warn text: ${bErr.slice(-400)}`); + + // Phase 3 — the STALE resident: A never re-read the file (its pool + // entry is resident), so A's counter (3+1=4) is behind disk (5). Its + // write must be REJECTED (#405) and A must converge (#2401). + await turn(A, 6); + await new Promise((r) => setTimeout(r, 300)); + const aErr1 = A.stderr.join(""); + assert.ok(aErr1.includes("rejected stale snapshot"), `stale write must be rejected (stderr tail: ${aErr1.slice(-500)})`); + + // Phase 4 — A's next write continues from the CONVERGED (disk) + // counter: 5+1=6, accepted; disk must show exactly 6. + await turn(A, 7); + const converged = await waitFor(sessionsDir, (v) => v.requests >= 6, "A's post-convergence write accepted"); + assert.equal(converged.requests, 6, "converged resident must continue the DISK counter, not its stale one"); + + // End-state: the session still serves through BOTH instances. B is now + // itself behind (its resident counter is 5, disk is 6 after A's accepted + // write), so B's turn lands on the documented TIE: equal request + // counters + equal nextBlockId → staleDiskPayload returns null → the + // write passes through (last-writer-wins on ties, persist.ts:76-87). + // Either way the file must stay valid and the counter non-decreasing — + // the reader loop below is the real teeth for that. + await turn(B, 8); + const final2 = await waitFor(sessionsDir, (v) => v.requests >= 6, "B serves after A's convergence"); + assert.equal(final2.requests, 6, "disk counter reflects the last accepted lineage (B tie-passthrough at 6)"); + + // Reader loop verdict. + clearInterval(reader); + assert.ok(reads > 10, `reader loop ran (reads=${reads})`); + assert.equal(readerBroken, null, `shared record violated the always-valid/monotonic contract: ${readerBroken}`); + + // Both children stayed alive through the whole interleave. + for (const c of children) { + assert.equal(c.proc.exitCode, null, "a child died mid-test"); + assert.equal(c.proc.signalCode, null, "a child was killed mid-test"); + } + } finally { + clearInterval(reader); + upstream.close(); + for (const c of children) { + c.proc.kill("SIGTERM"); + } + await new Promise((r) => setTimeout(r, 300)); + for (const c of children) { + if (c.proc.exitCode === null) c.proc.kill("SIGKILL"); + } + rmrf(sessionsDir); + rmrf(stateDir); + } +}); diff --git a/tests/multi-instance-persist.test.ts b/tests/multi-instance-persist.test.ts new file mode 100644 index 0000000000..e1654af32a --- /dev/null +++ b/tests/multi-instance-persist.test.ts @@ -0,0 +1,168 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { tmpdir } from "node:os"; +import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { createInitialState } from "acp-kernel"; +import { rmrf } from "./tmp-rm.ts"; + +// Unit-tier pins for the TWO-INSTANCE residuals documented in src/persist.ts +// (the design deliberately rejects an election/lock; one state dir should have +// one live instance — but two CAN share a dir, and their interleavings have +// contracts worth pinning). The end-to-end shape (two live proxies) is covered +// by tests/multi-instance-live.test.ts; these tests drive two SessionStore +// objects over one dir directly. +// +// These pins document CURRENT behavior. The catch-up re-clobber pin in +// particular is expected to turn RED when someone tightens the guard +// (persist.ts:81-84 documents the residual) — that is its job: the change +// should be conscious, not accidental. + +function makeSession(id: string): Session { + return { + id, + meta: { protocol: "openai", upstreamOrigin: "http://upstream" }, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, compressCreditTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +function withTempDir(name: string, fn: (dir: string) => Promise | void): Promise { + return test(name, { timeout: 60_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-multiinst-")); + try { + await fn(dir); + } finally { + rmrf(dir); + } + }) as unknown as Promise; +} + +function envelopeOf(dir: string, id: string): { file: string; payload: { stats: { requests: number }; metadata: Record } } { + for (const d of readdirSync(dir, { withFileTypes: true }).filter((e) => e.isDirectory())) { + for (const f of readdirSync(join(dir, d.name))) { + if (!f.endsWith(".json") || f.endsWith(".content-store.json") || f.includes(".tmp-")) continue; + const p = join(dir, d.name, f); + const parsed = JSON.parse(readFileSync(p, "utf8")); + if (parsed.id === id) return { file: p, payload: parsed.payload }; + } + } + throw new Error(`no envelope for ${id} under ${dir}`); +} + +await withTempDir("interleaved storm: file stays valid and never rolls back", async (dir) => { + const warnings: string[] = []; + const log = (level: string, msg: string): void => { + if (level === "warn") warnings.push(msg); + }; + const storeA = new SessionStore({ dir, debounceMs: 1, enabled: true, log }); + const storeB = new SessionStore({ dir, debounceMs: 1, enabled: true, log }); + const a = makeSession("storm"); + const b = makeSession("storm"); + b.stats.requests = 0; + + // A is the live writer; B is a lagging resident whose debounced writes + // land late. 120 interleave rounds; sample disk validity throughout. + let peak = 0; + for (let i = 1; i <= 120; i++) { + a.stats.requests = i; + a.metadata.peak = `a-${i}`; + await storeA.writeNow(a); + if (i % 6 === 0) { + b.stats.requests = Math.max(0, i - 40); // always behind + b.metadata.peak = `b-${i}`; + await storeB.writeNow(b); + } + if (i % 10 === 0) { + const v = envelopeOf(dir, "storm"); // throws if torn/unparseable + peak = Math.max(peak, v.payload.stats.requests); + assert.ok(v.payload.stats.requests >= 1, "storm: disk counter vanished"); + } + } + const final = envelopeOf(dir, "storm"); + assert.equal(final.payload.stats.requests, 120, "final disk counter must be the freshest lineage (A's 120)"); + assert.equal(final.payload.metadata.peak, "a-120"); + assert.ok(warnings.some((w) => w.includes("rejected stale snapshot")), "lagging writes must be rejected with the stale warn"); + assert.ok(peak > 0, "sampling actually observed the file"); + storeA.cancelAll(); + storeB.cancelAll(); +}); + +await withTempDir("documented residual: counter catch-up re-clobbers newer content (persist.ts:81-84) — pin, not endorsement", async (dir) => { + const warnings: string[] = []; + const log = (level: string, msg: string): void => { + if (level === "warn") warnings.push(msg); + }; + const storeA = new SessionStore({ dir, debounceMs: 1, enabled: true, log }); + const storeB = new SessionStore({ dir, debounceMs: 1, enabled: true, log }); + + // Instance A writes the newer lineage at counter 10. + const a = makeSession("reclob"); + a.stats.requests = 10; + a.metadata.lineage = "A-newer"; + await storeA.writeNow(a); + + // Instance B hydrated BEFORE A's write (its view is older) and only now + // catches up on the COUNTER — its content is still the old lineage. + const b = makeSession("reclob"); + b.stats.requests = 10; // caught up numerically + b.metadata.lineage = "B-older"; + await storeB.writeNow(b); + + // Tie on requests AND nextBlockId → staleDiskPayload returns null → the + // write passes through and B's OLDER content replaces A's newer payload. + // persist.ts:81-84 documents this residual; the guard cannot see content + // lineage, only counters. This pin documents the CURRENT contract so a + // future tightening (content fingerprints / writer epochs) flips it + // intentionally instead of silently. + const after = envelopeOf(dir, "reclob"); + assert.equal(after.payload.metadata.lineage, "B-older", "CURRENT behavior: tie passthrough clobbers newer content"); + assert.equal(after.payload.stats.requests, 10, "counter itself does not roll back"); + assert.ok(!warnings.some((w) => w.includes("rejected stale snapshot")), "the clobbering write is silently accepted — that is the residual"); + storeA.cancelAll(); + storeB.cancelAll(); +}); + +await withTempDir("content-store companion has no guard: last writer wins (documented degradation)", async (dir) => { + const storeA = new SessionStore({ dir, debounceMs: 1, enabled: true, log: () => {} }); + const storeB = new SessionStore({ dir, debounceMs: 1, enabled: true, log: () => {} }); + + const mkEntry = (rawId: string): { hash: string; rawId: string; kind: string; tokens: number; chars: number; head: string } => ({ hash: `h-${rawId}`, rawId, kind: "shell output", tokens: 3, chars: 9, head: "A payload" }); + const mkStore = (rawId: string): NonNullable => ({ version: 1, byHash: { [`h-${rawId}`]: `payload ${rawId}` }, byRef: { [rawId]: mkEntry(rawId) } }); + const a = makeSession("ccs"); + a.stats.requests = 5; + a.contentStore = mkStore("r1"); + a.contentStoreDirty = true; + await storeA.writeNow(a); + + const b = makeSession("ccs"); + b.stats.requests = 6; // fresher main record + b.contentStore = mkStore("r2"); + b.contentStoreDirty = true; + await storeB.writeNow(b); + + // The main envelope guard sees requests 6 > 5 → fine. The CONTENT-STORE + // companion file (persist.ts saveContentStore) has no lineage guard at + // all: B's byRef REPLACES A's — refs only A stored are now missing from + // disk (degraded retrieves, not main-record corruption). Pinned as-is. + let companion: string | null = null; + for (const d of readdirSync(dir, { withFileTypes: true }).filter((e) => e.isDirectory())) { + for (const f of readdirSync(join(dir, d.name))) { + if (f.endsWith(".content-store.json")) companion = join(dir, d.name, f); + } + } + assert.ok(companion, "content-store companion file must exist"); + const parsed = JSON.parse(readFileSync(companion!, "utf8")) as { byRef: Record }; + assert.deepEqual(Object.keys(parsed.byRef).sort(), ["r2"], "CURRENT behavior: companion is last-writer-wins, A's r1 is gone"); + storeA.cancelAll(); + storeB.cancelAll(); +});