diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index 1b703de801..d585e2027e 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -71,12 +71,108 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno } return out; } + return canonicalScalar(value, key); +} + +function canonicalScalar(value: unknown, key: string | undefined): unknown { if (key === "biliVersion" && typeof value === "string") return "__VER__"; if (typeof value === "number" && Number.isFinite(value) && value > 1_600_000_000_000) return "__TS__"; if (typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) return "__UUID__"; return value; } +// Spec array index: "0" .. 2^32-2, no leading zeros. +function isArrayIndexKey(k: string): boolean { + return /^(0|[1-9]\d*)$/.test(k) && Number(k) <= 4_294_967_294; +} + export function sha256(s: string): string { return createHash("sha256").update(s, "utf8").digest("hex"); } + +/** Hash a storage view without materializing its canonical copy or JSON string. */ +export function canonicalDigest(value: unknown): string { + if (value === undefined || typeof value === "function" || typeof value === "symbol") return sha256(JSON.stringify(value)); + const hash = createHash("sha256"); + const chunkSize = 64 * 1024; + let pending = ""; + function write(text: string): void { + pending += text; + if (pending.length >= chunkSize) { + hash.update(pending, "utf8"); + pending = ""; + } + } + function writeString(text: string): void { + write('"'); + for (let start = 0; start < text.length;) { + let end = Math.min(start + chunkSize, text.length); + const last = text.charCodeAt(end - 1); + // Keep a surrogate pair together so JSON escaping and UTF-8 stay exact. + if (end < text.length && last >= 0xd800 && last <= 0xdbff) end--; + write(JSON.stringify(text.slice(start, end)).slice(1, -1)); + start = end; + } + write('"'); + } + function visit(current: unknown, key?: string): void { + if (Array.isArray(current)) { + write("["); + for (let i = 0; i < current.length; i++) { + if (i) write(","); + visit(current[i]); + } + write("]"); + } else if (current instanceof Map) { + write('{"__map__":['); + let separator = ""; + for (const [k, v] of current) { + write(separator + "["); + // canonicalize leaves Map keys untouched. + write(JSON.stringify(k) ?? "null"); + write(","); + visit(v); + write("]"); + separator = ","; + } + write("]}"); + } else if (current instanceof Set) { + write('{"__set__":['); + let separator = ""; + for (const v of current) { + write(separator); + visit(v); + separator = ","; + } + write("]}"); + } else if (current && typeof current === "object") { + write("{"); + let separator = ""; + // Effective order of JSON.stringify(canonicalize(v)): array-index keys + // first, ascending numeric (spec enumeration overrides canonicalize's + // lexicographic sort), then the remaining keys lexicographic. + const keys = Object.keys(current); + const orderedKeys = keys.filter(isArrayIndexKey).sort((a, b) => Number(a) - Number(b)) + .concat(keys.filter((k) => !isArrayIndexKey(k)).sort()); + for (const k of orderedKeys) { + // Assigning __proto__ to canonicalize's plain output creates no JSON property. + if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k) || k === "__proto__") continue; + const v = (current as Record)[k]; + if (v === undefined || typeof v === "function" || typeof v === "symbol") continue; + write(separator); + writeString(k); + write(":"); + visit(v, k); + separator = ","; + } + write("}"); + } else { + const normalized = canonicalScalar(current, key); + if (typeof normalized === "string") writeString(normalized); + else write(JSON.stringify(normalized) ?? "null"); + } + } + visit(value); + hash.update(pending, "utf8"); + return hash.digest("hex"); +} diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index f7d66cc8d5..f9e79e746e 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -29,7 +29,7 @@ import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSu import { renameSync } from "node:fs"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; -import { canonicalize, sha256 } from "./canonical.ts"; +import { canonicalDigest, canonicalize, sha256 } from "./canonical.ts"; import { recordSidecarInstance, recordSidecarOpenFailure } from "./runtime-status.ts"; import type { SqliteEngineName, SqliteEngineSource } from "./driver.ts"; import type { MessageRow, MessageRefContentRow } from "./store.js"; @@ -1243,7 +1243,7 @@ function warnTailGaps(id: string, view: { tailGaps?: number }): void { } function shadowDigest(view: unknown): string { - return sha256(JSON.stringify(canonicalize(view))); + return canonicalDigest(view); } // #3407: the parity digest must compare blockContents CONTENTS, not insertion order — diff --git a/src/web/blogfeed.ts b/src/web/blogfeed.ts index 494e7627dc..54839ae78a 100644 --- a/src/web/blogfeed.ts +++ b/src/web/blogfeed.ts @@ -106,10 +106,11 @@ export function _setBlogRootForTest(root: string | null): void { /** Mirror of the website tools' frontmatter parser (key: value between --- fences). */ function parseFrontmatter(text: string): { fm: Record; body: string } { const fm: Record = {}; - let body = text; - // Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so - // the exact-match fences below survive Windows checkouts (#4599). - const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text; + // Git-for-Windows autocrlf checks .md posts out with CRLF; normalize + // UNCONDITIONALLY so the exact-match fences survive Windows checkouts + // (#4599) and non-frontmatter bodies cannot leak \r into rendered HTML. + const src = text.replace(/\r\n/g, "\n"); + let body = src; if (src.startsWith("---\n")) { const end = src.indexOf("\n---\n", 4); if (end !== -1) { diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index 13bdee7bf5..598e8295d9 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -228,37 +228,68 @@ async function waitForStatus(port: number, timeoutMs: number): Promise async function killChild(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill("SIGTERM"); - await Promise.race([ - new Promise((resolve) => child.once("exit", () => resolve())), - new Promise((resolve) => setTimeout(resolve, 10_000)), - ]); - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + child.kill("SIGTERM"); + }); } /** A TCP listener that accepts connections and then says nothing — invisible * to /__bili/health probes (they time out as "no evidence") but still owns * the socket, so binders get EADDRINUSE. Deterministically forces the LATE * EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */ -function silentTcpListener(port: number): Promise { +function silentTcpListener(port: number): Promise<{ close(): Promise }> { return new Promise((resolve, reject) => { + const sockets = new Set(); const server = net.createServer((sock) => { + sockets.add(sock); + sock.once("close", () => sockets.delete(sock)); // accept and hang — never answer HTTP; drain the probe's bytes so peer // EOF is actually delivered (a paused socket buffers them and never sees // EOF) and release the connection, or a dead prober pins server.close() // with CLOSE-WAIT sockets sock.resume(); - // A killed prober (Windows kill() is a hard terminate) RSTs us mid-probe; - // without an error listener that surfaces as an uncaughtException AFTER - // the test ends and fails the whole file (win-only #5050 follow-up). - sock.on("error", () => {}); sock.on("end", () => sock.destroy()); + // An aborted health probe may reset instead of ending the connection, + // or a killed prober (Windows kill() is a hard terminate) RSTs mid-probe; + // without an error listener that surfaces as an uncaughtException AFTER + // the test ends and fails the whole file (#5050). Destroying keeps the + // fixture clean and owns the socket. + sock.on("error", () => sock.destroy()); }); server.once("error", reject); - server.listen(port, "127.0.0.1", () => resolve(server)); + server.listen(port, "127.0.0.1", () => resolve({ + close: () => new Promise((done) => { + server.close(() => done()); + for (const socket of sockets) socket.destroy(); + }), + })); }); } +test("silent TCP fixture survives an aborted health probe", { timeout: 5_000 }, async () => { + const port = await freePort(); + const fixture = await silentTcpListener(port); + const client = net.connect({ host: "127.0.0.1", port }); + try { + await new Promise((resolve, reject) => { + client.once("connect", resolve); + client.once("error", reject); + }); + client.write("GET /__bili/health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + await new Promise((resolve) => setTimeout(resolve, 25)); + client.resetAndDestroy(); + await new Promise((resolve) => setTimeout(resolve, 25)); + } finally { + client.destroy(); + await fixture.close(); + } +}); + function tcpRefused(port: number): Promise { return new Promise((resolve) => { const s = net.connect({ port, host: "127.0.0.1" }); @@ -323,6 +354,9 @@ function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: { async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise { let port = 0; await waitUntil(async () => { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`${label} exited before becoming healthy`); + } const m = readMarkers(stateHome).find((e) => e.pid === child.pid); if (!m || typeof m.port !== "number") return false; port = m.port as number; @@ -339,7 +373,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders let p1Ref: ReturnType | null = null; let p2Ref: ReturnType | null = null; let heldRef: net.Socket | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // P1: the serving proxy that will enter graceful shutdown. const p1 = spawnBili(sandbox, port); @@ -415,7 +449,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders heldRef?.destroy(); if (p1Ref) await killChild(p1Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); @@ -426,7 +460,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n const port = await freePort(); await assertPortDead(port, { label: "4982 unverified-twin port" }); let p2Ref: ReturnType | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // A registry marker claiming `port` with a LIVE pid and full identity // match — but NO portReleasedAt stamp (pre-fix build, or a failed @@ -476,7 +510,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n } finally { if (p2Ref) await killChild(p2Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 878505d0fe..0b22b273de 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -115,7 +115,7 @@ async function harness(persist = false, seedParent = true, compressExtra: Record assert.equal(r.status, 200); await r.text(); } - return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; + return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.closeQuietly(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; } test("plain proxy panel snapshots do not change raw-history persistence", { timeout: 60_000 }, async () => { @@ -885,82 +885,85 @@ for (const corruption of ["missing-ref", "missing-payload", "orphan-payload", "w }); } -test("HTTP fork revision and originals survive a new proxy process", { timeout: 60_000 }, async () => { - const h = await harness(true); - let child: ReturnType | undefined; - try { - await compress(h); - // CI-only race (#4599): the compress fold lands in the live session - // state asynchronously after the tool response (#3573 replay - // semantics). On slow runners flushSync below would otherwise persist - // the pre-fold state and the cold load reports "0 block(s) - // (compressions=0)". Bounded-wait for the fold first. - for (let i = 0; i < 100 && !(resolveConversation("parent").session?.state.blocks?.length ?? 0); i++) { - await new Promise((r) => setTimeout(r, 50)); - } - const parent = resolveConversation("parent").session!; - parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; - parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); - parent.contentStoreDirty = true; - assert(h.store.flushSync(parent)); - // #4316 PR3/3: cold load runs the one-time ledger alignment, but only - // when the unified store is available — config-clean machines (CI) run - // legacy mode where the alignment is an honest skip and the loaded - // state never carries refLedgerAligned. Mirror the mode in the - // expectation instead of asserting the flag unconditionally. - const persistedState = JSON.parse(JSON.stringify(parent.state)); - if (h.store.replicasAvailable()) persistedState.refLedgerAligned = true; - assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); - const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; - const req = forkRequest(snapshot); - const fork = await h.request("/__bili/plugin/fork", req); - assert.equal(fork.status, 201); - const childSnapshot = (await h.request("/__bili/plugin/snapshot?conversationId=child")).body; - const code = `import {startServer} from './src/server.ts'; import {SessionStore,_setStoreForTest} from './src/persist.ts'; import {defaultConfig} from 'acp-kernel'; import {_setForTest} from './src/registry.ts'; _setForTest({}); _setStoreForTest(new SessionStore({dir:${JSON.stringify(h.dir + "/sessions")},enabled:true})); const s=await startServer({port:0,host:'127.0.0.1',upstream:${JSON.stringify(h.upstreamUrl)},modelContextLimit:400000,routes:{},kernelConfig:defaultConfig(400000),compress:{injectTool:true,injectNudge:true},promptCache:{routing:'auto'},sessionHeader:'x-acp-session',log:false,debug:false,passthrough:false,autoUpdate:false,mitm:{enabled:false,domains:[]}}); const ready=()=>process.stdout.write(JSON.stringify({port:s.address().port})+'\\n'); if(s.listening)ready();else s.once('listening',ready);`; - child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], { cwd: fileURLToPath(new URL("../", import.meta.url)), env: { ...process.env, NODE_ENV: "test" }, stdio: ["ignore", "pipe", "pipe"] }); - let stderr = ""; - child.stderr!.on("data", (b) => { stderr += b.toString(); }); - const port = await new Promise((resolve, reject) => { - let stdout = ""; - const timer = setTimeout(() => reject(new Error("restart timeout: " + stderr + stdout)), 15000); - child!.stdout!.on("data", (b) => { - stdout += b.toString(); - const port = stdout.match(/\{"port":(\d+)\}/); - if (port) { clearTimeout(timer); resolve(Number(port[1])); } +for (const storageMode of ["legacy", "full"] as const) { + test(`HTTP fork revision and originals survive a new proxy process (${storageMode})`, { timeout: 60_000 }, async () => { + const previousMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = storageMode; + const h = await harness(true); + let child: ReturnType | undefined; + try { + await compress(h); + // CI-only race (#4599): the compress fold lands in the live session + // state asynchronously after the tool response (#3573 replay + // semantics). On slow runners flushSync below would otherwise persist + // the pre-fold state and the cold load reports "0 block(s) + // (compressions=0)". Bounded-wait for the fold first. + for (let i = 0; i < 100 && !(resolveConversation("parent").session?.state.blocks?.length ?? 0); i++) { + await new Promise((r) => setTimeout(r, 50)); + } + const parent = resolveConversation("parent").session!; + parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; + parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); + parent.contentStoreDirty = true; + assert(h.store.flushSync(parent)); + // #4316 PR3/3: only a unified cold load runs the one-time ledger + // alignment; the CI legacy baseline retains the original state. + const persistedState = JSON.parse(JSON.stringify(parent.state)); + if (storageMode === "full") persistedState.refLedgerAligned = true; + assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); + const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; + const req = forkRequest(snapshot); + const fork = await h.request("/__bili/plugin/fork", req); + assert.equal(fork.status, 201); + const childSnapshot = (await h.request("/__bili/plugin/snapshot?conversationId=child")).body; + const code = `import {startServer} from './src/server.ts'; import {SessionStore,_setStoreForTest} from './src/persist.ts'; import {defaultConfig} from 'acp-kernel'; import {_setForTest} from './src/registry.ts'; _setForTest({}); _setStoreForTest(new SessionStore({dir:${JSON.stringify(h.dir + "/sessions")},enabled:true})); const s=await startServer({port:0,host:'127.0.0.1',upstream:${JSON.stringify(h.upstreamUrl)},modelContextLimit:400000,routes:{},kernelConfig:defaultConfig(400000),compress:{injectTool:true,injectNudge:true},promptCache:{routing:'auto'},sessionHeader:'x-acp-session',log:false,debug:false,passthrough:false,autoUpdate:false,mitm:{enabled:false,domains:[]}}); const ready=()=>process.stdout.write(JSON.stringify({port:s.address().port})+'\\n'); if(s.listening)ready();else s.once('listening',ready);`; + child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", code], { cwd: fileURLToPath(new URL("../", import.meta.url)), env: { ...process.env, NODE_ENV: "test" }, stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr!.on("data", (b) => { stderr += b.toString(); }); + const port = await new Promise((resolve, reject) => { + let stdout = ""; + const timer = setTimeout(() => reject(new Error("restart timeout: " + stderr + stdout)), 15000); + child!.stdout!.on("data", (b) => { + stdout += b.toString(); + const port = stdout.match(/\{"port":(\d+)\}/); + if (port) { clearTimeout(timer); resolve(Number(port[1])); } + }); + child!.once("exit", (code) => { clearTimeout(timer); reject(new Error(`restart exited ${code}: ${stderr}`)); }); + child!.once("error", (error) => { clearTimeout(timer); reject(error); }); }); - child!.once("exit", (code) => { clearTimeout(timer); reject(new Error(`restart exited ${code}: ${stderr}`)); }); - child!.once("error", (error) => { clearTimeout(timer); reject(error); }); - }); - const call = async (path: Path, body?: unknown) => { - const r = await fetch(`http://127.0.0.1:${port}` + path, body === undefined ? {} : { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); - return { status: r.status, body: await responseBody(r, path) }; - }; - const restored = await call("/__bili/plugin/snapshot?conversationId=child"); - const restoredStatus = await call("/__bili/plugin/status?conversationId=parent&fallback=latest"); - assert.equal(restoredStatus.status, 200); - assert.equal(restoredStatus.body.sessionId, "parent"); - assert.equal(restoredStatus.body.fallback, undefined); - assert.equal(restoredStatus.body.sessionRevision, snapshot.parentRevision); - assert.equal(restoredStatus.body.contextTokensSource, "estimate"); - assert.equal(restored.status, 200); - assert.equal(restored.body.parentRevision, childSnapshot.parentRevision); - assert.deepEqual(restored.body.orderedMessages, childSnapshot.orderedMessages); - assert.equal((await call("/__bili/plugin/snapshot?conversationId=parent")).body.parentRevision, snapshot.parentRevision); - const replay = await call("/__bili/plugin/fork", req); - assert.equal(replay.status, 200); - assert.equal(replay.body.replayed, true); - const retrieved = await call("/__bili/plugin/tool", { conversationId: "child", tool: "acp_retrieve", args: { ref: "m00001" } }); - assert.equal(retrieved.status, 200); - assert.match(retrieved.body.result, /CCR original retained independently after restart/); - const original = await call("/__bili/plugin/tool", { conversationId: "child", tool: "decompress", args: { blockId: "b1", full: true } }); - assert.match(original.body.result, /first original/); - assert.match(original.body.result, /second original/); - assert(!original.body.result.includes("tail original")); - } finally { - if (child?.pid && child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await once(child, "exit"); } - await h.close(); - } -}); + const call = async (path: Path, body?: unknown) => { + const r = await fetch(`http://127.0.0.1:${port}` + path, body === undefined ? {} : { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + return { status: r.status, body: await responseBody(r, path) }; + }; + const restored = await call("/__bili/plugin/snapshot?conversationId=child"); + const restoredStatus = await call("/__bili/plugin/status?conversationId=parent&fallback=latest"); + assert.equal(restoredStatus.status, 200); + assert.equal(restoredStatus.body.sessionId, "parent"); + assert.equal(restoredStatus.body.fallback, undefined); + assert.equal(restoredStatus.body.sessionRevision, snapshot.parentRevision); + assert.equal(restoredStatus.body.contextTokensSource, "estimate"); + assert.equal(restored.status, 200); + assert.equal(restored.body.parentRevision, childSnapshot.parentRevision); + assert.deepEqual(restored.body.orderedMessages, childSnapshot.orderedMessages); + assert.equal((await call("/__bili/plugin/snapshot?conversationId=parent")).body.parentRevision, snapshot.parentRevision); + const replay = await call("/__bili/plugin/fork", req); + assert.equal(replay.status, 200); + assert.equal(replay.body.replayed, true); + const retrieved = await call("/__bili/plugin/tool", { conversationId: "child", tool: "acp_retrieve", args: { ref: "m00001" } }); + assert.equal(retrieved.status, 200); + assert.match(retrieved.body.result, /CCR original retained independently after restart/); + const original = await call("/__bili/plugin/tool", { conversationId: "child", tool: "decompress", args: { blockId: "b1", full: true } }); + assert.match(original.body.result, /first original/); + assert.match(original.body.result, /second original/); + assert(!original.body.result.includes("tail original")); + } finally { + if (child?.pid && child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await once(child, "exit"); } + await h.close(); + if (previousMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previousMode; + } + }); +} test("HTTP status separates real usage from a manual-compression estimate and changing generation", { timeout: 60_000 }, async () => { const h = await harness(); diff --git a/tests/storage-canonical-digest.test.ts b/tests/storage-canonical-digest.test.ts new file mode 100644 index 0000000000..0fea66a29e --- /dev/null +++ b/tests/storage-canonical-digest.test.ts @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { canonicalDigest, canonicalize, sha256 } from "../src/storage/canonical.ts"; + +function matchesLegacy(value: unknown): void { + assert.equal(canonicalDigest(value), sha256(JSON.stringify(canonicalize(value)))); +} + +test("streaming digest preserves storage normalization and collection order", () => { + const uuid = "12345678-1234-1234-1234-123456789abc"; + matchesLegacy({ + metadata: { biliVersion: "0.1.204", storageMode: "full", savedAt: 1_700_000_000_000 }, + tailGaps: 3, + state: { nextBlockId: 42, ids: new Set([uuid, "b2"]) }, + blockContents: new Map([ + [uuid, { full: { text: "original" }, traceId: uuid }], + [1_700_000_000_000, [null, true, false, -0, NaN, Infinity]], + [{ storageMode: "raw map key", traceId: uuid }, new Map([["b1", "summary"]])], + ]), + }); + assert.notEqual(canonicalDigest(new Map([["b1", 1], ["b2", 2]])), + canonicalDigest(new Map([["b2", 2], ["b1", 1]]))); +}); + +test("streaming digest preserves JSON omissions, sparse arrays and property order", () => { + matchesLegacy({ z: undefined, f: () => 1, s: Symbol("ignored"), a: [undefined, , Symbol("null")], "10": "ten", "2": "two" }); + matchesLegacy(JSON.parse('{"__proto__":{"ignored":true},"constructor":"kept","value":1}')); + for (const value of [null, true, 0, "text", [], {}, new Map(), new Set()]) matchesLegacy(value); +}); + +test("streaming digest escapes long strings without splitting surrogate pairs", () => { + matchesLegacy({ + text: "x".repeat(65_535) + "\uD83D\uDE80" + "\n\t\u0000\"\\\u4E2D\u6587".repeat(20_000), + loneSurrogates: "\uD800" + "y".repeat(65_534) + "\uDC00", + ["key\"\\\n\uD83D\uDE80"]: "escaped key", + }); +}); + +test("streaming digest changes when retained history changes", () => { + const before = { blockContents: new Map([["b1", { full: { text: "original" } }]]) }; + const after = { blockContents: new Map([["b1", { full: { text: "changed" } }]]) }; + matchesLegacy(before); + matchesLegacy(after); + assert.notEqual(canonicalDigest(before), canonicalDigest(after)); +}); + +test("streaming digest stays byte-identical across many 64 KiB chunks with pairs on the boundary (#4529)", () => { + const chunk = 64 * 1024; + const unit = "a".repeat(chunk - 1) + "\uD83D\uDE80"; + matchesLegacy({ text: unit }); + matchesLegacy({ text: unit.repeat(5) }); + const seam = "z".repeat(chunk - 6) + "\"\\\n\t\u0000\u4E2D文".repeat(20); + matchesLegacy({ text: seam }); +}); + +test("streaming digest stays byte-identical over an MB-scale retained history (#4529)", () => { + const body = "line \uD83D\uDE80 \u4E2D\u6587 \"quoted\\\" \n\t".repeat(2_000); + const blockContents = new Map(); + for (let i = 0; i < 40; i++) { + blockContents.set("blk-" + i, { full: { text: body + i }, traceId: "aaaaaaaa-bbbb-cccc-dddd-" + i.toString(16).padStart(12, "0") }); + } + matchesLegacy({ + id: "sess-4529", + metadata: { biliVersion: "0.1.206", savedAt: 1_700_000_000_000 }, + state: { nextBlockId: 41, ids: [...blockContents.keys()] }, + blockContents, + }); +}); + +test("streaming digest stays byte-identical under deep nesting and wide fan-out (#4529)", () => { + let nested: unknown = "leaf"; + for (let i = 0; i < 400; i++) nested = i % 2 === 0 ? [nested] : { w: nested }; + matchesLegacy(nested); + const wide: Record = {}; + for (let i = 0; i < 5_000; i++) wide["k" + i] = i % 3 === 0 ? "v" + i : i; + matchesLegacy(wide); + const big = "\uD83D\uDE80\n\"\\中文".repeat(40_000); + matchesLegacy(new Map([ + [{ nested: { deep: big } }, new Set([big, 1_700_000_000_000])], + ["plain", big], + ])); +}); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index d9eb3334b2..2ef31e8605 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -22,8 +22,8 @@ * IMPORT GUARD — a hand-aged legacy file (older savedAt than the * index watermark) must not re-import over newer index * state on load. - * ROLLBACK — unset the env after running full → the legacy file - * loads with the exact digest. + * ROLLBACK — explicitly select legacy after persisting the full + * load's migration → the file loads the exact digest. * SELF-DISABLE — 3 consecutive sidecar failures disable it; reads flip * back to legacy; the legacy file stays readable. * CORRUPT INDEX — a corrupt index.db degrades the store to legacy-only @@ -281,10 +281,10 @@ test("storage engine: load-side import guard skips a hand-aged legacy file (inde }); // --------------------------------------------------------------------------- -// ROLLBACK: full → unset env → legacy file authoritative, digest exact. +// ROLLBACK: full → explicit legacy → legacy file authoritative, digest exact. // --------------------------------------------------------------------------- -test("storage engine: rollback contract — unset the env and the legacy read path serves the identical state", async () => { +test("storage engine: rollback contract — explicit legacy mode serves the identical persisted state", async () => { const dir = tmpdir("bili-engproof-rollback-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { @@ -305,8 +305,10 @@ test("storage engine: rollback contract — unset the env and the legacy read pa // merely deleting the env would read unified and vacate this leg. const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - storeFull.closeQuietly(); assert.ok(viaFull, "full-mode reload returned null"); + // Persist the one-time cold-load alignment before exercising rollback. + await storeFull.writeNow(viaFull); + storeFull.closeQuietly(); const digestFull = viewDigest(toLegacyLike(viaFull)); process.env.BILI_STORAGE_UNIFIED = "legacy"; @@ -378,8 +380,10 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki // fallback must reproduce exactly from the legacy file. const clean = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = clean.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - clean.closeQuietly(); assert.ok(viaFull, "full-mode read returned null"); + // Both replicas must contain the migration before corrupting the index. + await clean.writeNow(viaFull); + clean.closeQuietly(); // #4338: everything past this point (corrupt the header → degrade // construction → fallback read → rename guards → write → close) runs diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index 9fd6f13b0a..ac2d9e08d7 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -75,6 +75,8 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", const alignedRef = recovered.state.messageRefs.byRaw["raw-new"]; assert.ok(alignedRef, "raw-new must keep a citation number after recovery"); assert.equal(recovered.state.messageRefs.byRef[alignedRef], "raw-new"); + // The pre-alignment number is recorded as superseded, not silently dropped. + assert.equal(recovered.restampSuperseded?.get("m00003"), "raw-new"); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); diff --git a/tests/web-blogfeed.test.ts b/tests/web-blogfeed.test.ts index fe9a00387c..8f8a61c840 100644 --- a/tests/web-blogfeed.test.ts +++ b/tests/web-blogfeed.test.ts @@ -10,7 +10,7 @@ */ import { test, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { mkdtempSync, rmSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { ServerResponse } from "node:http"; @@ -94,6 +94,21 @@ test("rendering is escape-first: script tags survive only escaped; tables become assert.ok(alpha.html.includes("two")); }); +test("CRLF markdown preserves bilingual frontmatter and rendered content", () => { + const expected = getBlogFeed(); + for (const relative of [ + "website/blog/posts/2026-10-08_demo.md", + "website/blog/posts/2026-10-08_demo.zh.md", + "paper/a.md", + "paper/b.md", + ]) { + const file = join(pkgRoot, relative); + writeFileSync(file, readFileSync(file, "utf8").replace(/\n/g, "\r\n")); + } + _setBlogRootForTest(pkgRoot); + assert.deepEqual(getBlogFeed(), expected); +}); + test("image refs: shipped files reroute to the panel asset prefix; traversal/unshipped refs drop silently", () => { const data = getBlogFeed(); const demo = data.items.find((i) => i.id === "demo")!;