diff --git a/src/persist.ts b/src/persist.ts index 5aa2e6f80f..aca170c9a2 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2855,7 +2855,14 @@ export class SessionStore { this.drainFlushWaiters(id, this.lastSaveLanded.get(id) === true); }); }, this.debounceMs); - timer.unref?.(); + // #4788: the debounce timer is host-owned periodic upkeep and must not + // pin a process open on its own -- so it is unref'd. But `afterNextFlush` + // makes it a HOST: a caller that awaits a verdict has no other handle + // guaranteed to fire, and an unref'd timer is not a reason for Node to + // stay alive. Awaiting then exits the loop with the promise unresolved + // (`Promise resolution is still pending but the event loop has already + // resolved`). Keep the timer referenced exactly while someone waits. + if (!this.flushWaiters.has(id)) timer.unref?.(); this.saveTimers.set(id, timer); } @@ -2876,6 +2883,10 @@ export class SessionStore { const ws = this.flushWaiters.get(sessionId) ?? []; ws.push((ok: boolean): void => { resolve(ok); }); this.flushWaiters.set(sessionId, ws); + // #4788: the timer was armed unref'd (no waiter existed then). The + // waiter now holds the promise open, so the timer must hold the loop + // open too -- otherwise the process drains and the verdict is lost. + this.saveTimers.get(sessionId)?.ref?.(); }); } diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index db9efa522b..43c13f3ba7 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -329,12 +329,12 @@ export class UnifiedShadow { : "; delete the file to reset the index"; loggerLog( "warn", - `[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${tail}`, + `[storage] unified sidecar failed to open ${path.join(dir, "index.db")} (${errStr}) — staying legacy-only${tail}`, ); recordSidecarOpenFailure(errStr); return null; } - const shadow = new UnifiedShadow(mode, store, legacyLoad ?? (() => null), undefined, `${dir}/index.db`); + const shadow = new UnifiedShadow(mode, store, legacyLoad ?? (() => null), undefined, path.join(dir, "index.db")); recordSidecarInstance(shadow); const engineTag = ` [engine: ${store.engine}/${store.engineSource}]`; const walTag = ` [wal-cap: ${walLimit}]`; @@ -755,13 +755,13 @@ export class UnifiedShadow { } /** Absolute-seq tip of the session's message log (+1 = next free seq). - * Used as a checkpoint's seq_at. */ + * Used as a checkpoint's seq_at. #4739 follow-up: one MAX(seq) aggregate + * over live rows (#4730's lastLiveSeq) instead of materializing the whole + * log (rows + CAS reads + media hydration) just to reduce it to a scalar. */ sessionTipSeq(sessionId: string): number { if (this.sync === null) return 0; this.assertLive(); - let tip = 0; - for (const m of this.sync.getMessages(sessionId)) if (m.seq + 1 > tip) tip = m.seq + 1; - return tip; + return (this.sync.lastLiveSeq(sessionId) ?? -1) + 1; } /** #3301: message rows for specific ids (+ seq±1 neighbors) — stored @@ -1220,9 +1220,8 @@ export class UnifiedShadow { sessionTipSeqAsync(sessionId: string): Promise { return this.viaAsync(0, () => this.sessionTipSeq(sessionId), async () => { - let tip = 0; - for (const m of await this.backend!.getMessages(sessionId)) if (m.seq + 1 > tip) tip = m.seq + 1; - return tip; + const max = await this.backend!.lastLiveSeq(sessionId); + return (max ?? -1) + 1; }); } diff --git a/tests/inc-canonical-pointers.test.ts b/tests/inc-canonical-pointers.test.ts index e0210b8d4f..bdc730e143 100644 --- a/tests/inc-canonical-pointers.test.ts +++ b/tests/inc-canonical-pointers.test.ts @@ -105,7 +105,11 @@ test("cold canonical rebuild: identical resync keeps the shared prefix; delta as // #5155: the wrap-up defers the durable flush (debounced) — let the // rows settle before dropping the resident caches, or the cold // rebuild reads a one-turn-stale table and honestly 409s. - await new Promise((resolve) => setTimeout(resolve, 20)); + // Await the settle itself (afterNextFlush resolves once the armed save + // lands) instead of a fixed sleep: the settle is a real condition, and + // a magic millisecond budget is a deadline a loaded windows-latest + // runner can miss — the 409 flake this replaces. + assert.equal(await getStore().afterNextFlush("conv-ptrs"), true, "the turn's durable save must land before the cold reset"); __resetIncrementalLaneForTests(); const delta = await h.turn({ sinceSeq: 3, messages: [core("u4", "four")] }); @@ -135,7 +139,7 @@ test("cold canonical rebuild: edited-tail resync keeps the shared prefix, drops await h.turn({ sinceSeq: 0, resync: true, messages: [core("u1", "one"), core("u2", "two-EDITED")] }); await __incrementalPtrChainForTests("conv-ptrs"); // #5155: settle the debounced flush before the cold reset (see above). - await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(await getStore().afterNextFlush("conv-ptrs"), true, "the resync's durable save must land before the cold reset"); __resetIncrementalLaneForTests(); const delta = await h.turn({ sinceSeq: 2, messages: [core("u3", "three")] }); assert.equal(delta.status, 200); diff --git a/tests/issue4739-bounded-session-load.test.ts b/tests/issue4739-bounded-session-load.test.ts index bc9906b5f0..bf91303252 100644 --- a/tests/issue4739-bounded-session-load.test.ts +++ b/tests/issue4739-bounded-session-load.test.ts @@ -139,7 +139,7 @@ test("full mode: viewFor decodes only pointer rows and matches the full-log deco assert.deepEqual(via, reference, "bounded decode byte-equal to the full-log decode"); assert.equal(delta, expectedPointers, `viewFor read ${delta} message rows for ${expectedPointers} pointer seqs (full log = ${liveSeqs.size})`); } finally { - store.cancelAll(); + store.closeQuietly(); if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prev; rmrf(dir); @@ -178,7 +178,7 @@ test("full mode: dropped metadata restores from the index alone (narrow read, no assert.ok(again, "envelope rewritten by later save"); assert.deepEqual(Object.keys(JSON.parse(fs.readFileSync(again, "utf8")).payload.metadata as object).sort(), persistedKeys); } finally { - store.cancelAll(); + store.closeQuietly(); if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prev; rmrf(dir); diff --git a/tests/issue4788-resident-minimal.test.ts b/tests/issue4788-resident-minimal.test.ts index 5f20c83aa8..c96e38ec54 100644 --- a/tests/issue4788-resident-minimal.test.ts +++ b/tests/issue4788-resident-minimal.test.ts @@ -86,6 +86,10 @@ async function turn(store: SessionStore, s: Session, midFlight?: boolean): Promi if (midFlight) s.inFlight = 1; s.revisionEpoch = (s.revisionEpoch ?? 0) + 1; store.scheduleSave(s); + // #4788: while a caller awaits afterNextFlush, persist.ts keeps the pending + // debounce timer ref'd, so this settle is deterministic even in a node:test + // child whose only surviving handles are unref'd sockets — no test-side + // keep-alive needed here anymore. const landed = await store.afterNextFlush(s.id); if (midFlight) s.inFlight = 0; return landed; @@ -129,7 +133,12 @@ test("(a) between-turns drop: every landed request-end boundary discharges dropp assert.ok((s.blockContents as LazyBlockContents).isDropped, "still folded after the second boundary"); assert.ok(isDroppedMetadata(s.metadata)); } finally { - store.cancelAll(); + // #4739/#5140 family: the suite arms a real SessionStorage under + // BILI_STORAGE_UNIFIED=full, so the sidecar handle is still open here. + // cancelAll() clears timers and the build map but never closes the + // shadow; Windows then refuses to unlink the directory (EBUSY/EPERM). + // Close the short-lived store first: cancel, then close. + store.closeQuietly(); rmrf(dir); } }); @@ -164,7 +173,7 @@ test("(b) narrow-read reconstruction deepEquals the pre-drop reference", async ( const rec = (parsed.payload ?? parsed) as Record; assert.equal(Object.keys((rec.blockContents ?? {}) as object).length, Object.keys(refBlocks).length, "full set round-trips through the envelope"); } finally { - store.cancelAll(); + store.closeQuietly(); rmrf(dir); } }); @@ -203,7 +212,7 @@ test("(c) hard cap under sustained activity: boundaries drop, in-flight skips, w process.env.BILI_SESSION_RESIDENT_HARD_MAX_BYTES = saved; } } finally { - store.cancelAll(); + store.closeQuietly(); rmrf(dir); } }); @@ -246,7 +255,7 @@ test("(d) envelope-frozen routing is unchanged AND gets equivalent drops", async assert.equal(s.blockContents.size, 21, "all entries resident again after reconstruction"); } finally { process.env.BILI_STORAGE_UNIFIED = "legacy"; - store.cancelAll(); + store.closeQuietly(); rmrf(dir); } }); diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 96d9dfeb0f..44db0d8a8d 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -1477,6 +1477,10 @@ test("HTTP a mid-history rewrite replaces the snapshot wholesale with an exact b }); test("HTTP a cold-restored non-retained parent serves its intact raw tail immediately (#5140)", { timeout: 60_000 }, async () => { + // Pin the mode this raw-tail proof requires: under the runner's legacy + // baseline no unified rows exist and forkSnapshot fails closed (#4599). + const oldMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; const h = await harness(true); try { const parent = resolveConversation("parent").session!; @@ -1495,7 +1499,11 @@ test("HTTP a cold-restored non-retained parent serves its intact raw tail immedi await sendModel(h, "parent", h.messages); assert.equal((await h.request("/__bili/plugin/snapshot?conversationId=parent")).status, 200); assert.equal(resolveConversation("parent").session!.pluginSnapshot!.length, 3); - } finally { await h.close(); } + } finally { + await h.close(); + if (oldMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = oldMode; + } }); test("fork protocol v2: snapshot serves the light revision, a stale v1 whole-store revision is still accepted (#2671 Phase 2b)", { timeout: 60_000 }, async () => { diff --git a/tests/storage-tipseq-narrow-read.test.ts b/tests/storage-tipseq-narrow-read.test.ts new file mode 100644 index 0000000000..649b43b709 --- /dev/null +++ b/tests/storage-tipseq-narrow-read.test.ts @@ -0,0 +1,84 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { snapshotMessages } from "../src/session.ts"; +import { createInitialState, type CoreMessage } from "acp-kernel"; +import { UnifiedStore, messageRowReads } from "../src/storage/store.ts"; +import type { UnifiedShadow } from "../src/storage/shadow.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// #4739 follow-up (adjacent finding from the #4765 review): the checkpoint +// tip-seq read must be ONE MAX(seq) aggregate over live rows — not a full +// materialization of the message log (rows + content-CAS reads + media +// hydration) reduced to a scalar. The old scan cost O(N) per checkpoint +// insert on the write path. + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-tipseq-hermetic-")), + "no-config.json", +); +process.env.BILI_STORAGE_UNIFIED = "full"; + +function makeSession(id: string): Session { + return { + id, + meta: { label: id, protocol: "openai", upstreamOrigin: "http://up:1" }, + stats: { requests: 3, tokensSaved: 0, inputTokens: 10, cachedTokens: 0, outputTokens: 5, cacheSamples: 0, lastInputTokens: 10, contextTokens: 1000, compressCreditTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now() - 1000, + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +function rawTail(n: number): CoreMessage[] { + const out: CoreMessage[] = []; + for (let i = 0; i < n; i++) { + out.push({ id: `tip${i}`, contentType: "text", role: i % 2 === 0 ? "user" : "assistant", text: `tip-seq probe message number ${i} with filler words to spend tokens` }); + } + return out; +} + +test("sessionTipSeq is max live seq + 1 and materializes zero message rows", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-tipseq-")); + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + try { + const s = makeSession("tip-1"); + const N = 300; + snapshotMessages(s, rawTail(N)); + await store.writeNow(s); + + const sh = (store as unknown as { shadow?: UnifiedShadow }).shadow; + assert.ok(sh && sh.readsUnified, "full-mode sidecar live"); + + // Reference value computed with the OLD algorithm's input set (live + // rows only) — taken BEFORE the narrowness window opens. + const direct = UnifiedStore.open(path.join(dir, "index.db")); + const rows = direct.getMessages("tip-1"); + assert.equal(rows.length, N, "generation 1 ingested the full history"); + const expectedTip = rows[rows.length - 1]!.seq + 1; + direct.close(); + + assert.equal(sh.sessionTipSeq("no-such-session"), 0, "no live rows → tip 0 (as before)"); + + const before = messageRowReads.count; + assert.equal(sh.sessionTipSeq("tip-1"), expectedTip, "tip = max live seq + 1"); + assert.equal(messageRowReads.count - before, 0, `no message rows materialized (old scan read ${N})`); + + assert.equal(await sh.sessionTipSeqAsync("tip-1"), expectedTip, "async twin agrees"); + } finally { + // release the unified index.db handle before rmrf — Windows cannot + // unlink an open SQLite file (the #4599 tmp-rm EBUSY family) + store.closeQuietly(); + rmrf(dir); + } +});