From 24f781e81cb2d6710d4977173a3a8e24c76c301a Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:22:21 -0700 Subject: [PATCH 1/5] fix: bound session cold loads and narrow lazy field reads (#4739) --- src/persist.ts | 82 +++++- src/storage/backend.ts | 25 +- src/storage/driver.ts | 45 ++-- src/storage/ingest.ts | 254 ++++++++++-------- src/storage/mysql.ts | 82 ++++-- src/storage/shadow.ts | 100 +++++-- src/storage/store.ts | 87 +++++- tests/issue4739-storage-read-snapshot.test.ts | 191 +++++++++++++ tests/persist-lazy-narrow-reads.test.ts | 208 ++++++++++++++ tests/persist-rollback-guard.test.ts | 2 +- tests/storage-backend-contract.test.ts | 9 + tests/storage-bounded-load.test.ts | 222 +++++++++++++++ 12 files changed, 1110 insertions(+), 197 deletions(-) create mode 100644 tests/issue4739-storage-read-snapshot.test.ts create mode 100644 tests/persist-lazy-narrow-reads.test.ts create mode 100644 tests/storage-bounded-load.test.ts diff --git a/src/persist.ts b/src/persist.ts index bdf9abc0d1..e2a4876f61 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -379,6 +379,11 @@ export class SessionStore { // #3931 step 3: assigned asynchronously for network backends (mysql) — // see the constructor and ready(). private shadow?: UnifiedShadow; + /** Only alive while one synchronous record build hydrates both lazy fields. */ + private residentFieldsForBuild?: { + id: string; + fields: { metadata?: Record; blockContents?: Record } | null; + }; /** #2960 ②: sessions whose fork_base checkpoint has been cut (or was * found already present) — memoized so steady-state writes of receipt * sessions do not re-query the checkpoints table every flush. */ @@ -667,7 +672,7 @@ export class SessionStore { // #408 one-time migration: the in-memory value is already // clamped by buildSession — rewrite the stale file so the // negative value is gone from disk. - await this.store.writeNow(id, () => buildRecord(session)); + await this.store.writeNow(id, () => this.buildRecordForSave(session)); clamped++; // #3049: the clamp rewrite writes a DIFFERENT payload than any // ingested record (clamped stats) without a sidecar ingest — @@ -1147,7 +1152,7 @@ export class SessionStore { continue; } session.id = label; - await this.store.writeNow(label, () => buildRecord(session)); + await this.store.writeNow(label, () => this.buildRecordForSave(session)); // #3049: direct write bypassing guardedBuild — the boot rekey // rewrites envelopes without the sidecar ingest, so any prior // ownership token for this id is stale. @@ -1625,6 +1630,60 @@ export class SessionStore { return this.shadow ? this.shadow.viewFor(id) : null; } + /** Restore only the dropped fields, with the same source preference as loadSync. + * A field read must not restamp refs, import rows, or schedule another save. */ + private loadResidentFields( + id: string, + meta: { protocol?: string; upstreamOrigin?: string }, + fields: { metadata?: boolean; blockContents?: boolean }, + ): { metadata?: Record; blockContents?: Record } | null { + if (this.residentFieldsForBuild?.id === id) return this.residentFieldsForBuild.fields; + if (!this.enabled) return null; + const indexed = this.shadow?.readsUnified ? this.shadow.readResidentFieldsFor(id, fields) : null; + const envelope = this.loadEnvelope(id, meta); + let useEnvelope = indexed === null; + if (indexed && envelope) { + if (this.shadow?.driftedSessions.has(id)) { + const envActive = countActiveBlocks(envelope.payload); + const envRefs = Object.keys(envelope.payload.state?.messageRefs?.byRef ?? {}).length; + useEnvelope = indexed.header.activeBlockCount <= envActive && indexed.header.mappedRefCount <= envRefs; + } else { + const requests = Number(envelope.payload.stats?.requests ?? 0); + useEnvelope = requests > indexed.header.requests || + (requests === indexed.header.requests && Number(envelope.payload.state?.nextBlockId ?? 0) > indexed.header.nextBlockId); + } + } + if (useEnvelope) { + if (!envelope) return null; + return { + ...(fields.metadata ? { metadata: envelope.payload.metadata ?? {} } : {}), + ...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(Object.entries(envelope.payload.blockContents ?? {}))) } : {}), + }; + } + if (!indexed) return null; + return { + ...(fields.metadata ? { metadata: indexed.metadata } : {}), + ...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(indexed.blockContents ?? [])) } : {}), + }; + } + + private buildRecordForSave(session: Session, storageMode: "legacy" | "shadow" | "full" = effectiveFileStorageMode()): PersistedSession { + const bothDropped = isDroppedMetadata(session.metadata) && + session.blockContents instanceof LazyBlockContents && session.blockContents.isDropped; + if (!bothDropped) return buildRecord(session, storageMode); + // Metadata and contents used to each rebuild a whole Session here. Share + // one projection and envelope arbitration for this build, never a cache + // that keeps another session alive after the synchronous build returns. + const previous = this.residentFieldsForBuild; + const fields = this.loadResidentFields(session.id, session.meta ?? {}, { metadata: true, blockContents: true }); + this.residentFieldsForBuild = { id: session.id, fields }; + try { + return buildRecord(session, storageMode); + } finally { + this.residentFieldsForBuild = previous; + } + } + /** Read-only state load for cross-session search (#841): unlike loadSync, * never schedules a save (no #408 clamp-rewrite side effect). */ loadStateForSearch(id: string): CompressionState | null { @@ -1997,7 +2056,7 @@ export class SessionStore { this.forkBaseCut.add(session.id); return; } - const record = buildRecord(session); + const record = this.buildRecordForSave(session); const activeBlockIds = session.state.blocks .filter((b) => (b as { active?: boolean }).active !== false) .map((b) => b.blockId) @@ -2306,7 +2365,7 @@ export class SessionStore { // the JSON envelope, and the stamp must say legacy for that copy. const stamp = this.liveStorageStamp; return () => { - const record = buildRecord(session, stamp); + const record = this.buildRecordForSave(session, stamp); const disk = this.staleDiskPayload(record); if (disk === null) { // #3301: external-converter import payload rides exactly ONE @@ -2588,8 +2647,7 @@ export class SessionStore { // loader degrades to the id-only lookup, same as a protocol-less file. const meta = session.meta ?? undefined; session.metadata = lazyMetadata(session.id, { protocol: meta?.protocol, upstreamOrigin: meta?.upstreamOrigin }, (id, m) => { - const loaded = this.loadSync(id, m); - return loaded ? loaded.metadata : null; + return this.loadResidentFields(id, m, { metadata: true })?.metadata ?? null; }); } @@ -2607,8 +2665,7 @@ export class SessionStore { } if (existing.size === 0) return; const lazy = new LazyBlockContents(session.id, { protocol: session.meta.protocol, upstreamOrigin: session.meta.upstreamOrigin }, (id, meta) => { - const loaded = this.loadSync(id, meta); - return loaded ? Object.fromEntries(loaded.blockContents) : null; + return this.loadResidentFields(id, meta, { blockContents: true })?.blockContents ?? null; }); lazy.drop(); session.blockContents = lazy; @@ -2864,9 +2921,9 @@ function restoreCalibrationRing(v: unknown): { origin: string; values: number[] return recent.length > 0 ? { origin: ring.origin, values: recent } : undefined; } -function buildSession(parsed: PersistedSession): Session { +function restoreBlockContents(entries: Iterable): Map { const blockContents = new Map(); - for (const [bid, content] of Object.entries(parsed.blockContents ?? {})) { + for (const [bid, content] of entries) { if (!content || typeof content !== "object") continue; const full = (content as Record).full; if (!isBlockView(full)) continue; @@ -2879,6 +2936,11 @@ function buildSession(parsed: PersistedSession): Session { full, }); } + return blockContents; +} + +function buildSession(parsed: PersistedSession): Session { + const blockContents = restoreBlockContents(Object.entries(parsed.blockContents ?? {})); // Read grouped shape (v2+); fall back to flat fields for v1 files. const meta = parsed.meta ?? {}; const stats = parsed.stats ?? {}; diff --git a/src/storage/backend.ts b/src/storage/backend.ts index dd8473893a..4341a2ec45 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -73,10 +73,12 @@ export interface StorageBackend { seqByRaw(sessionId: string): Promise | null>; /** #4316 PR3/3: fork adoption row clone (idempotent). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise; - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]>; + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; getRefs(sessionId: string): Promise; + /** All reads, including CAS hydration, use one short read-only snapshot. */ + readSnapshot(read: (reader: StorageReadView) => Promise): Promise; getCcr(sessionId: string): Promise; getCcrEntries(sessionId: string): Promise; @@ -139,6 +141,8 @@ export interface StorageBackend { init?(): Promise; } +export type StorageReadView = Pick; + /** The sqlite backend: the existing UnifiedStore behind the async contract. * Synchronous under the hood; `Promise.resolve` wrapping only. */ export class SqliteBackend implements StorageBackend { @@ -186,11 +190,11 @@ export class SqliteBackend implements StorageBackend { this.store.cloneSessionRows(parentId, childId, throughSeq); return Promise.resolve(true); } - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { - return Promise.resolve(this.store.getMessages(sessionId)); + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]> { + return Promise.resolve(this.store.getMessages(sessionId, opts)); } - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { - return Promise.resolve(this.store.getBlocks(sessionId)); + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + return Promise.resolve(this.store.getBlocks(sessionId, opts)); } getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { return Promise.resolve(this.store.getDeadRefs(sessionId)); @@ -198,6 +202,15 @@ export class SqliteBackend implements StorageBackend { getRefs(sessionId: string): Promise { return Promise.resolve(this.store.getRefs(sessionId)); } + readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + return this.store.readSnapshotAsync((store) => read({ + getSessionRow: async (id) => store.getSessionRow(id), + getMessages: async (id, opts) => store.getMessages(id, opts), + getBlocks: async (id, opts) => store.getBlocks(id, opts), + getDeadRefs: async (id) => store.getDeadRefs(id), + getRefs: async (id) => store.getRefs(id), + })); + } getCcr(sessionId: string): Promise { return Promise.resolve(this.store.getCcr(sessionId)); } diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 6bc44be0c7..63799382d4 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -40,6 +40,8 @@ export interface SqliteDatabase { prepare(sql: string): SqliteStatement; /** Runs fn inside BEGIN/COMMIT; rolls back on throw (including nested calls via savepoints). */ transaction(fn: () => T): T; + /** Deferred snapshot: readers do not reserve the writer lock. */ + readTransaction(fn: () => T): T; close(): void; readonly open: boolean; } @@ -87,6 +89,7 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { // cannot resolve. Taking the write lock up front makes concurrent // writers queue on busy_timeout instead (#2690 review blocking 5). transaction: (fn) => db.transaction(fn).immediate(), + readTransaction: (fn) => db.transaction(fn).deferred(), close: () => void db.close(), get open() { return db.open; @@ -104,31 +107,33 @@ function openNodeSqlite(path: string): SqliteDatabase { }; const db = new DatabaseSync(path); let depth = 0; - const begin = () => db.exec(depth === 0 ? "BEGIN IMMEDIATE" : `SAVEPOINT sp${depth}`); // IMMEDIATE at top level: multi-process SQLITE_BUSY_SNAPSHOT guard (see openBetterSqlite3) + const begin = (readOnly: boolean) => db.exec(depth === 0 ? (readOnly ? "BEGIN DEFERRED" : "BEGIN IMMEDIATE") : `SAVEPOINT sp${depth}`); const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`); const rollback = () => db.exec(depth === 1 ? "ROLLBACK" : `ROLLBACK TO sp${depth - 1}`); + const transaction = (fn: () => T, readOnly: boolean): T => { + if (db.getAutoCommit?.() === false && depth === 0) { + // Already inside an outer transaction driven directly by + // DatabaseSync — just run inline; the outer COMMIT covers us. + return fn(); + } + begin(readOnly); + depth++; + try { + const out = fn(); + commit(); + return out; + } catch (err) { + rollback(); + throw err; + } finally { + depth--; + } + }; return { exec: (sql) => void db.exec(sql), prepare: (sql) => adaptNodeSqliteStatement(db.prepare(sql)), - transaction: (fn: () => T): T => { - if (db.getAutoCommit?.() === false && depth === 0) { - // Already inside an outer transaction driven directly by - // DatabaseSync — just run inline; the outer COMMIT covers us. - return fn(); - } - begin(); - depth++; - try { - const out = fn(); - commit(); - return out; - } catch (err) { - rollback(); - throw err; - } finally { - depth--; - } - }, + transaction: (fn) => transaction(fn, false), + readTransaction: (fn) => transaction(fn, true), close: () => void db.close(), get open() { return db.open; diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index c3a84aff22..54a319a8c8 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -487,28 +487,28 @@ export function completeBlockContents( return out; } -export function loadLegacyView( - rows: { - session: { - sessionId: string; - createdAt: number; - updatedAt: number; - nextBlockId: number; - nextRunId: number; - metaJson: string; - statsJson: string; - metadataJson: string; - kstateJson: string; - tailJson: string; - ccrJson: string | null; - orphansJson?: string | null; - }; - messages: { seq: number; msgId: string; ref: string | null; content: string }[]; - blocks: { blockId: string; summary: string; one: string | null; payloadJson: string }[]; - deadRefs: { ord: number; ref: string }[]; - refs: { ord: number; rawId: string; ref: string }[]; - }, -): { +interface LegacyViewRows { + session: { + sessionId: string; + createdAt: number; + updatedAt: number; + nextBlockId: number; + nextRunId: number; + metaJson: string; + statsJson: string; + metadataJson: string; + kstateJson: string; + tailJson: string; + ccrJson: string | null; + orphansJson?: string | null; + }; + messages: { seq: number; msgId: string; ref: string | null; content: string }[]; + blocks: { blockId: string; summary: string; one: string | null; payloadJson: string }[]; + deadRefs: { ord: number; ref: string }[]; + refs: { ord: number; rawId: string; ref: string }[]; +} + +interface LegacyView { id: string; createdAt: number; lastSeen: number; @@ -523,105 +523,147 @@ export function loadLegacyView( contentStore?: LegacySessionLike["contentStore"]; /** #4390: tail/pluginSnapshot pointers whose backing row is missing. */ tailGaps?: number; -} { - const s = rows.session; - const kstate = JSON.parse(s.kstateJson) as Record; - const tail = JSON.parse(s.tailJson) as { - pluginSnapshotSeqs: number[] | null; - tailSeqs: number[] | null; - tailFolded: boolean; - }; - - // messageRefs rebuilt from the ref ledger verbatim: ord preserves the - // persisted map's key order (digest-sensitive), and ghost ids survive - // even though no message row backs them (#2362 Kernel Contract). - const byRaw: Record = {}; - const byRef: Record = {}; - for (const r of rows.refs) { - byRaw[r.rawId] = r.ref; - // "BLOCKED" is a byRaw-only token: the in-memory map keeps the - // inverse projection strictly mNNNNN→rawId. Admitting it into byRef - // mints a key the persisted state never had (#2671 soak: sub-agent - // sessions with protected messages drifted on every save). - if (r.ref !== "BLOCKED" && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId; - } +} - const state: Record = {}; - for (const field of MERGE_STATE_FIELDS) { - switch (field) { - case "blocks": - state.blocks = rows.blocks.map((b) => JSON.parse(b.payloadJson)); - break; - case "messageRefs": - state.messageRefs = { byRaw, byRef }; - break; - case "deadRefs": - state.deadRefs = rows.deadRefs.length > 0 ? rows.deadRefs.map((d) => d.ref) : undefined; - break; - case "nextBlockId": - state.nextBlockId = s.nextBlockId; - break; - case "nextRunId": - state.nextRunId = s.nextRunId; - break; - default: - if (kstate[field] !== undefined) state[field] = kstate[field]; - break; - } - } +type BlockContents = LegacyView["blockContents"]; - const blockContents = new Map(); - for (const b of rows.blocks) { - blockContents.set(b.blockId, { +/** Shared by full-view and resident-field reads; each page is released after parsing. */ +export function appendBlockContents(contents: BlockContents, rows: Pick[]): void { + for (const b of rows) { + contents.set(b.blockId, { one: b.one === null ? null : (JSON.parse(b.one) as BlockView), full: JSON.parse(b.summary) as BlockView, }); } - // #3841 reap journal: journal'd orphan contents (reaped blocks) ride - // blockContents so the derived-decompress fallback survives restarts — - // WITHOUT resurrecting state.blocks (the reaped entries stay reaped). - if (s.orphansJson !== null && s.orphansJson !== undefined && s.orphansJson !== "") { +} + +export function mergeOrphanBlockContents(contents: BlockContents, orphansJson: string | null | undefined): void { + // Reaped originals survive without resurrecting their state.blocks rows. + if (orphansJson !== null && orphansJson !== undefined && orphansJson !== "") { try { - const orphans = JSON.parse(s.orphansJson) as Record; + const orphans = JSON.parse(orphansJson) as Record; for (const [blockId, view] of Object.entries(orphans)) { - if (!blockContents.has(blockId)) blockContents.set(blockId, view); + if (!contents.has(blockId)) contents.set(blockId, view); } } catch { // corrupt journal → skip; the JSON envelope still carries the text } } +} - const messagesBySeq = new Map(rows.messages.map((m) => [m.seq, JSON.parse(m.content) as CoreMessage])); - // #4390: a tail pointer whose row is gone used to be silently filtered - // out — the view shrank with no trace. Count the gaps so callers can - // warn (persist logs them); the surviving members still load. - let tailGaps = 0; - const resolveSeqs = (seqs: number[]): CoreMessage[] => { - const out: CoreMessage[] = []; - for (const seq of seqs) { - const m = messagesBySeq.get(seq); - if (m === undefined) tailGaps++; - else out.push(m); +/** Incremental cold-load assembly keeps raw CAS strings to one page at a time. */ +export class LegacyViewBuilder { + readonly messageSeqs: number[]; + private readonly tail: { + pluginSnapshotSeqs: number[] | null; + tailSeqs: number[] | null; + tailFolded: boolean; + }; + private readonly state: Record = {}; + private readonly blocks: unknown[] = []; + private readonly blockContents: BlockContents = new Map(); + private readonly messagesBySeq = new Map(); + + constructor( + private readonly session: LegacyViewRows["session"], + refs: LegacyViewRows["refs"], + deadRefs: LegacyViewRows["deadRefs"], + ) { + const kstate = JSON.parse(session.kstateJson) as Record; + this.tail = JSON.parse(session.tailJson); + this.messageSeqs = [...new Set([...(this.tail.tailSeqs ?? []), ...(this.tail.pluginSnapshotSeqs ?? [])])]; + + // messageRefs rebuilt from the ref ledger verbatim: ord preserves the + // persisted map's key order (digest-sensitive), and ghost ids survive + // even though no message row backs them (#2362 Kernel Contract). + const byRaw: Record = {}; + const byRef: Record = {}; + for (const r of refs) { + byRaw[r.rawId] = r.ref; + // "BLOCKED" is a byRaw-only token: the in-memory map keeps the + // inverse projection strictly mNNNNN→rawId. Admitting it into byRef + // mints a key the persisted state never had (#2671 soak: sub-agent + // sessions with protected messages drifted on every save). + if (r.ref !== "BLOCKED" && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId; + } + + const state = this.state; + for (const field of MERGE_STATE_FIELDS) { + switch (field) { + case "blocks": + state.blocks = this.blocks; + break; + case "messageRefs": + state.messageRefs = { byRaw, byRef }; + break; + case "deadRefs": + state.deadRefs = deadRefs.length > 0 ? deadRefs.map((d) => d.ref) : undefined; + break; + case "nextBlockId": + state.nextBlockId = session.nextBlockId; + break; + case "nextRunId": + state.nextRunId = session.nextRunId; + break; + default: + if (kstate[field] !== undefined) state[field] = kstate[field]; + break; + } } + } + + addBlocks(rows: LegacyViewRows["blocks"]): void { + for (const b of rows) this.blocks.push(JSON.parse(b.payloadJson)); + appendBlockContents(this.blockContents, rows); + } + + addMessages(rows: LegacyViewRows["messages"]): void { + for (const m of rows) this.messagesBySeq.set(m.seq, JSON.parse(m.content) as CoreMessage); + } + + finish(): LegacyView { + const s = this.session; + const tail = this.tail; + mergeOrphanBlockContents(this.blockContents, s.orphansJson); + // #4390: a tail pointer whose row is gone used to be silently filtered + // out — the view shrank with no trace. Count the gaps so callers can + // warn (persist logs them); the surviving members still load. + let tailGaps = 0; + const resolveSeqs = (seqs: number[]): CoreMessage[] => { + const out: CoreMessage[] = []; + for (const seq of seqs) { + const m = this.messagesBySeq.get(seq); + if (m === undefined) tailGaps++; + else out.push(m); + } + return out; + }; + const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; + const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; + + const out: LegacyView = { + id: s.sessionId, + createdAt: s.createdAt, + lastSeen: s.updatedAt, + tailGaps, + meta: JSON.parse(s.metaJson), + stats: JSON.parse(s.statsJson), + metadata: JSON.parse(s.metadataJson), + state: this.state, + blockContents: this.blockContents, + lastMessages, + lastMessagesFolded: tail.tailFolded, + }; + if (pluginSnapshot) out.pluginSnapshot = pluginSnapshot; + if (s.ccrJson !== null) out.contentStore = JSON.parse(s.ccrJson); return out; - }; - const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; - const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; + } +} - const out: ReturnType = { - id: s.sessionId, - createdAt: s.createdAt, - lastSeen: s.updatedAt, - tailGaps, - meta: JSON.parse(s.metaJson), - stats: JSON.parse(s.statsJson), - metadata: JSON.parse(s.metadataJson), - state, - blockContents, - lastMessages, - lastMessagesFolded: tail.tailFolded, - }; - if (pluginSnapshot) out.pluginSnapshot = pluginSnapshot; - if (s.ccrJson !== null) out.contentStore = JSON.parse(s.ccrJson); - return out; +/** Compatibility wrapper for callers that already hold all rows. */ +export function loadLegacyView(rows: LegacyViewRows): LegacyView { + const builder = new LegacyViewBuilder(rows.session, rows.refs, rows.deadRefs); + builder.addBlocks(rows.blocks); + builder.addMessages(rows.messages); + return builder.finish(); } diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index f3c46ea373..8137503866 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -34,7 +34,7 @@ import { log as loggerLog } from "../logger.js"; import { sha256Hex } from "./store.js"; import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; -import type { StorageBackend } from "./backend.js"; +import type { StorageBackend, StorageReadView } from "./backend.js"; import type { SessionRow, MessageRow, @@ -972,7 +972,32 @@ export class MysqlBackend implements StorageBackend { } async getSessionRow(sessionId: string): Promise { - const [rows] = (await this.pool.query( + return this.getSessionRowOn(this.pool, sessionId); + } + + async readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + const conn = await this.pool.getConnection(); + try { + await conn.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + await conn.query("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"); + return await read({ + getSessionRow: (id) => this.getSessionRowOn(conn, id), + getMessages: (id, opts) => this.getMessagesOn(conn, id, opts), + getBlocks: (id, opts) => this.getBlocksOn(conn, id, opts), + getDeadRefs: (id) => this.getDeadRefsOn(conn, id), + getRefs: (id) => this.getRefsOn(conn, id), + }); + } finally { + try { + await conn.rollback(); + } finally { + conn.release(); + } + } + } + + private async getSessionRowOn(conn: Pool | PoolConnection, sessionId: string): Promise { + const [rows] = (await conn.query( `SELECT session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin, status, created_at, updated_at, next_block_id, next_run_id, meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq, orphans_json @@ -1007,13 +1032,18 @@ export class MysqlBackend implements StorageBackend { }; } - async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + async getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + return this.getMessagesOn(this.pool, sessionId, opts); + } + + private async getMessagesOn(conn: Pool | PoolConnection, sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { // #4390: retired rows stay out of the replay by default. - const [rows] = (await this.pool.query( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", - [sessionId], + if (opts?.seqs?.length === 0) return []; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(",")})` : ""; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; + const [rows] = (await conn.query( + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, + [sessionId, ...(opts?.seqs ?? [])], )) as unknown as Row[][]; const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = []; for (const r of rows) { @@ -1027,20 +1057,20 @@ export class MysqlBackend implements StorageBackend { kind: str(r.kind), contentHash, retiredAt: nullableNum(r.retired_at), - content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? ""), + content: await this.hydrateMediaAsync((await this.getContent(conn, contentHash)) ?? "", conn), }); } return out; } - private async hydrateMediaAsync(content: string): Promise { + private async hydrateMediaAsync(content: string, conn: Pool | PoolConnection): Promise { let out = content; if (out.includes('"$media"')) { try { const parsed = JSON.parse(out) as { imageBase64?: unknown }; const slot = parsed.imageBase64; if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { - const bytes = await this.getContent(this.pool, (slot as { $media: string }).$media); + const bytes = await this.getContent(conn, (slot as { $media: string }).$media); if (typeof bytes === "string" && bytes.length > 0) { parsed.imageBase64 = bytes; out = JSON.stringify(parsed); @@ -1059,7 +1089,7 @@ export class MysqlBackend implements StorageBackend { if (hits.length > 0) { const cache = new Map(); for (const h of new Set(hits.map((x) => x.hash))) { - cache.set(h, await this.getContent(this.pool, h)); + cache.set(h, await this.getContent(conn, h)); } out = out.replace(markerRe, (all: string, mime: string, hash: string) => { const bytes = cache.get(hash); @@ -1070,12 +1100,16 @@ export class MysqlBackend implements StorageBackend { return out; } - async getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { - const [rows] = (await this.pool.query( + async getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + return this.getBlocksOn(this.pool, sessionId, opts); + } + + private async getBlocksOn(conn: Pool | PoolConnection, sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + const [rows] = (await conn.query( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, - [sessionId], + FROM blocks WHERE session_id = ? ORDER BY ord${opts ? ", block_id LIMIT ? OFFSET ?" : ""}`, + [sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])], )) as unknown as Row[][]; const out: (BlockRow & { summaryHash: string; oneHash: string | null })[] = []; for (const r of rows) { @@ -1089,8 +1123,8 @@ export class MysqlBackend implements StorageBackend { tier: num(r.tier), summaryHash: str(r.summary_hash), oneHash: nullableStr(r.one_hash), - summary: (await this.getContent(this.pool, str(r.summary_hash))) ?? "", - one: r.one_hash === null || r.one_hash === undefined ? null : ((await this.getContent(this.pool, str(r.one_hash))) ?? null), + summary: (await this.getContent(conn, str(r.summary_hash))) ?? "", + one: r.one_hash === null || r.one_hash === undefined ? null : ((await this.getContent(conn, str(r.one_hash))) ?? null), supersedesBlockId: nullableStr(r.supersedes_block_id), active: num(r.active) === 1, createdAt: num(r.created_at), @@ -1102,7 +1136,11 @@ export class MysqlBackend implements StorageBackend { } async getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { - const [rows] = (await this.pool.query( + return this.getDeadRefsOn(this.pool, sessionId); + } + + private async getDeadRefsOn(conn: Pool | PoolConnection, sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { + const [rows] = (await conn.query( "SELECT ord, ref, ref_num, recorded_at FROM dead_refs WHERE session_id = ? ORDER BY ord", [sessionId], )) as unknown as Row[][]; @@ -1115,7 +1153,11 @@ export class MysqlBackend implements StorageBackend { } async getRefs(sessionId: string): Promise { - const [rows] = (await this.pool.query( + return this.getRefsOn(this.pool, sessionId); + } + + private async getRefsOn(conn: Pool | PoolConnection, sessionId: string): Promise { + const [rows] = (await conn.query( "SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord", [sessionId], )) as unknown as Row[][]; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d527be96c4..be28d5ac6f 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -22,7 +22,7 @@ */ import { log as loggerLog } from "../logger.js"; -import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; +import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import type { BlockView } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; @@ -37,6 +37,15 @@ import type { CoreMessage, MessageContentStore } from "acp-kernel"; export const MAX_CONSECUTIVE_FAILURES = 3; +/** Bound transient CAS strings during normal cold loads, not the retained history. */ +const VIEW_READ_BATCH_SIZE = 32; + +interface ResidentFields { + header: { requests: number; nextBlockId: number; activeBlockCount: number; mappedRefCount: number }; + metadata?: Record; + blockContents?: Map; +} + /** #3133: read-back verification cadence — the verify leg (full read-back + * double shadowDigest) is O(log) per save, while the drift it guards is a * migration-parity property, not a per-commit one. The FIRST ingest of a @@ -506,22 +515,27 @@ export class UnifiedShadow { /** Loads the unified view of one session (full mode read path). * Returns null on miss — the caller falls back to the legacy file. */ viewFor(id: string): LegacySessionLike | null { - if (this.sync === null) return null; + const store = this.sync; + if (store === null) return null; if (this.stats.disabled) return null; try { - const sessionRow = this.sync.getSessionRow(id); - if (!sessionRow) return null; - const rows = { - session: sessionRow, - messages: this.sync.getMessages(id), - blocks: this.sync.getBlocks(id), - deadRefs: this.sync.getDeadRefs(id), - refs: this.sync.getRefs(id), - }; - // The view's `state` object carries every MERGE_STATE_FIELDS key by - // construction; the declared Record just needs a - // widening cast to reach LegacySessionLike. - const unified = loadLegacyView(rows) as LegacySessionLike; + const unified = store.readSnapshot(() => { + const sessionRow = store.getSessionRow(id); + if (!sessionRow) return null; + const builder = new LegacyViewBuilder(sessionRow, store.getRefs(id), store.getDeadRefs(id)); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = store.getBlocks(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + builder.addBlocks(blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { + builder.addMessages(store.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); + } + // The builder preserves every MERGE_STATE_FIELDS key; its + // generic state declaration only needs widening here. + return builder.finish() as LegacySessionLike; + }); + if (!unified) return null; warnTailGaps(id, unified as unknown as { tailGaps?: number }); return unified; } catch (err) { @@ -530,6 +544,38 @@ export class UnifiedShadow { } } + /** Lazy resident fields share the same source header without rebuilding a session. */ + readResidentFieldsFor(id: string, fields: { metadata?: boolean; blockContents?: boolean }): ResidentFields | null { + const store = this.sync; + if (store === null || this.stats.disabled) return null; + try { + return store.readSnapshot(() => { + const header = store.getLoadHeader(id); + if (!header) return null; + const result: ResidentFields = { header }; + if (fields.metadata) { + const json = store.getMetadataJson(id); + if (json === null) return null; + result.metadata = JSON.parse(json) as Record; + } + if (fields.blockContents) { + const contents: NonNullable = new Map(); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = store.getBlockContents(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + appendBlockContents(contents, blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + mergeOrphanBlockContents(contents, store.getOrphansJson(id)); + result.blockContents = contents; + } + return result; + }); + } catch (err) { + loggerLog("warn", `[storage] resident field load failed for ${id} (${String(err)}) — falling back to the legacy file`); + return null; + } + } + /** #4323: true once this session's drift escalated and no clean sample * has since re-admitted it. Sqlite lanes only — network backends keep * the envelope as their sync read path, where freezing would strand @@ -971,15 +1017,21 @@ export class UnifiedShadow { if (this.stats.disabled) return null; try { const backend = this.backend!; - const sessionRow = await backend.getSessionRow(id); - if (!sessionRow) return null; - const loaded = loadLegacyView({ - session: sessionRow, - messages: await backend.getMessages(id), - blocks: await backend.getBlocks(id), - deadRefs: await backend.getDeadRefs(id), - refs: await backend.getRefs(id), - }) as LegacySessionLike; + const loaded = await backend.readSnapshot(async (reader) => { + const sessionRow = await reader.getSessionRow(id); + if (!sessionRow) return null; + const builder = new LegacyViewBuilder(sessionRow, await reader.getRefs(id), await reader.getDeadRefs(id)); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = await reader.getBlocks(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + builder.addBlocks(blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { + builder.addMessages(await reader.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); + } + return builder.finish() as LegacySessionLike; + }); + if (!loaded) return null; warnTailGaps(id, loaded as unknown as { tailGaps?: number }); return loaded; } catch (err) { diff --git a/src/storage/store.ts b/src/storage/store.ts index a13acf4edb..1cfae29b83 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -336,7 +336,7 @@ export class UnifiedStore { private readonly foreignLockHolder: { pid: number } | null; private readonly lockPath: string | null; - private constructor(db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null) { + private constructor(db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null, private readonly dbPath: string) { this.db = db; this.engine = engine; this.engineSource = engineSource; @@ -438,7 +438,25 @@ export class UnifiedStore { } catch (err) { loggerLog("warn", `[storage] WAL checkpoint on open failed for ${path}: ${String(err)}`); } - return new UnifiedStore(db, used, source, lockPath, foreignLockHolder); + return new UnifiedStore(db, used, source, lockPath, foreignLockHolder, path); + } + + /** Keep paged reads on one generation without taking the writer lock. */ + readSnapshot(read: () => T): T { + return this.db.readTransaction(read); + } + + /** Async callers need a separate connection: an await must not let a + * normal save on this store join the reader's transaction. */ + async readSnapshotAsync(read: (reader: UnifiedStore) => Promise): Promise { + const { db, engine, source } = openSqliteDatabase(this.dbPath, this.engine); + try { + db.exec("PRAGMA query_only = ON; BEGIN DEFERRED"); + const reader = new UnifiedStore(db, engine, source, null, null, this.dbPath); + return await read(reader); + } finally { + db.close(); // closing rolls back the read-only transaction + } } close(): void { @@ -2135,6 +2153,54 @@ export class UnifiedStore { return Buffer.from(row.bytes).toString("utf8"); } + /** Small arbitration fields for lazy reads; no serialized session surfaces. */ + getLoadHeader(sessionId: string): { requests: number; nextBlockId: number; activeBlockCount: number; mappedRefCount: number } | null { + const row = this.db.prepare(` + SELECT next_block_id, stats_json, + (SELECT COUNT(*) FROM blocks WHERE session_id = ? AND active = 1) AS active_blocks, + (SELECT COUNT(DISTINCT ref) FROM refs WHERE session_id = ? AND ref <> 'BLOCKED') AS mapped_refs + FROM sessions WHERE session_id = ? + `).get(sessionId, sessionId, sessionId) as { + next_block_id: number; stats_json: string; active_blocks: number; mapped_refs: number; + } | undefined; + if (!row) return null; + try { + const stats = JSON.parse(row.stats_json) as { requests?: unknown }; + return { + requests: Number(stats.requests ?? 0), + nextBlockId: row.next_block_id, + activeBlockCount: row.active_blocks, + mappedRefCount: row.mapped_refs, + }; + } catch { + return null; + } + } + + getMetadataJson(sessionId: string): string | null { + const row = this.db.prepare("SELECT metadata_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { metadata_json: string } | undefined; + return row?.metadata_json ?? null; + } + + getOrphansJson(sessionId: string): string | null { + const row = this.db.prepare("SELECT orphans_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { orphans_json: string | null } | undefined; + return row?.orphans_json ?? null; + } + + /** Contents for active AND inactive blocks, without their CompressionBlock payload. */ + getBlockContents(sessionId: string, opts: { limit: number; offset?: number }): Pick[] { + const rows = this.db.prepare(`SELECT block_id, summary_hash, one_hash FROM blocks + WHERE session_id = ? ORDER BY ord, block_id LIMIT ? OFFSET ?`) + .all(sessionId, opts.limit, opts.offset ?? 0) as { block_id: string; summary_hash: string; one_hash: string | null }[]; + return rows.map((row) => ({ + blockId: row.block_id, + summary: this.getContent(row.summary_hash) ?? "", + one: row.one_hash ? this.getContent(row.one_hash) : null, + })); + } + getSessionRow(sessionId: string): SessionRow | null { const row = this.db .prepare( @@ -2308,17 +2374,18 @@ export class UnifiedStore { })); } - getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { // #4390: retired rows (rewound/undo history) stay out of the unified // replay by default — they are history, not view. Callers that need // the full historian record (export/diagnostics) opt in. + if (opts?.seqs?.length === 0) return []; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(",")})` : ""; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; const rows = this.db .prepare( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts?.seqs ?? [])) as Record[]; return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string, @@ -2374,14 +2441,14 @@ export class UnifiedStore { return row ? row.refcount : 0; } - getBlocks(sessionId: string): (BlockRow & { summaryHash: string; oneHash: string | null })[] { + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): (BlockRow & { summaryHash: string; oneHash: string | null })[] { const rows = this.db .prepare( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, + FROM blocks WHERE session_id = ? ORDER BY ord${opts ? ", block_id LIMIT ? OFFSET ?" : ""}`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])) as Record[]; return rows.map((r) => ({ blockId: r.block_id as string, ord: r.ord as number, diff --git a/tests/issue4739-storage-read-snapshot.test.ts b/tests/issue4739-storage-read-snapshot.test.ts new file mode 100644 index 0000000000..5d77b9cc8c --- /dev/null +++ b/tests/issue4739-storage-read-snapshot.test.ts @@ -0,0 +1,191 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { UnifiedStore, type IngestSessionInput } from "../src/storage/store.ts"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; +import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; + +function fixture(requests = 1): IngestSessionInput { + return { + session: { + sessionId: "snapshot", createdAt: 1, updatedAt: requests, nextBlockId: 4, nextRunId: 1, + metaJson: "{}", statsJson: JSON.stringify({ requests }), metadataJson: JSON.stringify({ requests }), + kstateJson: "{}", tailJson: "{}", ccrJson: null, + orphansJson: JSON.stringify({ orphan: { one: null, full: { text: "orphan", count: 1 } } }), + }, + messages: [0, 1, 2].map((seq) => ({ + seq, msgId: `m${seq}`, ref: null, refNum: null, role: "user", kind: "message", + content: JSON.stringify({ id: `m${seq}`, role: "user", text: `text${seq}` }), + })), + blocks: [0, 1, 2].map((ord) => ({ + blockId: `b${ord}`, ord, startSeq: ord, endSeq: ord, startRef: null, endRef: null, tier: 0, + summary: JSON.stringify({ text: `full${ord}`, count: 1 }), + one: JSON.stringify({ text: `one${ord}`, count: 1 }), supersedesBlockId: null, + active: ord !== 1, createdAt: 1, coveredHash: "[]", payloadJson: JSON.stringify({ blockId: `b${ord}` }), + })), + refs: [ + { ord: 0, rawId: "a", ref: "m00001", refNum: 1 }, + { ord: 1, rawId: "b", ref: "m00001", refNum: 1 }, + { ord: 2, rawId: "c", ref: "BLOCKED", refNum: null }, + ], + deadRefs: [], ccrEntries: [], + orphansJson: JSON.stringify({ orphan: { one: null, full: { text: "orphan", count: 1 } } }), + }; +} + +for (const engine of ["node:sqlite", "better-sqlite3"] as const) { + test(`${engine}: selected rows, page boundaries and narrow reads preserve their contracts`, (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const file = join(dir, "index.db"); + let store: UnifiedStore | undefined; + try { + try { store = UnifiedStore.open(file, engine); } catch (err) { + if (engine === "better-sqlite3" && String(err).includes("[storage:better-sqlite3]")) return t.skip(String(err)); + throw err; + } + store.ingestSession(fixture()); + const db = openSqliteDatabase(file, engine).db; + try { + db.prepare("UPDATE messages SET retired_at = 1 WHERE session_id = ? AND seq = 1").run("snapshot"); + assert.deepEqual(store.getMessages("snapshot").map((row) => row.seq), [0, 2]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [2, 1, 0, 2, 99] }).map((row) => row.seq), [0, 2]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [1], includeRetired: true }).map((row) => row.seq), [1]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [] }), []); + assert.deepEqual(store.getBlocks("snapshot", { limit: 1, offset: 1 }).map((row) => row.blockId), ["b1"]); + assert.deepEqual(store.getBlocks("snapshot", { limit: 2, offset: 3 }), []); + assert.equal(store.getBlocks("snapshot").length, 3); + + // Invalid unrelated payloads must not be decoded by the lazy field readers. + db.prepare("UPDATE sessions SET tail_json = 'bad', kstate_json = 'bad', ccr_json = 'bad' WHERE session_id = ?").run("snapshot"); + db.prepare("UPDATE blocks SET payload_json = 'bad' WHERE session_id = ?").run("snapshot"); + assert.deepEqual(store.getLoadHeader("snapshot"), { requests: 1, nextBlockId: 4, activeBlockCount: 2, mappedRefCount: 1 }); + db.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":"2"}', "snapshot"); + assert.equal(store.getLoadHeader("snapshot")!.requests, 2, "old numeric-string counter matches full-load arbitration"); + assert.equal(store.getMetadataJson("snapshot"), '{"requests":1}'); + assert.deepEqual(JSON.parse(store.getOrphansJson("snapshot")!), JSON.parse(fixture().orphansJson!)); + const contents = store.getBlockContents("snapshot", { limit: 2, offset: 1 }); + assert.deepEqual(contents.map((row) => row.blockId), ["b1", "b2"]); + assert.deepEqual(JSON.parse(contents[0]!.summary), { text: "full1", count: 1 }); + assert.deepEqual(JSON.parse(contents[0]!.one!), { text: "one1", count: 1 }); + assert.equal(store.getLoadHeader("missing"), null); + assert.equal(store.getMetadataJson("missing"), null); + db.prepare("UPDATE blocks SET ord = 0 WHERE session_id = ?").run("snapshot"); + assert.equal(store.getBlocks("snapshot", { limit: 1, offset: 1 })[0]!.blockId, "b1", "equal ord uses a stable page boundary"); + } finally { db.close(); } + } finally { + store?.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + test(`${engine}: sync snapshot permits another writer and keeps the earlier generation`, (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const file = join(dir, "index.db"); + let store: UnifiedStore | undefined; + try { + try { store = UnifiedStore.open(file, engine); } catch (err) { + if (engine === "better-sqlite3" && String(err).includes("[storage:better-sqlite3]")) return t.skip(String(err)); + throw err; + } + store.ingestSession(fixture()); + const writer = openSqliteDatabase(file, engine).db; + const reader = store; + try { + reader.readSnapshot(() => { + assert.equal(reader.getLoadHeader("snapshot")!.requests, 1); + writer.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":2}', "snapshot"); + writer.prepare("UPDATE blocks SET active = 0 WHERE session_id = ?").run("snapshot"); + assert.equal(reader.getLoadHeader("snapshot")!.requests, 1); + assert.equal(reader.getBlocks("snapshot", { limit: 1, offset: 2 })[0]!.active, true); + }); + assert.equal(reader.getLoadHeader("snapshot")!.requests, 2); + assert.throws(() => reader.readSnapshot(() => { throw new Error("reader failed"); }), /reader failed/); + writer.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":3}', "snapshot"); + assert.equal(reader.readSnapshot(() => reader.getLoadHeader("snapshot")!.requests), 3); + } finally { writer.close(); } + } finally { + store?.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); +} + +test("async sqlite snapshot isolates normal writes across await and closes on callback failure", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const backend = SqliteBackend.open(join(dir, "index.db")); + try { + await backend.ingestSession(fixture()); + await backend.readSnapshot(async (reader) => { + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 1); + await backend.ingestSession(fixture(2)); + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 1); + assert.deepEqual((await reader.getMessages("snapshot", { seqs: [2] })).map((row) => row.seq), [2]); + }); + assert.equal(JSON.parse((await backend.getSessionRow("snapshot"))!.statsJson).requests, 2); + await assert.rejects(backend.readSnapshot(async (reader) => { + await reader.getSessionRow("snapshot"); + throw new Error("async reader failed"); + }), /async reader failed/); + await backend.ingestSession(fixture(3)); + await backend.readSnapshot(async (reader) => { + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 3); + }); + } finally { + await backend.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("mysql snapshot pins SQL, CAS and media hydration to its read-only connection and releases on error", async () => { + const queries: { sql: string; values?: unknown[] }[] = []; + let releases = 0; + let rollbacks = 0; + const mediaHash = "a".repeat(64); + const conn = { + async query(sql: string, values?: unknown[]) { + queries.push({ sql, values }); + if (sql.includes("FROM messages")) return [[{ seq: 2, msg_id: "m2", role: "user", kind: "message", content_hash: "msg" }]]; + if (sql.includes("FROM blocks")) return [[{ block_id: "b1", summary_hash: "summary", one_hash: "one", active: 1 }]]; + if (sql.includes("FROM content")) { + const bytes = values?.[0] === "msg" ? JSON.stringify({ imageBase64: { $media: mediaHash } }) + : values?.[0] === mediaHash ? "image-bytes" : '{"text":"summary","count":1}'; + return [[{ bytes }]]; + } + return [[]]; + }, + async rollback() { rollbacks++; }, + release() { releases++; }, + }; + const driver = { createPool: () => ({ + getConnection: async () => conn, + query: async () => { throw new Error("snapshot escaped to pool"); }, + end: async () => {}, + }) } as unknown as Mysql2Module; + const backend = new MysqlBackend("mysql://snapshot-test", driver); + await backend.readSnapshot(async (reader) => { + const before = queries.length; + assert.deepEqual(await reader.getMessages("snapshot", { seqs: [] }), []); + assert.equal(queries.length, before); + const rows = await reader.getMessages("snapshot", { seqs: [2] }); + assert.equal(JSON.parse(rows[0]!.content).imageBase64, "image-bytes"); + assert.equal((await reader.getBlocks("snapshot", { limit: 1, offset: 2 }))[0]!.blockId, "b1"); + await reader.getSessionRow("snapshot"); + await reader.getRefs("snapshot"); + await reader.getDeadRefs("snapshot"); + }); + assert.equal(queries[0]!.sql, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + assert.equal(queries[1]!.sql, "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"); + const selected = queries.find((query) => query.sql.includes("FROM messages"))!; + assert.match(selected.sql, /retired_at IS NULL AND seq IN \(\?\)/); + assert.deepEqual(selected.values, ["snapshot", 2]); + const paged = queries.find((query) => query.sql.includes("FROM blocks"))!; + assert.match(paged.sql, /ORDER BY ord, block_id LIMIT \? OFFSET \?/); + assert.deepEqual(paged.values, ["snapshot", 1, 2]); + await assert.rejects(backend.readSnapshot(async () => { throw new Error("mysql reader failed"); }), /mysql reader failed/); + assert.equal(rollbacks, 2); + assert.equal(releases, 2); + await backend.close(); +}); diff --git a/tests/persist-lazy-narrow-reads.test.ts b/tests/persist-lazy-narrow-reads.test.ts new file mode 100644 index 0000000000..e295ad018c --- /dev/null +++ b/tests/persist-lazy-narrow-reads.test.ts @@ -0,0 +1,208 @@ +import test, { type TestContext } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createInitialState, type CompressionBlock } from "acp-kernel"; +import { SessionStore } from "../src/persist.ts"; +import { isDroppedMetadata, LazyBlockContents, type BlockContent, type Session } from "../src/session.ts"; +import type { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { UnifiedStore } from "../src/storage/store.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const ROOT = mkdtempSync(path.join(tmpdir(), "bili-lazy-narrow-")); +process.env.BILI_CONFIG_FILE = path.join(ROOT, "no-config.json"); +process.env.BILI_PERSIST_ZSTD = "0"; +delete process.env.BILI_STORAGE_JSON_WRITER; +test.after(() => rmrf(ROOT)); + +type SavedFields = { metadata: Record; blockContents: Record }; +type StoreInternals = { + shadow: UnifiedShadow; + loadEnvelope(id: string, meta?: Session["meta"]): { payload: SavedFields } | null; + buildRecordForSave(session: Session): SavedFields; +}; + +function makeBlock(blockId: string, active = true): CompressionBlock { + return { + blockId, runId: "r0", tier: 1, summary: `summary ${blockId}`, + directMessageIds: [], effectiveMessageIds: [], directBlockIds: [], + compressedTokens: 0, createdAt: 1, survivedCount: 0, generation: "young", active, + }; +} + +function makeSession(id: string, source: string): Session { + return { + id, meta: { protocol: "openai", upstreamOrigin: "http://up:1", label: id }, + stats: { + requests: 5, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, + cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, compressCreditTokens: 0, + retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, + storeBytesSaved: 0, rangeRestores: 0, + }, + metadata: { source }, state: createInitialState(), + createdAt: 1, lastSeen: 2, inFlight: 0, persisted: false, pendingRetrievals: [], + blockContents: new Map([["orphan", { one: null, full: { text: `${source} orphan original`, count: 2 } }]]), + }; +} + +function fixture(t: TestContext, mode: "full" | "legacy"): { store: SessionStore; dir: string } { + process.env.BILI_STORAGE_UNIFIED = mode; + const dir = mkdtempSync(path.join(ROOT, `${mode}-`)); + const store = new SessionStore({ dir, enabled: true, debounceMs: 60_000 }); + t.after(() => store.closeQuietly()); + return { store, dir }; +} + +function internals(store: SessionStore): StoreInternals { + return store as unknown as StoreInternals; +} + +function stateFile(dir: string): string { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) { + const nested = stateFile(file); + if (nested) return nested; + } else if (entry.name.endsWith(".json") && !entry.name.endsWith(".content-store.json")) { + return file; + } + } + return ""; +} + +function assertDropped(session: Session): void { + assert.equal(isDroppedMetadata(session.metadata), true); + assert.ok(session.blockContents instanceof LazyBlockContents); + assert.equal(session.blockContents.isDropped, true); +} + +test("full-mode lazy getters read only their requested resident field and keep all originals", async (t) => { + const { store } = fixture(t, "full"); + const session = makeSession("narrow-full", "index"); + session.state.blocks = [makeBlock("active"), makeBlock("archived", false)]; + session.blockContents.set("active", { + one: { text: "short view", count: 1 }, full: { text: "full active original", count: 4 }, + }); + session.blockContents.set("archived", { + one: { text: "same archived original", count: 2 }, full: { text: "same archived original", count: 2 }, + }); + await store.writeNow(session); + assertDropped(session); + + const shadow = internals(store).shadow; + const driver = (shadow as unknown as { sync: UnifiedStore }).sync; + assert.ok(driver, "full mode must exercise the SQLite lane"); + const wholeLoads = t.mock.method(store, "loadSync"); + const wholeViews = t.mock.method(shadow, "viewFor"); + const messages = t.mock.method(driver, "getMessages"); + const blockPayloads = t.mock.method(driver, "getBlocks"); + const sessionRows = t.mock.method(driver, "getSessionRow"); + const metadata = t.mock.method(driver, "getMetadataJson"); + const contents = t.mock.method(driver, "getBlockContents"); + const narrow = t.mock.method(shadow, "readResidentFieldsFor"); + const imports = t.mock.method(shadow, "importRecord"); + const saves = t.mock.method(store, "scheduleSave"); + + assert.equal(session.metadata.source, "index"); + assert.equal(metadata.mock.callCount(), 1); + assert.equal(contents.mock.callCount(), 0, "metadata access does not fetch any originals"); + assert.equal((session.blockContents as LazyBlockContents).isDropped, true); + assert.deepEqual(narrow.mock.calls[0].arguments[1], { metadata: true }); + + assert.deepEqual(session.blockContents.get("active"), { + one: { text: "short view", count: 1 }, full: { text: "full active original", count: 4 }, + }); + assert.deepEqual(session.blockContents.get("archived"), { + one: null, full: { text: "same archived original", count: 2 }, + }, "inactive originals survive and duplicate one/full pairs normalize"); + assert.deepEqual(session.blockContents.get("orphan"), { one: null, full: { text: "index orphan original", count: 2 } }); + assert.equal(session.blockContents.size, 3); + assert.equal(metadata.mock.callCount(), 1, "contents access does not re-fetch metadata"); + assert.deepEqual(narrow.mock.calls[1].arguments[1], { blockContents: true }); + for (const spy of [wholeLoads, wholeViews, messages, blockPayloads, sessionRows, imports, saves]) { + assert.equal(spy.mock.callCount(), 0, "a lazy getter must not rebuild, import or schedule a whole session"); + } +}); + +test("legacy save builder shares one envelope read for both dropped fields without retaining an old projection", async (t) => { + const { store, dir } = fixture(t, "legacy"); + const session = makeSession("narrow-legacy", "first"); + await store.writeNow(session); + assertDropped(session); + const envelopes = t.mock.method(internals(store), "loadEnvelope"); + const wholeLoads = t.mock.method(store, "loadSync"); + + const saved = internals(store).buildRecordForSave(session); + assert.equal(saved.metadata.source, "first"); + assert.equal(saved.blockContents.orphan.full.text, "first orphan original"); + assert.equal(envelopes.mock.callCount(), 1, "one build shares a single source read across both lazy getters"); + assert.equal(wholeLoads.mock.callCount(), 0); + await store.writeNow(session); + assertDropped(session); + const priorReads = envelopes.mock.callCount(); + + // An external writer changes only the two resident fields. The next read + // must consult disk again after the synchronous save has finished. + const file = stateFile(dir); + assert.ok(file); + const envelope = JSON.parse(readFileSync(file, "utf8")) as { payload: SavedFields }; + envelope.payload.metadata.source = "external"; + envelope.payload.blockContents.orphan.full.text = "external original"; + writeFileSync(file, JSON.stringify(envelope)); + assert.equal(session.metadata.source, "external"); + assert.equal(session.blockContents.get("orphan")?.full.text, "external original"); + assert.equal(envelopes.mock.callCount(), priorReads + 2, "independent touches outside a save each select their current source"); + assert.equal(wholeLoads.mock.callCount(), 0); +}); + +test("lazy source arbitration matches cold-load progress and drift rules", async (t) => { + type Generation = { requests: number; nextBlockId: number; active: number; refs: number }; + const base: Generation = { requests: 5, nextBlockId: 10, active: 1, refs: 1 }; + const cases: { name: string; latch: boolean; index?: Partial; envelope: Partial; expected: string }[] = [ + { name: "strictly more envelope requests", latch: false, envelope: { requests: 6 }, expected: "envelope" }, + { name: "equal requests and ahead envelope fold cursor", latch: false, envelope: { nextBlockId: 11 }, expected: "envelope" }, + { name: "equal progress keeps index", latch: false, envelope: {}, expected: "index" }, + { name: "drift prefers greater index active coverage", latch: true, index: { active: 2 }, envelope: { requests: 99 }, expected: "index" }, + { name: "drift prefers greater index mapped refs", latch: true, index: { refs: 3 }, envelope: { requests: 99 }, expected: "index" }, + { name: "drift ties keep envelope", latch: true, envelope: {}, expected: "envelope" }, + ]; + for (const [i, c] of cases.entries()) { + await t.test(c.name, async (st) => { + const { store } = fixture(st, "full"); + const id = `narrow-arbitration-${i}`; + const generation = (source: string, values: Generation): Session => { + const session = makeSession(id, source); + session.stats.requests = values.requests; + session.state.nextBlockId = values.nextBlockId; + session.state.blocks = Array.from({ length: values.active }, (_, n) => makeBlock(`b${n + 1}`)); + for (let n = 1; n <= values.refs; n++) { + const ref = `m${String(n).padStart(5, "0")}`; + session.state.messageRefs.byRaw[`raw-${n}`] = ref; + session.state.messageRefs.byRef[ref] = `raw-${n}`; + } + return session; + }; + await store.writeNow(generation("index", { ...base, ...c.index })); + const shadow = internals(store).shadow; + const ingest = st.mock.method(shadow, "record", () => {}); + const envelope = generation("envelope", { ...base, ...c.envelope }); + await store.writeNow(envelope); + ingest.mock.restore(); + if (c.latch) shadow.driftedSessions.add(id); + assertDropped(envelope); + + const loads = st.mock.method(store, "loadSync"); + const views = st.mock.method(shadow, "viewFor"); + assert.equal(envelope.metadata.source, c.expected); + assert.equal(envelope.blockContents.get("orphan")?.full.text, `${c.expected} orphan original`); + assert.equal(loads.mock.callCount(), 0); + assert.equal(views.mock.callCount(), 0); + loads.mock.restore(); + views.mock.restore(); + const cold = store.loadSync(id); + assert.equal(cold?.metadata.source, c.expected, "the public cold loader selects the same generation"); + assert.equal(cold?.blockContents.get("orphan")?.full.text, envelope.blockContents.get("orphan")?.full.text); + }); + } +}); diff --git a/tests/persist-rollback-guard.test.ts b/tests/persist-rollback-guard.test.ts index 6441a048a5..ae6357bec0 100644 --- a/tests/persist-rollback-guard.test.ts +++ b/tests/persist-rollback-guard.test.ts @@ -36,7 +36,7 @@ function withTempStore(name: string, fn: (store: SessionStore, dir: string, warn try { await fn(store, dir, warnings); } finally { - store.cancelAll(); + store.closeQuietly(); rmrf(dir); } }) as unknown as Promise; diff --git a/tests/storage-backend-contract.test.ts b/tests/storage-backend-contract.test.ts index ba36a1a71b..762e0aa7cc 100644 --- a/tests/storage-backend-contract.test.ts +++ b/tests/storage-backend-contract.test.ts @@ -248,6 +248,15 @@ test("mysql lane: contract surface against MySQL 8.0", { skip: MYSQL_DSN.length const rows2 = await backend.getMessages(sid); assert.equal(rows2.length, 2, "no duplicate rows on identical re-ingest"); + await backend.readSnapshot(async (reader) => { + assert.equal((await reader.getSessionRow(sid))!.sessionId, sid); + assert.deepEqual(await reader.getMessages(sid, { seqs: [] }), []); + const selected = await reader.getMessages(sid, { seqs: [rows2[1]!.seq] }); + assert.equal(selected.length, 1); + assert.equal(selected[0]!.content, rows2[1]!.content); + assert.deepEqual(await reader.getBlocks(sid, { limit: 1 }), []); + }); + // reap journal await backend.reapBlocks(sid, new Set(["b2"]), new Map([["b2", { one: null, full: { text: "orphaned", count: 0 } }]])); await backend.reapBlocks(sid, new Set(["b2"]), new Map(), ); diff --git a/tests/storage-bounded-load.test.ts b/tests/storage-bounded-load.test.ts new file mode 100644 index 0000000000..a412e8457f --- /dev/null +++ b/tests/storage-bounded-load.test.ts @@ -0,0 +1,222 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { planIngest, loadLegacyView, LegacyViewBuilder, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import type { SqliteDatabase } from "../src/storage/driver.ts"; + +function fixture(): LegacySessionLike { + const messages: NonNullable = Array.from({ length: 80 }, (_, i) => ({ + id: `msg${i}`, role: "user", contentType: "text", text: `message ${i}`, + })); + const blocks: LegacySessionLike["state"]["blocks"] = Array.from({ length: 70 }, (_, i) => ({ + blockId: `b${i + 1}`, runId: "r1", tier: 1, summary: `summary ${i + 1}`, + directMessageIds: [`msg${i}`], effectiveMessageIds: [`msg${i}`], directBlockIds: [], + compressedTokens: 10, createdAt: 1000 + i, survivedCount: 0, generation: "young", active: i % 2 === 0, + })); + return { + id: "bounded", createdAt: 1000, lastSeen: 2000, + meta: { protocol: "openai" }, stats: { requests: 7 }, metadata: { pluginAgent: "codex", note: "retained" }, + state: { + blocks, nextBlockId: 71, nextRunId: 2, + messageRefs: { byRaw: { msg0: "m00001", alias: "m00001", ghost: "m99999", blocked: "BLOCKED" }, byRef: { m00001: "msg0", m99999: "ghost" } }, + deadRefs: ["m90001", "m90002"], + }, + blockContents: new Map(blocks.map((block) => [block.blockId, { + one: { text: `one ${block.blockId}`, count: 1 }, full: { text: `original ${block.blockId}`, count: 2 }, + }])), + decompositionLog: messages, + pluginSnapshot: messages.slice(0, 36), + lastMessages: [messages[65], messages[33], messages[65]], + lastMessagesFolded: true, + contentStore: { version: 1, byHash: { h: "retained CCR" }, byRef: {} }, + }; +} + +function seed(store: UnifiedStore): void { + const plan = planIngest(fixture()); + plan.session.orphansJson = JSON.stringify({ + b1: { one: null, full: { text: "stale journal duplicate", count: 1 } }, + reaped: { one: null, full: { text: "reaped original", count: 1 } }, + }); + store.ingestSession(plan); + // Simulate missing historical backing rows after the ingest remapper. + const tail = JSON.parse(store.getSessionRow("bounded")!.tailJson) as { tailSeqs: number[]; pluginSnapshotSeqs: number[] }; + tail.tailSeqs.push(9999); + tail.pluginSnapshotSeqs.push(9999); + (store as unknown as { db: SqliteDatabase }).db.prepare("UPDATE sessions SET tail_json = ? WHERE session_id = ?").run(JSON.stringify(tail), "bounded"); +} + +function allRows(store: UnifiedStore) { + return { + session: store.getSessionRow("bounded")!, messages: store.getMessages("bounded"), + blocks: store.getBlocks("bounded"), refs: store.getRefs("bounded"), deadRefs: store.getDeadRefs("bounded"), + }; +} + +function assertHistory(view: LegacySessionLike | null): asserts view is LegacySessionLike { + assert.ok(view); + assert.equal(view.state.blocks.length, 70); + assert.equal(view.state.blocks.filter((block) => !block.active).length, 35, "inactive ancestors remain available to decompress"); + assert.deepEqual(view.state.blocks.map((block) => block.blockId), fixture().state.blocks.map((block) => block.blockId)); + assert.equal(view.blockContents.get("b1")!.full.text, "original b1", "block row wins over a stale orphan journal entry"); + assert.equal(view.blockContents.get("reaped")!.full.text, "reaped original"); + assert.equal(view.blockContents.size, 71); + assert.deepEqual(view.lastMessages?.map((message) => message.id), ["msg65", "msg33", "msg65"]); + assert.strictEqual(view.lastMessages![0], view.lastMessages![2], "repeated pointers retain shared message identity"); + assert.strictEqual(view.lastMessages![1], view.pluginSnapshot![33], "tail and snapshot overlap shares one parsed message"); + assert.deepEqual(view.state.messageRefs, fixture().state.messageRefs, "ghost and BLOCKED refs survive verbatim"); + assert.deepEqual(view.state.deadRefs, ["m90001", "m90002"]); + assert.deepEqual(view.contentStore, fixture().contentStore); + assert.equal((view as unknown as { tailGaps: number }).tailGaps, 2); +} + +test("ordinary sync cold load reads only unique tail/snapshot seqs and bounded block pages", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-load-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const expected = loadLegacyView(allRows(store)); + const getMessages = store.getMessages.bind(store); + const getBlocks = store.getBlocks.bind(store); + const requested: number[] = []; + const blockPages: number[] = []; + let snapshots = 0; + const snapshot = store.readSnapshot.bind(store); + store.readSnapshot = (fn) => { snapshots++; return snapshot(fn); }; + store.getMessages = (id, opts) => { + assert.ok(opts?.seqs && opts.seqs.length <= 32, "cold load must never hydrate the whole message log"); + requested.push(...opts.seqs); + return getMessages(id, opts); + }; + store.getBlocks = (id, opts) => { + assert.equal(opts?.limit, 32); + blockPages.push(opts?.offset ?? 0); + return getBlocks(id, opts); + }; + const view = shadow.viewFor("bounded"); + assertHistory(view); + assert.deepEqual(view, expected); + assert.equal(snapshots, 1, "header and every page share one read snapshot"); + assert.deepEqual(blockPages, [0, 32, 64]); + assert.equal(requested.length, new Set(requested).size); + assert.deepEqual(new Set(requested), new Set([65, 33, 9999, ...Array.from({ length: 36 }, (_, i) => i)])); + assert.equal(shadow.viewFor("absent"), null); +}); + +test("ordinary async cold load uses the snapshot reader for each bounded page", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-async-")); + const backend = SqliteBackend.open(join(dir, "index.db")); + t.after(async () => { await backend.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(backend.store); + const expected = loadLegacyView(allRows(backend.store)); + const readSnapshot = backend.readSnapshot.bind(backend); + const offsets: number[] = []; + const requested: number[] = []; + let snapshots = 0; + backend.readSnapshot = (fn) => readSnapshot(async (reader) => { + snapshots++; + return fn({ + ...reader, + getMessages: async (id, opts) => { + assert.ok(opts?.seqs && opts.seqs.length <= 32); + requested.push(...opts.seqs); + return reader.getMessages(id, opts); + }, + getBlocks: async (id, opts) => { + assert.equal(opts?.limit, 32); + offsets.push(opts?.offset ?? 0); + return reader.getBlocks(id, opts); + }, + }); + }); + backend.getMessages = async () => { throw new Error("load bypassed the snapshot reader"); }; + backend.getBlocks = async () => { throw new Error("load bypassed the snapshot reader"); }; + const shadow = Reflect.construct(UnifiedShadow, ["full", null, () => null, backend]) as UnifiedShadow; + const view = await shadow.viewForAsync("bounded"); + assertHistory(view); + assert.deepEqual(view, expected); + assert.equal(snapshots, 1); + assert.deepEqual(offsets, [0, 32, 64]); + assert.equal(requested.length, new Set(requested).size); + assert.deepEqual(new Set(requested), new Set([65, 33, 9999, ...Array.from({ length: 36 }, (_, i) => i)])); +}); + +test("a failed block page falls back and releases the read snapshot for the next load", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-failure-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const getBlocks = store.getBlocks.bind(store); + store.getBlocks = (id, opts) => { + if (opts?.offset === 32) throw new Error("injected block page failure"); + return getBlocks(id, opts); + }; + assert.equal(shadow.viewFor("bounded"), null, "partial views must never escape on a failed page"); + store.getBlocks = getBlocks; + assertHistory(shadow.viewFor("bounded")); +}); + +test("resident field reads avoid unrelated JSON, messages, refs, and block payloads", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-resident-fields-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const expected = loadLegacyView(allRows(store)); + for (const key of ["getSessionRow", "getMessages", "getBlocks", "getRefs", "getDeadRefs"] as const) { + store[key] = () => { throw new Error(`resident field read called ${key}`); }; + } + let metadataReads = 0; + let contentReads = 0; + let orphanReads = 0; + let snapshots = 0; + const snapshot = store.readSnapshot.bind(store); + store.readSnapshot = (fn) => { snapshots++; return snapshot(fn); }; + const metadata = store.getMetadataJson.bind(store); + const contents = store.getBlockContents.bind(store); + const orphans = store.getOrphansJson.bind(store); + store.getMetadataJson = (id) => { metadataReads++; return metadata(id); }; + store.getBlockContents = (id, opts) => { contentReads++; assert.equal(opts?.limit, 32); return contents(id, opts); }; + store.getOrphansJson = (id) => { orphanReads++; return orphans(id); }; + const meta = shadow.readResidentFieldsFor("bounded", { metadata: true }); + assert.deepEqual(meta, { header: { requests: 7, nextBlockId: 71, activeBlockCount: 35, mappedRefCount: 2 }, metadata: expected.metadata }); + assert.equal(contentReads, 0); + assert.equal(orphanReads, 0); + const blocks = shadow.readResidentFieldsFor("bounded", { blockContents: true }); + assert.deepEqual(blocks?.blockContents, expected.blockContents); + assert.equal(metadataReads, 1, "content getter does not read metadata"); + const both = shadow.readResidentFieldsFor("bounded", { metadata: true, blockContents: true }); + assert.deepEqual(both?.metadata, expected.metadata); + assert.deepEqual(both?.blockContents, expected.blockContents); + assert.equal(snapshots, 3, "combined fields use one snapshot"); + assert.equal(contentReads, 6); + assert.equal(orphanReads, 2); +}); + +test("incremental assembly preserves empty versus absent pointers and skips a corrupt orphan journal", () => { + const plan = planIngest(fixture()); + plan.session.tailJson = JSON.stringify({ tailSeqs: [], pluginSnapshotSeqs: null, tailFolded: false }); + plan.session.orphansJson = "broken journal"; + const builder = new LegacyViewBuilder(plan.session, plan.refs!, plan.deadRefs); + assert.deepEqual(builder.messageSeqs, []); + builder.addBlocks(plan.blocks); + const view = builder.finish(); + assert.deepEqual(view.lastMessages, []); + assert.equal(view.pluginSnapshot, undefined); + assert.equal(view.tailGaps, 0); + assert.equal(view.blockContents.size, 70); + plan.session.tailJson = JSON.stringify({ tailSeqs: null, pluginSnapshotSeqs: [], tailFolded: false }); + const other = new LegacyViewBuilder(plan.session, [], []).finish(); + assert.equal(other.lastMessages, undefined); + assert.deepEqual(other.pluginSnapshot, []); +}); From 66297ff97e131681a510d1c96d26c9f8e29963de Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:09:26 -0700 Subject: [PATCH 2/5] test: honor Windows cmd wrapper for inline OpenCode launch --- tests/launcher-opencode-inline.test.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 7ff04b2ace..fd5452a332 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -28,6 +28,7 @@ interface SpawnRecord { command: string; args: string[]; env: NodeJS.ProcessEnv | undefined; + windowsVerbatimArguments?: boolean; } // `bili opencode-inline` (#4669): the launcher must start NO proxy, hand the @@ -37,16 +38,12 @@ interface SpawnRecord { // clone (plugin injection only; 2.x directory wrapper when the host is 2.x). test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attach/proxy env stripped, plugin rides OPENCODE_CONFIG", async () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ocinline-")); - const prevHome = process.env.HOME; - const prevUserProfile = process.env.USERPROFILE; const prevXdg = process.env.XDG_CONFIG_HOME; const prevOcConfig = process.env.OPENCODE_CONFIG; const prevClientBin = process.env.BILI_CLIENT_BIN; const prevBcp = process.env.BILLION_CONTEXT_PROXY; const prevHttpsProxy = process.env.HTTPS_PROXY; const prevMitmHosts = process.env.BILI_MITM_HOSTS; - process.env.HOME = home; - if (prevUserProfile !== undefined) process.env.USERPROFILE = home; // Isolated config root: no user opencode.json, no ambient OPENCODE_CONFIG. process.env.XDG_CONFIG_HOME = path.join(home, ".config"); delete process.env.OPENCODE_CONFIG; @@ -75,7 +72,7 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, windowsVerbatimArguments: options?.windowsVerbatimArguments }); const child = makeFakeChild(0); const orig = child.on!.bind(child); (child as { on: SpawnChild["on"] }).on = (event, listener) => { @@ -109,8 +106,16 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac { fetchImpl, fetchHealthInfo, spawnImpl, sleep: () => Promise.resolve() }, ); assert.equal(spawns.length, 1, "exactly one child (opencode itself) — no proxy spawn"); - assert.equal(spawns[0].command, fakeOc); const env = spawns[0].env!; + if (process.platform === "win32") { + assert.equal(spawns[0].command, env.COMSPEC?.trim() ? env.COMSPEC : "cmd.exe"); + const quotedOc = /\s/.test(fakeOc) ? `"${fakeOc}"` : fakeOc; + assert.deepEqual(spawns[0].args, ["/d", "/s", "/c", `"${quotedOc}"`]); + assert.equal(spawns[0].windowsVerbatimArguments, true); + } else { + assert.equal(spawns[0].command, fakeOc); + assert.deepEqual(spawns[0].args, []); + } assert.equal(env.BILI_OPENCODE_INLINE, "1"); assert.equal(env.BILLION_CONTEXT_PROXY, undefined, "inherited BILLION_CONTEXT_PROXY must be stripped (attach would win)"); assert.equal(env.HTTPS_PROXY, undefined, "inherited HTTPS_PROXY must be stripped (no MITM lane)"); @@ -134,9 +139,6 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac assert.equal(clone.compaction?.auto, false, "host auto-compaction stays off (ACP owns compression)"); } finally { process.exit = prevExit; - process.env.HOME = prevHome; - if (prevUserProfile === undefined) delete process.env.USERPROFILE; - else process.env.USERPROFILE = prevUserProfile; if (prevXdg === undefined) delete process.env.XDG_CONFIG_HOME; else process.env.XDG_CONFIG_HOME = prevXdg; if (prevOcConfig === undefined) delete process.env.OPENCODE_CONFIG; From 610b164e170dfa43ad4839af841075952c205acf Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:14:31 -0700 Subject: [PATCH 3/5] test: close fork session stores before Windows cleanup --- tests/plugin-fork.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 1c4801377d..e6ea8b84de 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -114,7 +114,7 @@ async function harness(persist = false, seedParent = true, compressExtra: Record assert.equal(r.status, 200); await r.text(); } - return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; + return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.closeQuietly(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; } test("plain proxy panel snapshots do not change raw-history persistence", { timeout: 60_000 }, async () => { From 8c03ff1e6ef1560870d9ab07858b4d4be5d54f33 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:47:58 -0700 Subject: [PATCH 4/5] test: fix Windows graceful shutdown and fixture teardown Windows SIGTERM terminates the child instead of running its graceful shutdown handler. Trigger the existing loopback shutdown route on Windows so the draining-marker regression exercises the same lifecycle as POSIX. Wait for child close before deleting its database, destroy sockets owned by the silent TCP fixture, and stop health polling when the child exits. Keep both regression scenarios and their behavioral assertions intact. Validation: the isolated CI-preloaded serial test file passes 6/6 and exits naturally in 11.58 seconds; npm run typecheck and npm run build pass. --- tests/issue4982-draining-twin-ladder.test.ts | 50 +++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index 71ea29a279..d0eb94783e 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -228,21 +228,26 @@ async function waitForStatus(port: number, timeoutMs: number): Promise async function killChild(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill("SIGTERM"); - await Promise.race([ - new Promise((resolve) => child.once("exit", () => resolve())), - new Promise((resolve) => setTimeout(resolve, 10_000)), - ]); - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + child.kill("SIGTERM"); + }); } /** A TCP listener that accepts connections and then says nothing — invisible * to /__bili/health probes (they time out as "no evidence") but still owns * the socket, so binders get EADDRINUSE. Deterministically forces the LATE * EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */ -function silentTcpListener(port: number): Promise { +function silentTcpListener(port: number): Promise<{ close(): Promise }> { return new Promise((resolve, reject) => { + const sockets = new Set(); const server = net.createServer((sock) => { + sockets.add(sock); + sock.once("close", () => sockets.delete(sock)); // accept and hang — never answer HTTP; drain the probe's bytes so peer // EOF is actually delivered (a paused socket buffers them and never sees // EOF) and release the connection, or a dead prober pins server.close() @@ -251,7 +256,12 @@ function silentTcpListener(port: number): Promise { sock.on("end", () => sock.destroy()); }); server.once("error", reject); - server.listen(port, "127.0.0.1", () => resolve(server)); + server.listen(port, "127.0.0.1", () => resolve({ + close: () => new Promise((done) => { + server.close(() => done()); + for (const socket of sockets) socket.destroy(); + }), + })); }); } @@ -319,6 +329,9 @@ function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: { async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise { let port = 0; await waitUntil(async () => { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`${label} exited before becoming healthy`); + } const m = readMarkers(stateHome).find((e) => e.pid === child.pid); if (!m || typeof m.port !== "number") return false; port = m.port as number; @@ -335,7 +348,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders let p1Ref: ReturnType | null = null; let p2Ref: ReturnType | null = null; let heldRef: net.Socket | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // P1: the serving proxy that will enter graceful shutdown. const p1 = spawnBili(sandbox, port); @@ -353,7 +366,18 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders }); heldRef = held; - p1.child.kill("SIGTERM"); + if (process.platform === "win32") { + // Windows SIGTERM terminates immediately instead of invoking the + // handler. The loopback admin route enters the same graceful path. + const response = await fetch(`http://127.0.0.1:${port}/__bili/shutdown`, { + method: "POST", + signal: AbortSignal.timeout(2_000), + }); + assert.equal(response.status, 202, "P1 must accept the graceful shutdown request"); + await response.arrayBuffer(); + } else { + p1.child.kill("SIGTERM"); + } const p1MarkerFile = path.join(stateHome, "billion-context", "instances", `${p1Marker.instanceId}.json`); await waitUntil(async () => { let stamped = false; @@ -395,7 +419,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders heldRef?.destroy(); if (p1Ref) await killChild(p1Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); @@ -406,7 +430,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n const port = await freePort(); await assertPortDead(port, { label: "4982 unverified-twin port" }); let p2Ref: ReturnType | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // A registry marker claiming `port` with a LIVE pid and full identity // match — but NO portReleasedAt stamp (pre-fix build, or a failed @@ -456,7 +480,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n } finally { if (p2Ref) await killChild(p2Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); From aec79ef4181a2f477d65b527ce6ac8326a932ff9 Mon Sep 17 00:00:00 2001 From: z0d0dOr <128966342+z0d0dOr@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:11:26 -0700 Subject: [PATCH 5/5] test: handle reset health probes in silent TCP fixture Windows CI can reset an aborted probe connection instead of ending it. The silent TCP fixture did not handle errors on accepted sockets, so ECONNRESET became an uncaught test exception after the probe finished. Destroy reset sockets and add a real resetAndDestroy regression against the same fixture. Keep the existing port-owner and ladder assertions. Validation: the new regression reproduces ECONNRESET before the fix; the isolated CI-preloaded serial test file then passes 7/7 and exits naturally. Pure tsc and git diff --check pass. --- tests/issue4982-draining-twin-ladder.test.ts | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index d0eb94783e..5c8e178cce 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -254,6 +254,9 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { // with CLOSE-WAIT sockets sock.resume(); sock.on("end", () => sock.destroy()); + // An aborted health probe may reset instead of ending the connection. + // Socket errors belong to this silent fixture, not the enclosing test. + sock.on("error", () => sock.destroy()); }); server.once("error", reject); server.listen(port, "127.0.0.1", () => resolve({ @@ -265,6 +268,25 @@ function silentTcpListener(port: number): Promise<{ close(): Promise }> { }); } +test("silent TCP fixture survives an aborted health probe", { timeout: 5_000 }, async () => { + const port = await freePort(); + const fixture = await silentTcpListener(port); + const client = net.connect({ host: "127.0.0.1", port }); + try { + await new Promise((resolve, reject) => { + client.once("connect", resolve); + client.once("error", reject); + }); + client.write("GET /__bili/health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + await new Promise((resolve) => setTimeout(resolve, 25)); + client.resetAndDestroy(); + await new Promise((resolve) => setTimeout(resolve, 25)); + } finally { + client.destroy(); + await fixture.close(); + } +}); + function tcpRefused(port: number): Promise { return new Promise((resolve) => { const s = net.connect({ port, host: "127.0.0.1" });