diff --git a/src/persist.ts b/src/persist.ts index 37fb45048d..71523056cd 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -381,6 +381,11 @@ export class SessionStore { // #3931 step 3: assigned asynchronously for network backends (mysql) — // see the constructor and ready(). private shadow?: UnifiedShadow; + /** Only alive while one synchronous record build hydrates both lazy fields. */ + private residentFieldsForBuild?: { + id: string; + fields: { metadata?: Record; blockContents?: Record } | null; + }; /** #2960 ②: sessions whose fork_base checkpoint has been cut (or was * found already present) — memoized so steady-state writes of receipt * sessions do not re-query the checkpoints table every flush. */ @@ -669,7 +674,7 @@ export class SessionStore { // #408 one-time migration: the in-memory value is already // clamped by buildSession — rewrite the stale file so the // negative value is gone from disk. - await this.store.writeNow(id, () => buildRecord(session)); + await this.store.writeNow(id, () => this.buildRecordForSave(session)); clamped++; // #3049: the clamp rewrite writes a DIFFERENT payload than any // ingested record (clamped stats) without a sidecar ingest — @@ -1149,7 +1154,7 @@ export class SessionStore { continue; } session.id = label; - await this.store.writeNow(label, () => buildRecord(session)); + await this.store.writeNow(label, () => this.buildRecordForSave(session)); // #3049: direct write bypassing guardedBuild — the boot rekey // rewrites envelopes without the sidecar ingest, so any prior // ownership token for this id is stale. @@ -1672,6 +1677,60 @@ export class SessionStore { return this.shadow ? this.shadow.viewFor(id) : null; } + /** Restore only the dropped fields, with the same source preference as loadSync. + * A field read must not restamp refs, import rows, or schedule another save. */ + private loadResidentFields( + id: string, + meta: { protocol?: string; upstreamOrigin?: string }, + fields: { metadata?: boolean; blockContents?: boolean }, + ): { metadata?: Record; blockContents?: Record } | null { + if (this.residentFieldsForBuild?.id === id) return this.residentFieldsForBuild.fields; + if (!this.enabled) return null; + const indexed = this.shadow?.readsUnified ? this.shadow.readResidentFieldsFor(id, fields) : null; + const envelope = this.loadEnvelope(id, meta); + let useEnvelope = indexed === null; + if (indexed && envelope) { + if (this.shadow?.driftedSessions.has(id)) { + const envActive = countActiveBlocks(envelope.payload); + const envRefs = Object.keys(envelope.payload.state?.messageRefs?.byRef ?? {}).length; + useEnvelope = indexed.header.activeBlockCount <= envActive && indexed.header.mappedRefCount <= envRefs; + } else { + const requests = Number(envelope.payload.stats?.requests ?? 0); + useEnvelope = requests > indexed.header.requests || + (requests === indexed.header.requests && Number(envelope.payload.state?.nextBlockId ?? 0) > indexed.header.nextBlockId); + } + } + if (useEnvelope) { + if (!envelope) return null; + return { + ...(fields.metadata ? { metadata: envelope.payload.metadata ?? {} } : {}), + ...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(Object.entries(envelope.payload.blockContents ?? {}))) } : {}), + }; + } + if (!indexed) return null; + return { + ...(fields.metadata ? { metadata: indexed.metadata } : {}), + ...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(indexed.blockContents ?? [])) } : {}), + }; + } + + private buildRecordForSave(session: Session, storageMode: "legacy" | "shadow" | "full" = effectiveFileStorageMode()): PersistedSession { + const bothDropped = isDroppedMetadata(session.metadata) && + session.blockContents instanceof LazyBlockContents && session.blockContents.isDropped; + if (!bothDropped) return buildRecord(session, storageMode); + // Metadata and contents used to each rebuild a whole Session here. Share + // one projection and envelope arbitration for this build, never a cache + // that keeps another session alive after the synchronous build returns. + const previous = this.residentFieldsForBuild; + const fields = this.loadResidentFields(session.id, session.meta ?? {}, { metadata: true, blockContents: true }); + this.residentFieldsForBuild = { id: session.id, fields }; + try { + return buildRecord(session, storageMode); + } finally { + this.residentFieldsForBuild = previous; + } + } + /** Read-only state load for cross-session search (#841): unlike loadSync, * never schedules a save (no #408 clamp-rewrite side effect). */ loadStateForSearch(id: string): CompressionState | null { @@ -2084,7 +2143,7 @@ export class SessionStore { this.forkBaseCut.add(session.id); return; } - const record = buildRecord(session); + const record = this.buildRecordForSave(session); const activeBlockIds = session.state.blocks .filter((b) => (b as { active?: boolean }).active !== false) .map((b) => b.blockId) @@ -2393,7 +2452,7 @@ export class SessionStore { // the JSON envelope, and the stamp must say legacy for that copy. const stamp = this.liveStorageStamp; return () => { - const record = buildRecord(session, stamp); + const record = this.buildRecordForSave(session, stamp); const disk = this.staleDiskPayload(record); if (disk === null) { // #3301: external-converter import payload rides exactly ONE @@ -2675,8 +2734,7 @@ export class SessionStore { // loader degrades to the id-only lookup, same as a protocol-less file. const meta = session.meta ?? undefined; session.metadata = lazyMetadata(session.id, { protocol: meta?.protocol, upstreamOrigin: meta?.upstreamOrigin }, (id, m) => { - const loaded = this.loadSync(id, m); - return loaded ? loaded.metadata : null; + return this.loadResidentFields(id, m, { metadata: true })?.metadata ?? null; }); } @@ -2694,8 +2752,7 @@ export class SessionStore { } if (existing.size === 0) return; const lazy = new LazyBlockContents(session.id, { protocol: session.meta.protocol, upstreamOrigin: session.meta.upstreamOrigin }, (id, meta) => { - const loaded = this.loadSync(id, meta); - return loaded ? Object.fromEntries(loaded.blockContents) : null; + return this.loadResidentFields(id, meta, { blockContents: true })?.blockContents ?? null; }); lazy.drop(); session.blockContents = lazy; @@ -2993,9 +3050,9 @@ function restoreCalibrationRing(v: unknown): { origin: string; values: number[] return recent.length > 0 ? { origin: ring.origin, values: recent } : undefined; } -function buildSession(parsed: PersistedSession): Session { +function restoreBlockContents(entries: Iterable): Map { const blockContents = new Map(); - for (const [bid, content] of Object.entries(parsed.blockContents ?? {})) { + for (const [bid, content] of entries) { if (!content || typeof content !== "object") continue; const full = (content as Record).full; if (!isBlockView(full)) continue; @@ -3008,6 +3065,11 @@ function buildSession(parsed: PersistedSession): Session { full, }); } + return blockContents; +} + +function buildSession(parsed: PersistedSession): Session { + const blockContents = restoreBlockContents(Object.entries(parsed.blockContents ?? {})); // Read grouped shape (v2+); fall back to flat fields for v1 files. const meta = parsed.meta ?? {}; const stats = parsed.stats ?? {}; diff --git a/src/storage/backend.ts b/src/storage/backend.ts index 4af7c27859..f924ae626a 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -75,10 +75,12 @@ export interface StorageBackend { seqByRaw(sessionId: string): Promise | null>; /** #4316 PR3/3: fork adoption row clone (idempotent). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number, initial?: IngestSessionInput): Promise; - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]>; + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; getRefs(sessionId: string): Promise; + /** All reads, including CAS hydration, use one short read-only snapshot. */ + readSnapshot(read: (reader: StorageReadView) => Promise): Promise; getCcr(sessionId: string): Promise; getCcrEntries(sessionId: string): Promise; @@ -141,6 +143,8 @@ export interface StorageBackend { init?(): Promise; } +export type StorageReadView = Pick; + /** The sqlite backend: the existing UnifiedStore behind the async contract. * Synchronous under the hood; `Promise.resolve` wrapping only. */ export class SqliteBackend implements StorageBackend { @@ -190,11 +194,11 @@ export class SqliteBackend implements StorageBackend { cloneSessionRows(parentId: string, childId: string, throughSeq: number, initial?: IngestSessionInput): Promise { return Promise.resolve(this.store.cloneSessionRows(parentId, childId, throughSeq, initial)); } - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { - return Promise.resolve(this.store.getMessages(sessionId)); + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]> { + return Promise.resolve(this.store.getMessages(sessionId, opts)); } - getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { - return Promise.resolve(this.store.getBlocks(sessionId)); + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + return Promise.resolve(this.store.getBlocks(sessionId, opts)); } getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { return Promise.resolve(this.store.getDeadRefs(sessionId)); @@ -202,6 +206,15 @@ export class SqliteBackend implements StorageBackend { getRefs(sessionId: string): Promise { return Promise.resolve(this.store.getRefs(sessionId)); } + readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + return this.store.readSnapshotAsync((store) => read({ + getSessionRow: async (id) => store.getSessionRow(id), + getMessages: async (id, opts) => store.getMessages(id, opts), + getBlocks: async (id, opts) => store.getBlocks(id, opts), + getDeadRefs: async (id) => store.getDeadRefs(id), + getRefs: async (id) => store.getRefs(id), + })); + } getCcr(sessionId: string): Promise { return Promise.resolve(this.store.getCcr(sessionId)); } diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 6bc44be0c7..63799382d4 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -40,6 +40,8 @@ export interface SqliteDatabase { prepare(sql: string): SqliteStatement; /** Runs fn inside BEGIN/COMMIT; rolls back on throw (including nested calls via savepoints). */ transaction(fn: () => T): T; + /** Deferred snapshot: readers do not reserve the writer lock. */ + readTransaction(fn: () => T): T; close(): void; readonly open: boolean; } @@ -87,6 +89,7 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { // cannot resolve. Taking the write lock up front makes concurrent // writers queue on busy_timeout instead (#2690 review blocking 5). transaction: (fn) => db.transaction(fn).immediate(), + readTransaction: (fn) => db.transaction(fn).deferred(), close: () => void db.close(), get open() { return db.open; @@ -104,31 +107,33 @@ function openNodeSqlite(path: string): SqliteDatabase { }; const db = new DatabaseSync(path); let depth = 0; - const begin = () => db.exec(depth === 0 ? "BEGIN IMMEDIATE" : `SAVEPOINT sp${depth}`); // IMMEDIATE at top level: multi-process SQLITE_BUSY_SNAPSHOT guard (see openBetterSqlite3) + const begin = (readOnly: boolean) => db.exec(depth === 0 ? (readOnly ? "BEGIN DEFERRED" : "BEGIN IMMEDIATE") : `SAVEPOINT sp${depth}`); const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`); const rollback = () => db.exec(depth === 1 ? "ROLLBACK" : `ROLLBACK TO sp${depth - 1}`); + const transaction = (fn: () => T, readOnly: boolean): T => { + if (db.getAutoCommit?.() === false && depth === 0) { + // Already inside an outer transaction driven directly by + // DatabaseSync — just run inline; the outer COMMIT covers us. + return fn(); + } + begin(readOnly); + depth++; + try { + const out = fn(); + commit(); + return out; + } catch (err) { + rollback(); + throw err; + } finally { + depth--; + } + }; return { exec: (sql) => void db.exec(sql), prepare: (sql) => adaptNodeSqliteStatement(db.prepare(sql)), - transaction: (fn: () => T): T => { - if (db.getAutoCommit?.() === false && depth === 0) { - // Already inside an outer transaction driven directly by - // DatabaseSync — just run inline; the outer COMMIT covers us. - return fn(); - } - begin(); - depth++; - try { - const out = fn(); - commit(); - return out; - } catch (err) { - rollback(); - throw err; - } finally { - depth--; - } - }, + transaction: (fn) => transaction(fn, false), + readTransaction: (fn) => transaction(fn, true), close: () => void db.close(), get open() { return db.open; diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index c3a84aff22..54a319a8c8 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -487,28 +487,28 @@ export function completeBlockContents( return out; } -export function loadLegacyView( - rows: { - session: { - sessionId: string; - createdAt: number; - updatedAt: number; - nextBlockId: number; - nextRunId: number; - metaJson: string; - statsJson: string; - metadataJson: string; - kstateJson: string; - tailJson: string; - ccrJson: string | null; - orphansJson?: string | null; - }; - messages: { seq: number; msgId: string; ref: string | null; content: string }[]; - blocks: { blockId: string; summary: string; one: string | null; payloadJson: string }[]; - deadRefs: { ord: number; ref: string }[]; - refs: { ord: number; rawId: string; ref: string }[]; - }, -): { +interface LegacyViewRows { + session: { + sessionId: string; + createdAt: number; + updatedAt: number; + nextBlockId: number; + nextRunId: number; + metaJson: string; + statsJson: string; + metadataJson: string; + kstateJson: string; + tailJson: string; + ccrJson: string | null; + orphansJson?: string | null; + }; + messages: { seq: number; msgId: string; ref: string | null; content: string }[]; + blocks: { blockId: string; summary: string; one: string | null; payloadJson: string }[]; + deadRefs: { ord: number; ref: string }[]; + refs: { ord: number; rawId: string; ref: string }[]; +} + +interface LegacyView { id: string; createdAt: number; lastSeen: number; @@ -523,105 +523,147 @@ export function loadLegacyView( contentStore?: LegacySessionLike["contentStore"]; /** #4390: tail/pluginSnapshot pointers whose backing row is missing. */ tailGaps?: number; -} { - const s = rows.session; - const kstate = JSON.parse(s.kstateJson) as Record; - const tail = JSON.parse(s.tailJson) as { - pluginSnapshotSeqs: number[] | null; - tailSeqs: number[] | null; - tailFolded: boolean; - }; - - // messageRefs rebuilt from the ref ledger verbatim: ord preserves the - // persisted map's key order (digest-sensitive), and ghost ids survive - // even though no message row backs them (#2362 Kernel Contract). - const byRaw: Record = {}; - const byRef: Record = {}; - for (const r of rows.refs) { - byRaw[r.rawId] = r.ref; - // "BLOCKED" is a byRaw-only token: the in-memory map keeps the - // inverse projection strictly mNNNNN→rawId. Admitting it into byRef - // mints a key the persisted state never had (#2671 soak: sub-agent - // sessions with protected messages drifted on every save). - if (r.ref !== "BLOCKED" && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId; - } +} - const state: Record = {}; - for (const field of MERGE_STATE_FIELDS) { - switch (field) { - case "blocks": - state.blocks = rows.blocks.map((b) => JSON.parse(b.payloadJson)); - break; - case "messageRefs": - state.messageRefs = { byRaw, byRef }; - break; - case "deadRefs": - state.deadRefs = rows.deadRefs.length > 0 ? rows.deadRefs.map((d) => d.ref) : undefined; - break; - case "nextBlockId": - state.nextBlockId = s.nextBlockId; - break; - case "nextRunId": - state.nextRunId = s.nextRunId; - break; - default: - if (kstate[field] !== undefined) state[field] = kstate[field]; - break; - } - } +type BlockContents = LegacyView["blockContents"]; - const blockContents = new Map(); - for (const b of rows.blocks) { - blockContents.set(b.blockId, { +/** Shared by full-view and resident-field reads; each page is released after parsing. */ +export function appendBlockContents(contents: BlockContents, rows: Pick[]): void { + for (const b of rows) { + contents.set(b.blockId, { one: b.one === null ? null : (JSON.parse(b.one) as BlockView), full: JSON.parse(b.summary) as BlockView, }); } - // #3841 reap journal: journal'd orphan contents (reaped blocks) ride - // blockContents so the derived-decompress fallback survives restarts — - // WITHOUT resurrecting state.blocks (the reaped entries stay reaped). - if (s.orphansJson !== null && s.orphansJson !== undefined && s.orphansJson !== "") { +} + +export function mergeOrphanBlockContents(contents: BlockContents, orphansJson: string | null | undefined): void { + // Reaped originals survive without resurrecting their state.blocks rows. + if (orphansJson !== null && orphansJson !== undefined && orphansJson !== "") { try { - const orphans = JSON.parse(s.orphansJson) as Record; + const orphans = JSON.parse(orphansJson) as Record; for (const [blockId, view] of Object.entries(orphans)) { - if (!blockContents.has(blockId)) blockContents.set(blockId, view); + if (!contents.has(blockId)) contents.set(blockId, view); } } catch { // corrupt journal → skip; the JSON envelope still carries the text } } +} - const messagesBySeq = new Map(rows.messages.map((m) => [m.seq, JSON.parse(m.content) as CoreMessage])); - // #4390: a tail pointer whose row is gone used to be silently filtered - // out — the view shrank with no trace. Count the gaps so callers can - // warn (persist logs them); the surviving members still load. - let tailGaps = 0; - const resolveSeqs = (seqs: number[]): CoreMessage[] => { - const out: CoreMessage[] = []; - for (const seq of seqs) { - const m = messagesBySeq.get(seq); - if (m === undefined) tailGaps++; - else out.push(m); +/** Incremental cold-load assembly keeps raw CAS strings to one page at a time. */ +export class LegacyViewBuilder { + readonly messageSeqs: number[]; + private readonly tail: { + pluginSnapshotSeqs: number[] | null; + tailSeqs: number[] | null; + tailFolded: boolean; + }; + private readonly state: Record = {}; + private readonly blocks: unknown[] = []; + private readonly blockContents: BlockContents = new Map(); + private readonly messagesBySeq = new Map(); + + constructor( + private readonly session: LegacyViewRows["session"], + refs: LegacyViewRows["refs"], + deadRefs: LegacyViewRows["deadRefs"], + ) { + const kstate = JSON.parse(session.kstateJson) as Record; + this.tail = JSON.parse(session.tailJson); + this.messageSeqs = [...new Set([...(this.tail.tailSeqs ?? []), ...(this.tail.pluginSnapshotSeqs ?? [])])]; + + // messageRefs rebuilt from the ref ledger verbatim: ord preserves the + // persisted map's key order (digest-sensitive), and ghost ids survive + // even though no message row backs them (#2362 Kernel Contract). + const byRaw: Record = {}; + const byRef: Record = {}; + for (const r of refs) { + byRaw[r.rawId] = r.ref; + // "BLOCKED" is a byRaw-only token: the in-memory map keeps the + // inverse projection strictly mNNNNN→rawId. Admitting it into byRef + // mints a key the persisted state never had (#2671 soak: sub-agent + // sessions with protected messages drifted on every save). + if (r.ref !== "BLOCKED" && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId; + } + + const state = this.state; + for (const field of MERGE_STATE_FIELDS) { + switch (field) { + case "blocks": + state.blocks = this.blocks; + break; + case "messageRefs": + state.messageRefs = { byRaw, byRef }; + break; + case "deadRefs": + state.deadRefs = deadRefs.length > 0 ? deadRefs.map((d) => d.ref) : undefined; + break; + case "nextBlockId": + state.nextBlockId = session.nextBlockId; + break; + case "nextRunId": + state.nextRunId = session.nextRunId; + break; + default: + if (kstate[field] !== undefined) state[field] = kstate[field]; + break; + } } + } + + addBlocks(rows: LegacyViewRows["blocks"]): void { + for (const b of rows) this.blocks.push(JSON.parse(b.payloadJson)); + appendBlockContents(this.blockContents, rows); + } + + addMessages(rows: LegacyViewRows["messages"]): void { + for (const m of rows) this.messagesBySeq.set(m.seq, JSON.parse(m.content) as CoreMessage); + } + + finish(): LegacyView { + const s = this.session; + const tail = this.tail; + mergeOrphanBlockContents(this.blockContents, s.orphansJson); + // #4390: a tail pointer whose row is gone used to be silently filtered + // out — the view shrank with no trace. Count the gaps so callers can + // warn (persist logs them); the surviving members still load. + let tailGaps = 0; + const resolveSeqs = (seqs: number[]): CoreMessage[] => { + const out: CoreMessage[] = []; + for (const seq of seqs) { + const m = this.messagesBySeq.get(seq); + if (m === undefined) tailGaps++; + else out.push(m); + } + return out; + }; + const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; + const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; + + const out: LegacyView = { + id: s.sessionId, + createdAt: s.createdAt, + lastSeen: s.updatedAt, + tailGaps, + meta: JSON.parse(s.metaJson), + stats: JSON.parse(s.statsJson), + metadata: JSON.parse(s.metadataJson), + state: this.state, + blockContents: this.blockContents, + lastMessages, + lastMessagesFolded: tail.tailFolded, + }; + if (pluginSnapshot) out.pluginSnapshot = pluginSnapshot; + if (s.ccrJson !== null) out.contentStore = JSON.parse(s.ccrJson); return out; - }; - const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; - const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; + } +} - const out: ReturnType = { - id: s.sessionId, - createdAt: s.createdAt, - lastSeen: s.updatedAt, - tailGaps, - meta: JSON.parse(s.metaJson), - stats: JSON.parse(s.statsJson), - metadata: JSON.parse(s.metadataJson), - state, - blockContents, - lastMessages, - lastMessagesFolded: tail.tailFolded, - }; - if (pluginSnapshot) out.pluginSnapshot = pluginSnapshot; - if (s.ccrJson !== null) out.contentStore = JSON.parse(s.ccrJson); - return out; +/** Compatibility wrapper for callers that already hold all rows. */ +export function loadLegacyView(rows: LegacyViewRows): LegacyView { + const builder = new LegacyViewBuilder(rows.session, rows.refs, rows.deadRefs); + builder.addBlocks(rows.blocks); + builder.addMessages(rows.messages); + return builder.finish(); } diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 42c9e2ddd9..e8d826b326 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -42,7 +42,7 @@ import { log as loggerLog } from "../logger.js"; import { sha256Hex } from "./store.js"; import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; -import type { StorageBackend } from "./backend.js"; +import type { StorageBackend, StorageReadView } from "./backend.js"; import type { SessionRow, MessageRow, @@ -1331,7 +1331,32 @@ export class MysqlBackend implements StorageBackend { } async getSessionRow(sessionId: string): Promise { - const [rows] = (await this.pool.query( + return this.getSessionRowOn(this.pool, sessionId); + } + + async readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + const conn = await this.pool.getConnection(); + try { + await conn.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + await conn.query("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"); + return await read({ + getSessionRow: (id) => this.getSessionRowOn(conn, id), + getMessages: (id, opts) => this.getMessagesOn(conn, id, opts), + getBlocks: (id, opts) => this.getBlocksOn(conn, id, opts), + getDeadRefs: (id) => this.getDeadRefsOn(conn, id), + getRefs: (id) => this.getRefsOn(conn, id), + }); + } finally { + try { + await conn.rollback(); + } finally { + conn.release(); + } + } + } + + private async getSessionRowOn(conn: Pool | PoolConnection, sessionId: string): Promise { + const [rows] = (await conn.query( `SELECT session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin, status, created_at, updated_at, next_block_id, next_run_id, meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq, orphans_json @@ -1366,13 +1391,18 @@ export class MysqlBackend implements StorageBackend { }; } - async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + async getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + return this.getMessagesOn(this.pool, sessionId, opts); + } + + private async getMessagesOn(conn: Pool | PoolConnection, sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { // #4390: retired rows stay out of the replay by default. - const [rows] = (await this.pool.query( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", - [sessionId], + if (opts?.seqs?.length === 0) return []; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(",")})` : ""; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; + const [rows] = (await conn.query( + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, + [sessionId, ...(opts?.seqs ?? [])], )) as unknown as Row[][]; const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = []; for (const r of rows) { @@ -1386,20 +1416,20 @@ export class MysqlBackend implements StorageBackend { kind: str(r.kind), contentHash, retiredAt: nullableNum(r.retired_at), - content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? ""), + content: await this.hydrateMediaAsync((await this.getContent(conn, contentHash)) ?? "", conn), }); } return out; } - private async hydrateMediaAsync(content: string): Promise { + private async hydrateMediaAsync(content: string, conn: Pool | PoolConnection): Promise { let out = content; if (out.includes('"$media"')) { try { const parsed = JSON.parse(out) as { imageBase64?: unknown }; const slot = parsed.imageBase64; if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { - const bytes = await this.getContent(this.pool, (slot as { $media: string }).$media); + const bytes = await this.getContent(conn, (slot as { $media: string }).$media); if (typeof bytes === "string" && bytes.length > 0) { parsed.imageBase64 = bytes; out = JSON.stringify(parsed); @@ -1418,7 +1448,7 @@ export class MysqlBackend implements StorageBackend { if (hits.length > 0) { const cache = new Map(); for (const h of new Set(hits.map((x) => x.hash))) { - cache.set(h, await this.getContent(this.pool, h)); + cache.set(h, await this.getContent(conn, h)); } out = out.replace(markerRe, (all: string, mime: string, hash: string) => { const bytes = cache.get(hash); @@ -1429,12 +1459,16 @@ export class MysqlBackend implements StorageBackend { return out; } - async getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { - const [rows] = (await this.pool.query( + async getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + return this.getBlocksOn(this.pool, sessionId, opts); + } + + private async getBlocksOn(conn: Pool | PoolConnection, sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + const [rows] = (await conn.query( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, - [sessionId], + FROM blocks WHERE session_id = ? ORDER BY ord${opts ? ", block_id LIMIT ? OFFSET ?" : ""}`, + [sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])], )) as unknown as Row[][]; const out: (BlockRow & { summaryHash: string; oneHash: string | null })[] = []; for (const r of rows) { @@ -1448,8 +1482,8 @@ export class MysqlBackend implements StorageBackend { tier: num(r.tier), summaryHash: str(r.summary_hash), oneHash: nullableStr(r.one_hash), - summary: (await this.getContent(this.pool, str(r.summary_hash))) ?? "", - one: r.one_hash === null || r.one_hash === undefined ? null : ((await this.getContent(this.pool, str(r.one_hash))) ?? null), + summary: (await this.getContent(conn, str(r.summary_hash))) ?? "", + one: r.one_hash === null || r.one_hash === undefined ? null : ((await this.getContent(conn, str(r.one_hash))) ?? null), supersedesBlockId: nullableStr(r.supersedes_block_id), active: num(r.active) === 1, createdAt: num(r.created_at), @@ -1461,7 +1495,11 @@ export class MysqlBackend implements StorageBackend { } async getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { - const [rows] = (await this.pool.query( + return this.getDeadRefsOn(this.pool, sessionId); + } + + private async getDeadRefsOn(conn: Pool | PoolConnection, sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> { + const [rows] = (await conn.query( "SELECT ord, ref, ref_num, recorded_at FROM dead_refs WHERE session_id = ? ORDER BY ord", [sessionId], )) as unknown as Row[][]; @@ -1474,7 +1512,11 @@ export class MysqlBackend implements StorageBackend { } async getRefs(sessionId: string): Promise { - const [rows] = (await this.pool.query( + return this.getRefsOn(this.pool, sessionId); + } + + private async getRefsOn(conn: Pool | PoolConnection, sessionId: string): Promise { + const [rows] = (await conn.query( "SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord", [sessionId], )) as unknown as Row[][]; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 2589148c46..bf869cea7a 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -22,7 +22,7 @@ */ import { log as loggerLog } from "../logger.js"; -import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; +import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import type { BlockView } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; @@ -37,6 +37,15 @@ import type { CoreMessage, MessageContentStore } from "acp-kernel"; export const MAX_CONSECUTIVE_FAILURES = 3; +/** Bound transient CAS strings during normal cold loads, not the retained history. */ +const VIEW_READ_BATCH_SIZE = 32; + +interface ResidentFields { + header: { requests: number; nextBlockId: number; activeBlockCount: number; mappedRefCount: number }; + metadata?: Record; + blockContents?: Map; +} + /** #3133: read-back verification cadence — the verify leg (full read-back + * double shadowDigest) is O(log) per save, while the drift it guards is a * migration-parity property, not a per-commit one. The FIRST ingest of a @@ -555,22 +564,27 @@ export class UnifiedShadow { /** Loads the unified view of one session (full mode read path). * Returns null on miss — the caller falls back to the legacy file. */ viewFor(id: string): LegacySessionLike | null { - if (this.sync === null) return null; + const store = this.sync; + if (store === null) return null; if (this.stats.disabled) return null; try { - const sessionRow = this.sync.getSessionRow(id); - if (!sessionRow) return null; - const rows = { - session: sessionRow, - messages: this.sync.getMessages(id), - blocks: this.sync.getBlocks(id), - deadRefs: this.sync.getDeadRefs(id), - refs: this.sync.getRefs(id), - }; - // The view's `state` object carries every MERGE_STATE_FIELDS key by - // construction; the declared Record just needs a - // widening cast to reach LegacySessionLike. - const unified = loadLegacyView(rows) as LegacySessionLike; + const unified = store.readSnapshot(() => { + const sessionRow = store.getSessionRow(id); + if (!sessionRow) return null; + const builder = new LegacyViewBuilder(sessionRow, store.getRefs(id), store.getDeadRefs(id)); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = store.getBlocks(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + builder.addBlocks(blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { + builder.addMessages(store.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); + } + // The builder preserves every MERGE_STATE_FIELDS key; its + // generic state declaration only needs widening here. + return builder.finish() as LegacySessionLike; + }); + if (!unified) return null; warnTailGaps(id, unified as unknown as { tailGaps?: number }); return unified; } catch (err) { @@ -579,6 +593,38 @@ export class UnifiedShadow { } } + /** Lazy resident fields share the same source header without rebuilding a session. */ + readResidentFieldsFor(id: string, fields: { metadata?: boolean; blockContents?: boolean }): ResidentFields | null { + const store = this.sync; + if (store === null || this.stats.disabled) return null; + try { + return store.readSnapshot(() => { + const header = store.getLoadHeader(id); + if (!header) return null; + const result: ResidentFields = { header }; + if (fields.metadata) { + const json = store.getMetadataJson(id); + if (json === null) return null; + result.metadata = JSON.parse(json) as Record; + } + if (fields.blockContents) { + const contents: NonNullable = new Map(); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = store.getBlockContents(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + appendBlockContents(contents, blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + mergeOrphanBlockContents(contents, store.getOrphansJson(id)); + result.blockContents = contents; + } + return result; + }); + } catch (err) { + loggerLog("warn", `[storage] resident field load failed for ${id} (${String(err)}) — falling back to the legacy file`); + return null; + } + } + /** #4323: true once this session's drift escalated and no clean sample * has since re-admitted it. Sqlite lanes only — network backends keep * the envelope as their sync read path, where freezing would strand @@ -1041,15 +1087,21 @@ export class UnifiedShadow { if (this.stats.disabled) return null; try { const backend = this.backend!; - const sessionRow = await backend.getSessionRow(id); - if (!sessionRow) return null; - const loaded = loadLegacyView({ - session: sessionRow, - messages: await backend.getMessages(id), - blocks: await backend.getBlocks(id), - deadRefs: await backend.getDeadRefs(id), - refs: await backend.getRefs(id), - }) as LegacySessionLike; + const loaded = await backend.readSnapshot(async (reader) => { + const sessionRow = await reader.getSessionRow(id); + if (!sessionRow) return null; + const builder = new LegacyViewBuilder(sessionRow, await reader.getRefs(id), await reader.getDeadRefs(id)); + for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { + const blocks = await reader.getBlocks(id, { limit: VIEW_READ_BATCH_SIZE, offset }); + builder.addBlocks(blocks); + if (blocks.length < VIEW_READ_BATCH_SIZE) break; + } + for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { + builder.addMessages(await reader.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); + } + return builder.finish() as LegacySessionLike; + }); + if (!loaded) return null; warnTailGaps(id, loaded as unknown as { tailGaps?: number }); return loaded; } catch (err) { diff --git a/src/storage/store.ts b/src/storage/store.ts index 09c2e509b2..c637b190c1 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -515,6 +515,24 @@ export class UnifiedStore { this.lastIntegrityOk = true; } + /** Keep paged reads on one generation without taking the writer lock. */ + readSnapshot(read: () => T): T { + return this.db.readTransaction(read); + } + + /** Async callers need a separate connection: an await must not let a + * normal save on this store join the reader's transaction. */ + async readSnapshotAsync(read: (reader: UnifiedStore) => Promise): Promise { + const { db, engine, source } = openSqliteDatabase(this.dbPath, this.engine); + try { + db.exec("PRAGMA query_only = ON; BEGIN DEFERRED"); + const reader = new UnifiedStore(db, engine, source, null, null, this.dbPath); + return await read(reader); + } finally { + db.close(); // closing rolls back the read-only transaction + } + } + close(): void { // #3250: converge the WAL before releasing the handle so a clean exit // leaves nothing for the next boot's recovery to replay. Same isolation @@ -2442,6 +2460,54 @@ export class UnifiedStore { return Buffer.from(row.bytes).toString("utf8"); } + /** Small arbitration fields for lazy reads; no serialized session surfaces. */ + getLoadHeader(sessionId: string): { requests: number; nextBlockId: number; activeBlockCount: number; mappedRefCount: number } | null { + const row = this.db.prepare(` + SELECT next_block_id, stats_json, + (SELECT COUNT(*) FROM blocks WHERE session_id = ? AND active = 1) AS active_blocks, + (SELECT COUNT(DISTINCT ref) FROM refs WHERE session_id = ? AND ref <> 'BLOCKED') AS mapped_refs + FROM sessions WHERE session_id = ? + `).get(sessionId, sessionId, sessionId) as { + next_block_id: number; stats_json: string; active_blocks: number; mapped_refs: number; + } | undefined; + if (!row) return null; + try { + const stats = JSON.parse(row.stats_json) as { requests?: unknown }; + return { + requests: Number(stats.requests ?? 0), + nextBlockId: row.next_block_id, + activeBlockCount: row.active_blocks, + mappedRefCount: row.mapped_refs, + }; + } catch { + return null; + } + } + + getMetadataJson(sessionId: string): string | null { + const row = this.db.prepare("SELECT metadata_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { metadata_json: string } | undefined; + return row?.metadata_json ?? null; + } + + getOrphansJson(sessionId: string): string | null { + const row = this.db.prepare("SELECT orphans_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { orphans_json: string | null } | undefined; + return row?.orphans_json ?? null; + } + + /** Contents for active AND inactive blocks, without their CompressionBlock payload. */ + getBlockContents(sessionId: string, opts: { limit: number; offset?: number }): Pick[] { + const rows = this.db.prepare(`SELECT block_id, summary_hash, one_hash FROM blocks + WHERE session_id = ? ORDER BY ord, block_id LIMIT ? OFFSET ?`) + .all(sessionId, opts.limit, opts.offset ?? 0) as { block_id: string; summary_hash: string; one_hash: string | null }[]; + return rows.map((row) => ({ + blockId: row.block_id, + summary: this.getContent(row.summary_hash) ?? "", + one: row.one_hash ? this.getContent(row.one_hash) : null, + })); + } + getSessionRow(sessionId: string): SessionRow | null { const row = this.db .prepare( @@ -2659,17 +2725,18 @@ export class UnifiedStore { })); } - getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { // #4390: retired rows (rewound/undo history) stay out of the unified // replay by default — they are history, not view. Callers that need // the full historian record (export/diagnostics) opt in. + if (opts?.seqs?.length === 0) return []; + const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(",")})` : ""; + const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL"; const rows = this.db .prepare( - opts?.includeRetired === true - ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" - : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts?.seqs ?? [])) as Record[]; return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string, @@ -2725,14 +2792,14 @@ export class UnifiedStore { return row ? row.refcount : 0; } - getBlocks(sessionId: string): (BlockRow & { summaryHash: string; oneHash: string | null })[] { + getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): (BlockRow & { summaryHash: string; oneHash: string | null })[] { const rows = this.db .prepare( `SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json - FROM blocks WHERE session_id = ? ORDER BY ord`, + FROM blocks WHERE session_id = ? ORDER BY ord${opts ? ", block_id LIMIT ? OFFSET ?" : ""}`, ) - .all(sessionId) as Record[]; + .all(sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])) as Record[]; return rows.map((r) => ({ blockId: r.block_id as string, ord: r.ord as number, diff --git a/tests/issue4739-storage-read-snapshot.test.ts b/tests/issue4739-storage-read-snapshot.test.ts new file mode 100644 index 0000000000..5d77b9cc8c --- /dev/null +++ b/tests/issue4739-storage-read-snapshot.test.ts @@ -0,0 +1,191 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { UnifiedStore, type IngestSessionInput } from "../src/storage/store.ts"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; +import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; + +function fixture(requests = 1): IngestSessionInput { + return { + session: { + sessionId: "snapshot", createdAt: 1, updatedAt: requests, nextBlockId: 4, nextRunId: 1, + metaJson: "{}", statsJson: JSON.stringify({ requests }), metadataJson: JSON.stringify({ requests }), + kstateJson: "{}", tailJson: "{}", ccrJson: null, + orphansJson: JSON.stringify({ orphan: { one: null, full: { text: "orphan", count: 1 } } }), + }, + messages: [0, 1, 2].map((seq) => ({ + seq, msgId: `m${seq}`, ref: null, refNum: null, role: "user", kind: "message", + content: JSON.stringify({ id: `m${seq}`, role: "user", text: `text${seq}` }), + })), + blocks: [0, 1, 2].map((ord) => ({ + blockId: `b${ord}`, ord, startSeq: ord, endSeq: ord, startRef: null, endRef: null, tier: 0, + summary: JSON.stringify({ text: `full${ord}`, count: 1 }), + one: JSON.stringify({ text: `one${ord}`, count: 1 }), supersedesBlockId: null, + active: ord !== 1, createdAt: 1, coveredHash: "[]", payloadJson: JSON.stringify({ blockId: `b${ord}` }), + })), + refs: [ + { ord: 0, rawId: "a", ref: "m00001", refNum: 1 }, + { ord: 1, rawId: "b", ref: "m00001", refNum: 1 }, + { ord: 2, rawId: "c", ref: "BLOCKED", refNum: null }, + ], + deadRefs: [], ccrEntries: [], + orphansJson: JSON.stringify({ orphan: { one: null, full: { text: "orphan", count: 1 } } }), + }; +} + +for (const engine of ["node:sqlite", "better-sqlite3"] as const) { + test(`${engine}: selected rows, page boundaries and narrow reads preserve their contracts`, (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const file = join(dir, "index.db"); + let store: UnifiedStore | undefined; + try { + try { store = UnifiedStore.open(file, engine); } catch (err) { + if (engine === "better-sqlite3" && String(err).includes("[storage:better-sqlite3]")) return t.skip(String(err)); + throw err; + } + store.ingestSession(fixture()); + const db = openSqliteDatabase(file, engine).db; + try { + db.prepare("UPDATE messages SET retired_at = 1 WHERE session_id = ? AND seq = 1").run("snapshot"); + assert.deepEqual(store.getMessages("snapshot").map((row) => row.seq), [0, 2]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [2, 1, 0, 2, 99] }).map((row) => row.seq), [0, 2]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [1], includeRetired: true }).map((row) => row.seq), [1]); + assert.deepEqual(store.getMessages("snapshot", { seqs: [] }), []); + assert.deepEqual(store.getBlocks("snapshot", { limit: 1, offset: 1 }).map((row) => row.blockId), ["b1"]); + assert.deepEqual(store.getBlocks("snapshot", { limit: 2, offset: 3 }), []); + assert.equal(store.getBlocks("snapshot").length, 3); + + // Invalid unrelated payloads must not be decoded by the lazy field readers. + db.prepare("UPDATE sessions SET tail_json = 'bad', kstate_json = 'bad', ccr_json = 'bad' WHERE session_id = ?").run("snapshot"); + db.prepare("UPDATE blocks SET payload_json = 'bad' WHERE session_id = ?").run("snapshot"); + assert.deepEqual(store.getLoadHeader("snapshot"), { requests: 1, nextBlockId: 4, activeBlockCount: 2, mappedRefCount: 1 }); + db.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":"2"}', "snapshot"); + assert.equal(store.getLoadHeader("snapshot")!.requests, 2, "old numeric-string counter matches full-load arbitration"); + assert.equal(store.getMetadataJson("snapshot"), '{"requests":1}'); + assert.deepEqual(JSON.parse(store.getOrphansJson("snapshot")!), JSON.parse(fixture().orphansJson!)); + const contents = store.getBlockContents("snapshot", { limit: 2, offset: 1 }); + assert.deepEqual(contents.map((row) => row.blockId), ["b1", "b2"]); + assert.deepEqual(JSON.parse(contents[0]!.summary), { text: "full1", count: 1 }); + assert.deepEqual(JSON.parse(contents[0]!.one!), { text: "one1", count: 1 }); + assert.equal(store.getLoadHeader("missing"), null); + assert.equal(store.getMetadataJson("missing"), null); + db.prepare("UPDATE blocks SET ord = 0 WHERE session_id = ?").run("snapshot"); + assert.equal(store.getBlocks("snapshot", { limit: 1, offset: 1 })[0]!.blockId, "b1", "equal ord uses a stable page boundary"); + } finally { db.close(); } + } finally { + store?.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + test(`${engine}: sync snapshot permits another writer and keeps the earlier generation`, (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const file = join(dir, "index.db"); + let store: UnifiedStore | undefined; + try { + try { store = UnifiedStore.open(file, engine); } catch (err) { + if (engine === "better-sqlite3" && String(err).includes("[storage:better-sqlite3]")) return t.skip(String(err)); + throw err; + } + store.ingestSession(fixture()); + const writer = openSqliteDatabase(file, engine).db; + const reader = store; + try { + reader.readSnapshot(() => { + assert.equal(reader.getLoadHeader("snapshot")!.requests, 1); + writer.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":2}', "snapshot"); + writer.prepare("UPDATE blocks SET active = 0 WHERE session_id = ?").run("snapshot"); + assert.equal(reader.getLoadHeader("snapshot")!.requests, 1); + assert.equal(reader.getBlocks("snapshot", { limit: 1, offset: 2 })[0]!.active, true); + }); + assert.equal(reader.getLoadHeader("snapshot")!.requests, 2); + assert.throws(() => reader.readSnapshot(() => { throw new Error("reader failed"); }), /reader failed/); + writer.prepare("UPDATE sessions SET stats_json = ? WHERE session_id = ?").run('{"requests":3}', "snapshot"); + assert.equal(reader.readSnapshot(() => reader.getLoadHeader("snapshot")!.requests), 3); + } finally { writer.close(); } + } finally { + store?.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); +} + +test("async sqlite snapshot isolates normal writes across await and closes on callback failure", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-read-snapshot-")); + const backend = SqliteBackend.open(join(dir, "index.db")); + try { + await backend.ingestSession(fixture()); + await backend.readSnapshot(async (reader) => { + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 1); + await backend.ingestSession(fixture(2)); + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 1); + assert.deepEqual((await reader.getMessages("snapshot", { seqs: [2] })).map((row) => row.seq), [2]); + }); + assert.equal(JSON.parse((await backend.getSessionRow("snapshot"))!.statsJson).requests, 2); + await assert.rejects(backend.readSnapshot(async (reader) => { + await reader.getSessionRow("snapshot"); + throw new Error("async reader failed"); + }), /async reader failed/); + await backend.ingestSession(fixture(3)); + await backend.readSnapshot(async (reader) => { + assert.equal(JSON.parse((await reader.getSessionRow("snapshot"))!.statsJson).requests, 3); + }); + } finally { + await backend.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("mysql snapshot pins SQL, CAS and media hydration to its read-only connection and releases on error", async () => { + const queries: { sql: string; values?: unknown[] }[] = []; + let releases = 0; + let rollbacks = 0; + const mediaHash = "a".repeat(64); + const conn = { + async query(sql: string, values?: unknown[]) { + queries.push({ sql, values }); + if (sql.includes("FROM messages")) return [[{ seq: 2, msg_id: "m2", role: "user", kind: "message", content_hash: "msg" }]]; + if (sql.includes("FROM blocks")) return [[{ block_id: "b1", summary_hash: "summary", one_hash: "one", active: 1 }]]; + if (sql.includes("FROM content")) { + const bytes = values?.[0] === "msg" ? JSON.stringify({ imageBase64: { $media: mediaHash } }) + : values?.[0] === mediaHash ? "image-bytes" : '{"text":"summary","count":1}'; + return [[{ bytes }]]; + } + return [[]]; + }, + async rollback() { rollbacks++; }, + release() { releases++; }, + }; + const driver = { createPool: () => ({ + getConnection: async () => conn, + query: async () => { throw new Error("snapshot escaped to pool"); }, + end: async () => {}, + }) } as unknown as Mysql2Module; + const backend = new MysqlBackend("mysql://snapshot-test", driver); + await backend.readSnapshot(async (reader) => { + const before = queries.length; + assert.deepEqual(await reader.getMessages("snapshot", { seqs: [] }), []); + assert.equal(queries.length, before); + const rows = await reader.getMessages("snapshot", { seqs: [2] }); + assert.equal(JSON.parse(rows[0]!.content).imageBase64, "image-bytes"); + assert.equal((await reader.getBlocks("snapshot", { limit: 1, offset: 2 }))[0]!.blockId, "b1"); + await reader.getSessionRow("snapshot"); + await reader.getRefs("snapshot"); + await reader.getDeadRefs("snapshot"); + }); + assert.equal(queries[0]!.sql, "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + assert.equal(queries[1]!.sql, "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY"); + const selected = queries.find((query) => query.sql.includes("FROM messages"))!; + assert.match(selected.sql, /retired_at IS NULL AND seq IN \(\?\)/); + assert.deepEqual(selected.values, ["snapshot", 2]); + const paged = queries.find((query) => query.sql.includes("FROM blocks"))!; + assert.match(paged.sql, /ORDER BY ord, block_id LIMIT \? OFFSET \?/); + assert.deepEqual(paged.values, ["snapshot", 1, 2]); + await assert.rejects(backend.readSnapshot(async () => { throw new Error("mysql reader failed"); }), /mysql reader failed/); + assert.equal(rollbacks, 2); + assert.equal(releases, 2); + await backend.close(); +}); diff --git a/tests/issue4982-draining-twin-ladder.test.ts b/tests/issue4982-draining-twin-ladder.test.ts index 13bdee7bf5..f65d48d491 100644 --- a/tests/issue4982-draining-twin-ladder.test.ts +++ b/tests/issue4982-draining-twin-ladder.test.ts @@ -228,21 +228,26 @@ async function waitForStatus(port: number, timeoutMs: number): Promise async function killChild(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill("SIGTERM"); - await Promise.race([ - new Promise((resolve) => child.once("exit", () => resolve())), - new Promise((resolve) => setTimeout(resolve, 10_000)), - ]); - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + child.kill("SIGTERM"); + }); } /** A TCP listener that accepts connections and then says nothing — invisible * to /__bili/health probes (they time out as "no evidence") but still owns * the socket, so binders get EADDRINUSE. Deterministically forces the LATE * EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */ -function silentTcpListener(port: number): Promise { +function silentTcpListener(port: number): Promise<{ close(): Promise }> { return new Promise((resolve, reject) => { + const sockets = new Set(); const server = net.createServer((sock) => { + sockets.add(sock); + sock.once("close", () => sockets.delete(sock)); // accept and hang — never answer HTTP; drain the probe's bytes so peer // EOF is actually delivered (a paused socket buffers them and never sees // EOF) and release the connection, or a dead prober pins server.close() @@ -253,12 +258,39 @@ function silentTcpListener(port: number): Promise { // the test ends and fails the whole file (win-only #5050 follow-up). sock.on("error", () => {}); sock.on("end", () => sock.destroy()); + // An aborted health probe may reset instead of ending the connection. + // Socket errors belong to this silent fixture, not the enclosing test. + sock.on("error", () => sock.destroy()); }); server.once("error", reject); - server.listen(port, "127.0.0.1", () => resolve(server)); + server.listen(port, "127.0.0.1", () => resolve({ + close: () => new Promise((done) => { + server.close(() => done()); + for (const socket of sockets) socket.destroy(); + }), + })); }); } +test("silent TCP fixture survives an aborted health probe", { timeout: 5_000 }, async () => { + const port = await freePort(); + const fixture = await silentTcpListener(port); + const client = net.connect({ host: "127.0.0.1", port }); + try { + await new Promise((resolve, reject) => { + client.once("connect", resolve); + client.once("error", reject); + }); + client.write("GET /__bili/health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + await new Promise((resolve) => setTimeout(resolve, 25)); + client.resetAndDestroy(); + await new Promise((resolve) => setTimeout(resolve, 25)); + } finally { + client.destroy(); + await fixture.close(); + } +}); + function tcpRefused(port: number): Promise { return new Promise((resolve) => { const s = net.connect({ port, host: "127.0.0.1" }); @@ -323,6 +355,9 @@ function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: { async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise { let port = 0; await waitUntil(async () => { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`${label} exited before becoming healthy`); + } const m = readMarkers(stateHome).find((e) => e.pid === child.pid); if (!m || typeof m.port !== "number") return false; port = m.port as number; @@ -339,7 +374,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders let p1Ref: ReturnType | null = null; let p2Ref: ReturnType | null = null; let heldRef: net.Socket | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // P1: the serving proxy that will enter graceful shutdown. const p1 = spawnBili(sandbox, port); @@ -415,7 +450,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders heldRef?.destroy(); if (p1Ref) await killChild(p1Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); @@ -426,7 +461,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n const port = await freePort(); await assertPortDead(port, { label: "4982 unverified-twin port" }); let p2Ref: ReturnType | null = null; - let squatterRef: net.Server | null = null; + let squatterRef: Awaited> | null = null; try { // A registry marker claiming `port` with a LIVE pid and full identity // match — but NO portReleasedAt stamp (pre-fix build, or a failed @@ -476,7 +511,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n } finally { if (p2Ref) await killChild(p2Ref.child); const squatter = squatterRef; - if (squatter) await new Promise((resolve) => squatter.close(() => resolve())); + if (squatter) await squatter.close(); fs.rmSync(sandbox, { recursive: true, force: true }); } }); diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 09f433c205..0474cecdbb 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -38,16 +38,12 @@ interface SpawnRecord { // clone (plugin injection only; 2.x directory wrapper when the host is 2.x). test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attach/proxy env stripped, plugin rides OPENCODE_CONFIG", async () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ocinline-")); - const prevHome = process.env.HOME; - const prevUserProfile = process.env.USERPROFILE; const prevXdg = process.env.XDG_CONFIG_HOME; const prevOcConfig = process.env.OPENCODE_CONFIG; const prevClientBin = process.env.BILI_CLIENT_BIN; const prevBcp = process.env.BILLION_CONTEXT_PROXY; const prevHttpsProxy = process.env.HTTPS_PROXY; const prevMitmHosts = process.env.BILI_MITM_HOSTS; - process.env.HOME = home; - if (prevUserProfile !== undefined) process.env.USERPROFILE = home; // Isolated config root: no user opencode.json, no ambient OPENCODE_CONFIG. process.env.XDG_CONFIG_HOME = path.join(home, ".config"); delete process.env.OPENCODE_CONFIG; @@ -122,7 +118,8 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac const childComspec = ce.COMSPEC ?? ce.ComSpec ?? ce.comspec; const expectedComspec = typeof childComspec === "string" && childComspec.trim().length > 0 ? childComspec : "cmd.exe"; assert.equal(cap.command, expectedComspec, "#679 comspec wrap uses the child env's COMSPEC"); - assert.deepEqual(cap.args.slice(0, 3), ["/d", "/s", "/c"]); + const quotedOc = /\s/.test(fakeOc) ? `"${fakeOc}"` : fakeOc; + assert.deepEqual(cap.args, ["/d", "/s", "/c", `"${quotedOc}"`]); assert.equal(cap.verbatim, true, "#679 verbatim argv keeps cmd.exe from re-splitting"); assert.match(cap.args[3] ?? "", /^".*\.cmd"$/, "comspec line wraps the quoted .cmd shim"); } else { @@ -153,9 +150,6 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac assert.equal(clone.compaction?.auto, false, "host auto-compaction stays off (ACP owns compression)"); } finally { process.exit = prevExit; - process.env.HOME = prevHome; - if (prevUserProfile === undefined) delete process.env.USERPROFILE; - else process.env.USERPROFILE = prevUserProfile; if (prevXdg === undefined) delete process.env.XDG_CONFIG_HOME; else process.env.XDG_CONFIG_HOME = prevXdg; if (prevOcConfig === undefined) delete process.env.OPENCODE_CONFIG; diff --git a/tests/persist-lazy-narrow-reads.test.ts b/tests/persist-lazy-narrow-reads.test.ts new file mode 100644 index 0000000000..e295ad018c --- /dev/null +++ b/tests/persist-lazy-narrow-reads.test.ts @@ -0,0 +1,208 @@ +import test, { type TestContext } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createInitialState, type CompressionBlock } from "acp-kernel"; +import { SessionStore } from "../src/persist.ts"; +import { isDroppedMetadata, LazyBlockContents, type BlockContent, type Session } from "../src/session.ts"; +import type { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { UnifiedStore } from "../src/storage/store.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const ROOT = mkdtempSync(path.join(tmpdir(), "bili-lazy-narrow-")); +process.env.BILI_CONFIG_FILE = path.join(ROOT, "no-config.json"); +process.env.BILI_PERSIST_ZSTD = "0"; +delete process.env.BILI_STORAGE_JSON_WRITER; +test.after(() => rmrf(ROOT)); + +type SavedFields = { metadata: Record; blockContents: Record }; +type StoreInternals = { + shadow: UnifiedShadow; + loadEnvelope(id: string, meta?: Session["meta"]): { payload: SavedFields } | null; + buildRecordForSave(session: Session): SavedFields; +}; + +function makeBlock(blockId: string, active = true): CompressionBlock { + return { + blockId, runId: "r0", tier: 1, summary: `summary ${blockId}`, + directMessageIds: [], effectiveMessageIds: [], directBlockIds: [], + compressedTokens: 0, createdAt: 1, survivedCount: 0, generation: "young", active, + }; +} + +function makeSession(id: string, source: string): Session { + return { + id, meta: { protocol: "openai", upstreamOrigin: "http://up:1", label: id }, + stats: { + requests: 5, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, + cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, compressCreditTokens: 0, + retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, + storeBytesSaved: 0, rangeRestores: 0, + }, + metadata: { source }, state: createInitialState(), + createdAt: 1, lastSeen: 2, inFlight: 0, persisted: false, pendingRetrievals: [], + blockContents: new Map([["orphan", { one: null, full: { text: `${source} orphan original`, count: 2 } }]]), + }; +} + +function fixture(t: TestContext, mode: "full" | "legacy"): { store: SessionStore; dir: string } { + process.env.BILI_STORAGE_UNIFIED = mode; + const dir = mkdtempSync(path.join(ROOT, `${mode}-`)); + const store = new SessionStore({ dir, enabled: true, debounceMs: 60_000 }); + t.after(() => store.closeQuietly()); + return { store, dir }; +} + +function internals(store: SessionStore): StoreInternals { + return store as unknown as StoreInternals; +} + +function stateFile(dir: string): string { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) { + const nested = stateFile(file); + if (nested) return nested; + } else if (entry.name.endsWith(".json") && !entry.name.endsWith(".content-store.json")) { + return file; + } + } + return ""; +} + +function assertDropped(session: Session): void { + assert.equal(isDroppedMetadata(session.metadata), true); + assert.ok(session.blockContents instanceof LazyBlockContents); + assert.equal(session.blockContents.isDropped, true); +} + +test("full-mode lazy getters read only their requested resident field and keep all originals", async (t) => { + const { store } = fixture(t, "full"); + const session = makeSession("narrow-full", "index"); + session.state.blocks = [makeBlock("active"), makeBlock("archived", false)]; + session.blockContents.set("active", { + one: { text: "short view", count: 1 }, full: { text: "full active original", count: 4 }, + }); + session.blockContents.set("archived", { + one: { text: "same archived original", count: 2 }, full: { text: "same archived original", count: 2 }, + }); + await store.writeNow(session); + assertDropped(session); + + const shadow = internals(store).shadow; + const driver = (shadow as unknown as { sync: UnifiedStore }).sync; + assert.ok(driver, "full mode must exercise the SQLite lane"); + const wholeLoads = t.mock.method(store, "loadSync"); + const wholeViews = t.mock.method(shadow, "viewFor"); + const messages = t.mock.method(driver, "getMessages"); + const blockPayloads = t.mock.method(driver, "getBlocks"); + const sessionRows = t.mock.method(driver, "getSessionRow"); + const metadata = t.mock.method(driver, "getMetadataJson"); + const contents = t.mock.method(driver, "getBlockContents"); + const narrow = t.mock.method(shadow, "readResidentFieldsFor"); + const imports = t.mock.method(shadow, "importRecord"); + const saves = t.mock.method(store, "scheduleSave"); + + assert.equal(session.metadata.source, "index"); + assert.equal(metadata.mock.callCount(), 1); + assert.equal(contents.mock.callCount(), 0, "metadata access does not fetch any originals"); + assert.equal((session.blockContents as LazyBlockContents).isDropped, true); + assert.deepEqual(narrow.mock.calls[0].arguments[1], { metadata: true }); + + assert.deepEqual(session.blockContents.get("active"), { + one: { text: "short view", count: 1 }, full: { text: "full active original", count: 4 }, + }); + assert.deepEqual(session.blockContents.get("archived"), { + one: null, full: { text: "same archived original", count: 2 }, + }, "inactive originals survive and duplicate one/full pairs normalize"); + assert.deepEqual(session.blockContents.get("orphan"), { one: null, full: { text: "index orphan original", count: 2 } }); + assert.equal(session.blockContents.size, 3); + assert.equal(metadata.mock.callCount(), 1, "contents access does not re-fetch metadata"); + assert.deepEqual(narrow.mock.calls[1].arguments[1], { blockContents: true }); + for (const spy of [wholeLoads, wholeViews, messages, blockPayloads, sessionRows, imports, saves]) { + assert.equal(spy.mock.callCount(), 0, "a lazy getter must not rebuild, import or schedule a whole session"); + } +}); + +test("legacy save builder shares one envelope read for both dropped fields without retaining an old projection", async (t) => { + const { store, dir } = fixture(t, "legacy"); + const session = makeSession("narrow-legacy", "first"); + await store.writeNow(session); + assertDropped(session); + const envelopes = t.mock.method(internals(store), "loadEnvelope"); + const wholeLoads = t.mock.method(store, "loadSync"); + + const saved = internals(store).buildRecordForSave(session); + assert.equal(saved.metadata.source, "first"); + assert.equal(saved.blockContents.orphan.full.text, "first orphan original"); + assert.equal(envelopes.mock.callCount(), 1, "one build shares a single source read across both lazy getters"); + assert.equal(wholeLoads.mock.callCount(), 0); + await store.writeNow(session); + assertDropped(session); + const priorReads = envelopes.mock.callCount(); + + // An external writer changes only the two resident fields. The next read + // must consult disk again after the synchronous save has finished. + const file = stateFile(dir); + assert.ok(file); + const envelope = JSON.parse(readFileSync(file, "utf8")) as { payload: SavedFields }; + envelope.payload.metadata.source = "external"; + envelope.payload.blockContents.orphan.full.text = "external original"; + writeFileSync(file, JSON.stringify(envelope)); + assert.equal(session.metadata.source, "external"); + assert.equal(session.blockContents.get("orphan")?.full.text, "external original"); + assert.equal(envelopes.mock.callCount(), priorReads + 2, "independent touches outside a save each select their current source"); + assert.equal(wholeLoads.mock.callCount(), 0); +}); + +test("lazy source arbitration matches cold-load progress and drift rules", async (t) => { + type Generation = { requests: number; nextBlockId: number; active: number; refs: number }; + const base: Generation = { requests: 5, nextBlockId: 10, active: 1, refs: 1 }; + const cases: { name: string; latch: boolean; index?: Partial; envelope: Partial; expected: string }[] = [ + { name: "strictly more envelope requests", latch: false, envelope: { requests: 6 }, expected: "envelope" }, + { name: "equal requests and ahead envelope fold cursor", latch: false, envelope: { nextBlockId: 11 }, expected: "envelope" }, + { name: "equal progress keeps index", latch: false, envelope: {}, expected: "index" }, + { name: "drift prefers greater index active coverage", latch: true, index: { active: 2 }, envelope: { requests: 99 }, expected: "index" }, + { name: "drift prefers greater index mapped refs", latch: true, index: { refs: 3 }, envelope: { requests: 99 }, expected: "index" }, + { name: "drift ties keep envelope", latch: true, envelope: {}, expected: "envelope" }, + ]; + for (const [i, c] of cases.entries()) { + await t.test(c.name, async (st) => { + const { store } = fixture(st, "full"); + const id = `narrow-arbitration-${i}`; + const generation = (source: string, values: Generation): Session => { + const session = makeSession(id, source); + session.stats.requests = values.requests; + session.state.nextBlockId = values.nextBlockId; + session.state.blocks = Array.from({ length: values.active }, (_, n) => makeBlock(`b${n + 1}`)); + for (let n = 1; n <= values.refs; n++) { + const ref = `m${String(n).padStart(5, "0")}`; + session.state.messageRefs.byRaw[`raw-${n}`] = ref; + session.state.messageRefs.byRef[ref] = `raw-${n}`; + } + return session; + }; + await store.writeNow(generation("index", { ...base, ...c.index })); + const shadow = internals(store).shadow; + const ingest = st.mock.method(shadow, "record", () => {}); + const envelope = generation("envelope", { ...base, ...c.envelope }); + await store.writeNow(envelope); + ingest.mock.restore(); + if (c.latch) shadow.driftedSessions.add(id); + assertDropped(envelope); + + const loads = st.mock.method(store, "loadSync"); + const views = st.mock.method(shadow, "viewFor"); + assert.equal(envelope.metadata.source, c.expected); + assert.equal(envelope.blockContents.get("orphan")?.full.text, `${c.expected} orphan original`); + assert.equal(loads.mock.callCount(), 0); + assert.equal(views.mock.callCount(), 0); + loads.mock.restore(); + views.mock.restore(); + const cold = store.loadSync(id); + assert.equal(cold?.metadata.source, c.expected, "the public cold loader selects the same generation"); + assert.equal(cold?.blockContents.get("orphan")?.full.text, envelope.blockContents.get("orphan")?.full.text); + }); + } +}); diff --git a/tests/persist-rollback-guard.test.ts b/tests/persist-rollback-guard.test.ts index 6441a048a5..ae6357bec0 100644 --- a/tests/persist-rollback-guard.test.ts +++ b/tests/persist-rollback-guard.test.ts @@ -36,7 +36,7 @@ function withTempStore(name: string, fn: (store: SessionStore, dir: string, warn try { await fn(store, dir, warnings); } finally { - store.cancelAll(); + store.closeQuietly(); rmrf(dir); } }) as unknown as Promise; diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 878505d0fe..6dcedc88f5 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -115,7 +115,7 @@ async function harness(persist = false, seedParent = true, compressExtra: Record assert.equal(r.status, 200); await r.text(); } - return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; + return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.closeQuietly(); await closeServers([proxy, upstream], "plugin-fork harness"); } }; } test("plain proxy panel snapshots do not change raw-history persistence", { timeout: 60_000 }, async () => { diff --git a/tests/storage-backend-contract.test.ts b/tests/storage-backend-contract.test.ts index 1870beca80..479a9e936e 100644 --- a/tests/storage-backend-contract.test.ts +++ b/tests/storage-backend-contract.test.ts @@ -296,6 +296,15 @@ test("mysql lane: contract surface against MySQL 8.0", { skip: MYSQL_DSN.length const rows2 = await backend.getMessages(sid); assert.equal(rows2.length, 2, "no duplicate rows on identical re-ingest"); + await backend.readSnapshot(async (reader) => { + assert.equal((await reader.getSessionRow(sid))!.sessionId, sid); + assert.deepEqual(await reader.getMessages(sid, { seqs: [] }), []); + const selected = await reader.getMessages(sid, { seqs: [rows2[1]!.seq] }); + assert.equal(selected.length, 1); + assert.equal(selected[0]!.content, rows2[1]!.content); + assert.deepEqual(await reader.getBlocks(sid, { limit: 1 }), []); + }); + // fork clone (#4316 PR3/3, #4911): the same contract surface the shadow // async twins route into on the real fork lane — semantics must match // the sqlite lane's dedicated test line for line. diff --git a/tests/storage-bounded-load.test.ts b/tests/storage-bounded-load.test.ts new file mode 100644 index 0000000000..a412e8457f --- /dev/null +++ b/tests/storage-bounded-load.test.ts @@ -0,0 +1,222 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { planIngest, loadLegacyView, LegacyViewBuilder, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import type { SqliteDatabase } from "../src/storage/driver.ts"; + +function fixture(): LegacySessionLike { + const messages: NonNullable = Array.from({ length: 80 }, (_, i) => ({ + id: `msg${i}`, role: "user", contentType: "text", text: `message ${i}`, + })); + const blocks: LegacySessionLike["state"]["blocks"] = Array.from({ length: 70 }, (_, i) => ({ + blockId: `b${i + 1}`, runId: "r1", tier: 1, summary: `summary ${i + 1}`, + directMessageIds: [`msg${i}`], effectiveMessageIds: [`msg${i}`], directBlockIds: [], + compressedTokens: 10, createdAt: 1000 + i, survivedCount: 0, generation: "young", active: i % 2 === 0, + })); + return { + id: "bounded", createdAt: 1000, lastSeen: 2000, + meta: { protocol: "openai" }, stats: { requests: 7 }, metadata: { pluginAgent: "codex", note: "retained" }, + state: { + blocks, nextBlockId: 71, nextRunId: 2, + messageRefs: { byRaw: { msg0: "m00001", alias: "m00001", ghost: "m99999", blocked: "BLOCKED" }, byRef: { m00001: "msg0", m99999: "ghost" } }, + deadRefs: ["m90001", "m90002"], + }, + blockContents: new Map(blocks.map((block) => [block.blockId, { + one: { text: `one ${block.blockId}`, count: 1 }, full: { text: `original ${block.blockId}`, count: 2 }, + }])), + decompositionLog: messages, + pluginSnapshot: messages.slice(0, 36), + lastMessages: [messages[65], messages[33], messages[65]], + lastMessagesFolded: true, + contentStore: { version: 1, byHash: { h: "retained CCR" }, byRef: {} }, + }; +} + +function seed(store: UnifiedStore): void { + const plan = planIngest(fixture()); + plan.session.orphansJson = JSON.stringify({ + b1: { one: null, full: { text: "stale journal duplicate", count: 1 } }, + reaped: { one: null, full: { text: "reaped original", count: 1 } }, + }); + store.ingestSession(plan); + // Simulate missing historical backing rows after the ingest remapper. + const tail = JSON.parse(store.getSessionRow("bounded")!.tailJson) as { tailSeqs: number[]; pluginSnapshotSeqs: number[] }; + tail.tailSeqs.push(9999); + tail.pluginSnapshotSeqs.push(9999); + (store as unknown as { db: SqliteDatabase }).db.prepare("UPDATE sessions SET tail_json = ? WHERE session_id = ?").run(JSON.stringify(tail), "bounded"); +} + +function allRows(store: UnifiedStore) { + return { + session: store.getSessionRow("bounded")!, messages: store.getMessages("bounded"), + blocks: store.getBlocks("bounded"), refs: store.getRefs("bounded"), deadRefs: store.getDeadRefs("bounded"), + }; +} + +function assertHistory(view: LegacySessionLike | null): asserts view is LegacySessionLike { + assert.ok(view); + assert.equal(view.state.blocks.length, 70); + assert.equal(view.state.blocks.filter((block) => !block.active).length, 35, "inactive ancestors remain available to decompress"); + assert.deepEqual(view.state.blocks.map((block) => block.blockId), fixture().state.blocks.map((block) => block.blockId)); + assert.equal(view.blockContents.get("b1")!.full.text, "original b1", "block row wins over a stale orphan journal entry"); + assert.equal(view.blockContents.get("reaped")!.full.text, "reaped original"); + assert.equal(view.blockContents.size, 71); + assert.deepEqual(view.lastMessages?.map((message) => message.id), ["msg65", "msg33", "msg65"]); + assert.strictEqual(view.lastMessages![0], view.lastMessages![2], "repeated pointers retain shared message identity"); + assert.strictEqual(view.lastMessages![1], view.pluginSnapshot![33], "tail and snapshot overlap shares one parsed message"); + assert.deepEqual(view.state.messageRefs, fixture().state.messageRefs, "ghost and BLOCKED refs survive verbatim"); + assert.deepEqual(view.state.deadRefs, ["m90001", "m90002"]); + assert.deepEqual(view.contentStore, fixture().contentStore); + assert.equal((view as unknown as { tailGaps: number }).tailGaps, 2); +} + +test("ordinary sync cold load reads only unique tail/snapshot seqs and bounded block pages", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-load-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const expected = loadLegacyView(allRows(store)); + const getMessages = store.getMessages.bind(store); + const getBlocks = store.getBlocks.bind(store); + const requested: number[] = []; + const blockPages: number[] = []; + let snapshots = 0; + const snapshot = store.readSnapshot.bind(store); + store.readSnapshot = (fn) => { snapshots++; return snapshot(fn); }; + store.getMessages = (id, opts) => { + assert.ok(opts?.seqs && opts.seqs.length <= 32, "cold load must never hydrate the whole message log"); + requested.push(...opts.seqs); + return getMessages(id, opts); + }; + store.getBlocks = (id, opts) => { + assert.equal(opts?.limit, 32); + blockPages.push(opts?.offset ?? 0); + return getBlocks(id, opts); + }; + const view = shadow.viewFor("bounded"); + assertHistory(view); + assert.deepEqual(view, expected); + assert.equal(snapshots, 1, "header and every page share one read snapshot"); + assert.deepEqual(blockPages, [0, 32, 64]); + assert.equal(requested.length, new Set(requested).size); + assert.deepEqual(new Set(requested), new Set([65, 33, 9999, ...Array.from({ length: 36 }, (_, i) => i)])); + assert.equal(shadow.viewFor("absent"), null); +}); + +test("ordinary async cold load uses the snapshot reader for each bounded page", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-async-")); + const backend = SqliteBackend.open(join(dir, "index.db")); + t.after(async () => { await backend.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(backend.store); + const expected = loadLegacyView(allRows(backend.store)); + const readSnapshot = backend.readSnapshot.bind(backend); + const offsets: number[] = []; + const requested: number[] = []; + let snapshots = 0; + backend.readSnapshot = (fn) => readSnapshot(async (reader) => { + snapshots++; + return fn({ + ...reader, + getMessages: async (id, opts) => { + assert.ok(opts?.seqs && opts.seqs.length <= 32); + requested.push(...opts.seqs); + return reader.getMessages(id, opts); + }, + getBlocks: async (id, opts) => { + assert.equal(opts?.limit, 32); + offsets.push(opts?.offset ?? 0); + return reader.getBlocks(id, opts); + }, + }); + }); + backend.getMessages = async () => { throw new Error("load bypassed the snapshot reader"); }; + backend.getBlocks = async () => { throw new Error("load bypassed the snapshot reader"); }; + const shadow = Reflect.construct(UnifiedShadow, ["full", null, () => null, backend]) as UnifiedShadow; + const view = await shadow.viewForAsync("bounded"); + assertHistory(view); + assert.deepEqual(view, expected); + assert.equal(snapshots, 1); + assert.deepEqual(offsets, [0, 32, 64]); + assert.equal(requested.length, new Set(requested).size); + assert.deepEqual(new Set(requested), new Set([65, 33, 9999, ...Array.from({ length: 36 }, (_, i) => i)])); +}); + +test("a failed block page falls back and releases the read snapshot for the next load", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-bounded-failure-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const getBlocks = store.getBlocks.bind(store); + store.getBlocks = (id, opts) => { + if (opts?.offset === 32) throw new Error("injected block page failure"); + return getBlocks(id, opts); + }; + assert.equal(shadow.viewFor("bounded"), null, "partial views must never escape on a failed page"); + store.getBlocks = getBlocks; + assertHistory(shadow.viewFor("bounded")); +}); + +test("resident field reads avoid unrelated JSON, messages, refs, and block payloads", (t) => { + const dir = mkdtempSync(join(tmpdir(), "bili-resident-fields-")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + const store = (shadow as unknown as { sync: UnifiedStore }).sync; + t.after(() => { shadow.close(); rmSync(dir, { recursive: true, force: true }); }); + seed(store); + const expected = loadLegacyView(allRows(store)); + for (const key of ["getSessionRow", "getMessages", "getBlocks", "getRefs", "getDeadRefs"] as const) { + store[key] = () => { throw new Error(`resident field read called ${key}`); }; + } + let metadataReads = 0; + let contentReads = 0; + let orphanReads = 0; + let snapshots = 0; + const snapshot = store.readSnapshot.bind(store); + store.readSnapshot = (fn) => { snapshots++; return snapshot(fn); }; + const metadata = store.getMetadataJson.bind(store); + const contents = store.getBlockContents.bind(store); + const orphans = store.getOrphansJson.bind(store); + store.getMetadataJson = (id) => { metadataReads++; return metadata(id); }; + store.getBlockContents = (id, opts) => { contentReads++; assert.equal(opts?.limit, 32); return contents(id, opts); }; + store.getOrphansJson = (id) => { orphanReads++; return orphans(id); }; + const meta = shadow.readResidentFieldsFor("bounded", { metadata: true }); + assert.deepEqual(meta, { header: { requests: 7, nextBlockId: 71, activeBlockCount: 35, mappedRefCount: 2 }, metadata: expected.metadata }); + assert.equal(contentReads, 0); + assert.equal(orphanReads, 0); + const blocks = shadow.readResidentFieldsFor("bounded", { blockContents: true }); + assert.deepEqual(blocks?.blockContents, expected.blockContents); + assert.equal(metadataReads, 1, "content getter does not read metadata"); + const both = shadow.readResidentFieldsFor("bounded", { metadata: true, blockContents: true }); + assert.deepEqual(both?.metadata, expected.metadata); + assert.deepEqual(both?.blockContents, expected.blockContents); + assert.equal(snapshots, 3, "combined fields use one snapshot"); + assert.equal(contentReads, 6); + assert.equal(orphanReads, 2); +}); + +test("incremental assembly preserves empty versus absent pointers and skips a corrupt orphan journal", () => { + const plan = planIngest(fixture()); + plan.session.tailJson = JSON.stringify({ tailSeqs: [], pluginSnapshotSeqs: null, tailFolded: false }); + plan.session.orphansJson = "broken journal"; + const builder = new LegacyViewBuilder(plan.session, plan.refs!, plan.deadRefs); + assert.deepEqual(builder.messageSeqs, []); + builder.addBlocks(plan.blocks); + const view = builder.finish(); + assert.deepEqual(view.lastMessages, []); + assert.equal(view.pluginSnapshot, undefined); + assert.equal(view.tailGaps, 0); + assert.equal(view.blockContents.size, 70); + plan.session.tailJson = JSON.stringify({ tailSeqs: null, pluginSnapshotSeqs: [], tailFolded: false }); + const other = new LegacyViewBuilder(plan.session, [], []).finish(); + assert.equal(other.lastMessages, undefined); + assert.deepEqual(other.pluginSnapshot, []); +});