diff --git a/src/server.ts b/src/server.ts index 7059a79cbd..281463c60e 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4622,11 +4622,15 @@ function formatBytes(n: number): string { // bytes are counted with Buffer.byteLength since undici sends UTF-8 bytes even // for string bodies (.length would count UTF-16 chars and undercount any // non-ASCII injection). Omitted on paths that fail fast without forwarding. -// #4528: inbound histories past this size pay an O(messages) synchronous scan -// pass per turn on the main thread, large enough to starve the event loop and +// #4528: bili's own per-request processing past this WALL-CLOCK budget means +// synchronous main-thread work large enough to starve the event loop and // freeze every endpoint on the same listener — warn so a pathological session // shows up in the log instead of being found through a frozen UI. -const SLOW_SESSION_MSG_WARN = 1000; +// #5744: the trigger used to be message COUNT (>1000), which cannot separate +// cost from size — one reporter session saved at 0.8s AND 14.5s at the same +// ~3k msgs. Wall clock IS the cost; the info line above keeps reporting the +// count for correlation. +const SLOW_SESSION_MS_WARN = 1000; export function logRequestCost(log: (level: string, msg: string) => void, sessionId: string, msgs: number | null, inboundBytes: number, t0: number, outbound?: string | Buffer): void { const ms = Math.max(0, Math.round(performance.now() - t0)); const outboundField = outbound !== undefined ? `, outbound=${formatBytes(Buffer.byteLength(outbound))}` : ""; @@ -4635,8 +4639,8 @@ export function logRequestCost(log: (level: string, msg: string) => void, sessio // #5408: this line is THE observation point — a hot-at-attach session that // never produces one is silent compression loss (the incident signature). noteRequest(sessionId); - if (msgs !== null && msgs > SLOW_SESSION_MSG_WARN) { - log("warn", `[${sessionId}] slow-session: ${msgs} msgs / inbound=${formatBytes(inboundBytes)} took local=${ms}ms — per-turn O(messages) synchronous scans are starving the event loop (#4528); consider closing/restarting the session or enabling compression`); + if (ms > SLOW_SESSION_MS_WARN) { + log("warn", `[${sessionId}] slow-session: ${msgs ?? "?"} msgs / inbound=${formatBytes(inboundBytes)} took local=${ms}ms — synchronous per-turn work is starving the event loop (#4528/#5744); consider closing/restarting the session or enabling compression`); } } diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 3b54cd6092..907fc103d2 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -403,6 +403,8 @@ function finishPlan( const views = completeBlockContents(session.blockContents, session.state.blocks).get(b.blockId); const full = views?.full ?? { text: b.summary, count: 0 }; const one = views?.one ?? null; + const summaryJson = JSON.stringify(full); + const oneJson = one === null ? null : JSON.stringify(one); const covered = { effectiveMessageIds: b.effectiveMessageIds, directMessageIds: b.directMessageIds }; let startSeq = -1; let endSeq = -1; @@ -420,8 +422,13 @@ function finishPlan( startRef: b.startRef !== undefined ? refToIndex(b.startRef) : null, endRef: b.endRef !== undefined ? refToIndex(b.endRef) : null, tier: b.tier, - summary: JSON.stringify(full), - one: one === null ? null : JSON.stringify(one), + summary: summaryJson, + one: oneJson, + // #5744: carry the digests so ingestSession can compare against the + // stored row BEFORE any CAS op (putContent with a precomputed hash + // recomputes nothing — same bytes, same utf8, same sha256). + summaryHash: sha256Hex(summaryJson), + oneHash: oneJson === null ? null : sha256Hex(oneJson), supersedesBlockId: null, active: b.active, createdAt: b.createdAt, @@ -451,11 +458,10 @@ function finishPlan( })); const ccrEntries: CcrEntryRow[] = session.contentStore - ? Object.entries(session.contentStore.byRef).map(([key, value], ord) => ({ - ord, - key, - content: JSON.stringify(value), - })) + ? Object.entries(session.contentStore.byRef).map(([key, value], ord) => { + const content = JSON.stringify(value); + return { ord, key, content, contentHash: sha256Hex(content) }; + }) : []; const kstate: Record = {}; diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 1dbf6d91d3..c426bbc54f 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -92,6 +92,10 @@ const MIGRATION_LOCK_POLL_S = 30; /** PR #4725 N3 (mirror of store.ts): covered-seq span point queries chunk the * msg_id IN (…) list at the same 500 batch size for cross-backend parity. */ const SEQ_SPAN_QUERY_BATCH = 500; +/** Max bound parameters per generated statement (refs/ccr batching, #5744) — + * mirror of the sqlite twin's constant; well under MySQL's max_allowed_packet + * and driver placeholder limits. */ +const REF_STMT_MAX_PARAMS = 700; /** #4796: rows per batch of gc()'s zero-ref CAS sweep (mysql counterpart of * GC_ZERO_REF_BATCH_SIZE in store.ts): one giant DELETE holds every row @@ -800,16 +804,28 @@ export class MysqlBackend implements StorageBackend { } } } - const incomingBlockIds = new Set(input.blocks.map((b) => b.blockId)); - const staleBlocks = (await c.query( - "SELECT block_id, summary_hash, one_hash FROM blocks WHERE session_id = ?", + // #5744 (mirror of store.ts): ONE full-row read serves both the + // stale sweep and the unchanged-row skip below. + type StoredBlockRow = { + block_id: string | number; ord: number | string; start_seq: number | string; end_seq: number | string; + start_ref: number | null; end_ref: number | null; tier: number | string; + summary_hash: string | number; one_hash: string | null; supersedes_block_id: string | null; + active: number | string; created_at: number | string; covered_hash: string | number; payload_json: string | number; + }; + const storedBlocks = new Map(); + const storedBlockRows = (await c.query( + "SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, " + + "summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json " + + "FROM blocks WHERE session_id = ?", [s.sessionId], - )) as unknown as Row[][]; - for (const stale of staleBlocks[0]) { - if (incomingBlockIds.has(str(stale.block_id))) continue; + )) as unknown as StoredBlockRow[][]; + for (const row of storedBlockRows[0]) storedBlocks.set(str(row.block_id), row); + for (const [blockId, stale] of storedBlocks) { + if (input.blocks.some((b) => b.blockId === blockId)) continue; + // referencing row first, then release (#4242 delete-on-zero) + await c.query("DELETE FROM blocks WHERE session_id = ? AND block_id = ?", [s.sessionId, blockId]); await this.releaseContent(c, str(stale.summary_hash)); if (stale.one_hash !== null && stale.one_hash !== undefined) await this.releaseContent(c, str(stale.one_hash)); - await c.query("DELETE FROM blocks WHERE session_id = ? AND block_id = ?", [s.sessionId, stale.block_id]); } // #4316 layer 3 (mirror of store.ts): resolve coverage to canonical // seq bounds from the append-only transcript; unresolvable stores @@ -866,18 +882,35 @@ export class MysqlBackend implements StorageBackend { }; for (const b of input.blocks) { const [bStartSeq, bEndSeq] = boundsForCovered(b.payloadJson); - const summaryHash = await this.putContent(c, b.summary); - const oneHash = b.one === null ? null : await this.putContent(c, b.one); - const [existingRows] = (await c.query( - "SELECT summary_hash, one_hash FROM blocks WHERE session_id = ? AND block_id = ?", - [s.sessionId, b.blockId], - )) as unknown as Row[][]; - const existing = existingRows[0]; - if (existing) { - if (str(existing.summary_hash) === summaryHash) await this.releaseContent(c, summaryHash); - else await this.releaseContent(c, str(existing.summary_hash)); - if (existing.one_hash !== null && existing.one_hash !== undefined) { - const exOne = str(existing.one_hash); + // #5744 (mirror of store.ts): skip the whole write + CAS dance + // when the effective post-upsert row equals the stored row — + // steady-state saves re-send every block byte-identical. The + // -1/unknown bound keeps its stored value via the IF(>=0) + // guard below, so "incoming < 0" counts as unchanged. + const ex = storedBlocks.get(b.blockId); + if (ex && + num(ex.ord) === b.ord && + (bStartSeq < 0 || bStartSeq === num(ex.start_seq)) && + (bEndSeq < 0 || bEndSeq === num(ex.end_seq)) && + (ex.start_ref === null ? null : num(ex.start_ref)) === b.startRef && + (ex.end_ref === null ? null : num(ex.end_ref)) === b.endRef && + num(ex.tier) === b.tier && + str(ex.summary_hash) === b.summaryHash && + (ex.one_hash === null || ex.one_hash === undefined ? null : str(ex.one_hash)) === b.oneHash && + (ex.supersedes_block_id === null || ex.supersedes_block_id === undefined ? null : str(ex.supersedes_block_id)) === b.supersedesBlockId && + num(ex.active) === (b.active ? 1 : 0) && + num(ex.created_at) === b.createdAt && + str(ex.covered_hash) === b.coveredHash && + str(ex.payload_json) === b.payloadJson) { + continue; + } + const summaryHash = await this.putContent(c, b.summary, b.summaryHash); + const oneHash = b.one === null ? null : await this.putContent(c, b.one, b.oneHash ?? undefined); + if (ex) { + if (str(ex.summary_hash) === summaryHash) await this.releaseContent(c, summaryHash); + else await this.releaseContent(c, str(ex.summary_hash)); + if (ex.one_hash !== null && ex.one_hash !== undefined) { + const exOne = str(ex.one_hash); if (exOne === oneHash && oneHash !== null) await this.releaseContent(c, oneHash); else await this.releaseContent(c, exOne); } @@ -924,22 +957,88 @@ export class MysqlBackend implements StorageBackend { [s.sessionId, d.ord, d.ref, d.refNum, d.recordedAt], ); } - await c.query("DELETE FROM refs WHERE session_id = ?", [s.sessionId]); + // #5744 (mirror of store.ts): diff against one indexed read of the + // stored set instead of DELETE-all + per-row INSERT — the network + // round-trip cost of a full-history rewrite is worse than sqlite's. + const oldRefs = new Map(); + const storedRefRows = (await c.query( + "SELECT raw_id, ord, ref, ref_num FROM refs WHERE session_id = ?", + [s.sessionId], + )) as unknown as Row[][]; + for (const row of storedRefRows[0]) { + oldRefs.set(str(row.raw_id), { ord: num(row.ord), ref: str(row.ref), refNum: row.ref_num === null || row.ref_num === undefined ? null : num(row.ref_num) }); + } + const goneRefIds: string[] = []; + const touchedRefs: RefRow[] = []; for (const r of input.refs) { - await c.query( - "INSERT INTO refs (session_id, ord, raw_id, ref, ref_num) VALUES (?, ?, ?, ?, ?) " + - "ON DUPLICATE KEY UPDATE ord = VALUES(ord), ref = VALUES(ref), ref_num = VALUES(ref_num)", - [s.sessionId, r.ord, r.rawId, r.ref, r.refNum], - ); + const o = oldRefs.get(r.rawId); + if (!o || o.ord !== r.ord || o.ref !== r.ref || o.refNum !== r.refNum) { + touchedRefs.push(r); + } else { + oldRefs.delete(r.rawId); + } } - const oldCcr = (await c.query( - "SELECT content_hash FROM ccr_entries WHERE session_id = ?", + for (const rawId of oldRefs.keys()) goneRefIds.push(rawId); + if (goneRefIds.length > 0) { + const perChunk = REF_STMT_MAX_PARAMS - 1; + for (let i = 0; i < goneRefIds.length; i += perChunk) { + const ids = goneRefIds.slice(i, i + perChunk); + await c.query( + "DELETE FROM refs WHERE session_id = ? AND raw_id IN (" + ids.map(() => "?").join(", ") + ")", + [s.sessionId, ...ids], + ); + } + } + if (touchedRefs.length > 0) { + const colsPerRow = 5; + const perChunk = Math.floor(REF_STMT_MAX_PARAMS / colsPerRow); + for (let i = 0; i < touchedRefs.length; i += perChunk) { + const rows = touchedRefs.slice(i, i + perChunk); + const params: unknown[] = []; + for (const r of rows) params.push(s.sessionId, r.ord, r.rawId, r.ref, r.refNum); + await c.query( + "INSERT INTO refs (session_id, ord, raw_id, ref, ref_num) VALUES " + + rows.map(() => "(?, ?, ?, ?, ?)").join(", ") + + " ON DUPLICATE KEY UPDATE ord = VALUES(ord), ref = VALUES(ref), ref_num = VALUES(ref_num)", + params, + ); + } + } + // #5744 (mirror of store.ts): diff instead of DELETE-all + + // re-INSERT-all; unchanged entries skip the CAS round-trips. + const incomingCcr = new Map(); + for (const e of input.ccrEntries) incomingCcr.set(e.key, e); + const storedCcrRows = (await c.query( + "SELECT ord, ccr_key, content_hash FROM ccr_entries WHERE session_id = ?", [s.sessionId], )) as unknown as Row[][]; - for (const row of oldCcr[0]) await this.releaseContent(c, str(row.content_hash)); - await c.query("DELETE FROM ccr_entries WHERE session_id = ?", [s.sessionId]); + const changedCcrKeys: string[] = []; + const unchangedCcrKeys = new Set(); + const releasedOldCcrHashes: string[] = []; + for (const row of storedCcrRows[0]) { + const key = str(row.ccr_key); + const inc = incomingCcr.get(key); + if (!inc) { changedCcrKeys.push(key); releasedOldCcrHashes.push(str(row.content_hash)); continue; } + if (num(row.ord) !== inc.ord || str(row.content_hash) !== inc.contentHash) { + changedCcrKeys.push(key); + releasedOldCcrHashes.push(str(row.content_hash)); + } else { + unchangedCcrKeys.add(key); + } + } + if (changedCcrKeys.length > 0) { + for (let i = 0; i < changedCcrKeys.length; i += SEQ_SPAN_QUERY_BATCH) { + const keys = changedCcrKeys.slice(i, i + SEQ_SPAN_QUERY_BATCH); + await c.query( + "DELETE FROM ccr_entries WHERE session_id = ? AND ccr_key IN (" + keys.map(() => "?").join(", ") + ")", + [s.sessionId, ...keys], + ); + } + } + for (const hash of releasedOldCcrHashes) await this.releaseContent(c, hash); for (const e of input.ccrEntries) { - const hash = await this.putContent(c, e.content); + if (unchangedCcrKeys.has(e.key)) continue; + const hash = await this.putContent(c, e.content, e.contentHash); await c.query( "INSERT INTO ccr_entries (session_id, ord, ccr_key, content_hash) VALUES (?, ?, ?, ?)", [s.sessionId, e.ord, e.key, hash], @@ -1650,7 +1749,7 @@ export class MysqlBackend implements StorageBackend { )) as unknown as Row[][]; const out: CcrEntryRow[] = []; for (const r of rows) { - out.push({ ord: num(r.ord), key: str(r.ccr_key), content: (await this.getContent(this.pool, str(r.content_hash))) ?? "" }); + out.push({ ord: num(r.ord), key: str(r.ccr_key), content: (await this.getContent(this.pool, str(r.content_hash))) ?? "", contentHash: str(r.content_hash) }); } return out; } diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 6b77bf7ffc..273ebdb068 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -64,6 +64,11 @@ interface ResidentFields { * session always verifies (cold-path full check), then every N-th. */ export const VERIFY_EVERY_INGESTS = 20; +/** #5744: wall-clock threshold for the slow-ingest warn (ms). The old + * request-level warn keyed on message count, which cannot separate a fast + * big session from a slow small one. */ +const SLOW_INGEST_WARN_MS = 500; + /** #4323: consecutive sampled drifts required before escalating. A one-off * drift is transient external churn (the watermark clear below heals it); * N in a row proves the views are STRUCTURALLY divergent — the multi-writer @@ -482,8 +487,15 @@ export class UnifiedShadow { // full path below; the delta itself re-verifies the boundary // inside the ingest transaction, so a racing writer degrades to // a full ingest, never a corrupt one. + // #5744: wall-clock guard on the ingest itself — message count is + // a bad proxy (one reporter session: 0.8s and 14.5s saves at the + // SAME ~3k msgs). Names the path so a spike is attributable + // (delta fast path vs full fallback) without a profiler. + const tIngestStart = performance.now(); + let ingestPath = "delta"; phase = "ingest-delta"; if (!this.ingestDelta(legacy, hasRawTail)) { + ingestPath = "full"; phase = "plan-ingest"; const plan = planIngest(legacy); phase = "ingest-session"; @@ -501,6 +513,14 @@ export class UnifiedShadow { this.sync.ingestSession(plan, undefined, { completeLog }); } this.stats.ingests++; + const ingestMs = Math.round(performance.now() - tIngestStart); + if (ingestMs > SLOW_INGEST_WARN_MS) { + // #5744: attach WAL forensics so a stall spike can be attributed + // to a checkpoint burst (multi-second tick inside the window) + // instead of per-row statement cost — read-only, warn path only. + const walSuffix = this.sync.walDiagnosticsSuffix() ?? ""; + loggerLog("warn", `[storage] slow-ingest: ${legacy.id} path=${ingestPath} took=${ingestMs}ms${walSuffix} — synchronous per-save storage work is starving the event loop (#5744); path=full means the delta fast path fell back (watermark missing/cleared or tail shape changed)`); + } // #3133 sampled verification: ingest integrity is already guarded // by the SQLite write path; the read-back digest is a migration // parity oracle, so it runs on the first ingest of a session and diff --git a/src/storage/store.ts b/src/storage/store.ts index 73beaa345b..78c3b7bd82 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -42,6 +42,10 @@ const WAL_CHECKPOINT_EVERY_INGESTS = 20; /** PR #4725 N3: covered-seq span point queries chunk the msg_id IN (…) list * to stay under SQLite's 999 bound-parameter limit (1 session id + 500). */ const SEQ_SPAN_QUERY_BATCH = 500; +/** Max bound parameters per generated statement (refs batching, #5744). + * Well under the conservative 999-parameter default some SQLite builds + * still ship with — see sqlite3.MAX_BIND_PARAMS. */ +const REF_STMT_MAX_PARAMS = 700; /** #4796: rows per batch of gc()'s zero-ref CAS sweep. Sized so one batch's * page churn stays modest even when the store is dominated by multi-MB @@ -166,8 +170,14 @@ export interface BlockRow { tier: number; /** JSON of the full BlockView {text, count}. */ summary: string; + /** #5744: sha256(summary) computed by the planner — lets ingest + * compare against the stored row BEFORE any CAS op, so an + * unchanged block costs neither a re-hash nor a refcount dance. */ + summaryHash: string; /** JSON of the one-line BlockView, or null (legacy #401 collapse). */ one: string | null; + /** #5744: sha256(one), or null with `one`. */ + oneHash: string | null; supersedesBlockId: string | null; active: boolean; createdAt: number; @@ -188,6 +198,9 @@ export interface CcrEntryRow { ord: number; key: string; content: string; + /** #5744: sha256(content) from the planner — unchanged entries skip the + * CAS bump/release round-trips entirely on steady-state ingests. */ + contentHash: string; } /** One checkpoint/replica row (Phase 2, #2671). `state_hash` never leaves @@ -421,6 +434,10 @@ export class UnifiedStore { private readonly db: SqliteDatabase; /** #3250: successful-ingest counter driving the WAL_CHECKPOINT_EVERY_INGESTS cadence. */ private ingestCount = 0; + /** #5744: last periodic checkpoint result (plus its wall time) for the + * slow-ingest warn line — lets an operator attribute a stall spike to a + * checkpoint burst instead of guessing. Null until the first tick. */ + private lastWalTick: { busy: number; log: number; checkpointed: number; ms: number } | null = null; /** #4175: the live foreign pid found holding .lock at open, if any. */ private readonly foreignLockHolder: { pid: number } | null; private readonly lockPath: string | null; @@ -1095,14 +1112,29 @@ export class UnifiedStore { // via reapDestroyedSubstrate) — they would resurrect on every // unified load. Drop absent ids first, releasing their content // refs inside the same transaction (deleteSession discipline). - const incomingBlockIds = new Set(input.blocks.map((b) => b.blockId)); - const staleBlocks = this.db - .prepare("SELECT block_id, summary_hash, one_hash FROM blocks WHERE session_id = ?") - .all(s.sessionId) as Array<{ block_id: string; summary_hash: string; one_hash: string | null }>; - for (const stale of staleBlocks) { - if (incomingBlockIds.has(stale.block_id)) continue; + // #5744: ONE full-row read serves both the stale sweep and the + // unchanged-row skip below (previously: 1 hash-only sweep read + + // 1 point read per block per save). + type StoredBlockRow = { + block_id: string; ord: number; start_seq: number; end_seq: number; + start_ref: number | null; end_ref: number | null; tier: number; + summary_hash: string; one_hash: string | null; supersedes_block_id: string | null; + active: number; created_at: number; covered_hash: string; payload_json: string; + }; + const storedBlocks = new Map(); + for (const row of this.db + .prepare( + "SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, " + + "summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json " + + "FROM blocks WHERE session_id = ?", + ) + .all(s.sessionId) as StoredBlockRow[]) { + storedBlocks.set(row.block_id, row); + } + for (const [blockId, stale] of storedBlocks) { + if (input.blocks.some((b) => b.blockId === blockId)) continue; // referencing row first, then release (#4242 delete-on-zero) - this.db.prepare("DELETE FROM blocks WHERE session_id = ? AND block_id = ?").run(s.sessionId, stale.block_id); + this.db.prepare("DELETE FROM blocks WHERE session_id = ? AND block_id = ?").run(s.sessionId, blockId); this.releaseContent(stale.summary_hash); if (stale.one_hash !== null) this.releaseContent(stale.one_hash); } @@ -1163,109 +1195,90 @@ export class UnifiedStore { } catch { /* malformed payload: keep the no-claim sentinel */ } return hi < 0 ? [-1, -1] : [lo, hi]; }; + const blocksUpsert = this.db.prepare( + `INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, + summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, block_id) DO UPDATE SET + ord = excluded.ord, + start_seq = CASE WHEN excluded.start_seq >= 0 THEN excluded.start_seq ELSE blocks.start_seq END, + end_seq = CASE WHEN excluded.end_seq >= 0 THEN excluded.end_seq ELSE blocks.end_seq END, + start_ref = excluded.start_ref, end_ref = excluded.end_ref, tier = excluded.tier, + summary_hash = excluded.summary_hash, one_hash = excluded.one_hash, + supersedes_block_id = excluded.supersedes_block_id, active = excluded.active, + created_at = excluded.created_at, covered_hash = excluded.covered_hash, + payload_json = excluded.payload_json`, + ); for (const b of input.blocks) { const [bStartSeq, bEndSeq] = boundsForCovered(b.payloadJson); - const summaryHash = this.putContent(b.summary); - const oneHash = b.one === null ? null : this.putContent(b.one); + // #5744: steady-state saves re-send every block byte-identical. + // When the effective post-upsert row equals the stored row the + // whole write (and its CAS bump/release dance, which nets to + // zero anyway) is skipped — the ingest then costs no statement + // per unchanged block. #4608 bound preservation makes "the + // incoming -1 keeps the stored value" part of the equality. + const ex = storedBlocks.get(b.blockId); + if (ex && + ex.ord === b.ord && + (bStartSeq < 0 || bStartSeq === ex.start_seq) && + (bEndSeq < 0 || bEndSeq === ex.end_seq) && + (ex.start_ref ?? null) === (b.startRef ?? null) && + (ex.end_ref ?? null) === (b.endRef ?? null) && + ex.tier === b.tier && + ex.summary_hash === b.summaryHash && + (ex.one_hash ?? null) === (b.oneHash ?? null) && + (ex.supersedes_block_id ?? null) === b.supersedesBlockId && + ex.active === (b.active ? 1 : 0) && + ex.created_at === b.createdAt && + ex.covered_hash === b.coveredHash && + ex.payload_json === b.payloadJson) { + continue; + } + const summaryHash = this.putContent(b.summary, b.summaryHash); + const oneHash = b.one === null ? null : this.putContent(b.one, b.oneHash ?? undefined); // Re-ingest of the same block id replaces the row (dry-run // idempotence). Refcount math must distinguish identical // re-ingest (bump then undo) from a true replacement (release // the old hash, keep the new bump) — otherwise identical // re-ingest drives the surviving refcount to zero. - const existing = this.db - .prepare("SELECT summary_hash, one_hash FROM blocks WHERE session_id = ? AND block_id = ?") - .get(s.sessionId, b.blockId) as { summary_hash: string; one_hash: string | null } | undefined; - if (existing) { - // #4608: preserve interval bounds across resaves. A later batch that - // no longer contains the covered raws (a post-fold client resend sees - // the folded view, not the raw tail) re-derives -1/-1 "unknown" from - // its own log; once real evidence exists it is durable — the upsert's - // CASE keeps the stored bounds when the incoming side is unknown. - // #4242: redirect the row BEFORE the undo/release so a - // released-to-zero old hash is no longer FK-referenced by - // this blocks row when the content DELETE fires. The - // identical-re-ingest undo (same hash) is a pure -1 - // against the +1 bump and cannot reach zero while the row - // itself holds the reference. - // Re-saves bind the TRANSCRIPT-resolved bounds (same source - // as the fresh-insert branch below), never the plan-local - // b.startSeq/endSeq: those are positions in THIS plan's log, - // and once a covered id's old generation is tombstoned (an - // in-place edit lands the new one past max(seq)) they no - // longer coincide with absolute seqs — writing them would - // clobber the durable interval with plan-relative numbers. - this.db - .prepare( - `INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, - summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id, block_id) DO UPDATE SET - ord = excluded.ord, - start_seq = CASE WHEN excluded.start_seq >= 0 THEN excluded.start_seq ELSE blocks.start_seq END, - end_seq = CASE WHEN excluded.end_seq >= 0 THEN excluded.end_seq ELSE blocks.end_seq END, - start_ref = excluded.start_ref, end_ref = excluded.end_ref, tier = excluded.tier, - summary_hash = excluded.summary_hash, one_hash = excluded.one_hash, - supersedes_block_id = excluded.supersedes_block_id, active = excluded.active, - created_at = excluded.created_at, covered_hash = excluded.covered_hash, - payload_json = excluded.payload_json`, - ) - .run( - b.blockId, - s.sessionId, - b.ord, - bStartSeq, - bEndSeq, - b.startRef, - b.endRef, - b.tier, - summaryHash, - oneHash, - b.supersedesBlockId, - b.active ? 1 : 0, - b.createdAt, - b.coveredHash, - b.payloadJson, - ); - if (existing.summary_hash === summaryHash) this.releaseContent(summaryHash); - else this.releaseContent(existing.summary_hash); - if (existing.one_hash !== null) { - if (existing.one_hash === oneHash && oneHash !== null) this.releaseContent(oneHash); - else this.releaseContent(existing.one_hash); + // #4242: redirect the row BEFORE the undo/release so a + // released-to-zero old hash is no longer FK-referenced by + // this blocks row when the content DELETE fires. The + // identical-re-ingest undo (same hash) is a pure -1 + // against the +1 bump and cannot reach zero while the row + // itself holds the reference. + // Re-saves bind the TRANSCRIPT-resolved bounds, never the + // plan-local b.startSeq/endSeq: those are positions in THIS + // plan's log, and once a covered id's old generation is + // tombstoned (an in-place edit lands the new one past + // max(seq)) they no longer coincide with absolute seqs — + // writing them would clobber the durable interval with + // plan-relative numbers. + blocksUpsert.run( + b.blockId, + s.sessionId, + b.ord, + bStartSeq, + bEndSeq, + b.startRef, + b.endRef, + b.tier, + summaryHash, + oneHash, + b.supersedesBlockId, + b.active ? 1 : 0, + b.createdAt, + b.coveredHash, + b.payloadJson, + ); + if (ex) { + if (ex.summary_hash === summaryHash) this.releaseContent(summaryHash); + else this.releaseContent(ex.summary_hash); + if (ex.one_hash !== null) { + if (ex.one_hash === oneHash && oneHash !== null) this.releaseContent(oneHash); + else this.releaseContent(ex.one_hash); } - continue; } - this.db - .prepare( - `INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, - summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id, block_id) DO UPDATE SET - ord = excluded.ord, - start_seq = CASE WHEN excluded.start_seq >= 0 THEN excluded.start_seq ELSE blocks.start_seq END, - end_seq = CASE WHEN excluded.end_seq >= 0 THEN excluded.end_seq ELSE blocks.end_seq END, - start_ref = excluded.start_ref, end_ref = excluded.end_ref, tier = excluded.tier, - summary_hash = excluded.summary_hash, one_hash = excluded.one_hash, - supersedes_block_id = excluded.supersedes_block_id, active = excluded.active, - created_at = excluded.created_at, covered_hash = excluded.covered_hash, - payload_json = excluded.payload_json`, - ) - .run( - b.blockId, - s.sessionId, - b.ord, - bStartSeq, - bEndSeq, - b.startRef, - b.endRef, - b.tier, - summaryHash, - oneHash, - b.supersedesBlockId, - b.active ? 1 : 0, - b.createdAt, - b.coveredHash, - b.payloadJson, - ); } // dead_refs: tombstones can be LIFTED (backing message @@ -1282,31 +1295,119 @@ export class UnifiedStore { // refs: the incoming array is AUTHORITATIVE — native compaction // and the #2695 zombie reap renumber byRaw/byRef (survivors move - // to lower ords, pruned ids vanish). Upserting onto that shape - // either hits the (session_id, ord) PK (ingest rolls back, 3-strike - // self-disable — review #2690 blocking 1) or leaves ghost rows that - // make the loaded map strictly larger than legacy (permanent - // DIGEST DRIFT spam — blocking 2). Replace the whole set. - this.db.prepare("DELETE FROM refs WHERE session_id = ?").run(s.sessionId); + // to lower ords, pruned ids vanish). A blind upsert onto that shape + // would hit the (session_id, ord) PK (review #2690 blocking 1) or + // leave ghost rows (blocking 2), which is why this used to + // DELETE-all + re-insert EVERY row on EVERY save. That made the + // store leg of an O(new messages) delta save O(full history): + // ~H native statement executions per turn on a long session + // (~97% of all per-save statements; the CDP profile behind #5744) + // while touching zero history rows in the append-only steady + // state. Now: one indexed read of the stored set, an in-memory + // diff, and writes ONLY for what changed — vanished raw_ids are + // deleted (ghosts can never outlive one save, so the loaded map + // stays exactly the legacy one), untouched survivors are skipped, + // and added/moved rows go out as multi-row upsert batches (a + // first ingest of a 3k-message session costs ~20 statements, + // not 3001; the compaction storm that motivated the old + // replace-all now costs the same bounded batch budget). + // + // Load-bearing precondition (guaranteed by the producers, not + // checked here): a surviving raw_id's ord never INCREASES between + // saves — appends take higher ords, and the only renumberers + // (native-compaction archive #395 and zombie reap #2695, both via + // pruneRefsToLiveIds) preserve survivor key order and only drop + // ids. The incremental upsert depends on that: a new ord crossing a + // still-live neighbour's ord violates the (session_id, ord) PK + // mid-batch and rolls the whole ingest back. If a future producer + // can reorder survivors upward, add a delete-all fallback around + // the write phase below. + const oldRefs = new Map(); + for (const row of this.db + .prepare("SELECT raw_id, ord, ref, ref_num FROM refs WHERE session_id = ?") + .all(s.sessionId) as Array<{ raw_id: string; ord: number; ref: string; ref_num: number | null }>) { + oldRefs.set(row.raw_id, { ord: row.ord, ref: row.ref, refNum: row.ref_num }); + } + const goneRefIds: string[] = []; + const touchedRefs: RefRow[] = []; for (const r of input.refs) { - this.db - .prepare( - "INSERT INTO refs (session_id, ord, raw_id, ref, ref_num) VALUES (?, ?, ?, ?, ?) ON CONFLICT(session_id, raw_id) DO UPDATE SET ord = excluded.ord, ref = excluded.ref, ref_num = excluded.ref_num", - ) - .run(s.sessionId, r.ord, r.rawId, r.ref, r.refNum); + const o = oldRefs.get(r.rawId); + if (!o || o.ord !== r.ord || o.ref !== r.ref || o.refNum !== r.refNum) { + touchedRefs.push(r); + } else { + oldRefs.delete(r.rawId); // matched and unmodified — consume it + } + } + for (const rawId of oldRefs.keys()) goneRefIds.push(rawId); + if (goneRefIds.length > 0) { + const perChunk = REF_STMT_MAX_PARAMS - 1; // reserve the leading session_id slot + for (let i = 0; i < goneRefIds.length; i += perChunk) { + const ids = goneRefIds.slice(i, i + perChunk); + this.db + .prepare( + "DELETE FROM refs WHERE session_id = ? AND raw_id IN (" + ids.map(() => "?").join(", ") + ")", + ) + .run(s.sessionId, ...ids); + } + } + if (touchedRefs.length > 0) { + const colsPerRow = 5; + const perChunk = Math.floor(REF_STMT_MAX_PARAMS / colsPerRow); + for (let i = 0; i < touchedRefs.length; i += perChunk) { + const rows = touchedRefs.slice(i, i + perChunk); + const params: unknown[] = []; + for (const r of rows) params.push(s.sessionId, r.ord, r.rawId, r.ref, r.refNum); + this.db + .prepare( + "INSERT INTO refs (session_id, ord, raw_id, ref, ref_num) VALUES " + + rows.map(() => "(?, ?, ?, ?, ?)").join(", ") + + " ON CONFLICT(session_id, raw_id) DO UPDATE SET ord = excluded.ord, ref = excluded.ref, ref_num = excluded.ref_num", + ) + .run(...params); + } } // ccr_entries: the content store is a map whose keys can be - // dropped — full-replace keeps the table authoritative. Release - // every old hash first, then re-insert (idempotent: identical - // sets cancel out bump+release). - const oldCcr = this.db - .prepare("SELECT content_hash FROM ccr_entries WHERE session_id = ?") - .all(s.sessionId) as { content_hash: string }[]; - this.db.prepare("DELETE FROM ccr_entries WHERE session_id = ?").run(s.sessionId); - for (const row of oldCcr) this.releaseContent(row.content_hash); + // dropped — the final state must be EXACTLY the incoming set. + // #5744: steady state no longer pays DELETE-all + re-INSERT-all + // (which also destroyed and recreated every CAS blob via + // release-to-zero each save). Unchanged entries (same key → same + // ord AND same content hash) are skipped entirely — no CAS + // round-trips. Changed/absent rows are deleted FIRST (frees their + // PK ords so renumbered swaps cannot collide), their old hashes + // released while nothing references them anymore (#4242 + // redirect-then-release), then the changed+new entries are + // put+inserted. + const incomingCcr = new Map(); + for (const e of input.ccrEntries) incomingCcr.set(e.key, e); + const storedCcr = this.db + .prepare("SELECT ord, ccr_key, content_hash FROM ccr_entries WHERE session_id = ?") + .all(s.sessionId) as Array<{ ord: number; ccr_key: string; content_hash: string }>; + const changedCcrKeys: string[] = []; + const unchangedCcrKeys = new Set(); + const releasedOldCcrHashes: string[] = []; + for (const row of storedCcr) { + const inc = incomingCcr.get(row.ccr_key); + if (!inc) { changedCcrKeys.push(row.ccr_key); releasedOldCcrHashes.push(row.content_hash); continue; } + if (row.ord !== inc.ord || row.content_hash !== inc.contentHash) { + changedCcrKeys.push(row.ccr_key); + releasedOldCcrHashes.push(row.content_hash); + } else { + unchangedCcrKeys.add(row.ccr_key); + } + } + if (changedCcrKeys.length > 0) { + for (let i = 0; i < changedCcrKeys.length; i += SEQ_SPAN_QUERY_BATCH) { + const keys = changedCcrKeys.slice(i, i + SEQ_SPAN_QUERY_BATCH); + this.db + .prepare(`DELETE FROM ccr_entries WHERE session_id = ? AND ccr_key IN (${keys.map(() => "?").join(", ")})`) + .run(s.sessionId, ...keys); + } + } + for (const hash of releasedOldCcrHashes) this.releaseContent(hash); for (const e of input.ccrEntries) { - const hash = this.putContent(e.content); + if (unchangedCcrKeys.has(e.key)) continue; + const hash = this.putContent(e.content, e.contentHash); this.db .prepare("INSERT INTO ccr_entries (session_id, ord, ccr_key, content_hash) VALUES (?, ?, ?, ?)") .run(s.sessionId, e.ord, e.key, hash); @@ -1326,16 +1427,38 @@ export class UnifiedStore { try { // Log the result: operators tuning storage.walJournalSizeLimitBytes // need to see whether ticks are actually reclaiming frames. + const t0 = performance.now(); const row = this.db.prepare("PRAGMA wal_checkpoint(PASSIVE)").get() as { busy: number; log: number; checkpointed: number }; + const ms = Math.round(performance.now() - t0); + this.lastWalTick = { busy: row.busy, log: row.log, checkpointed: row.checkpointed, ms }; loggerLog( "debug", - `[storage] periodic WAL checkpoint at ingest ${this.ingestCount}: busy=${row.busy} log=${row.log} checkpointed=${row.checkpointed}`, + `[storage] periodic WAL checkpoint at ingest ${this.ingestCount}: busy=${row.busy} log=${row.log} checkpointed=${row.checkpointed} took=${ms}ms`, ); } catch (err) { loggerLog("debug", `[storage] periodic WAL checkpoint failed at ingest ${this.ingestCount}: ${String(err)}`); } } + /** #5744: read-only WAL forensics for the slow-ingest warn line — current + * WAL size plus the last periodic checkpoint's (busy, log, checkpointed) + * triple and duration. A slow save whose window contains a multi-second + * tick is a checkpoint burst, not per-row statement cost; the two must be + * told apart from the log alone. Returns null when there is nothing to + * report (no WAL file). Never throws into the caller. */ + walDiagnosticsSuffix(): string | null { + let walBytes: number; + try { + walBytes = statSync(`${this.dbPath}-wal`).size; + } catch { + return null; + } + const tick = this.lastWalTick + ? ` lastCheckpoint=(busy=${this.lastWalTick.busy} log=${this.lastWalTick.log} checkpointed=${this.lastWalTick.checkpointed} took=${this.lastWalTick.ms}ms)` + : ""; + return `, ${formatDbSize(walBytes)} WAL${tick}`; + } + /** Full-ingest path of ingestSession, extracted for readability: the * absolute-seq remap plus the #3133 watermark alignment decision. * Mutates input.session.tailJson (pointer sequences become absolute). */ @@ -3232,6 +3355,7 @@ export class UnifiedStore { ord: r.ord as number, key: r.ccr_key as string, content: this.getContent(r.content_hash as string) ?? "", + contentHash: r.content_hash as string, })); } } diff --git a/tests/issue4739-storage-read-snapshot.test.ts b/tests/issue4739-storage-read-snapshot.test.ts index 5d77b9cc8c..ad259e4717 100644 --- a/tests/issue4739-storage-read-snapshot.test.ts +++ b/tests/issue4739-storage-read-snapshot.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { mkdtempSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; -import { UnifiedStore, type IngestSessionInput } from "../src/storage/store.ts"; +import { UnifiedStore, sha256Hex, type IngestSessionInput } from "../src/storage/store.ts"; import { SqliteBackend } from "../src/storage/backend.ts"; import { openSqliteDatabase } from "../src/storage/driver.ts"; import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; @@ -20,12 +20,15 @@ function fixture(requests = 1): IngestSessionInput { seq, msgId: `m${seq}`, ref: null, refNum: null, role: "user", kind: "message", content: JSON.stringify({ id: `m${seq}`, role: "user", text: `text${seq}` }), })), - blocks: [0, 1, 2].map((ord) => ({ - blockId: `b${ord}`, ord, startSeq: ord, endSeq: ord, startRef: null, endRef: null, tier: 0, - summary: JSON.stringify({ text: `full${ord}`, count: 1 }), - one: JSON.stringify({ text: `one${ord}`, count: 1 }), supersedesBlockId: null, - active: ord !== 1, createdAt: 1, coveredHash: "[]", payloadJson: JSON.stringify({ blockId: `b${ord}` }), - })), + blocks: [0, 1, 2].map((ord) => { + const summary = JSON.stringify({ text: `full${ord}`, count: 1 }); + const one = JSON.stringify({ text: `one${ord}`, count: 1 }); + return { + blockId: `b${ord}`, ord, startSeq: ord, endSeq: ord, startRef: null, endRef: null, tier: 0, + summary, one, summaryHash: sha256Hex(summary), oneHash: sha256Hex(one), supersedesBlockId: null, + active: ord !== 1, createdAt: 1, coveredHash: "[]", payloadJson: JSON.stringify({ blockId: `b${ord}` }), + }; + }), refs: [ { ord: 0, rawId: "a", ref: "m00001", refNum: 1 }, { ord: 1, rawId: "b", ref: "m00001", refNum: 1 }, diff --git a/tests/issue5744-incremental-refs.test.ts b/tests/issue5744-incremental-refs.test.ts new file mode 100644 index 0000000000..42e7a66741 --- /dev/null +++ b/tests/issue5744-incremental-refs.test.ts @@ -0,0 +1,239 @@ +/** + * #5744: refs-table maintenance must cost O(changed rows) per save, not O(full history). + * + * The `refs` table holds one row per message the session has ever seen (the + * byRaw ledger, verbatim). It used to be maintained by DELETE-all + one-row + * INSERT on EVERY save, so a steady-state delta save of 2 new messages into a + * 3k-message session executed ~3001 native SQLite statements and rewrote the + * whole set's WAL footprint — 96.5% of self time in node:sqlite + * StatementSync.run on a Windows instance, event-loop starvation behind it + * (#5744). The store now diffs against one indexed read of the stored set and + * writes only what changed (vanished raw_ids deleted, untouched survivors + * skipped, added/moved rows emitted as multi-row upsert batches), so the + * append-only steady state costs ~2 statements regardless of history length. + * + * This suite pins: + * 1. end-state equality with the legacy byRaw map after appends and after a + * compaction-shaped renumber (pruned ids vanish, survivors shift to lower + * ords) — the replace-all contract from review #2690 must still hold; + * 2. the statement budget: a save touching the `refs` table executes at most + * a handful of statements even when the stored set is large (the old + * code would execute H+1); + * 3. no ghost rows survive an open/close cycle. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import type { CoreMessage } from "acp-kernel"; + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: a developer machine with +// storage.unified in its real config must not leak a tier into these tests. +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5744-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +function tmpdir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +const refToken = (i: number): string => `m${String(i + 1).padStart(5, "0")}`; + +function msg(i: number): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text: `hello ${i}` } as CoreMessage; +} + +/** Full-history presentation: the anthropic shape where every request re-sends + * the whole conversation; byRaw covers exactly the presented ids in order. */ +function mkLegacy(id: string, count: number, dropPrefix = 0): LegacySessionLike { + const byRaw: Record = {}; + const lastMessages: CoreMessage[] = []; + for (let i = dropPrefix; i < count; i++) { + byRaw[`h_msg${i}`] = refToken(dropPrefix === 0 ? i : i - dropPrefix); + lastMessages.push(msg(i)); + } + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "anthropic" }, + stats: { requests: 1 }, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + }; +} + +function openPair(dir: string): { shadow: UnifiedShadow; store: UnifiedStore } { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow mode sidecar must open"); + return { shadow, store }; +} + +interface StatementMeter { + /** Statements executed since the last reset() whose SQL touches the refs table. */ + refsTouching: () => number; + reset: () => void; + dispose: () => void; +} + +/** Counts statement executions whose SQL touches the refs table. \brefs\b + * matches the refs table only (never dead_refs — no word boundary before it). */ +function instrument(store: UnifiedStore): StatementMeter { + const db = (store as unknown as { + db: { prepare(q: string): unknown } & Record; + }).db; + const orig = (db.prepare as (q: string) => { run(...a: unknown[]): unknown; get(...a: unknown[]): unknown; all(...a: unknown[]): unknown }).bind(db); + let touching = 0; + db.prepare = ((q: string) => { + const stmt = orig(q); + const isRefs = /\brefs\b/.test(q); + return { + run: (...args: unknown[]) => { + if (isRefs) touching += 1; + return stmt.run(...args); + }, + get: (...args: unknown[]) => { + if (isRefs) touching += 1; + return stmt.get(...args); + }, + all: (...args: unknown[]) => { + if (isRefs) touching += 1; + return stmt.all(...args); + }, + }; + }) as (q: string) => unknown; + return { + refsTouching: () => touching, + reset: () => { + touching = 0; + }, + dispose: () => { + db.prepare = orig as never; + }, + }; +} + +test("#5744: steady-state saves touch O(new) refs rows, never the full history", () => { + const dir = tmpdir("bili-5744-steady-"); + try { + const { shadow, store } = openPair(dir); + // record() writes through the shadow's OWN store instance — a second + // connection to the same file. A meter on the standalone handle above + // would count zero statements and make every budget below vacuous. + const meter = instrument((shadow as unknown as { sync: UnifiedStore }).sync); + try { + const n0 = 100; + let legacy = mkLegacy("steady", n0); + meter.reset(); + shadow.record(legacy); + assert.equal(store.getRefs("steady").length, n0, "first save materializes the full ledger once"); + + // Steady state: +2 messages, everything else unchanged. The old + // replace-all code executed n+1 refs statements here (~102 for + // this size, ~3001 on the reporter's sessions); the diff path may + // spend at most the old-set read + one batched upsert of the new + // rows (and nothing else touches the table). + for (let turn = 1; turn <= 5; turn++) { + legacy = mkLegacy("steady", n0 + turn * 2); + meter.reset(); + shadow.record(legacy); + assert.equal( + store.getRefs("steady").length, + n0 + turn * 2, + `turn ${turn}: ledger must track the session`, + ); + // >= 2: the old-set read plus the upsert batch must actually + // execute — a zero count means the meter watches no writes. + assert.ok( + meter.refsTouching() >= 2 && meter.refsTouching() <= 3, + `turn ${turn}: ${meter.refsTouching()} refs statements for +2 rows (old code: ~${n0 + turn * 2 + 1})`, + ); + } + + // End-state parity: every stored row equals the legacy map, with + // ord = key order and ref_num decoded from the token. + const expected = new Map(Object.entries(legacy.state.messageRefs.byRaw).map(([rawId, ref], ord) => [rawId, { ord, ref }])); + const actual = store.getRefs("steady").map((r) => ({ rawId: r.rawId, ord: r.ord, ref: r.ref })); + assert.equal(actual.length, expected.size); + for (const row of actual) { + const want = expected.get(row.rawId); + assert.ok(want, `unexpected ghost raw_id ${row.rawId}`); + assert.equal(row.ord, want.ord); + assert.equal(row.ref, want.ref); + } + } finally { + meter.dispose(); + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("#5744: compaction-shaped renumber reconciles in bounded statements without ghosts", () => { + const dir = tmpdir("bili-5744-renumber-"); + try { + const { shadow, store } = openPair(dir); + const meter = instrument((shadow as unknown as { sync: UnifiedStore }).sync); + try { + const n = 120; + const pruned = 20; // first 20 ids are "reaped": survivors shift down + shadow.record(mkLegacy("renumber", n)); + assert.equal(store.getRefs("renumber").length, n); + + meter.reset(); + shadow.record(mkLegacy("renumber", n, pruned)); + const rows = store.getRefs("renumber"); + assert.equal(rows.length, n - pruned, "pruned ids must leave the table — no ghosts"); + // Survivors renumbered to lower ords in map order. + rows.forEach((r, i) => assert.equal(r.ord, i, `survivor at position ${i} keeps its shifted ord`)); + for (const r of rows) assert.ok(!/^h_msg\d+$/.test(r.rawId) || Number(r.rawId.slice(5)) >= pruned, "reaped id leaked back"); + // Bounded write: 1 delete (IN-list) + the read + batched upserts + // of the ~100 moved rows (≤ ceil(100/160) statements) — and NOT + // the old n+1 replace storm. + assert.ok( + meter.refsTouching() >= 2 && meter.refsTouching() <= 6, + `${meter.refsTouching()} refs statements for a full renumber (old code: ${n + 1})`, + ); + + // Ghost check across an open/close cycle: reload reads the table + // straight from disk. + shadow.close(); + store.close(); + const reopened = UnifiedStore.open(path.join(dir, "index.db")); + try { + assert.equal(reopened.getRefs("renumber").length, n - pruned, "reopen sees exactly the live ledger"); + } finally { + reopened.close(); + } + } finally { + meter.dispose(); + } + } finally { + rmBestEffort(dir); + } +}); diff --git a/tests/issue5744-ingest-diff.test.ts b/tests/issue5744-ingest-diff.test.ts new file mode 100644 index 0000000000..bacb67e1c6 --- /dev/null +++ b/tests/issue5744-ingest-diff.test.ts @@ -0,0 +1,311 @@ +/** + * #5744 (companion to issue5744-incremental-refs.test.ts): the refs ledger was + * the dominant O(full history) leg, but the SAME full-replace shape existed on + * two more tables maintained by ingestSession — ccr_entries (DELETE-all + + * re-INSERT-all every save, which also destroyed and recreated every CAS blob + * via release-to-zero) and blocks (per-block re-hash + upsert + CAS + * bump/release dance even when the block was byte-identical). This suite pins + * the incremental treatment of those two legs: + * 1. statement budgets: steady-state saves touching ccr_entries / blocks cost + * a constant number of statements regardless of stored-set size (the old + * code paid ~4H and ~6B respectively); + * 2. end-state equality: add/change/delete of ccr keys lands EXACTLY the + * incoming set with honest refcounts (no zero-refcount rows, deleted blobs + * reclaimed); + * 3. convergence: a forced full-path save (watermark cleared) reproduces the + * exact same table state the diff path left behind — the "incoming set is + * authoritative" contract holds on both paths. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { UnifiedStore, sha256Hex } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import type { CoreMessage, CompressionBlock } from "../kernel/src/types.js"; + +process.env.BILI_PERSIST_ZSTD = "0"; +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5744c-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +function tmpdir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +const refToken = (i: number): string => `m${String(i + 1).padStart(5, "0")}`; + +function msg(i: number): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text: `hello ${i}` } as CoreMessage; +} + +const CCR_KEYS = 400; +const BLOCKS = 60; + +function ccrStore(vary: number) { + const byRef: Record = {}; + for (let i = 0; i < CCR_KEYS; i++) byRef[`m${String(i).padStart(5, "0")}`] = { text: `ccr payload ${i}`, vary }; + return { version: 1 as const, byHash: {} as Record, byRef }; +} + +function blockFixture(ord: number, mutate = false): CompressionBlock { + const coveredIds = [`h_msg${ord * 2}`, `h_msg${ord * 2 + 1}`]; + return { + blockId: `blk${ord}`, + runId: `run${ord}`, + summary: mutate ? `mutated summary ${ord}` : `summary body ${ord} ${"x".repeat(64)}`, + effectiveMessageIds: coveredIds, + directMessageIds: [], + directBlockIds: [], + compressedTokens: 100 + ord, + survivedCount: 2, + generation: "young", + active: true, + createdAt: 1_000 + ord, + tier: 1, + } as CompressionBlock; +} + +/** Full-history presentation; byRaw/byRef cover exactly the presented ids. */ +function mkLegacy(id: string, count: number, ccrVary: number, mutateBlock: number | null): LegacySessionLike { + const byRaw: Record = {}; + const byRef: Record = {}; + const lastMessages: CoreMessage[] = []; + for (let i = 0; i < count; i++) { + const ref = refToken(i); + byRaw[`h_msg${i}`] = ref; + byRef[ref] = `h_msg${i}`; + lastMessages.push(msg(i)); + } + const blocks: CompressionBlock[] = []; + const blockContents = new Map(); + for (let ord = 0; ord < BLOCKS; ord++) { + const b = blockFixture(ord, mutateBlock === ord); + blocks.push(b); + blockContents.set(`blk${ord}`, { + one: { text: `one ${ord}`, count: 1 }, + full: { text: b.summary as string, count: 2 }, + }); + } + return { + id, + createdAt: 1_000, + lastSeen: 2_000 + count, + meta: { protocol: "anthropic" }, + stats: { requests: count }, + metadata: {}, + state: { + blocks, + messageRefs: { byRaw, byRef }, + nextBlockId: BLOCKS + 1, + nextRunId: 1, + }, + blockContents, + lastMessages, + lastMessagesFolded: false, + contentStore: ccrStore(ccrVary), + } as LegacySessionLike; +} + +interface Meters { + ccr: () => number; + blocks: () => number; + reset: () => void; + dispose: () => void; +} + +function instrument(store: UnifiedStore): Meters { + const db = (store as unknown as { + db: { prepare(q: string): unknown } & Record; + }).db; + const orig = (db.prepare as (q: string) => { run(...a: unknown[]): unknown; get(...a: unknown[]): unknown; all(...a: unknown[]): unknown }).bind(db); + let ccr = 0; + let blocks = 0; + db.prepare = ((q: string) => { + const stmt = orig(q); + const isCcr = q.includes("ccr_entries"); + const isBlocks = /\bblocks\b/.test(q); + return { + run: (...args: unknown[]) => { + if (isCcr) ccr += 1; + if (isBlocks) blocks += 1; + return stmt.run(...args); + }, + get: (...args: unknown[]) => { + if (isCcr) ccr += 1; + if (isBlocks) blocks += 1; + return stmt.get(...args); + }, + all: (...args: unknown[]) => { + if (isCcr) ccr += 1; + if (isBlocks) blocks += 1; + return stmt.all(...args); + }, + }; + }) as typeof orig; + return { + ccr: () => ccr, + blocks: () => blocks, + reset: () => { ccr = 0; blocks = 0; }, + dispose: () => { db.prepare = orig; }, + }; +} + +function openPair(dir: string): { shadow: UnifiedShadow; store: UnifiedStore } { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow mode sidecar must open"); + return { shadow, store }; +} + +const SID = "sess-diff"; + +test("#5744: steady-state saves touch ccr_entries and blocks with a constant statement budget", (t) => { + const dir = tmpdir("bili-5744c-budget-"); + t.after(() => rmBestEffort(dir)); + const { shadow, store } = openPair(dir); + t.after(() => shadow.close()); + t.after(() => store.close()); + // Instrument the store the SHADOW actually writes through — openPair's + // standalone handle is a second connection to the same file, and a meter + // on it would read zero and make every budget below vacuous. + const m = instrument((shadow as unknown as { sync: UnifiedStore }).sync); + t.after(() => m.dispose()); + + // Turn 0: cold full ingest — allowed to pay whatever a first ingest costs. + shadow.record(mkLegacy(SID, 300, 0, null)); + m.reset(); + + // Steady state: append 2 messages, identical ccr store and blocks. + for (let turn = 1; turn <= 4; turn++) { + shadow.record(mkLegacy(SID, 300 + turn * 2, 0, null)); + // One indexed read of the stored set per table; nothing else may touch + // the table in steady state. (Old code: ccr ~4*CCR_KEYS, blocks ~6*BLOCKS.) + // The >= 1 floors prove the meter sees the write path at all. + assert.ok(m.ccr() >= 1 && m.ccr() <= 2, `turn ${turn}: ccr_entries touched ${m.ccr()} times, want 1..2`); + assert.ok(m.blocks() >= 1 && m.blocks() <= 3, `turn ${turn}: blocks touched ${m.blocks()} times, want 1..3`); + m.reset(); + } +}); + +test("#5744: ccr add/change/delete lands exactly the incoming set with honest refcounts", (t) => { + const dir = tmpdir("bili-5744c-ccr-"); + t.after(() => rmBestEffort(dir)); + const { shadow, store } = openPair(dir); + t.after(() => shadow.close()); + t.after(() => store.close()); + + shadow.record(mkLegacy(SID, 100, 0, null)); + + // Mutate the store: drop key idx 2, change value of idx 5, append a new key. + const mutated = ccrStore(7); + delete mutated.byRef[refToken(2)]; + (mutated.byRef[refToken(5)] as { text: string }).text = "changed value 5"; + mutated.byRef["mNEWKEY"] = { text: "brand new entry" }; + const legacy = mkLegacy(SID, 102, 0, null); + legacy.contentStore = mutated; + shadow.record(legacy); + + const rows = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): Array> } } }).db + .prepare("SELECT ord, ccr_key, content_hash FROM ccr_entries WHERE session_id = ? ORDER BY ord") + .all(SID); + assert.equal(rows.length, CCR_KEYS, "entry count survives the mutation (dropped one, added one)"); + for (let i = 0; i < rows.length; i++) { + assert.equal(rows[i].ord as number, i, `ord ${i} is dense`); + } + const byKey = new Map(rows.map((r) => [r.ccr_key as string, r])); + assert.equal(byKey.has(refToken(2)), false, "deleted key is gone"); + assert.equal(byKey.has("mNEWKEY"), true, "new key landed"); + const newHash = sha256Hex(JSON.stringify(mutated.byRef["mNEWKEY"])); + assert.equal(byKey.get("mNEWKEY")?.content_hash as string, newHash, "new key holds the digest of its value"); + const changedHash = sha256Hex(JSON.stringify((mutated.byRef[refToken(5)] as object))); + assert.equal(byKey.get(refToken(5))?.content_hash as string, changedHash, "changed key holds the NEW digest"); + + // Refcount honesty: no zero/negative-refcount rows anywhere; the dropped + // key's blob was reclaimed (delete-on-zero) since nothing else references it. + const bad = (store as unknown as { db: { prepare(q: string): { all(): Array> } } }).db + .prepare("SELECT COUNT(*) AS n FROM content WHERE refcount <= 0").all()[0]; + assert.equal(bad.n as number, 0, "no zero-refcount content rows"); + const droppedHash = sha256Hex(JSON.stringify({ text: "ccr payload 2", vary: 0 })); + const leftover = (store as unknown as { db: { prepare(q: string): { get(...a: unknown[]): unknown } } }).db + .prepare("SELECT 1 FROM content WHERE hash = ?").get(droppedHash); + assert.equal(leftover, undefined, "dropped key's blob was reclaimed"); +}); + +test("#5744: a changed block releases its old summary blob and keeps refcounts honest", (t) => { + const dir = tmpdir("bili-5744c-blocks-"); + t.after(() => rmBestEffort(dir)); + const { shadow, store } = openPair(dir); + t.after(() => shadow.close()); + t.after(() => store.close()); + + shadow.record(mkLegacy(SID, 100, 0, null)); + const oldSummaryJson = JSON.stringify({ text: blockFixture(3).summary as string, count: 2 }); + const oldHash = sha256Hex(oldSummaryJson); + const db = (store as unknown as { db: { prepare(q: string): { get(...a: unknown[]): unknown } } }).db; + assert.notEqual(db.prepare("SELECT 1 FROM content WHERE hash = ?").get(oldHash), undefined, "baseline summary blob present"); + + shadow.record(mkLegacy(SID, 102, 0, 3)); + + assert.equal(db.prepare("SELECT 1 FROM content WHERE hash = ?").get(oldHash), undefined, "replaced summary blob released to zero"); + const bad = (store as unknown as { db: { prepare(q: string): { all(): Array> } } }).db + .prepare("SELECT COUNT(*) AS n FROM content WHERE refcount <= 0").all()[0]; + assert.equal(bad.n as number, 0, "no zero-refcount content rows after the replacement"); +}); + +test("#5744: forced full-path save converges to the exact state the diff path left behind", (t) => { + const dir = tmpdir("bili-5744c-converge-"); + t.after(() => rmBestEffort(dir)); + const { shadow, store } = openPair(dir); + t.after(() => shadow.close()); + t.after(() => store.close()); + const rawDb = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): unknown[] } } }).db; + + shadow.record(mkLegacy(SID, 200, 0, null)); + shadow.record(mkLegacy(SID, 202, 0, null)); // delta turns + + const snapshot = (): string => JSON.stringify([ + rawDb.prepare("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord").all(SID), + rawDb.prepare("SELECT ord, ccr_key, content_hash FROM ccr_entries WHERE session_id = ? ORDER BY ord").all(SID), + rawDb.prepare("SELECT block_id, ord, start_seq, end_seq, tier, summary_hash, one_hash, active, covered_hash FROM blocks WHERE session_id = ? ORDER BY ord").all(SID), + ]); + const diffState = snapshot(); + + // Force the next save down the full path: drop the watermark, keep the raw + // tail (complete-log full ingest). The authoritative-set contract means the + // three tables must come out BYTE-IDENTICAL to what the diff path wrote. + (shadow as unknown as { sync: { clearIngestWatermark(id: string): void } }).sync.clearIngestWatermark(SID); + shadow.record(mkLegacy(SID, 202, 0, null)); + assert.equal(snapshot(), diffState, "full-path save reproduces the diff-path end state exactly"); +}); + +test("#5744: slow-ingest diagnostics expose WAL size and the last checkpoint triple", (t) => { + const dir = tmpdir("bili-5744c-waldiag-"); + t.after(() => rmBestEffort(dir)); + const { shadow, store } = openPair(dir); + t.after(() => shadow.close()); + t.after(() => store.close()); + + // Exactly 20 ingests so the periodic checkpoint tick (every 20th successful + // ingest) fires on the store the shadow writes through. + for (let turn = 0; turn < 20; turn++) { + shadow.record(mkLegacy(SID, 10 + turn, 0, null)); + } + const syncStore = (shadow as unknown as { sync: UnifiedStore }).sync; + const suffix = syncStore.walDiagnosticsSuffix(); + assert.ok(typeof suffix === "string" && suffix.length > 0, "WAL forensics available once a WAL file exists"); + assert.ok(suffix.includes("WAL"), `suffix carries the WAL size: ${suffix}`); + assert.ok(suffix.includes("lastCheckpoint=(busy="), `last tick triple present: ${suffix}`); + assert.ok(suffix.includes("checkpointed=") && suffix.includes("took="), `tick duration present: ${suffix}`); +});