From b6f9777d1646372204dd7446f6e3042b9dfd8c8e Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Wed, 14 Oct 2026 17:48:24 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20per-session=20ingest=20watermark=20?= =?UTF-8?q?=E2=80=94=20O(new=20messages)=20per=20save=20(#3133)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/storage/ingest.ts | 137 +++++- src/storage/schema.ts | 25 +- src/storage/shadow.ts | 50 ++- src/storage/store.ts | 232 ++++++++-- tests/storage-ext.test.ts | 6 +- tests/storage-ingest-watermark.test.ts | 590 +++++++++++++++++++++++++ tests/storage-unified.test.ts | 2 +- 7 files changed, 978 insertions(+), 64 deletions(-) create mode 100644 tests/storage-ingest-watermark.test.ts diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index dc5ac1483b..2e10bc53ee 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -82,6 +82,26 @@ interface IngestPlan extends IngestSessionInput { tailSeqs: number[] | null; } +/** #3133 test instrumentation: per-message JSON serializations performed by + * the planners in this process (intern pass, delta probe, delta new-tail + * pass). The watermark fix exists to make this O(new messages) per save + * instead of O(full log) — the storage watermark tests assert exactly + * that; production code never reads it. */ +export const ingestSerializations = { count: 0 }; + +/** One messages-log entry with its ABSOLUTE seq already assigned (intern + * assigns first-seen order; the delta planner continues after the trusted + * prefix). */ +interface LogEntry { + seq: number; + msgId: string; + content: string; + hash: string; + role: string; + kind: string; + ref: string | null; +} + /** Adapts any Session-shaped object (persist loadSync output) to the * ingest input type. Structural on purpose: no import of src/session.ts * (avoids a module cycle through persist.ts). */ @@ -124,12 +144,13 @@ export function planIngest(session: LegacySessionLike): IngestPlan { // Pass 1: ordered union of identities (msg_id + content hash). // #3133: no cross-save memo — the wire entry points replace the message // array with freshly parsed objects on every request, so an object- - // identity WeakMap never hits on the production save path (see the - // issue for the content/seq watermark design that actually cuts cost). - const log: { msgId: string; content: string; hash: string; role: string; kind: string; ref: string | null }[] = []; + // identity WeakMap never hits on the production save path. The cost cut + // lives in planIngestDelta + the sessions.last_ingested_seq watermark. + const log: LogEntry[] = []; const seqByIdentity = new Map(); const intern = (m: CoreMessage): number => { const content = JSON.stringify(m); + ingestSerializations.count++; const parsed = JSON.parse(content) as { role: string; contentType?: string }; const entry = { msgId: m.id, content, hash: sha256Hex(content), role: parsed.role, kind: parsed.contentType ?? "message" }; const key = `${m.id}\u0000${entry.hash}`; @@ -137,12 +158,116 @@ export function planIngest(session: LegacySessionLike): IngestPlan { if (existing !== undefined) return existing; const seq = log.length; seqByIdentity.set(key, seq); - log.push({ msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null }); + log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null }); return seq; }; const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null; const tailSeqs = session.lastMessages ? session.lastMessages.map(intern) : null; + return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); +} + +/** #3133 fast-path planner: build the ingest plan for a session whose + * stored tail is a VERIFIED trusted prefix — the sessions row's watermark + * names the absolute seq backing the LAST tail message at save time, and + * `trustedTailSeqs` is that save's persisted tailSeqs (same transaction + * wrote both, so they cannot disagree). Soundness of skipping rows 0..L-1: + * every one of them was part of the tail that set the watermark (rows + * written by that save), and the probe pins slot L-1 to row `watermark` + * with nothing above it — so this save only has to append tail[L..] at + * watermark+1, watermark+2, … Only messages PAST the trusted prefix pay + * stringify+hash; every other row family is byte-identical to + * planIngest's, so the resulting store state equals a full ingest's. + * + * Returns null (→ full ingest) when: a pluginSnapshot is present (its + * entries may dedup against the trusted prefix and renumber the log), + * the tail is absent or not longer than the trusted prefix (shrunk / + * rewound / folded-in-place), or the boundary identity disagrees with the + * tail slot at L-1 (edited/deleted/re-minted history). Interior edits + * BELOW the boundary are the watermark design's accepted trade-off — they + * never mis-attribute, only leave stale rows, and the sampled verify + * digest is the backstop. */ +export function planIngestDelta( + session: LegacySessionLike, + watermark: number, + trustedTailSeqs: number[], + boundary: { msgId: string; contentHash: string }, + hasRawTail: boolean, +): IngestPlan | null { + if (session.pluginSnapshot && session.pluginSnapshot.length > 0) return null; + const tail = session.lastMessages; + const trustedLen = trustedTailSeqs.length; + if ( + !tail || + trustedLen === 0 || + !Number.isSafeInteger(watermark) || + watermark < 0 || + // An equal-length tail is a VALID empty delta, but ONLY on the raw + // lineage: a caller passing the envelope's bounded folded RE-RENDER + // (no rawTail — e.g. the stale-convergence record(disk) path) may + // present the SAME identities in a DIFFERENT order (#2960 ⑤: the + // table holds the raw tail; the envelope re-renders it folded), and + // the boundary probe can match while the served view would stay on + // the raw ordering instead of the envelope render the disk record + // promised. Envelope renders must fall through to the full ingest, + // which rebuilds tailSeqs in the envelope's order. + (tail.length === trustedLen && !hasRawTail) || + tail.length < trustedLen || + trustedTailSeqs[trustedLen - 1] !== watermark + ) { + return null; + } + // Boundary probe, content side: ONE serialization+hash of the tail slot + // at the trusted prefix's end. Identities are content-derived and + // unique within one conversion pass, so a match proves this tail slot + // IS the message the watermark was set from. + const probe = JSON.stringify(tail[trustedLen - 1]); + ingestSerializations.count++; + if (tail[trustedLen - 1].id !== boundary.msgId || sha256Hex(probe) !== boundary.contentHash) return null; + const byRawRef = session.state.messageRefs.byRaw; + const log: LogEntry[] = []; + const appended: number[] = []; + for (let i = trustedLen; i < tail.length; i++) { + const m = tail[i]; + const content = JSON.stringify(m); + ingestSerializations.count++; + const parsed = JSON.parse(content) as { role: string; contentType?: string }; + // Placeholder seqs under the aligned assumption (no rows above the + // watermark); ingestSession renumbers from max(seq)+1 inside the + // transaction when a fold left rows above the trusted tip. + const seq = watermark + 1 + log.length; + appended.push(seq); + log.push({ + seq, + msgId: m.id, + content, + hash: sha256Hex(content), + role: parsed.role, + kind: parsed.contentType ?? "message", + ref: byRawRef[m.id] ?? null, + }); + } + // planIngest's snapshot ternary replicated on the ORIGINAL value: an + // array (even empty, as fork receipts carry) maps to [], null stays + // null — loadLegacyView distinguishes the two, so they must not collapse. + const pluginSnapshotSeqs = session.pluginSnapshot ? [] : null; + // tailSeqs stay ABSOLUTE (the trusted prefix's seqs are the stored ones, + // holes included — folding removes log positions from the middle); the + // delta path skips remapPlanToAbsolute by construction. + const tailSeqs = [...trustedTailSeqs, ...appended]; + return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); +} + +/** Shared plan tail (everything except the messages log): byte-identical + * output for both planners — the watermark fast path must not change what + * a full ingest would write for the same session. */ +function finishPlan( + session: LegacySessionLike, + byRawRef: Record, + log: LogEntry[], + pluginSnapshotSeqs: number[] | null, + tailSeqs: number[] | null, +): IngestPlan { // Ref ledger: verbatim, ordered by the persisted map's key order. // Ghost ids (dead refs) have rows here and nowhere else — by design. const refs: RefRow[] = Object.entries(byRawRef).map(([rawId, ref], ord) => ({ @@ -152,8 +277,8 @@ export function planIngest(session: LegacySessionLike): IngestPlan { refNum: refToIndex(ref), })); - const messages: MessageRow[] = log.map((e, seq) => ({ - seq, + const messages: MessageRow[] = log.map((e) => ({ + seq: e.seq, msgId: e.msgId, ref: e.ref, refNum: e.ref === null ? null : refToIndex(e.ref), diff --git a/src/storage/schema.ts b/src/storage/schema.ts index c4aa8e19b8..27ab713472 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -53,7 +53,17 @@ // these tables are outside the LegacySessionLike view). Fresh // databases create the v4 shape directly; the migration is two // idempotent CREATE TABLE IF NOT EXISTS statements. -const STORAGE_SCHEMA_VERSION = 4; +// - v5: per-session ingest watermark (#3133) — sessions gains +// `last_ingested_seq`, the absolute seq backing the last tail +// message of the most recent save (NULL = no trusted prefix, full +// ingest). The trusted prefix length is re-read from the existing +// tail_json tailSeqs written by the same transaction — no second +// column. Point lookups and MAX(seq) stay on the pre-existing +// PRIMARY KEY (session_id, seq) / idx_messages_session — no new +// index. Both directions compatible: an older build opening a v5 +// file no-ops at the version gate (extra column ignored); a newer +// build upgrades a v4 file with one idempotent ALTER TABLE. +const STORAGE_SCHEMA_VERSION = 5; export const STORAGE_PRAGMAS = [ "PRAGMA journal_mode = WAL", @@ -82,7 +92,8 @@ const STORAGE_DDL_V1 = `CREATE TABLE IF NOT EXISTS sessions ( metadata_json TEXT NOT NULL DEFAULT '{}', kstate_json TEXT NOT NULL DEFAULT '{}', tail_json TEXT NOT NULL DEFAULT '{}', - ccr_json TEXT + ccr_json TEXT, + last_ingested_seq INTEGER ); CREATE TABLE IF NOT EXISTS content ( @@ -259,6 +270,16 @@ export function migrateSchema(db: { db.exec("CREATE TABLE IF NOT EXISTS session_ext (\n session_id TEXT NOT NULL,\n key TEXT NOT NULL,\n value_json TEXT NOT NULL,\n updated_at INTEGER NOT NULL,\n PRIMARY KEY (session_id, key)\n ) WITHOUT ROWID"); db.exec("CREATE TABLE IF NOT EXISTS kv_store (\n namespace TEXT NOT NULL,\n key TEXT NOT NULL,\n value_json TEXT NOT NULL,\n updated_at INTEGER NOT NULL,\n PRIMARY KEY (namespace, key)\n ) WITHOUT ROWID"); } + if (from < 5) { + // v5 ingest watermark (#3133): additive NULLable column on + // sessions. Probe first — same duplicate-column discipline as v2 + // (a dev-window build that already carried it re-lands cleanly). + const cols = (db.prepare("PRAGMA table_info(sessions)").all() as unknown as { name?: unknown }[]) + .map((c) => (typeof c?.name === "string" ? c.name : "")); + if (!cols.includes("last_ingested_seq")) { + db.exec("ALTER TABLE sessions ADD COLUMN last_ingested_seq INTEGER"); + } + } db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`); }); return { from, to: STORAGE_SCHEMA_VERSION }; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 19250f93b6..788163d352 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -22,7 +22,7 @@ */ import { log as loggerLog } from "../logger.js"; -import { planIngest, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; +import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import { resolveStorageMode, type StorageMode } from "../knobs.ts"; import { UnifiedStore, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import { canonicalize, sha256 } from "./canonical.ts"; @@ -44,6 +44,7 @@ export { resolveStorageMode, type StorageMode } from "../knobs.ts"; interface ShadowStats { ingests: number; + deltaIngests: number; /** #3133: sampled read-back verifications RUN (verifyOk + verifyDrift * ≤ this). Sampling, not removal — drift stays observable. */ verifySamples: number; @@ -56,7 +57,7 @@ interface ShadowStats { export class UnifiedShadow { readonly mode: StorageMode; - readonly stats: ShadowStats = { ingests: 0, verifySamples: 0, verifyOk: 0, verifyDrift: 0, imports: 0, disabled: false, lastError: null }; + readonly stats: ShadowStats = { ingests: 0, deltaIngests: 0, verifySamples: 0, verifyOk: 0, verifyDrift: 0, imports: 0, disabled: false, lastError: null }; private readonly store: UnifiedStore; private readonly legacyLoad: (record: unknown) => LegacySessionLike | null; private consecutiveFailures = 0; @@ -133,7 +134,9 @@ export class UnifiedShadow { const loaded = this.legacyLoad(record); if (!loaded) return; let legacy = loaded; + let hasRawTail = false; if (rawTail && rawTail.lastMessages.length > 0) { + hasRawTail = true; legacy = { ...legacy, lastMessages: rawTail.lastMessages, lastMessagesFolded: rawTail.lastMessagesFolded === true }; } // #2960 ③: the live kernel content store when the caller has one @@ -142,8 +145,18 @@ export class UnifiedShadow { // stay empty for every plain compressed session. Both digest legs // flow through the same override. if (liveCcr) legacy = { ...legacy, contentStore: liveCcr }; - const plan = planIngest(legacy); - this.store.ingestSession(plan); + // #3133 watermark: pay the full O(log) ingest only when the + // trusted-prefix probe cannot vouch for the stored prefix — + // steady-state saves stringify+hash+INSERT just the NEW tail. + // Any doubt (no watermark, foreign edit, tail shorter than the + // watermark, snapshot-bearing session) falls back to the exact + // full path below; the delta itself re-verifies the boundary + // inside the ingest transaction, so a racing writer degrades to + // a full ingest, never a corrupt one. + if (!this.ingestDelta(legacy, hasRawTail)) { + const plan = planIngest(legacy); + this.store.ingestSession(plan); + } this.stats.ingests++; // #3133 sampled verification: ingest integrity is already guarded // by the SQLite write path; the read-back digest is a migration @@ -186,6 +199,35 @@ export class UnifiedShadow { } } + /** #3133 delta ingest attempt for one record's legacy view — returns + * true when the O(new-messages) fast path committed. Every doubt + * routes to `false` (→ the caller's full ingest); failures here must + * never throw into record(), whose catch counts toward disabling the + * shadow — a watermark hiccup is not a storage health problem. */ + private ingestDelta(legacy: LegacySessionLike, hasRawTail: boolean): boolean { + try { + const mark = this.store.ingestWatermark(legacy.id); + if (!mark) return false; + // The pre-transaction probe is an optimization (skip building a + // doomed plan); ingestSession re-verifies the boundary inside + // BEGIN IMMEDIATE, which is the authoritative check. + const boundary = this.store.probeIngestBoundary(legacy.id, mark.seq); + if (!boundary) return false; + const plan = planIngestDelta(legacy, mark.seq, mark.tailSeqs, { msgId: boundary.msgId, contentHash: boundary.contentHash }, hasRawTail); + if (!plan) return false; + const applied = this.store.ingestSession(plan, { + trustedSeq: mark.seq, + boundaryMsgId: boundary.msgId, + boundaryHash: boundary.contentHash, + trustedTailSeqs: mark.tailSeqs, + }); + if (applied) this.stats.deltaIngests++; + return applied; + } catch { + return false; + } + } + /** Loads the unified view of one session (full mode read path). * Returns null on miss — the caller falls back to the legacy file. */ viewFor(id: string): LegacySessionLike | null { diff --git a/src/storage/store.ts b/src/storage/store.ts index 4f04faf1ef..87d900e5b5 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -58,6 +58,11 @@ export interface SessionRow { tailJson: string; /** Verbatim CCR store JSON (dying subsystem, Phase 3 removes it). */ ccrJson: string | null; + /** #3133 ingest watermark: highest absolute message seq whose bytes this + * index has verified (NULL = no trusted prefix → next save ingests + * fully). Computed by ingestSession only (delta provenance or full-path + * alignment) — never by planners or callers; read via ingestWatermark(). */ + lastIngestedSeq?: number | null; } export interface MessageRow { @@ -246,8 +251,10 @@ export class UnifiedStore { /** Content CAS: insert-if-absent + refcount bump; returns hash. */ private putContent(content: string | Buffer, precomputedHash?: string): string { const bytes = typeof content === "string" ? Buffer.from(content, "utf8") : content; - // #3133: delta-interned plan rows already carry the digest — skip the - // per-message re-hash (same value: sha256 over the same utf8 bytes). + // #3133: planner-produced rows carry the digest from their intern or + // delta pass — skip the per-message re-hash here (same value: sha256 + // over the same utf8 bytes). Only direct callers without a planner + // pay the hash at this layer. const hash = precomputedHash ?? sha256Hex(bytes); const existing = this.db.prepare("SELECT refcount FROM content WHERE hash = ?").get(hash) as | { refcount: number } @@ -268,58 +275,88 @@ export class UnifiedStore { this.db.prepare("UPDATE content SET refcount = MAX(refcount - 1, 0) WHERE hash = ?").run(hash); } - ingestSession(input: IngestSessionInput): void { + /** Ingests one session plan as a single transaction. Full path (no + * `delta`): identity-based absolute-seq remap of every plan row; the + * watermark becomes the abs seq backing the plan's LAST tail message + * (NULL when the plan carries no tail) — the trusted tip need not be + * the table's max, because delta appends anchor at max(seq)+1, never + * at watermark+1, so rows above the tip (a fold's summary) cannot + * collide with them. + * + * Delta path (#3133, `delta` given): the plan carries ONLY messages + * past the trusted prefix, with placeholder seqs (renumbered here). + * The boundary row (msg_id + content_hash at seq trustedSeq) is + * RE-VERIFIED inside this transaction (BEGIN IMMEDIATE serializes + * writers, so the in-tx check is authoritative — the caller's pre-plan + * probe was just an optimization), every delta message's identity must + * be absent (strictly-append — a re-derived tail would otherwise be + * silently dropped by the UNIQUE constraint), and new rows take + * max(seq)+1.. so no slot can collide. On any disagreement this + * returns false WITHOUT writing anything; the caller falls back to the + * full plan. A stale or foreign-written watermark therefore costs one + * redundant full write — never wrong rows — and the watermark write + * itself rides the SAME transaction as the rows. */ + ingestSession( + input: IngestSessionInput, + delta?: { trustedSeq: number; boundaryMsgId: string; boundaryHash: string; trustedTailSeqs: number[] }, + ): boolean { const s = input.session; - this.db.transaction(() => { - // Absolute-seq remap (#2671 soak finding): planIngest assigns seq - // RELATIVE to the current snapshot's log (pluginSnapshot first, - // then tail). For a folded session whose tail slides every save - // (and no pluginSnapshot to pin the prefix), plan seq 0 re-bases - // to a DIFFERENT message each save — while messages is keyed - // (session_id, seq) with INSERT OR IGNORE, so the FIRST row wins - // forever and tail pointers resolve to stale turns. Remap to - // stable absolute seqs: same identity (msgId + content hash) - // keeps its old seq; new identities continue after max(seq). - const existing = this.db - .prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?") - .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; - const seqByIdentity = new Map(); - let nextSeq = 0; - for (const row of existing) { - seqByIdentity.set(`${row.msg_id}\u0000${row.content_hash}`, row.seq); - if (row.seq + 1 > nextSeq) nextSeq = row.seq + 1; - } - const planToAbs = new Map(); - const messages = input.messages.map((m) => { - // #3133: plan rows carry the intern-pass hash; only direct - // callers without a planner pay the sha256 here. - const hash = m.contentHash ?? sha256Hex(m.content); - const key = `${m.msgId}\u0000${hash}`; - let abs = seqByIdentity.get(key); - if (abs === undefined) { - abs = nextSeq++; - seqByIdentity.set(key, abs); + return this.db.transaction((): boolean => { + let messages: MessageRow[]; + let watermark: number | null; + if (delta) { + const probe = this.probeIngestBoundary(s.sessionId, delta.trustedSeq); + if (!probe || probe.msgId !== delta.boundaryMsgId || probe.contentHash !== delta.boundaryHash) { + return false; } - planToAbs.set(m.seq, abs); - return { ...m, seq: abs }; - }); - const remapSeqs = (seqs: number[] | null | undefined): number[] | null | undefined => - seqs === null || seqs === undefined ? seqs : seqs.map((p) => planToAbs.get(p) ?? p); - const pluginSnapshotSeqs = remapSeqs(input.pluginSnapshotSeqs); - const tailSeqs = remapSeqs(input.tailSeqs); - s.tailJson = JSON.stringify({ - ...(JSON.parse(s.tailJson) as Record), - ...(pluginSnapshotSeqs !== undefined ? { pluginSnapshotSeqs } : {}), - ...(tailSeqs !== undefined ? { tailSeqs } : {}), - }); + // Strictly-append guard: a delta message whose identity + // already has a row (rewound/re-derived tail) must not be + // silently skipped by the UNIQUE(session_id, msg_id, + // content_hash) constraint — its tailSeqs slot would point + // at nothing and the read view would drop it. One indexed + // probe per NEW message; any hit → full ingest instead. + const existingIdentity = this.db.prepare( + "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND content_hash = ?", + ); + for (const m of input.messages) { + if (!m.contentHash) return false; + if (existingIdentity.get(s.sessionId, m.msgId, m.contentHash)) return false; + } + // Placeholder→absolute renumber: the plan emitted + // watermark+1.. under the aligned assumption; anchoring at + // the CURRENT max(seq)+1 (re-read inside this transaction) + // is what makes slot collisions structurally impossible — + // identical to the full path's nextSeq discipline, and it + // keeps the delta valid even when a fold left rows above + // the trusted tip. + messages = input.messages.map((m, i) => ({ ...m, seq: probe.maxSeq + 1 + i })); + const tail = JSON.parse(s.tailJson) as { tailSeqs?: number[] | null }; + // Defense: the plan's tailSeqs prefix must be exactly the + // trusted prefix the watermark read produced (same-tick by + // construction; disagreement means a stale plan). + const prefixOk = + Array.isArray(tail.tailSeqs) && + tail.tailSeqs.length === delta.trustedTailSeqs.length + input.messages.length && + delta.trustedTailSeqs.every((v, i) => tail.tailSeqs?.[i] === v); + if (!prefixOk) return false; + s.tailJson = JSON.stringify({ + ...tail, + tailSeqs: [...delta.trustedTailSeqs, ...messages.map((m) => m.seq)], + }); + watermark = messages.length > 0 ? messages[messages.length - 1].seq : delta.trustedSeq; + } else { + const full = this.remapPlanToAbsolute(input); + messages = full.messages; + watermark = full.watermark; + } this.db .prepare( `INSERT INTO sessions (session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin, status, created_at, updated_at, next_block_id, next_run_id, - meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json) + meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq) VALUES (@sessionId, @parentSessionId, @forkPointSeq, @protocol, @label, @title, @hostTitle, @namespace, @origin, @status, @createdAt, @updatedAt, @nextBlockId, @nextRunId, - @metaJson, @statsJson, @metadataJson, @kstateJson, @tailJson, @ccrJson) + @metaJson, @statsJson, @metadataJson, @kstateJson, @tailJson, @ccrJson, @lastIngestedSeq) ON CONFLICT(session_id) DO UPDATE SET parent_session_id = excluded.parent_session_id, fork_point_seq = excluded.fork_point_seq, @@ -338,7 +375,8 @@ export class UnifiedStore { metadata_json = excluded.metadata_json, kstate_json = excluded.kstate_json, tail_json = excluded.tail_json, - ccr_json = excluded.ccr_json`, + ccr_json = excluded.ccr_json, + last_ingested_seq = excluded.last_ingested_seq`, ) .run({ sessionId: s.sessionId, @@ -361,6 +399,7 @@ export class UnifiedStore { kstateJson: s.kstateJson, tailJson: s.tailJson, ccrJson: s.ccrJson, + lastIngestedSeq: watermark, }); // Identity index (#2960 R2.1): claim (namespace, conversation_id) @@ -496,7 +535,103 @@ export class UnifiedStore { .prepare("INSERT INTO ccr_entries (session_id, ord, ccr_key, content_hash) VALUES (?, ?, ?, ?)") .run(s.sessionId, e.ord, e.key, hash); } + return true; + }); + } + + /** Full-ingest path of ingestSession, extracted for readability: the + * absolute-seq remap plus the #3133 watermark alignment decision. + * Mutates input.session.tailJson (pointer sequences become absolute). */ + private remapPlanToAbsolute(input: IngestSessionInput): { messages: MessageRow[]; watermark: number | null } { + const s = input.session; + // Absolute-seq remap (#2671 soak finding): planIngest assigns seq + // RELATIVE to the current snapshot's log (pluginSnapshot first, + // then tail). For a folded session whose tail slides every save + // (and no pluginSnapshot to pin the prefix), plan seq 0 re-bases + // to a DIFFERENT message each save — while messages is keyed + // (session_id, seq) with INSERT OR IGNORE, so the FIRST row wins + // forever and tail pointers resolve to stale turns. Remap to + // stable absolute seqs: same identity (msgId + content hash) + // keeps its old seq; new identities continue after max(seq). + const existing = this.db + .prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; + const seqByIdentity = new Map(); + let nextSeq = 0; + for (const row of existing) { + seqByIdentity.set(`${row.msg_id}\u0000${row.content_hash}`, row.seq); + if (row.seq + 1 > nextSeq) nextSeq = row.seq + 1; + } + // #3133 watermark gate (full path): the watermark names the abs seq + // backing the plan's LAST tail message. It need not be the table's + // max — a fold's summary lands above surviving old rows — because + // delta appends anchor at max(seq)+1, never at watermark+1. NULL + // when the plan carries no tail (envelope-only imports): the next + // save runs the full ingest. + const planToAbs = new Map(); + const messages = input.messages.map((m) => { + // #3133: plan rows carry the intern-pass hash; only direct + // callers without a planner pay the sha256 here. + const hash = m.contentHash ?? sha256Hex(m.content); + const key = `${m.msgId}\u0000${hash}`; + let abs = seqByIdentity.get(key); + if (abs === undefined) { + abs = nextSeq++; + seqByIdentity.set(key, abs); + } + planToAbs.set(m.seq, abs); + return { ...m, seq: abs }; + }); + const remapSeqs = (seqs: number[] | null | undefined): number[] | null | undefined => + seqs === null || seqs === undefined ? seqs : seqs.map((p) => planToAbs.get(p) ?? p); + const pluginSnapshotSeqs = remapSeqs(input.pluginSnapshotSeqs); + const tailSeqs = remapSeqs(input.tailSeqs); + const watermark = tailSeqs !== null && tailSeqs !== undefined && tailSeqs.length > 0 ? tailSeqs[tailSeqs.length - 1] : null; + s.tailJson = JSON.stringify({ + ...(JSON.parse(s.tailJson) as Record), + ...(pluginSnapshotSeqs !== undefined ? { pluginSnapshotSeqs } : {}), + ...(tailSeqs !== undefined ? { tailSeqs } : {}), }); + return { messages, watermark }; + } + + /** #3133: the session's ingest watermark — the absolute seq backing the + * LAST message of the tail as ingested by the most recent save, plus + * that save's trusted tailSeqs (same transaction wrote both). NULL (or + * a shape that disagrees with tail_json — foreign write, partial + * upgrade) means "no trusted prefix": the next save runs the full + * ingest. Point read on the sessions PK. */ + ingestWatermark(sessionId: string): { seq: number; tailSeqs: number[] } | null { + const row = this.db + .prepare("SELECT last_ingested_seq, tail_json FROM sessions WHERE session_id = ?") + .get(sessionId) as { last_ingested_seq: number | null; tail_json: string } | undefined; + if (!row) return null; + const seq = row.last_ingested_seq; + if (typeof seq !== "number" || !Number.isSafeInteger(seq) || seq < 0) return null; + const tail = JSON.parse(row.tail_json) as { tailSeqs?: number[] | null }; + const seqs = tail.tailSeqs; + if (!Array.isArray(seqs) || seqs.length === 0 || seqs[seqs.length - 1] !== seq) return null; + return { seq, tailSeqs: seqs }; + } + + /** #3133 boundary probe: the identity of the messages row at + * (session_id, seq) plus the session's current max seq — both lookups + * ride the PRIMARY KEY (session_id, seq) / idx_messages_session (no + * full existing-rows scan, no seqByIdentity map). null when the row is + * missing. The identity half is the trust decision; the max-seq half + * is the ANCHOR delta appends continue after (max+1, the same + * strictly-above discipline as the full path's remap, so appends can + * never collide with an existing slot — rows above the watermark, + * e.g. a fold's summary, are expected and harmless). */ + probeIngestBoundary(sessionId: string, seq: number): { msgId: string; contentHash: string; maxSeq: number } | null { + const row = this.db + .prepare("SELECT msg_id, content_hash FROM messages WHERE session_id = ? AND seq = ?") + .get(sessionId, seq) as { msg_id: string; content_hash: string } | undefined; + if (!row) return null; + const max = this.db + .prepare("SELECT MAX(seq) AS m FROM messages WHERE session_id = ?") + .get(sessionId) as { m: number | null }; + return { msgId: row.msg_id, contentHash: row.content_hash, maxSeq: max.m ?? -1 }; } /** Deletes one session and releases every CAS refcount it holds @@ -945,7 +1080,7 @@ export class UnifiedStore { .prepare( `SELECT session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin, status, created_at, updated_at, next_block_id, next_run_id, - meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json + meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq FROM sessions WHERE session_id = ?`, ) .get(sessionId) as Record | undefined; @@ -971,6 +1106,7 @@ export class UnifiedStore { kstateJson: row.kstate_json as string, tailJson: row.tail_json as string, ccrJson: (row.ccr_json as string | null) ?? null, + lastIngestedSeq: (row.last_ingested_seq as number | null | undefined) ?? null, }; } diff --git a/tests/storage-ext.test.ts b/tests/storage-ext.test.ts index 969785eefb..972434bc35 100644 --- a/tests/storage-ext.test.ts +++ b/tests/storage-ext.test.ts @@ -174,17 +174,17 @@ test("v3 databases migrate to v4 in place, idempotently", async () => { const migrated = UnifiedStore.open(dbPath); try { const ver = (migrated as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 4, "open() runs migrateSchema: v3 → v4"); + assert.equal(ver.user_version, 5, "open() runs migrateSchema: v3 → v5"); migrated.setKv("probe", "alive", true); assert.deepEqual(migrated.getKv("probe", "alive"), true); assert.deepEqual(migrated.listSessionIds(), [], "existing v3 content untouched"); } finally { migrated.close(); } - // Reopening a v4 database is a no-op (idempotent migration). + // Reopening a v5 database is a no-op (idempotent migration). const again = UnifiedStore.open(dbPath); const ver2 = (again as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver2.user_version, 4); + assert.equal(ver2.user_version, 5); again.close(); } finally { fs.rmSync(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts new file mode 100644 index 0000000000..6e7b0a6659 --- /dev/null +++ b/tests/storage-ingest-watermark.test.ts @@ -0,0 +1,590 @@ +/** + * #3133: per-session ingest watermark — O(new messages) per save. + * + * The full-log ingest cost (JSON.stringify + sha256 per message + a scan of + * every existing row) made each shadow/full save O(full log): ~25-40ms per + * MB, ~0.6s for a 14k-message session, synchronous on the save path. The + * watermark names, on the sessions row, the abs seq backing the LAST tail + * message of the most recent save; the trusted-prefix length is re-read from + * tail_json's tailSeqs (same transaction wrote both). A save whose tail still + * ends with that message at that slot appends ONLY the new messages + * (stringify+hash+INSERT each), anchored at max(seq)+1 — folds leave rows + * above the trusted tip and the anchor absorbs them. Every doubt (NULL + * watermark, tampered/missing boundary row, rewound tail, snapshot-bearing + * session) falls back to the byte-identical full ingest, so a wrong + * watermark can only ever cost one redundant full write, never wrong rows. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; +import { planIngest, planIngestDelta, loadLegacyView, ingestSerializations, type LegacySessionLike } from "../src/storage/ingest.ts"; +import type { CoreMessage } from "acp-kernel"; + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: a developer machine with +// storage.unified in its real config must not leak a tier into these tests +// (the shadow here is constructed directly with its own env object anyway). +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-watermark-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +function tmpdir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +function msg(i: number, text = `hello ${i}`): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text } as CoreMessage; +} + +function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"], pluginSnapshot?: LegacySessionLike["pluginSnapshot"]): LegacySessionLike { + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "anthropic" }, + stats: { requests: 1 }, + metadata: {}, + // Key order matters: loadLegacyView rebuilds state in MERGE_STATE_FIELDS + // order and the shadow digest is JSON.stringify-based (order-sensitive). + state: { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + pluginSnapshot, + }; +} + +interface StatsView { + ingests: number; + deltaIngests: number; + verifySamples: number; + verifyOk: number; + verifyDrift: number; +} + +function openShadow(dir: string): { shadow: UnifiedShadow; store: UnifiedStore } { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow mode sidecar must open"); + return { shadow, store }; +} + +function stats(shadow: UnifiedShadow): StatsView { + return (shadow as unknown as { stats: StatsView }).stats; +} + +function view(store: UnifiedStore, id: string): ReturnType { + const session = store.getSessionRow(id); + assert.ok(session, `session row must exist: ${id}`); + return loadLegacyView({ + session, + messages: store.getMessages(id), + blocks: store.getBlocks(id), + deadRefs: store.getDeadRefs(id), + refs: store.getRefs(id), + }); +} + +/** The equivalence oracle: whatever the delta path wrote must equal what a + * full re-ingest of the same legacy view would write (deep view equality — + * state, tail, pointers). */ +function assertDeltaMatchesFull(store: UnifiedStore, legacy: LegacySessionLike): void { + const before = view(store, legacy.id); + assert.equal(store.ingestSession(planIngest(legacy)), true, "full re-ingest must apply"); + assert.deepEqual(view(store, legacy.id), before, "delta-written rows must equal a full re-ingest's"); +} + +test("watermark: steady-state saves cost O(new messages), not O(full log)", () => { + const dir = tmpdir("bili-wm-steady-"); + try { + const { shadow, store } = openShadow(dir); + try { + const n = 12; + let legacy = mkLegacy("wm-steady", Array.from({ length: n }, (_, i) => msg(i))); + + ingestSerializations.count = 0; + shadow.record(legacy); + assert.equal(stats(shadow).deltaIngests, 0, "first save has no watermark yet — full ingest"); + assert.equal(ingestSerializations.count, n, "full ingest serializes every message once"); + assert.deepEqual(store.ingestWatermark("wm-steady"), { seq: n - 1, tailSeqs: Array.from({ length: n }, (_, i) => i) }); + + // +1 message: probe(1) + new(1) — 2 serializations, not 13. + legacy = mkLegacy("wm-steady", Array.from({ length: n + 1 }, (_, i) => msg(i))); + ingestSerializations.count = 0; + shadow.record(legacy); + assert.equal(stats(shadow).deltaIngests, 1, "second save must take the delta path"); + assert.equal(ingestSerializations.count, 2, "delta serializes only the boundary probe + the new message"); + assert.equal(store.getMessages("wm-steady").length, n + 1, "row count tracks the tail"); + assertDeltaMatchesFull(store, legacy); + + // +3 messages: probe(1) + new(3) — cost scales with NEW, not total. + legacy = mkLegacy("wm-steady", Array.from({ length: n + 4 }, (_, i) => msg(i))); + ingestSerializations.count = 0; + shadow.record(legacy); + assert.equal(stats(shadow).deltaIngests, 2); + assert.equal(ingestSerializations.count, 4); + const mark = store.ingestWatermark("wm-steady"); + assert.ok(mark, "watermark re-established after each delta"); + assert.equal(mark.seq, n + 3, "watermark names the last tail message's seq"); + assertDeltaMatchesFull(store, legacy); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: zero-new-message save stays on the delta path", () => { + const dir = tmpdir("bili-wm-idle-"); + try { + const { shadow, store } = openShadow(dir); + try { + const n = 10; + let legacy = mkLegacy("wm-idle", Array.from({ length: n }, (_, i) => msg(i))); + shadow.record(legacy); + assert.equal(stats(shadow).deltaIngests, 0, "first save is a full ingest"); + + // Same tail, changed metadata only (stats bump): the delta path + // must commit an EMPTY delta — sessions-row write, zero message + // serializations, not a full re-ingest. + const tailNow = legacy.lastMessages ?? []; + legacy = mkLegacy("wm-idle", tailNow); + legacy.stats = { requests: 2 }; + ingestSerializations.count = 0; + // Production metadata-only saves carry the raw tail (persist + // passes it whenever lastMessages is non-empty). + shadow.record(legacy, { lastMessages: tailNow }); + assert.equal(stats(shadow).deltaIngests, 1, "metadata-only save must not fall back to full"); + assert.equal(ingestSerializations.count, 1, "only the boundary probe serializes"); + assert.equal(store.getMessages("wm-idle").length, n, "no message rows change"); + assert.deepEqual(store.ingestWatermark("wm-idle"), { seq: n - 1, tailSeqs: Array.from({ length: n }, (_, i) => i) }); + assertDeltaMatchesFull(store, legacy); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: envelope re-render without rawTail never rides the delta (#2960 ⑤ order)", () => { + const dir = tmpdir("bili-wm-render-"); + try { + const { shadow, store } = openShadow(dir); + try { + // Raw-tail lineage: summary carrier first, then raw messages. + const msgs = [msg(0), msg(1), msg(2)]; + const carrier = msg(99); + carrier.id = "acp_summary_b1"; + const lineage = [carrier, ...msgs]; + let legacy = mkLegacy("wm-render", lineage); + shadow.record(legacy, { lastMessages: lineage }); + assert.equal(stats(shadow).deltaIngests, 0, "first save is a full ingest"); + const rawOrder = store.getMessages("wm-render").map((r) => r.msgId); + assert.deepEqual(rawOrder, ["acp_summary_b1", "h_msg0", "h_msg1", "h_msg2"]); + + // Stale-convergence path shape: record(disk) with NO rawTail, the + // envelope's folded re-render presenting the SAME identities in a + // DIFFERENT order and the same length. The empty delta must be + // refused (hasRawTail gate) so the full ingest rebuilds tailSeqs + // in the envelope's order — otherwise the served view stays on + // the raw ordering and diverges from the disk record. + const envRender = [msgs[0], carrier, msgs[1], msgs[2]]; + legacy = mkLegacy("wm-render", envRender); + shadow.record(legacy); + assert.equal(stats(shadow).deltaIngests, 0, "envelope render must not ride the delta"); + const served = view(store, "wm-render"); + assert.ok(served.lastMessages); + assert.deepEqual(served.lastMessages.map((m) => m.id), ["h_msg0", "acp_summary_b1", "h_msg1", "h_msg2"], "served view must follow the envelope render order"); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: delta survives a fold (holes above the trusted tip)", () => { + const dir = tmpdir("bili-wm-fold-"); + try { + const { shadow, store } = openShadow(dir); + try { + const tail0 = Array.from({ length: 8 }, (_, i) => msg(i)); + shadow.record(mkLegacy("wm-fold", tail0)); + assert.equal(stats(shadow).deltaIngests, 0); + + // Fold: m2..m5 collapse into a new summary message; the tail + // SHRINKS below the trusted length — full ingest, and the + // summary lands ABOVE the surviving old rows (abs 8). + const summary = { id: "h_summary", role: "user", contentType: "text", text: "summary of 2..5" } as CoreMessage; + const tail1 = [tail0[0], tail0[1], summary, tail0[6], tail0[7]]; + shadow.record(mkLegacy("wm-fold", tail1)); + assert.equal(stats(shadow).deltaIngests, 0, "shorter tail cannot delta"); + const mark1 = store.ingestWatermark("wm-fold"); + assert.ok(mark1, "full ingest re-establishes the watermark at the fold"); + assert.deepEqual(mark1.tailSeqs, [0, 1, 8, 6, 7], "tail pointers carry the holes verbatim"); + assert.equal(mark1.seq, 7, "watermark = seq of the LAST tail message, not the max"); + + // Post-fold append: delta must work IMMEDIATELY (anchored at + // max(seq)+1 = 9, skipping the summary's slot is fine). + const tail2 = [...tail1, msg(8), msg(9)]; + ingestSerializations.count = 0; + shadow.record(mkLegacy("wm-fold", tail2)); + assert.equal(stats(shadow).deltaIngests, 1, "post-fold append takes the delta path"); + assert.equal(ingestSerializations.count, 3, "probe + 2 new messages"); + const mark2 = store.ingestWatermark("wm-fold"); + assert.ok(mark2); + assert.deepEqual(mark2.tailSeqs, [0, 1, 8, 6, 7, 9, 10], "appends continue after the max, not the watermark"); + assert.deepEqual(view(store, "wm-fold").lastMessages, tail2, "the read view is the exact folded tail"); + assertDeltaMatchesFull(store, mkLegacy("wm-fold", tail2)); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: boundary tamper falls back to full ingest and re-establishes", () => { + const dir = tmpdir("bili-wm-tamper-"); + try { + const { shadow, store } = openShadow(dir); + try { + const n = 10; + const at = (k: number) => mkLegacy("wm-tamper", Array.from({ length: k }, (_, i) => msg(i))); + shadow.record(at(n)); + assert.equal(stats(shadow).deltaIngests, 0); + const wm = store.ingestWatermark("wm-tamper"); + assert.ok(wm && wm.seq === n - 1); + + // Tamper leg 1: foreign UPDATE of the boundary row's hash (the + // junk content row exists purely to satisfy the FK). + const conn = openSqliteDatabase(path.join(dir, "index.db")).db; + try { + const junkBytes = Buffer.from(JSON.stringify({ id: "h_tampered", role: "user", contentType: "text", text: "tampered" }), "utf8"); + conn.prepare("INSERT OR IGNORE INTO content (hash, byte_len, refcount, bytes) VALUES ('deadbeef', ?, 1, ?)").run(junkBytes.length, junkBytes); + conn.prepare("UPDATE messages SET content_hash = 'deadbeef' WHERE session_id = ? AND seq = ?").run("wm-tamper", wm.seq); + } finally { + conn.close(); + } + shadow.record(at(n + 1)); + assert.equal(stats(shadow).deltaIngests, 0, "tampered boundary must force the full path"); + // The tampered row survives as an orphan at its old seq (its + // identity no longer matches the tail's message), and the true + // message re-enters at nextSeq — so assert on the VIEW (the + // tailSeqs-referenced messages), not the physical row count. + assert.deepEqual(view(store, "wm-tamper").lastMessages, at(n + 1).lastMessages, "full fallback writes every tail message correctly"); + const wm2 = store.ingestWatermark("wm-tamper"); + assert.ok(wm2, "watermark re-established after the full re-ingest"); + assertDeltaMatchesFull(store, at(n + 1)); + + // Tamper leg 2: foreign DELETE of the boundary row. + const conn2 = openSqliteDatabase(path.join(dir, "index.db")).db; + try { + conn2.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?").run("wm-tamper", wm2.seq); + } finally { + conn2.close(); + } + shadow.record(at(n + 2)); + assert.equal(stats(shadow).deltaIngests, 0, "missing boundary row must force the full path"); + assert.deepEqual(view(store, "wm-tamper").lastMessages, at(n + 2).lastMessages); + assertDeltaMatchesFull(store, at(n + 2)); + + // Recovery: with the boundary intact again, the next save deltas. + shadow.record(at(n + 3)); + assert.equal(stats(shadow).deltaIngests, 1, "delta path resumes once the watermark is trustworthy"); + assertDeltaMatchesFull(store, at(n + 3)); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: foreign rows above the watermark do not corrupt the delta", () => { + const dir = tmpdir("bili-wm-foreign-"); + try { + const { shadow, store } = openShadow(dir); + try { + const n = 6; + const at = (k: number) => mkLegacy("wm-foreign", Array.from({ length: k }, (_, i) => msg(i))); + shadow.record(at(n)); + const wm = store.ingestWatermark("wm-foreign"); + assert.ok(wm); + + // A non-bili writer drops a junk row above the trusted tip. The + // boundary identity is intact, so the delta still runs — but its + // appends anchor at max(seq)+1, never colliding with the junk. + const conn = openSqliteDatabase(path.join(dir, "index.db")).db; + try { + const junk = Buffer.from(JSON.stringify({ id: "h_junk", role: "user", contentType: "text", text: "foreign" }), "utf8"); + conn.prepare("INSERT INTO content (hash, byte_len, refcount, bytes) VALUES ('junkhash', ?, 1, ?)").run(junk.length, junk); + conn + .prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, NULL, NULL, 'user', 'message', 'junkhash')") + .run("wm-foreign", wm.seq + 5, "h_junk"); + } finally { + conn.close(); + } + shadow.record(at(n + 1)); + assert.equal(stats(shadow).deltaIngests, 1, "intact boundary still deltas"); + const mark = store.ingestWatermark("wm-foreign"); + assert.ok(mark); + assert.equal(mark.seq, wm.seq + 6, "append lands above the foreign row, watermark advances to it"); + const rows = store.getMessages("wm-foreign"); + assert.equal(rows.length, n + 2, "tail rows + the junk row, nothing lost"); + assert.deepEqual(view(store, "wm-foreign").lastMessages, at(n + 1).lastMessages, "the read view never references the junk"); + assertDeltaMatchesFull(store, at(n + 1)); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: fresh session (NULL watermark) runs the full ingest", () => { + const dir = tmpdir("bili-wm-fresh-"); + try { + const { shadow, store } = openShadow(dir); + try { + assert.equal(store.ingestWatermark("never-seen"), null, "no sessions row → NULL watermark"); + shadow.record(mkLegacy("wm-fresh", [msg(0), msg(1), msg(2)])); + assert.equal(stats(shadow).deltaIngests, 0); + assert.deepEqual(store.ingestWatermark("wm-fresh"), { seq: 2, tailSeqs: [0, 1, 2] }); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: remove() clears it; importRecord re-establishes it", () => { + const dir = tmpdir("bili-wm-remove-"); + try { + const { shadow, store } = openShadow(dir); + try { + const at = (k: number) => mkLegacy("wm-remove", Array.from({ length: k }, (_, i) => msg(i))); + shadow.record(at(5)); + assert.equal(stats(shadow).deltaIngests, 0); + + shadow.remove("wm-remove"); + assert.equal(store.getSessionRow("wm-remove"), null, "deleteSession drops the row (watermark dies with it)"); + assert.equal(store.ingestWatermark("wm-remove"), null); + + shadow.record(at(6)); + assert.equal(stats(shadow).deltaIngests, 0, "post-remove save must re-ingest fully"); + assert.equal(store.getMessages("wm-remove").length, 6); + + // Re-import (organic full-mode import path): full ingest by + // construction, watermark re-established for later deltas. + shadow.remove("wm-remove"); + const imported = shadow.importRecord(at(6)); + assert.ok(imported, "importRecord returns the legacy view"); + assert.deepEqual(store.ingestWatermark("wm-remove"), { seq: 5, tailSeqs: [0, 1, 2, 3, 4, 5] }); + shadow.record(at(7)); + assert.equal(stats(shadow).deltaIngests, 1, "saves delta again after the re-import"); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: rewound/re-derived tail falls back to full (no silent drops)", () => { + const dir = tmpdir("bili-wm-rewind-"); + try { + const { shadow, store } = openShadow(dir); + try { + // Save [m0..m4]; then the client rewinds and RE-DERIVES m1/m2 + // byte-identically (same ids + content) appended after m4 — the + // identities already have rows at seqs 1/2. + const tail0 = Array.from({ length: 5 }, (_, i) => msg(i)); + shadow.record(mkLegacy("wm-rewind", tail0)); + const tail1 = [...tail0, msg(1), msg(2)]; + shadow.record(mkLegacy("wm-rewind", tail1)); + assert.equal(stats(shadow).deltaIngests, 0, "re-derived identities must not ride the delta"); + const v = view(store, "wm-rewind"); + assert.deepEqual(v.lastMessages, tail1, "the full path points the re-derived slots at their existing rows"); + assert.equal(store.getMessages("wm-rewind").length, 5, "no duplicate rows for re-derived identities"); + assertDeltaMatchesFull(store, mkLegacy("wm-rewind", tail1)); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: snapshot-bearing sessions stay on the full path", () => { + const dir = tmpdir("bili-wm-snapshot-"); + try { + const { shadow, store } = openShadow(dir); + try { + const snapshot = [msg(0), msg(1)]; + const tail = [msg(0), msg(1), msg(2)]; + shadow.record(mkLegacy("wm-snap", tail, snapshot)); + // The full path sets the watermark here too (the fact "seq W backs + // the last tail message" holds regardless of snapshot shape), but + // it stays INERT: the delta planner refuses snapshot-bearing + // sessions, so their saves never ride it. + assert.ok(store.ingestWatermark("wm-snap"), "watermark fact is recorded"); + shadow.record(mkLegacy("wm-snap", [...tail, msg(3)], snapshot)); + assert.equal(stats(shadow).deltaIngests, 0, "pluginSnapshot present → always full"); + // The pure planner gate agrees (defense in depth). + assert.equal(planIngestDelta(mkLegacy("wm-snap", [...tail, msg(4)], snapshot), 2, [0, 1, 2], { msgId: "h_msg2", contentHash: "x" }, true), null); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: sampled verification still catches drift through the delta path", async () => { + const dir = tmpdir("bili-wm-verify-"); + try { + const { shadow, store } = openShadow(dir); + try { + // 21 appends: save 1 full (watermark set), saves 2..21 all delta. + let k = 5; + shadow.record(mkLegacy("wm-verify", Array.from({ length: k }, (_, i) => msg(i)))); + const s0 = stats(shadow); + assert.equal(s0.verifySamples, 1, "first ingest verifies"); + + // Tamper the CAS body under a tail row the delta path will NOT + // rewrite (the trusted prefix) — only a sampled read-back can + // catch it, exactly as pre-#3133. + const conn = openSqliteDatabase(path.join(dir, "index.db")).db; + try { + const row = conn.prepare("SELECT content_hash FROM messages WHERE session_id = ? ORDER BY seq LIMIT 1").get("wm-verify") as { content_hash: string } | undefined; + assert.ok(row, "row to tamper must exist"); + conn + .prepare("UPDATE content SET bytes = ? WHERE hash = ?") + .run(Buffer.from(JSON.stringify({ id: "tampered", role: "user", contentType: "text", text: "tampered" }), "utf8"), row.content_hash); + } finally { + conn.close(); + } + + for (let i = 0; i < 20; i++) { + k += 1; + shadow.record(mkLegacy("wm-verify", Array.from({ length: k }, (_, j) => msg(j)))); + } + const s1 = stats(shadow); + assert.equal(s1.deltaIngests, 20, "every post-watermark save took the delta path"); + assert.equal(s1.verifySamples, 2, "verify runs on ingest 1 and 21 only"); + assert.equal(s1.verifyOk, 1, "only the pre-tamper sample may pass"); + assert.equal(s1.verifyDrift, 1, "the sampled save at n=21 must catch the CAS drift"); + assert.equal(store.getMessages("wm-verify").length, k, "rows kept growing through the deltas"); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("watermark: v4 database migrates to v5 additively; newer user_version no-ops", async () => { + const dir = tmpdir("bili-wm-migrate-"); + try { + const dbPath = path.join(dir, "index.db"); + // Craft the v4 shape: the current fresh-install DDL minus the v5 + // column, one live session with message rows, user_version=4. + // created_at/updated_at must match mkLegacy: the sessions upsert + // keeps created_at from the existing row (first-seen semantics), so + // a mismatch would show up as sampled-verify drift, not a migration + // problem. + const { STORAGE_DDL_V1_FOR_TESTS } = await import("../src/storage/schema.ts"); + const v4Ddl = STORAGE_DDL_V1_FOR_TESTS.replace(/^\s*last_ingested_seq INTEGER,$/m, ""); + const { DatabaseSync } = await import("node:sqlite"); + const raw = new DatabaseSync(dbPath); + raw.exec(v4Ddl); + raw.prepare( + "INSERT INTO sessions (session_id, created_at, updated_at, next_block_id, next_run_id, meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ).run("v4-sess", 1000, 2000, 1, 1, "{}", "{}", "{}", "{}", '{"pluginSnapshotSeqs":null,"tailSeqs":[0],"tailFolded":false}', null); + raw.prepare("INSERT INTO content (hash, byte_len, refcount, bytes) VALUES (?, ?, 1, ?)").run("c0", Buffer.byteLength(JSON.stringify(msg(0))), Buffer.from(JSON.stringify(msg(0)), "utf8")); + raw.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, 0, ?, NULL, NULL, 'user', 'message', ?)").run("v4-sess", msg(0).id, "c0"); + raw.exec("PRAGMA user_version = 4"); + raw.close(); + + const store = UnifiedStore.open(dbPath); + try { + const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; + assert.equal(ver.user_version, 5, "open() migrates v4 → v5"); + const row = store.getSessionRow("v4-sess"); + assert.ok(row, "legacy session row survives the migration"); + assert.equal(row!.lastIngestedSeq, null, "legacy rows read back with a NULL watermark (≡ full ingest)"); + const messages = store.getMessages("v4-sess"); + assert.equal(messages.length, 1, "legacy message rows readable"); + assert.equal(messages[0].msgId, msg(0).id); + + // First save on a migrated DB: NULL watermark → full ingest → + // watermark established; the second save deltas. + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow); + try { + const legacy = mkLegacy("v4-sess", [msg(0), msg(1)]); + shadow.record(legacy); + assert.equal(stats(shadow).deltaIngests, 0); + const tail2 = [msg(0), msg(1), msg(2)]; + shadow.record(mkLegacy("v4-sess", tail2)); + assert.equal(stats(shadow).deltaIngests, 1, "migrated DB deltas from the first re-ingest on"); + assertDeltaMatchesFull(store, mkLegacy("v4-sess", tail2)); + } finally { + shadow.close(); + } + } finally { + store.close(); + } + + // Version-gate no-op: a v5 DB rolled FORWARD past the code's target + // (an older build opening a newer file) must open without error and + // leave user_version alone — the additive-column contract. + const fwd = new DatabaseSync(dbPath); + fwd.exec("PRAGMA user_version = 6"); + fwd.close(); + const older = UnifiedStore.open(dbPath); + try { + const ver2 = (older as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; + assert.equal(ver2.user_version, 6, "newer user_version is left untouched"); + assert.ok(older.getSessionRow("v4-sess"), "rows stay readable"); + } finally { + older.close(); + } + } finally { + rmBestEffort(dir); + } +}); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 0a43b751f0..9c4e47d023 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -706,7 +706,7 @@ test("schema v2: dev-window v1 database upgrades in place (identity columns adde assert.equal(row!.namespace, "proxy", "namespace backfills with its constant default"); assert.equal(row!.origin, null, "origin backfills NULL"); const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 4, "schema version stamped to 4 (v2 columns + v3 identity index + v4 extension namespaces)"); + assert.equal(ver.user_version, 5, "schema version stamped to 5 (v2 columns + v3 identity index + v4 extension namespaces + v5 watermark column)"); // v3 backfill: every pre-existing sessions row claims its // (namespace, conversation_id) identity binding. const binding = (store as unknown as { lookupConversation(ns: string, id: string): string | null }).lookupConversation("proxy", "dev-window-1"); From b9e82f641ad47c369657251c2c719ca39faedafd Mon Sep 17 00:00:00 2001 From: awork Date: Wed, 14 Oct 2026 19:21:07 +0800 Subject: [PATCH 2/3] fix: sampled digest drift drops the ingest watermark instead of reporting only (#3166 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive review fix on top of 64b99bb6a. The sampled read-back digest is the backstop for prefix corruption the boundary probe cannot see (envelope re-renders or interior rewrites below the boundary, foreign writes). It DETECTED drift but never repaired it: the sessions row kept its watermark, so every subsequent same-length save re-took the unverified empty delta against the stale watermark and the divergence was permanent (full mode would keep serving the superseded bytes). The drift branch now clears last_ingested_seq (new UnifiedStore.clearIngestWatermark) so the next save re-points every tail slot through the exact full path — divergence window bounded to <= VERIFY_EVERY_INGESTS saves. The drift test gains pointer-level heal assertions (watermark null, post-drift save full, row appended); byte parity is deliberately NOT asserted because the CAS tamper corrupts bytes under a known hash and content addressing re-lands on them (CAS repair is a separate lane). --- src/storage/shadow.ts | 11 ++++++++++- src/storage/store.ts | 10 ++++++++++ tests/storage-ingest-watermark.test.ts | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 788163d352..1aeea1eede 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -183,7 +183,16 @@ export class UnifiedShadow { // Panels consume {disabled, lastError} — make the sampled // degradation visible there, not just in the counter/log. this.stats.lastError = `digest drift for ${legacy.id}: unified replay != legacy load view`; - loggerLog("error", `[storage] unified shadow DIGEST DRIFT for ${legacy.id}: unified replay != legacy load view — migration parity broken, report this`); + // #3166 review: the backstop must heal, not only report — + // the watermark still vouches for a prefix the digest just + // proved stale (e.g. an envelope re-render or interior + // rewrite below the boundary), and every same-length save + // would re-take the unverified empty delta against it. + // Drop the watermark so the next save re-points every + // tail slot through the exact full path (divergence + // window bounded to ≤ VERIFY_EVERY_INGESTS saves). + this.store.clearIngestWatermark(legacy.id); + loggerLog("error", `[storage] unified shadow DIGEST DRIFT for ${legacy.id}: unified replay != legacy load view — dropped the ingest watermark, the next save re-ingests the full tail`); } else { this.stats.verifyOk++; } diff --git a/src/storage/store.ts b/src/storage/store.ts index 87d900e5b5..e078860426 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -614,6 +614,16 @@ export class UnifiedStore { return { seq, tailSeqs: seqs }; } + /** #3166 review (heal half of the sampled-verify backstop): drop the + * durable watermark so the next save takes the exact full path and + * re-points every tail slot. Called when the read-back digest proves + * the stored prefix no longer matches the record of truth — without it + * every same-length save would re-take the unverified empty delta + * against a stale watermark and the divergence would be permanent. */ + clearIngestWatermark(sessionId: string): void { + this.db.prepare("UPDATE sessions SET last_ingested_seq = NULL WHERE session_id = ?").run(sessionId); + } + /** #3133 boundary probe: the identity of the messages row at * (session_id, seq) plus the session's current max seq — both lookups * ride the PRIMARY KEY (session_id, seq) / idx_messages_session (no diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 6e7b0a6659..151d3c937c 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -508,6 +508,21 @@ test("watermark: sampled verification still catches drift through the delta path assert.equal(s1.verifyOk, 1, "only the pre-tamper sample may pass"); assert.equal(s1.verifyDrift, 1, "the sampled save at n=21 must catch the CAS drift"); assert.equal(store.getMessages("wm-verify").length, k, "rows kept growing through the deltas"); + // #3166 review: the drift backstop must HEAL, not only report — + // the watermark still vouches for a prefix the digest just proved + // stale, so it is dropped and the NEXT save re-points every tail + // slot through the exact full path. No byte-parity assertion + // here: the tamper corrupted the CAS BYTES under a known hash, + // and content addressing means a re-ingest of the same logical + // message legitimately lands on those bytes again (CAS repair is + // a separate integrity lane). The heal contract pinned here is + // pointer-level: watermark gone, next save full, row appended. + assert.equal(store.ingestWatermark("wm-verify"), null, "drift must clear the durable watermark"); + k += 1; + shadow.record(mkLegacy("wm-verify", Array.from({ length: k }, (_, j) => msg(j)))); + const s2 = stats(shadow); + assert.equal(s2.deltaIngests, 20, "post-drift save takes the full path"); + assert.equal(store.getMessages("wm-verify").length, k, "full re-ingest appended the new row"); } finally { shadow.close(); store.close(); From 976f8e364e52a729ad8003a012a29bc2bd8a3597 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Wed, 14 Oct 2026 21:31:47 +0800 Subject: [PATCH 3/3] test: kill idle keep-alive sockets at plugin-fork harness teardown (refs #3181) --- tests/plugin-fork.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 297cf6d1bb..dd65a2f2a3 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -112,7 +112,7 @@ async function harness(persist = false, seedParent = true, compressExtra: Record assert.equal(r.status, 200); await r.text(); } - return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); proxy.close(); upstream.close(); await Promise.all([once(proxy, "close"), once(upstream, "close")]); } }; + return { request, messages, forwarded, dir, store, upstreamUrl, origin, close: async () => { store.cancelAll(); proxy.close(); upstream.close(); proxy.closeIdleConnections?.(); upstream.closeIdleConnections?.(); await Promise.all([once(proxy, "close"), once(upstream, "close")]); } }; } test("plain proxy panel snapshots do not change raw-history persistence", async () => {