Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 131 additions & 6 deletions src/storage/ingest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,26 @@ interface IngestPlan extends IngestSessionInput {
tailSeqs: number[] | null;
}

/** #3133 test instrumentation: per-message JSON serializations performed by
* the planners in this process (intern pass, delta probe, delta new-tail
* pass). The watermark fix exists to make this O(new messages) per save
* instead of O(full log) — the storage watermark tests assert exactly
* that; production code never reads it. */
export const ingestSerializations = { count: 0 };

/** One messages-log entry with its ABSOLUTE seq already assigned (intern
* assigns first-seen order; the delta planner continues after the trusted
* prefix). */
interface LogEntry {
seq: number;
msgId: string;
content: string;
hash: string;
role: string;
kind: string;
ref: string | null;
}

/** Adapts any Session-shaped object (persist loadSync output) to the
* ingest input type. Structural on purpose: no import of src/session.ts
* (avoids a module cycle through persist.ts). */
Expand Down Expand Up @@ -124,25 +144,130 @@ export function planIngest(session: LegacySessionLike): IngestPlan {
// Pass 1: ordered union of identities (msg_id + content hash).
// #3133: no cross-save memo — the wire entry points replace the message
// array with freshly parsed objects on every request, so an object-
// identity WeakMap never hits on the production save path (see the
// issue for the content/seq watermark design that actually cuts cost).
const log: { msgId: string; content: string; hash: string; role: string; kind: string; ref: string | null }[] = [];
// identity WeakMap never hits on the production save path. The cost cut
// lives in planIngestDelta + the sessions.last_ingested_seq watermark.
const log: LogEntry[] = [];
const seqByIdentity = new Map<string, number>();
const intern = (m: CoreMessage): number => {
const content = JSON.stringify(m);
ingestSerializations.count++;
const parsed = JSON.parse(content) as { role: string; contentType?: string };
const entry = { msgId: m.id, content, hash: sha256Hex(content), role: parsed.role, kind: parsed.contentType ?? "message" };
const key = `${m.id}\u0000${entry.hash}`;
const existing = seqByIdentity.get(key);
if (existing !== undefined) return existing;
const seq = log.length;
seqByIdentity.set(key, seq);
log.push({ msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null });
log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null });
return seq;
};
const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null;
const tailSeqs = session.lastMessages ? session.lastMessages.map(intern) : null;

return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs);
}

/** #3133 fast-path planner: build the ingest plan for a session whose
* stored tail is a VERIFIED trusted prefix — the sessions row's watermark
* names the absolute seq backing the LAST tail message at save time, and
* `trustedTailSeqs` is that save's persisted tailSeqs (same transaction
* wrote both, so they cannot disagree). Soundness of skipping rows 0..L-1:
* every one of them was part of the tail that set the watermark (rows
* written by that save), and the probe pins slot L-1 to row `watermark`
* with nothing above it — so this save only has to append tail[L..] at
* watermark+1, watermark+2, … Only messages PAST the trusted prefix pay
* stringify+hash; every other row family is byte-identical to
* planIngest's, so the resulting store state equals a full ingest's.
*
* Returns null (→ full ingest) when: a pluginSnapshot is present (its
* entries may dedup against the trusted prefix and renumber the log),
* the tail is absent or not longer than the trusted prefix (shrunk /
* rewound / folded-in-place), or the boundary identity disagrees with the
* tail slot at L-1 (edited/deleted/re-minted history). Interior edits
* BELOW the boundary are the watermark design's accepted trade-off — they
* never mis-attribute, only leave stale rows, and the sampled verify
* digest is the backstop. */
export function planIngestDelta(
session: LegacySessionLike,
watermark: number,
trustedTailSeqs: number[],
boundary: { msgId: string; contentHash: string },
hasRawTail: boolean,
): IngestPlan | null {
if (session.pluginSnapshot && session.pluginSnapshot.length > 0) return null;
const tail = session.lastMessages;
const trustedLen = trustedTailSeqs.length;
if (
!tail ||
trustedLen === 0 ||
!Number.isSafeInteger(watermark) ||
watermark < 0 ||
// An equal-length tail is a VALID empty delta, but ONLY on the raw
// lineage: a caller passing the envelope's bounded folded RE-RENDER
// (no rawTail — e.g. the stale-convergence record(disk) path) may
// present the SAME identities in a DIFFERENT order (#2960 ⑤: the
// table holds the raw tail; the envelope re-renders it folded), and
// the boundary probe can match while the served view would stay on
// the raw ordering instead of the envelope render the disk record
// promised. Envelope renders must fall through to the full ingest,
// which rebuilds tailSeqs in the envelope's order.
(tail.length === trustedLen && !hasRawTail) ||
tail.length < trustedLen ||
trustedTailSeqs[trustedLen - 1] !== watermark
) {
return null;
}
// Boundary probe, content side: ONE serialization+hash of the tail slot
// at the trusted prefix's end. Identities are content-derived and
// unique within one conversion pass, so a match proves this tail slot
// IS the message the watermark was set from.
const probe = JSON.stringify(tail[trustedLen - 1]);
ingestSerializations.count++;
if (tail[trustedLen - 1].id !== boundary.msgId || sha256Hex(probe) !== boundary.contentHash) return null;
const byRawRef = session.state.messageRefs.byRaw;
const log: LogEntry[] = [];
const appended: number[] = [];
for (let i = trustedLen; i < tail.length; i++) {
const m = tail[i];
const content = JSON.stringify(m);
ingestSerializations.count++;
const parsed = JSON.parse(content) as { role: string; contentType?: string };
// Placeholder seqs under the aligned assumption (no rows above the
// watermark); ingestSession renumbers from max(seq)+1 inside the
// transaction when a fold left rows above the trusted tip.
const seq = watermark + 1 + log.length;
appended.push(seq);
log.push({
seq,
msgId: m.id,
content,
hash: sha256Hex(content),
role: parsed.role,
kind: parsed.contentType ?? "message",
ref: byRawRef[m.id] ?? null,
});
}
// planIngest's snapshot ternary replicated on the ORIGINAL value: an
// array (even empty, as fork receipts carry) maps to [], null stays
// null — loadLegacyView distinguishes the two, so they must not collapse.
const pluginSnapshotSeqs = session.pluginSnapshot ? [] : null;
// tailSeqs stay ABSOLUTE (the trusted prefix's seqs are the stored ones,
// holes included — folding removes log positions from the middle); the
// delta path skips remapPlanToAbsolute by construction.
const tailSeqs = [...trustedTailSeqs, ...appended];
return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs);
}

/** Shared plan tail (everything except the messages log): byte-identical
* output for both planners — the watermark fast path must not change what
* a full ingest would write for the same session. */
function finishPlan(
session: LegacySessionLike,
byRawRef: Record<string, string>,
log: LogEntry[],
pluginSnapshotSeqs: number[] | null,
tailSeqs: number[] | null,
): IngestPlan {
// Ref ledger: verbatim, ordered by the persisted map's key order.
// Ghost ids (dead refs) have rows here and nowhere else — by design.
const refs: RefRow[] = Object.entries(byRawRef).map(([rawId, ref], ord) => ({
Expand All @@ -152,8 +277,8 @@ export function planIngest(session: LegacySessionLike): IngestPlan {
refNum: refToIndex(ref),
}));

const messages: MessageRow[] = log.map((e, seq) => ({
seq,
const messages: MessageRow[] = log.map((e) => ({
seq: e.seq,
msgId: e.msgId,
ref: e.ref,
refNum: e.ref === null ? null : refToIndex(e.ref),
Expand Down
25 changes: 23 additions & 2 deletions src/storage/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,17 @@
// these tables are outside the LegacySessionLike view). Fresh
// databases create the v4 shape directly; the migration is two
// idempotent CREATE TABLE IF NOT EXISTS statements.
const STORAGE_SCHEMA_VERSION = 4;
// - v5: per-session ingest watermark (#3133) — sessions gains
// `last_ingested_seq`, the absolute seq backing the last tail
// message of the most recent save (NULL = no trusted prefix, full
// ingest). The trusted prefix length is re-read from the existing
// tail_json tailSeqs written by the same transaction — no second
// column. Point lookups and MAX(seq) stay on the pre-existing
// PRIMARY KEY (session_id, seq) / idx_messages_session — no new
// index. Both directions compatible: an older build opening a v5
// file no-ops at the version gate (extra column ignored); a newer
// build upgrades a v4 file with one idempotent ALTER TABLE.
const STORAGE_SCHEMA_VERSION = 5;

export const STORAGE_PRAGMAS = [
"PRAGMA journal_mode = WAL",
Expand Down Expand Up @@ -82,7 +92,8 @@ const STORAGE_DDL_V1 = `CREATE TABLE IF NOT EXISTS sessions (
metadata_json TEXT NOT NULL DEFAULT '{}',
kstate_json TEXT NOT NULL DEFAULT '{}',
tail_json TEXT NOT NULL DEFAULT '{}',
ccr_json TEXT
ccr_json TEXT,
last_ingested_seq INTEGER
);

CREATE TABLE IF NOT EXISTS content (
Expand Down Expand Up @@ -259,6 +270,16 @@ export function migrateSchema(db: {
db.exec("CREATE TABLE IF NOT EXISTS session_ext (\n session_id TEXT NOT NULL,\n key TEXT NOT NULL,\n value_json TEXT NOT NULL,\n updated_at INTEGER NOT NULL,\n PRIMARY KEY (session_id, key)\n ) WITHOUT ROWID");
db.exec("CREATE TABLE IF NOT EXISTS kv_store (\n namespace TEXT NOT NULL,\n key TEXT NOT NULL,\n value_json TEXT NOT NULL,\n updated_at INTEGER NOT NULL,\n PRIMARY KEY (namespace, key)\n ) WITHOUT ROWID");
}
if (from < 5) {
// v5 ingest watermark (#3133): additive NULLable column on
// sessions. Probe first — same duplicate-column discipline as v2
// (a dev-window build that already carried it re-lands cleanly).
const cols = (db.prepare("PRAGMA table_info(sessions)").all() as unknown as { name?: unknown }[])
.map((c) => (typeof c?.name === "string" ? c.name : ""));
if (!cols.includes("last_ingested_seq")) {
db.exec("ALTER TABLE sessions ADD COLUMN last_ingested_seq INTEGER");
}
}
db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`);
});
return { from, to: STORAGE_SCHEMA_VERSION };
Expand Down
61 changes: 56 additions & 5 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
*/

import { log as loggerLog } from "../logger.js";
import { planIngest, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts";
import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts";
import { resolveStorageMode, type StorageMode } from "../knobs.ts";
import { UnifiedStore, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts";
import { canonicalize, sha256 } from "./canonical.ts";
Expand All @@ -44,6 +44,7 @@ export { resolveStorageMode, type StorageMode } from "../knobs.ts";

interface ShadowStats {
ingests: number;
deltaIngests: number;
/** #3133: sampled read-back verifications RUN (verifyOk + verifyDrift
* ≤ this). Sampling, not removal — drift stays observable. */
verifySamples: number;
Expand All @@ -56,7 +57,7 @@ interface ShadowStats {

export class UnifiedShadow {
readonly mode: StorageMode;
readonly stats: ShadowStats = { ingests: 0, verifySamples: 0, verifyOk: 0, verifyDrift: 0, imports: 0, disabled: false, lastError: null };
readonly stats: ShadowStats = { ingests: 0, deltaIngests: 0, verifySamples: 0, verifyOk: 0, verifyDrift: 0, imports: 0, disabled: false, lastError: null };
private readonly store: UnifiedStore;
private readonly legacyLoad: (record: unknown) => LegacySessionLike | null;
private consecutiveFailures = 0;
Expand Down Expand Up @@ -133,7 +134,9 @@ export class UnifiedShadow {
const loaded = this.legacyLoad(record);
if (!loaded) return;
let legacy = loaded;
let hasRawTail = false;
if (rawTail && rawTail.lastMessages.length > 0) {
hasRawTail = true;
legacy = { ...legacy, lastMessages: rawTail.lastMessages, lastMessagesFolded: rawTail.lastMessagesFolded === true };
}
// #2960 ③: the live kernel content store when the caller has one
Expand All @@ -142,8 +145,18 @@ export class UnifiedShadow {
// stay empty for every plain compressed session. Both digest legs
// flow through the same override.
if (liveCcr) legacy = { ...legacy, contentStore: liveCcr };
const plan = planIngest(legacy);
this.store.ingestSession(plan);
// #3133 watermark: pay the full O(log) ingest only when the
// trusted-prefix probe cannot vouch for the stored prefix —
// steady-state saves stringify+hash+INSERT just the NEW tail.
// Any doubt (no watermark, foreign edit, tail shorter than the
// watermark, snapshot-bearing session) falls back to the exact
// full path below; the delta itself re-verifies the boundary
// inside the ingest transaction, so a racing writer degrades to
// a full ingest, never a corrupt one.
if (!this.ingestDelta(legacy, hasRawTail)) {
const plan = planIngest(legacy);
this.store.ingestSession(plan);
}
this.stats.ingests++;
// #3133 sampled verification: ingest integrity is already guarded
// by the SQLite write path; the read-back digest is a migration
Expand All @@ -170,7 +183,16 @@ export class UnifiedShadow {
// Panels consume {disabled, lastError} — make the sampled
// degradation visible there, not just in the counter/log.
this.stats.lastError = `digest drift for ${legacy.id}: unified replay != legacy load view`;
loggerLog("error", `[storage] unified shadow DIGEST DRIFT for ${legacy.id}: unified replay != legacy load view — migration parity broken, report this`);
// #3166 review: the backstop must heal, not only report —
// the watermark still vouches for a prefix the digest just
// proved stale (e.g. an envelope re-render or interior
// rewrite below the boundary), and every same-length save
// would re-take the unverified empty delta against it.
// Drop the watermark so the next save re-points every
// tail slot through the exact full path (divergence
// window bounded to ≤ VERIFY_EVERY_INGESTS saves).
this.store.clearIngestWatermark(legacy.id);
loggerLog("error", `[storage] unified shadow DIGEST DRIFT for ${legacy.id}: unified replay != legacy load view — dropped the ingest watermark, the next save re-ingests the full tail`);
} else {
this.stats.verifyOk++;
}
Expand All @@ -186,6 +208,35 @@ export class UnifiedShadow {
}
}

/** #3133 delta ingest attempt for one record's legacy view — returns
* true when the O(new-messages) fast path committed. Every doubt
* routes to `false` (→ the caller's full ingest); failures here must
* never throw into record(), whose catch counts toward disabling the
* shadow — a watermark hiccup is not a storage health problem. */
private ingestDelta(legacy: LegacySessionLike, hasRawTail: boolean): boolean {
try {
const mark = this.store.ingestWatermark(legacy.id);
if (!mark) return false;
// The pre-transaction probe is an optimization (skip building a
// doomed plan); ingestSession re-verifies the boundary inside
// BEGIN IMMEDIATE, which is the authoritative check.
const boundary = this.store.probeIngestBoundary(legacy.id, mark.seq);
if (!boundary) return false;
const plan = planIngestDelta(legacy, mark.seq, mark.tailSeqs, { msgId: boundary.msgId, contentHash: boundary.contentHash }, hasRawTail);
if (!plan) return false;
const applied = this.store.ingestSession(plan, {
trustedSeq: mark.seq,
boundaryMsgId: boundary.msgId,
boundaryHash: boundary.contentHash,
trustedTailSeqs: mark.tailSeqs,
});
if (applied) this.stats.deltaIngests++;
return applied;
} catch {
return false;
}
}

/** Loads the unified view of one session (full mode read path).
* Returns null on miss — the caller falls back to the legacy file. */
viewFor(id: string): LegacySessionLike | null {
Expand Down
Loading
Loading