Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 81 additions & 13 deletions src/storage/ingest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import type { CoreMessage, CompressionBlock } from "../../kernel/src/types.js";
import type { BlockView } from "../session.js";
import { refToIndex } from "../../kernel/src/refs.js";
import { sha256Hex, type BlockRow, type CcrEntryRow, type IngestSessionInput, type MessageRow, type RefRow, type SessionRow } from "./store.js";
import { sha256Hex, type BlockRow, type CcrEntryRow, type DeltaRegistration, type IngestSessionInput, type MessageRow, type RefRow, type SeqState, type SessionRow } from "./store.js";

/** Field order of mergeState() output (src/persist.ts) — the loader must
* rebuild the state object in exactly this order. Keep in sync. */
Expand Down Expand Up @@ -80,6 +80,11 @@ interface IngestPlan extends IngestSessionInput {
/** Resolved log seqs for pluginSnapshot entries (null when absent). */
pluginSnapshotSeqs: number[] | null;
tailSeqs: number[] | null;
/** #3133 incremental mode: identities this plan allocated fresh seqs for
* (plan.messages, with their ref tokens). The caller commits them to the
* per-session ledger AFTER ingestSession's transaction succeeds — the
* ledger must never know about rows that did not land. */
newRegs?: DeltaRegistration[];
}

/** Adapts any Session-shaped object (persist loadSync output) to the
Expand Down Expand Up @@ -117,27 +122,79 @@ export function toLegacyLike(s: {

/** Builds rows + pointer sequences. Pure: no database touched, so tests can
* assert the mapping itself. seq assignment: first-seen over the ordered
* union (pluginSnapshot first — frozen raw prefix — then fresh tail ids). */
export function planIngest(session: LegacySessionLike): IngestPlan {
* union (pluginSnapshot first — frozen raw prefix — then fresh tail ids).
*
* #3133 incremental mode (`seqState` bound): the per-session ledger claims
* known messages WITHOUT serializing them, so a steady-state save costs O(new
* messages) instead of O(full log) — the wire entry points re-parse the whole
* history into fresh objects every request, so object identity never survives
* across saves and only content-keyed claims work. Claim ladder (each rung
* fails soft to the next):
* 1. ref-token claim — the token occurs exactly ONCE in this save's union
* (split tool-call cores share a token across distinct content parts, so
* a repeated token is ambiguous by construction) AND the ledger holds a
* sole-identity claim for it whose msgId matches — zero serialization;
* 2. known-by-bytes — stringify+hash, but the identity is already in the
* ledger (token invalidated or absent): claim its seq, emit no row;
* 3. new identity — allocate the next absolute seq, emit a row, register
* the pending registration for post-commit ledger update.
* A token whose claimed msgId no longer matches is de-ambiguated PERMANENTLY
* (variants counter) — shared sentinels like "BLOCKED" change holders over
* time, and a stale claim would replay another message's bytes under this
* one's pointer until the sampled digest verify notices. Full mode
* (no `seqState`) is byte-identical to the pre-existing behavior: relative
* seqs, every identity serialized, no registrations. */
export function planIngest(session: LegacySessionLike, seqState?: SeqState): IngestPlan {
const byRawRef = session.state.messageRefs.byRaw;
const st = seqState;
const delta = st !== undefined;

// Pass 0 (incremental only): ref-token occurrence counts over the ordered
// union — cheap string pass, no serialization. A token seen more than once
// in THIS save cannot be claimed (split cores / sentinel reuse).
const tokenCounts = new Map<string, number>();
if (delta) {
for (const m of [...(session.pluginSnapshot ?? []), ...(session.lastMessages ?? [])]) {
const t = byRawRef[m.id];
if (t !== undefined) tokenCounts.set(t, (tokenCounts.get(t) ?? 0) + 1);
}
}

// Pass 1: ordered union of identities (msg_id + content hash).
// #3133: no cross-save memo — the wire entry points replace the message
// array with freshly parsed objects on every request, so an object-
// identity WeakMap never hits on the production save path (see the
// issue for the content/seq watermark design that actually cuts cost).
const log: { msgId: string; content: string; hash: string; role: string; kind: string; ref: string | null }[] = [];
// seq travels with the entry: in full mode it equals the log index, but a
// delta plan logs ONLY the new rows, whose seqs are absolute cursor values.
const log: { msgId: string; content: string; hash: string; role: string; kind: string; ref: string | null; seq: number }[] = [];
const regs: DeltaRegistration[] = [];
const seqByIdentity = new Map<string, number>();
const intern = (m: CoreMessage): number => {
if (delta && st) {
const t = byRawRef[m.id];
if (t !== undefined && tokenCounts.get(t) === 1 && (st.tokenVariants.get(t) ?? 0) <= 1) {
const claim = st.byRef.get(t);
if (claim !== undefined) {
if (claim.msgId === m.id) return claim.seq;
st.tokenVariants.set(t, (st.tokenVariants.get(t) ?? 1) + 1);
st.byRef.delete(t);
}
}
}
const content = JSON.stringify(m);
const parsed = JSON.parse(content) as { role: string; contentType?: string };
const entry = { msgId: m.id, content, hash: sha256Hex(content), role: parsed.role, kind: parsed.contentType ?? "message" };
const key = `${m.id}\u0000${entry.hash}`;
const existing = seqByIdentity.get(key);
if (existing !== undefined) return existing;
const seq = log.length;
if (delta && st) {
const known = st.byIdentity.get(key);
if (known !== undefined) {
seqByIdentity.set(key, known);
return known;
}
}
const seq = delta && st ? st.nextSeq++ : log.length;
seqByIdentity.set(key, seq);
log.push({ msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null });
log.push({ msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null, seq });
if (delta) regs.push({ msgId: m.id, hash: entry.hash, seq, token: byRawRef[m.id] });
return seq;
};
const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null;
Expand All @@ -152,8 +209,8 @@ export function planIngest(session: LegacySessionLike): IngestPlan {
refNum: refToIndex(ref),
}));

const messages: MessageRow[] = log.map((e, seq) => ({
seq,
const messages: MessageRow[] = log.map((e) => ({
seq: e.seq,
msgId: e.msgId,
ref: e.ref,
refNum: e.ref === null ? null : refToIndex(e.ref),
Expand Down Expand Up @@ -236,7 +293,18 @@ export function planIngest(session: LegacySessionLike): IngestPlan {
ccrJson: session.contentStore ? JSON.stringify(session.contentStore) : null,
};

return { session: sessionRow, messages, blocks, deadRefs, refs, ccrEntries, pluginSnapshotSeqs, tailSeqs };
return {
session: sessionRow,
messages,
blocks,
deadRefs,
refs,
ccrEntries,
pluginSnapshotSeqs,
tailSeqs,
delta,
...(delta ? { newRegs: regs } : {}),
};
}

/** Rebuilds the legacy load view from unified rows alone. */
Expand Down
77 changes: 74 additions & 3 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,20 +49,31 @@ interface ShadowStats {
verifySamples: number;
verifyOk: number;
verifyDrift: number;
/** #3133: ingests that ran through the incremental (delta) path —
* observability for the steady-state cost claim, same semantics as
* verifySamples (counts RUNS, not sessions). */
deltaSaves: number;
imports: number;
disabled: boolean;
lastError: string | null;
}

export class UnifiedShadow {
readonly mode: StorageMode;
readonly stats: ShadowStats = { ingests: 0, verifySamples: 0, verifyOk: 0, verifyDrift: 0, imports: 0, disabled: false, lastError: null };
readonly stats: ShadowStats = { ingests: 0, verifySamples: 0, verifyOk: 0, verifyDrift: 0, deltaSaves: 0, imports: 0, disabled: false, lastError: null };
private readonly store: UnifiedStore;
private readonly legacyLoad: (record: unknown) => LegacySessionLike | null;
private consecutiveFailures = 0;
/** #3133: per-session ingest counters (verify sampling). Per-process
* only — nothing persisted, no schema change. */
private readonly ingestCounts = new Map<string, number>();
/** #3133 incremental mode: one-shot "skip the ledger" flags. Set after a
* sampled digest drift or a delta PK-conflict heal; the NEXT record for
* that session re-seeds a fresh ledger from the DB (full replay when the
* conflict path already dropped it) instead of trusting a possibly-stale
* claim map. Consumed on read — a session that never saves again keeps
* no state beyond the set entry. */
private readonly forceFullNext = new Set<string>();

private constructor(mode: StorageMode, store: UnifiedStore, legacyLoad: (record: unknown) => LegacySessionLike | null) {
this.mode = mode;
Expand Down Expand Up @@ -142,8 +153,35 @@ export class UnifiedShadow {
// stay empty for every plain compressed session. Both digest legs
// flow through the same override.
if (liveCcr) legacy = { ...legacy, contentStore: liveCcr };
const plan = planIngest(legacy);
this.store.ingestSession(plan);
// #3133 incremental ingest: bind the per-session seq ledger so
// known messages are claimed by token/identity instead of being
// re-serialized every save. A pending forceFull flag (drift or a
// previous conflict heal) skips the ledger for exactly this save —
// it is consumed on read, so the save after it resumes delta mode
// off a freshly seeded ledger.
const forceFull = this.forceFullNext.delete(legacy.id);
const st = forceFull ? undefined : this.store.seqStateFor(legacy.id) ?? undefined;
let plan = planIngest(legacy, st);
try {
this.store.ingestSession(plan);
} catch (err) {
// The only self-healable failure: a delta row collided with an
// externally written (session_id, seq). Drop the ledger, re-seed
// from the DB (which absorbs the foreign rows), and replay. Any
// other error propagates to the catch below (consecutive-failure
// accounting unchanged).
if (!(plan.delta && isConstraintError(err))) throw err;
loggerLog("warn", `[storage] unified shadow ${legacy.id}: delta ingest PK conflict — reseeding seq ledger and re-planning`);
this.store.dropSeqState(legacy.id);
this.forceFullNext.add(legacy.id);
plan = planIngest(legacy, this.store.seqStateFor(legacy.id) ?? undefined);
this.store.ingestSession(plan);
}
if (plan.delta) {
// Ledger commits only what the transaction actually wrote.
this.store.commitDelta(legacy.id, plan.newRegs ?? []);
this.stats.deltaSaves++;
}
this.stats.ingests++;
// #3133 sampled verification: ingest integrity is already guarded
// by the SQLite write path; the read-back digest is a migration
Expand All @@ -170,6 +208,12 @@ export class UnifiedShadow {
// Panels consume {disabled, lastError} — make the sampled
// degradation visible there, not just in the counter/log.
this.stats.lastError = `digest drift for ${legacy.id}: unified replay != legacy load view`;
// Drift means some claim may have pinned a stale identity —
// drop the ledger so the next save replays fully (and the
// one after resumes delta off a fresh seed), instead of
// continuing to serve possibly-wrong bytes under old seqs.
this.store.dropSeqState(legacy.id);
this.forceFullNext.add(legacy.id);
loggerLog("error", `[storage] unified shadow DIGEST DRIFT for ${legacy.id}: unified replay != legacy load view — migration parity broken, report this`);
} else {
this.stats.verifyOk++;
Expand Down Expand Up @@ -230,6 +274,10 @@ export class UnifiedShadow {
/** GC parity: remove the session's rows when the legacy files are deleted. */
remove(id: string): void {
this.ingestCounts.delete(id);
// A recreated session id must re-seed from scratch — reusing the
// deleted session's ledger would alias its old seqs.
this.store.dropSeqState(id);
this.forceFullNext.delete(id);
if (this.stats.disabled) return;
try {
this.store.deleteSession(id);
Expand Down Expand Up @@ -384,3 +432,26 @@ export class UnifiedShadow {
function shadowDigest(view: unknown): string {
return sha256(JSON.stringify(canonicalize(view)));
}

/** #3133 incremental mode: row-exists constraint violations are the only
* delta-ingest failure class that is self-healable (a seq stolen by an
* external writer, not corruption) — record() re-seeds and replays instead of
* counting a sidecar failure. The engines report them differently:
* better-sqlite3 carries a string `code` (SQLITE_CONSTRAINT_PRIMARYKEY /
* _UNIQUE); node:sqlite carries a numeric extended result code in `errcode`
* (base 19 = SQLITE_CONSTRAINT | sub-class << 8; a composite PK reports as
* UNIQUE, sub 6). Other constraint subclasses (FK/CHECK/NOTNULL) indicate a
* real bug and must propagate to the consecutive-failure accounting. */
function isConstraintError(err: unknown): boolean {
const e = err as { code?: unknown; errcode?: unknown } | null;
if (!e) return false;
// better-sqlite3 surfaces the sqlite result code as a string. node:sqlite
// ALSO sets .code — to a generic ERR_SQLITE_ERROR — so a string code only
// decides when it is itself a SQLITE_* code; otherwise the extended
// result code in errcode decides.
if (typeof e.code === "string" && e.code.startsWith("SQLITE_")) {
return e.code === "SQLITE_CONSTRAINT_PRIMARYKEY" || e.code === "SQLITE_CONSTRAINT_UNIQUE";
}
if (typeof e.errcode === "number") return e.errcode === (19 | (5 << 8)) || e.errcode === (19 | (6 << 8));
return false;
}
Loading
Loading