Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 72 additions & 10 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,11 @@ export class SessionStore {
// #3931 step 3: assigned asynchronously for network backends (mysql) —
// see the constructor and ready().
private shadow?: UnifiedShadow;
/** Only alive while one synchronous record build hydrates both lazy fields. */
private residentFieldsForBuild?: {
id: string;
fields: { metadata?: Record<string, unknown>; blockContents?: Record<string, BlockContent> } | null;
};
/** #2960 ②: sessions whose fork_base checkpoint has been cut (or was
* found already present) — memoized so steady-state writes of receipt
* sessions do not re-query the checkpoints table every flush. */
Expand Down Expand Up @@ -669,7 +674,7 @@ export class SessionStore {
// #408 one-time migration: the in-memory value is already
// clamped by buildSession — rewrite the stale file so the
// negative value is gone from disk.
await this.store.writeNow(id, () => buildRecord(session));
await this.store.writeNow(id, () => this.buildRecordForSave(session));
clamped++;
// #3049: the clamp rewrite writes a DIFFERENT payload than any
// ingested record (clamped stats) without a sidecar ingest —
Expand Down Expand Up @@ -1149,7 +1154,7 @@ export class SessionStore {
continue;
}
session.id = label;
await this.store.writeNow(label, () => buildRecord(session));
await this.store.writeNow(label, () => this.buildRecordForSave(session));
// #3049: direct write bypassing guardedBuild — the boot rekey
// rewrites envelopes without the sidecar ingest, so any prior
// ownership token for this id is stale.
Expand Down Expand Up @@ -1672,6 +1677,60 @@ export class SessionStore {
return this.shadow ? this.shadow.viewFor(id) : null;
}

/** Restore only the dropped fields, with the same source preference as loadSync.
* A field read must not restamp refs, import rows, or schedule another save. */
private loadResidentFields(
id: string,
meta: { protocol?: string; upstreamOrigin?: string },
fields: { metadata?: boolean; blockContents?: boolean },
): { metadata?: Record<string, unknown>; blockContents?: Record<string, BlockContent> } | null {
if (this.residentFieldsForBuild?.id === id) return this.residentFieldsForBuild.fields;
if (!this.enabled) return null;
const indexed = this.shadow?.readsUnified ? this.shadow.readResidentFieldsFor(id, fields) : null;
const envelope = this.loadEnvelope(id, meta);
let useEnvelope = indexed === null;
if (indexed && envelope) {
if (this.shadow?.driftedSessions.has(id)) {
const envActive = countActiveBlocks(envelope.payload);
const envRefs = Object.keys(envelope.payload.state?.messageRefs?.byRef ?? {}).length;
useEnvelope = indexed.header.activeBlockCount <= envActive && indexed.header.mappedRefCount <= envRefs;
} else {
const requests = Number(envelope.payload.stats?.requests ?? 0);
useEnvelope = requests > indexed.header.requests ||
(requests === indexed.header.requests && Number(envelope.payload.state?.nextBlockId ?? 0) > indexed.header.nextBlockId);
}
}
if (useEnvelope) {
if (!envelope) return null;
return {
...(fields.metadata ? { metadata: envelope.payload.metadata ?? {} } : {}),
...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(Object.entries(envelope.payload.blockContents ?? {}))) } : {}),
};
}
if (!indexed) return null;
return {
...(fields.metadata ? { metadata: indexed.metadata } : {}),
...(fields.blockContents ? { blockContents: Object.fromEntries(restoreBlockContents(indexed.blockContents ?? [])) } : {}),
};
}

private buildRecordForSave(session: Session, storageMode: "legacy" | "shadow" | "full" = effectiveFileStorageMode()): PersistedSession {
const bothDropped = isDroppedMetadata(session.metadata) &&
session.blockContents instanceof LazyBlockContents && session.blockContents.isDropped;
if (!bothDropped) return buildRecord(session, storageMode);
// Metadata and contents used to each rebuild a whole Session here. Share
// one projection and envelope arbitration for this build, never a cache
// that keeps another session alive after the synchronous build returns.
const previous = this.residentFieldsForBuild;
const fields = this.loadResidentFields(session.id, session.meta ?? {}, { metadata: true, blockContents: true });
this.residentFieldsForBuild = { id: session.id, fields };
try {
return buildRecord(session, storageMode);
} finally {
this.residentFieldsForBuild = previous;
}
}

/** Read-only state load for cross-session search (#841): unlike loadSync,
* never schedules a save (no #408 clamp-rewrite side effect). */
loadStateForSearch(id: string): CompressionState | null {
Expand Down Expand Up @@ -2084,7 +2143,7 @@ export class SessionStore {
this.forkBaseCut.add(session.id);
return;
}
const record = buildRecord(session);
const record = this.buildRecordForSave(session);
const activeBlockIds = session.state.blocks
.filter((b) => (b as { active?: boolean }).active !== false)
.map((b) => b.blockId)
Expand Down Expand Up @@ -2393,7 +2452,7 @@ export class SessionStore {
// the JSON envelope, and the stamp must say legacy for that copy.
const stamp = this.liveStorageStamp;
return () => {
const record = buildRecord(session, stamp);
const record = this.buildRecordForSave(session, stamp);
const disk = this.staleDiskPayload(record);
if (disk === null) {
// #3301: external-converter import payload rides exactly ONE
Expand Down Expand Up @@ -2675,8 +2734,7 @@ export class SessionStore {
// loader degrades to the id-only lookup, same as a protocol-less file.
const meta = session.meta ?? undefined;
session.metadata = lazyMetadata(session.id, { protocol: meta?.protocol, upstreamOrigin: meta?.upstreamOrigin }, (id, m) => {
const loaded = this.loadSync(id, m);
return loaded ? loaded.metadata : null;
return this.loadResidentFields(id, m, { metadata: true })?.metadata ?? null;
});
}

Expand All @@ -2694,8 +2752,7 @@ export class SessionStore {
}
if (existing.size === 0) return;
const lazy = new LazyBlockContents(session.id, { protocol: session.meta.protocol, upstreamOrigin: session.meta.upstreamOrigin }, (id, meta) => {
const loaded = this.loadSync(id, meta);
return loaded ? Object.fromEntries(loaded.blockContents) : null;
return this.loadResidentFields(id, meta, { blockContents: true })?.blockContents ?? null;
});
lazy.drop();
session.blockContents = lazy;
Expand Down Expand Up @@ -2993,9 +3050,9 @@ function restoreCalibrationRing(v: unknown): { origin: string; values: number[]
return recent.length > 0 ? { origin: ring.origin, values: recent } : undefined;
}

function buildSession(parsed: PersistedSession): Session {
function restoreBlockContents(entries: Iterable<readonly [string, unknown]>): Map<string, BlockContent> {
const blockContents = new Map<string, BlockContent>();
for (const [bid, content] of Object.entries(parsed.blockContents ?? {})) {
for (const [bid, content] of entries) {
if (!content || typeof content !== "object") continue;
const full = (content as Record<string, unknown>).full;
if (!isBlockView(full)) continue;
Expand All @@ -3008,6 +3065,11 @@ function buildSession(parsed: PersistedSession): Session {
full,
});
}
return blockContents;
}

function buildSession(parsed: PersistedSession): Session {
const blockContents = restoreBlockContents(Object.entries(parsed.blockContents ?? {}));
// Read grouped shape (v2+); fall back to flat fields for v1 files.
const meta = parsed.meta ?? {};
const stats = parsed.stats ?? {};
Expand Down
25 changes: 19 additions & 6 deletions src/storage/backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,12 @@ export interface StorageBackend {
seqByRaw(sessionId: string): Promise<Map<string, number> | null>;
/** #4316 PR3/3: fork adoption row clone (idempotent). */
cloneSessionRows(parentId: string, childId: string, throughSeq: number, initial?: IngestSessionInput): Promise<boolean>;
getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>;
getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>;
getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]>;
getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>;
getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>;
getRefs(sessionId: string): Promise<RefRow[]>;
/** All reads, including CAS hydration, use one short read-only snapshot. */
readSnapshot<T>(read: (reader: StorageReadView) => Promise<T>): Promise<T>;
getCcr(sessionId: string): Promise<MessageContentStore | null>;
getCcrEntries(sessionId: string): Promise<CcrEntryRow[]>;

Expand Down Expand Up @@ -141,6 +143,8 @@ export interface StorageBackend {
init?(): Promise<void>;
}

export type StorageReadView = Pick<StorageBackend, "getSessionRow" | "getMessages" | "getBlocks" | "getDeadRefs" | "getRefs">;

/** The sqlite backend: the existing UnifiedStore behind the async contract.
* Synchronous under the hood; `Promise.resolve` wrapping only. */
export class SqliteBackend implements StorageBackend {
Expand Down Expand Up @@ -190,18 +194,27 @@ export class SqliteBackend implements StorageBackend {
cloneSessionRows(parentId: string, childId: string, throughSeq: number, initial?: IngestSessionInput): Promise<boolean> {
return Promise.resolve(this.store.cloneSessionRows(parentId, childId, throughSeq, initial));
}
getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> {
return Promise.resolve(this.store.getMessages(sessionId));
getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]> {
return Promise.resolve(this.store.getMessages(sessionId, opts));
}
getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> {
return Promise.resolve(this.store.getBlocks(sessionId));
getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> {
return Promise.resolve(this.store.getBlocks(sessionId, opts));
}
getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> {
return Promise.resolve(this.store.getDeadRefs(sessionId));
}
getRefs(sessionId: string): Promise<RefRow[]> {
return Promise.resolve(this.store.getRefs(sessionId));
}
readSnapshot<T>(read: (reader: StorageReadView) => Promise<T>): Promise<T> {
return this.store.readSnapshotAsync((store) => read({
getSessionRow: async (id) => store.getSessionRow(id),
getMessages: async (id, opts) => store.getMessages(id, opts),
getBlocks: async (id, opts) => store.getBlocks(id, opts),
getDeadRefs: async (id) => store.getDeadRefs(id),
getRefs: async (id) => store.getRefs(id),
}));
}
getCcr(sessionId: string): Promise<MessageContentStore | null> {
return Promise.resolve(this.store.getCcr(sessionId));
}
Expand Down
45 changes: 25 additions & 20 deletions src/storage/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ export interface SqliteDatabase {
prepare(sql: string): SqliteStatement;
/** Runs fn inside BEGIN/COMMIT; rolls back on throw (including nested calls via savepoints). */
transaction<T>(fn: () => T): T;
/** Deferred snapshot: readers do not reserve the writer lock. */
readTransaction<T>(fn: () => T): T;
close(): void;
readonly open: boolean;
}
Expand Down Expand Up @@ -87,6 +89,7 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase {
// cannot resolve. Taking the write lock up front makes concurrent
// writers queue on busy_timeout instead (#2690 review blocking 5).
transaction: (fn) => db.transaction(fn).immediate(),
readTransaction: (fn) => db.transaction(fn).deferred(),
close: () => void db.close(),
get open() {
return db.open;
Expand All @@ -104,31 +107,33 @@ function openNodeSqlite(path: string): SqliteDatabase {
};
const db = new DatabaseSync(path);
let depth = 0;
const begin = () => db.exec(depth === 0 ? "BEGIN IMMEDIATE" : `SAVEPOINT sp${depth}`); // IMMEDIATE at top level: multi-process SQLITE_BUSY_SNAPSHOT guard (see openBetterSqlite3)
const begin = (readOnly: boolean) => db.exec(depth === 0 ? (readOnly ? "BEGIN DEFERRED" : "BEGIN IMMEDIATE") : `SAVEPOINT sp${depth}`);
const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`);
const rollback = () => db.exec(depth === 1 ? "ROLLBACK" : `ROLLBACK TO sp${depth - 1}`);
const transaction = <T>(fn: () => T, readOnly: boolean): T => {
if (db.getAutoCommit?.() === false && depth === 0) {
// Already inside an outer transaction driven directly by
// DatabaseSync — just run inline; the outer COMMIT covers us.
return fn();
}
begin(readOnly);
depth++;
try {
const out = fn();
commit();
return out;
} catch (err) {
rollback();
throw err;
} finally {
depth--;
}
};
return {
exec: (sql) => void db.exec(sql),
prepare: (sql) => adaptNodeSqliteStatement(db.prepare(sql)),
transaction: <T>(fn: () => T): T => {
if (db.getAutoCommit?.() === false && depth === 0) {
// Already inside an outer transaction driven directly by
// DatabaseSync — just run inline; the outer COMMIT covers us.
return fn();
}
begin();
depth++;
try {
const out = fn();
commit();
return out;
} catch (err) {
rollback();
throw err;
} finally {
depth--;
}
},
transaction: (fn) => transaction(fn, false),
readTransaction: (fn) => transaction(fn, true),
close: () => void db.close(),
get open() {
return db.open;
Expand Down
Loading
Loading