Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 43 additions & 9 deletions src/storage/ingest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -611,14 +611,29 @@ interface LegacyView {

type BlockContents = LegacyView["blockContents"];

/** Shared by full-view and resident-field reads; each page is released after parsing. */
export function appendBlockContents(contents: BlockContents, rows: Pick<LegacyViewRows["blocks"][number], "blockId" | "summary" | "one">[]): void {
/** Shared by full-view and resident-field reads; each page is released after
* parsing. #7337: a corrupt block summary/one CAS row is skipped and returned
* (not thrown) so one bad row cannot abort the whole load — before this it
* threw out of finish()/readResidentFieldsFor and the session fell back to the
* legacy file (or was discarded when none exists). */
export function appendBlockContents(
contents: BlockContents,
rows: Pick<LegacyViewRows["blocks"][number], "blockId" | "summary" | "one">[],
): { blockId: string; error: string }[] {
const skipped: { blockId: string; error: string }[] = [];
for (const b of rows) {
contents.set(b.blockId, {
one: b.one === null ? null : (JSON.parse(b.one) as BlockView),
full: JSON.parse(b.summary) as BlockView,
});
let one: BlockView | null;
let full: BlockView;
try {
one = b.one === null ? null : (JSON.parse(b.one) as BlockView);
full = JSON.parse(b.summary) as BlockView;
} catch (err) {
skipped.push({ blockId: b.blockId, error: String(err) });
continue;
}
contents.set(b.blockId, { one, full });
}
return skipped;
}

export function mergeOrphanBlockContents(contents: BlockContents, orphansJson: string | null | undefined): void {
Expand Down Expand Up @@ -647,6 +662,11 @@ export class LegacyViewBuilder {
private readonly blocks: unknown[] = [];
private readonly blockContents: BlockContents = new Map();
private readonly messagesBySeq = new Map<number, CoreMessage>();
/** #7335 poison-row isolation: rows whose stored bytes no longer parse as
* JSON are skipped and recorded here instead of aborting the whole load —
* before this, one bad row threw out of finish() and the session was
* rebuilt from scratch (all blocks and refs discarded). */
readonly skippedCorruptRows: { table: "messages" | "blocks"; key: string; error: string }[] = [];

constructor(
private readonly session: LegacyViewRows["session"],
Expand Down Expand Up @@ -697,12 +717,26 @@ export class LegacyViewBuilder {
}

addBlocks(rows: LegacyViewRows["blocks"]): void {
for (const b of rows) this.blocks.push(JSON.parse(b.payloadJson));
appendBlockContents(this.blockContents, rows);
for (const b of rows) {
try {
this.blocks.push(JSON.parse(b.payloadJson));
} catch (err) {
this.skippedCorruptRows.push({ table: "blocks", key: String(b.blockId), error: String(err) });
}
}
for (const s of appendBlockContents(this.blockContents, rows)) {
this.skippedCorruptRows.push({ table: "blocks", key: String(s.blockId), error: s.error });
}
}

addMessages(rows: LegacyViewRows["messages"]): void {
for (const m of rows) this.messagesBySeq.set(m.seq, JSON.parse(m.content) as CoreMessage);
for (const m of rows) {
try {
this.messagesBySeq.set(m.seq, JSON.parse(m.content) as CoreMessage);
} catch (err) {
this.skippedCorruptRows.push({ table: "messages", key: `seq=${m.seq}`, error: String(err) });
}
}
}

finish(): LegacyView {
Expand Down
10 changes: 9 additions & 1 deletion src/storage/mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,15 @@ export class MysqlBackend implements StorageBackend {
/** Atomic refcount-aware CAS insert-or-bump. */
private async putContent(c: PoolConnection | Pool, content: string, precomputedHash?: string): Promise<string> {
const bytes = Buffer.from(content, "utf8");
const hash = precomputedHash ?? sha256Hex(bytes);
// #7335: verify the caller-supplied digest against the bytes — the CAS
// key is what every reader resolves by, so a mismatched pair poisons
// the store (sqlite twin has the same guard).
const hash = sha256Hex(bytes);
if (precomputedHash !== undefined && precomputedHash !== hash) {
throw new Error(
`content CAS digest mismatch: caller supplied ${precomputedHash} but ${bytes.byteLength} bytes hash to ${hash} — refusing to persist an unverified row`,
);
}
await c.query(
"INSERT INTO content (hash, byte_len, refcount, algo, bytes) VALUES (?, ?, 1, 'sha256', ?) " +
"ON DUPLICATE KEY UPDATE refcount = refcount + 1",
Expand Down
44 changes: 42 additions & 2 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,7 @@ export class UnifiedShadow {
for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) {
builder.addMessages(store.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) }));
}
warnCorruptSkips(id, builder);
// The builder preserves every MERGE_STATE_FIELDS key; its
// generic state declaration only needs widening here.
return builder.finish() as LegacySessionLike;
Expand Down Expand Up @@ -756,12 +757,21 @@ export class UnifiedShadow {
}
if (fields.blockContents) {
const contents: NonNullable<ResidentFields["blockContents"]> = new Map();
const skipped: { blockId: string; error: string }[] = [];
for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) {
const page = store.getBlockContents(id, { limit: VIEW_READ_BATCH_SIZE, offset });
const blocks = Array.isArray(page) ? page : [];
appendBlockContents(contents, blocks);
for (const s of appendBlockContents(contents, blocks)) skipped.push(s);
if (blocks.length < VIEW_READ_BATCH_SIZE) break;
}
if (skipped.length > 0) {
loggerLog(
"warn",
`[storage] resident block contents degraded for ${id}: ${skipped.length} unparseable row(s) skipped (${skipped
.map((s) => `${s.blockId}: ${s.error}`)
.join("; ")}); loaded without them (#7337)`,
);
}
mergeOrphanBlockContents(contents, store.getOrphansJson(id));
result.blockContents = contents;
}
Expand Down Expand Up @@ -1312,6 +1322,7 @@ export class UnifiedShadow {
for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) {
builder.addMessages(await reader.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) }));
}
warnCorruptSkips(id, builder);
return builder.finish() as LegacySessionLike;
});
if (!loaded) return null;
Expand Down Expand Up @@ -1537,6 +1548,25 @@ function warnTailGaps(id: string, view: { tailGaps?: number }): void {
}
}

/** #7335: poison-row isolation report — unparseable rows the builder skipped
* so a single corrupt row cannot abort the whole load (previously one bad
* row threw out of finish(), the session was rebuilt from scratch and every
* block and ref was discarded). */
function warnCorruptSkips(
id: string,
builder: { skippedCorruptRows: { table: string; key: string; error: string }[] },
): void {
const rows = builder.skippedCorruptRows;
if (rows.length > 0) {
loggerLog(
"warn",
`[storage] unified view degraded for ${id}: ${rows.length} unparseable row(s) skipped (${rows
.map((r) => `${r.table} ${r.key}: ${r.error}`)
.join("; ")}); the session loaded without them (#7335)`,
);
}
}

function shadowDigest(view: unknown): string {
return canonicalDigest(view);
}
Expand Down Expand Up @@ -1620,7 +1650,17 @@ class BatchedViewVerification {

addMessages(rows: MessageRow[]): void {
for (const row of rows) {
const raw = JSON.parse(row.content);
let raw: unknown;
try {
raw = JSON.parse(row.content);
} catch {
// #7335: a poisoned row must not abort the whole save — the
// verify-sample throw retried on every subsequent save until
// the store was cleaned. Skip it and mark drift so the
// comparison surfaces the disagreement loudly instead.
this.changed("messages");
continue;
}
this.messages.set(row.seq, { sig: messageSignature(raw), raw });
}
}
Expand Down
16 changes: 11 additions & 5 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -959,11 +959,17 @@ export class UnifiedStore {
/** Content CAS: insert-if-absent + refcount bump; returns hash. */
private putContent(content: string | Buffer, precomputedHash?: string): string {
const bytes = typeof content === "string" ? Buffer.from(content, "utf8") : content;
// #3133: planner-produced rows carry the digest from their intern or
// delta pass — skip the per-message re-hash here (same value: sha256
// over the same utf8 bytes). Only direct callers without a planner
// pay the hash at this layer.
const hash = precomputedHash ?? sha256Hex(bytes);
// #3133 kept the planner digest as the CAS key to skip the per-message
// re-hash; #7335 turns the same digest into a VERIFIED one — the CAS
// key is what every reader resolves by, so a caller-supplied hash that
// does not match its bytes silently poisons the store (observed: 26
// mismatched rows from an older dev build killed an entire session).
const hash = sha256Hex(bytes);
if (precomputedHash !== undefined && precomputedHash !== hash) {
throw new Error(
`content CAS digest mismatch: caller supplied ${precomputedHash} but ${bytes.byteLength} bytes hash to ${hash} — refusing to persist an unverified row`,
);
}
const existing = this.db.prepare("SELECT refcount FROM content WHERE hash = ?").get(hash) as
| { refcount: number }
| undefined;
Expand Down
Loading
Loading