Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/storage/backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ export interface StorageBackend {
seqByRaw(sessionId: string): Promise<Map<string, number> | null>;
/** #4316 PR3/3: fork adoption row clone (idempotent). */
cloneSessionRows(parentId: string, childId: string, throughSeq: number, initial?: IngestSessionInput): Promise<boolean>;
getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>;
getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>;
getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]>;
getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>;
getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>;
getRefs(sessionId: string): Promise<RefRow[]>;
getCcr(sessionId: string): Promise<MessageContentStore | null>;
Expand Down Expand Up @@ -190,11 +190,11 @@ export class SqliteBackend implements StorageBackend {
cloneSessionRows(parentId: string, childId: string, throughSeq: number, initial?: IngestSessionInput): Promise<boolean> {
return Promise.resolve(this.store.cloneSessionRows(parentId, childId, throughSeq, initial));
}
getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> {
return Promise.resolve(this.store.getMessages(sessionId));
getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string })[]> {
return Promise.resolve(this.store.getMessages(sessionId, opts));
}
getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> {
return Promise.resolve(this.store.getBlocks(sessionId));
getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> {
return Promise.resolve(this.store.getBlocks(sessionId, opts));
}
getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]> {
return Promise.resolve(this.store.getDeadRefs(sessionId));
Expand Down
17 changes: 9 additions & 8 deletions src/storage/mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1336,13 +1336,14 @@ export class MysqlBackend implements StorageBackend {
};
}

async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> {
async getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> {
// #4390: retired rows stay out of the replay by default.
if (opts?.seqs?.length === 0) return [];
const retiredFilter = opts?.includeRetired === true ? "" : " AND retired_at IS NULL";
const seqFilter = opts?.seqs ? ` AND seq IN (${opts.seqs.map(() => "?").join(", ")})` : "";
const [rows] = (await this.pool.query(
opts?.includeRetired === true
? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq"
: "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq",
[sessionId],
`SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ?${retiredFilter}${seqFilter} ORDER BY seq`,
[sessionId, ...(opts?.seqs ?? [])],
)) as unknown as Row[][];
const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = [];
for (const r of rows) {
Expand Down Expand Up @@ -1399,12 +1400,12 @@ export class MysqlBackend implements StorageBackend {
return out;
}

async getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> {
async getBlocks(sessionId: string, opts?: { limit: number; offset?: number }): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> {
const [rows] = (await this.pool.query(
`SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash,
supersedes_block_id, active, created_at, covered_hash, payload_json
FROM blocks WHERE session_id = ? ORDER BY ord`,
[sessionId],
FROM blocks WHERE session_id = ? ORDER BY ord, block_id${opts ? " LIMIT ? OFFSET ?" : ""}`,
[sessionId, ...(opts ? [opts.limit, opts.offset ?? 0] : [])],
)) as unknown as Row[][];
const out: (BlockRow & { summaryHash: string; oneHash: string | null })[] = [];
for (const r of rows) {
Expand Down
185 changes: 149 additions & 36 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,11 @@ import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBac
import { canonicalize, sha256 } from "./canonical.ts";
import { recordSidecarInstance, recordSidecarOpenFailure } from "./runtime-status.ts";
import type { SqliteEngineName, SqliteEngineSource } from "./driver.ts";
import type { MessageRow, MessageRefContentRow } from "./store.js";
import type { MessageRow, BlockRow, SessionRow, RefRow, MessageRefContentRow } from "./store.js";
import type { CoreMessage, MessageContentStore } from "acp-kernel";

export const MAX_CONSECUTIVE_FAILURES = 3;
const VERIFY_BATCH_SIZE = 32;

/** #3133: read-back verification cadence — the verify leg (full read-back +
* double shadowDigest) is O(log) per save, while the drift it guards is a
Expand Down Expand Up @@ -377,20 +378,18 @@ export class UnifiedShadow {
this.stats.verifySamples++;
const sessionRow = this.sync.getSessionRow(legacy.id);
if (!sessionRow) throw new Error(`session row missing after ingest: ${legacy.id}`);
const rows = {
session: sessionRow,
messages: this.sync.getMessages(legacy.id),
blocks: this.sync.getBlocks(legacy.id),
deadRefs: this.sync.getDeadRefs(legacy.id),
refs: this.sync.getRefs(legacy.id),
};
const replayed = loadLegacyView(rows);
warnTailGaps(legacy.id, replayed);
const expectView = { ...legacy, blockContents: orderedBlockContents(completeBlockContents(legacy.blockContents, legacy.state.blocks)) };
const gotView = { ...replayed, blockContents: orderedBlockContents(replayed.blockContents) };
const expect = shadowDigest(expectView);
const got = shadowDigest(gotView);
if (got !== expect) {
const verifier = new BatchedViewVerification(legacy, sessionRow, this.sync.getRefs(legacy.id), this.sync.getDeadRefs(legacy.id));
for (let offset = 0; ; offset += VERIFY_BATCH_SIZE) {
const rows = this.sync.getBlocks(legacy.id, { limit: VERIFY_BATCH_SIZE, offset });
verifier.addBlocks(rows);
if (rows.length < VERIFY_BATCH_SIZE) break;
}
for (let i = 0; i < verifier.messageSeqs.length; i += VERIFY_BATCH_SIZE) {
verifier.addMessages(this.sync.getMessages(legacy.id, { seqs: verifier.messageSeqs.slice(i, i + VERIFY_BATCH_SIZE) }));
}
const verified = verifier.finish();
warnTailGaps(legacy.id, verified);
if (!verified.matches) {
this.stats.verifyDrift++;
// #3919: membership from BEFORE this add is the repeat
// signal — two consecutive sampled drifts mean the views
Expand All @@ -416,8 +415,8 @@ export class UnifiedShadow {
this.frozenEnvelopeIds.add(legacy.id);
let sides = "";
try {
const envBlocks = expectView.state.blocks.filter((b) => b.active).length;
const idxBlocks = (gotView.state.blocks as { active?: boolean }[] | null | undefined)?.filter((b) => b.active === true).length ?? 0;
const envBlocks = legacy.state.blocks.filter((b) => b.active).length;
const idxBlocks = verified.activeBlocks;
sides = ` [envelope ${envBlocks} active block(s) vs index ${idxBlocks}]`;
} catch {
// counts must never break the heal path
Expand All @@ -443,7 +442,7 @@ export class UnifiedShadow {
// The line prefix stays byte-identical for existing greps.
let diagnosis = "";
try {
diagnosis = formatDriftDiagnostics(describeViewDivergence(expectView, gotView));
diagnosis = formatDriftDiagnostics(verified.diagnostics);
} catch {
// diagnostics must never break the heal path
}
Expand Down Expand Up @@ -959,17 +958,18 @@ export class UnifiedShadow {
this.stats.verifySamples++;
const sessionRow = await backend.getSessionRow(legacy.id);
if (!sessionRow) throw new Error(`session row missing after ingest: ${legacy.id}`);
const rows = {
session: sessionRow,
messages: await backend.getMessages(legacy.id),
blocks: await backend.getBlocks(legacy.id),
deadRefs: await backend.getDeadRefs(legacy.id),
refs: await backend.getRefs(legacy.id),
};
const loaded = loadLegacyView(rows);
const expect = shadowDigest({ ...legacy, blockContents: orderedBlockContents(completeBlockContents(legacy.blockContents, legacy.state.blocks)) });
const got = shadowDigest({ ...loaded, blockContents: orderedBlockContents(loaded.blockContents) });
if (got !== expect) {
const verifier = new BatchedViewVerification(legacy, sessionRow, await backend.getRefs(legacy.id), await backend.getDeadRefs(legacy.id));
for (let offset = 0; ; offset += VERIFY_BATCH_SIZE) {
const rows = await backend.getBlocks(legacy.id, { limit: VERIFY_BATCH_SIZE, offset });
verifier.addBlocks(rows);
if (rows.length < VERIFY_BATCH_SIZE) break;
}
for (let i = 0; i < verifier.messageSeqs.length; i += VERIFY_BATCH_SIZE) {
verifier.addMessages(await backend.getMessages(legacy.id, { seqs: verifier.messageSeqs.slice(i, i + VERIFY_BATCH_SIZE) }));
}
const verified = verifier.finish();
warnTailGaps(legacy.id, verified);
if (!verified.matches) {
this.stats.verifyDrift++;
this.driftedSessions.add(legacy.id);
this.stats.lastError = `digest drift for ${legacy.id}: unified replay != legacy load view`;
Expand Down Expand Up @@ -1246,13 +1246,6 @@ function shadowDigest(view: unknown): string {
return sha256(JSON.stringify(canonicalize(view)));
}

// #3407: the parity digest must compare blockContents CONTENTS, not insertion order —
// the expect leg keeps the live Map's order (completeBlockContents appends synthesized
// entries at the end) while the got leg lays blocks out in state.blocks array order.
function orderedBlockContents<T>(m: Map<string, T>): Map<string, T> {
return new Map([...m.entries()].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)));
}

interface DriftSlotFingerprint {
id: string;
role: string;
Expand All @@ -1267,6 +1260,126 @@ interface DriftDiagnostics {
tailSlot?: { index: number; expect: DriftSlotFingerprint; got: DriftSlotFingerprint };
}

/** Compare the replay by component, retaining only message fingerprints across batches. */
class BatchedViewVerification {
readonly messageSeqs: number[];
private readonly tail: { tailSeqs: number[] | null; pluginSnapshotSeqs: number[] | null };
private readonly remainingContents: LegacySessionLike["blockContents"];
private readonly orphans: LegacySessionLike["blockContents"];
private readonly messages = new Map<number, ReturnType<typeof messageSignature>>();
private readonly keys = new Set<string>();
private drift = false;
private blockCount = 0;
private activeBlocks = 0;
private tailGaps = 0;
private tailSlot: DriftDiagnostics["tailSlot"];

constructor(private readonly legacy: LegacySessionLike, session: SessionRow, refs: RefRow[], deadRefs: { ord: number; ref: string }[]) {
const header = loadLegacyView({ session, refs, deadRefs, messages: [], blocks: [] });
this.orphans = header.blockContents;
this.remainingContents = new Map(completeBlockContents(legacy.blockContents, legacy.state.blocks));
this.tail = JSON.parse(session.tailJson);
this.messageSeqs = [...new Set([...(this.tail.tailSeqs ?? []), ...(this.tail.pluginSnapshotSeqs ?? [])])];
const withoutBulk = (view: LegacySessionLike | typeof header) => ({
...view,
state: { ...view.state, blocks: [] },
blockContents: new Map(),
lastMessages: view.lastMessages === undefined ? undefined : [],
pluginSnapshot: view.pluginSnapshot === undefined ? undefined : [],
});
const expect = withoutBulk(legacy);
const got = withoutBulk(header);
if (shadowDigest(expect) !== shadowDigest(got)) {
this.drift = true;
try {
for (const key of describeViewDivergence(expect, got).keys) this.keys.add(key);
} catch {
// Diagnostic detail must not suppress a proven drift.
}
}
}

addBlocks(rows: BlockRow[]): void {
for (const row of rows) {
const payload = JSON.parse(row.payloadJson);
if (payload?.active === true) this.activeBlocks++;
if (this.blockCount >= this.legacy.state.blocks.length || shadowDigest(payload) !== shadowDigest(this.legacy.state.blocks[this.blockCount])) this.changed("state");
this.blockCount++;
this.compareContents(row.blockId, { one: row.one === null ? null : JSON.parse(row.one), full: JSON.parse(row.summary) });
// A current block wins over an identically named journal entry.
this.orphans.delete(row.blockId);
}
}

addMessages(rows: MessageRow[]): void {
for (const row of rows) this.messages.set(row.seq, messageSignature(JSON.parse(row.content)));
}

private changed(key: string): void {
this.drift = true;
this.keys.add(key);
}

private compareContents(id: string, view: unknown): void {
if (!this.remainingContents.has(id) || shadowDigest(this.remainingContents.get(id)) !== shadowDigest(view)) this.changed("blockContents");
this.remainingContents.delete(id);
}

private compareMessages(key: "lastMessages" | "pluginSnapshot", seqs: number[] | null): void {
const expected = this.legacy[key];
let index = 0;
for (const seq of seqs ?? []) {
const got = this.messages.get(seq);
if (!got) {
this.tailGaps++;
continue;
}
const want = expected && index < expected.length ? messageSignature(expected[index]) : undefined;
if (!want || want.digest !== got.digest) {
this.changed(key);
if (key === "lastMessages" && expected && !this.tailSlot) {
this.tailSlot = { index, expect: want?.slot ?? missingSlot(), got: got.slot };
}
}
index++;
}
if (seqs && expected && index < expected.length) {
this.changed(key);
if (key === "lastMessages" && !this.tailSlot) this.tailSlot = { index, expect: messageSignature(expected[index]).slot, got: missingSlot() };
}
}

finish(): { matches: boolean; activeBlocks: number; tailGaps: number; diagnostics: DriftDiagnostics } {
if (this.blockCount !== this.legacy.state.blocks.length) this.changed("state");
for (const [id, view] of this.orphans) this.compareContents(id, view);
if (this.remainingContents.size > 0) this.changed("blockContents");
this.compareMessages("lastMessages", this.tail.tailSeqs);
this.compareMessages("pluginSnapshot", this.tail.pluginSnapshotSeqs);
const diagnostics: DriftDiagnostics = { keys: [...this.keys].sort() };
if (this.tailSlot) diagnostics.tailSlot = this.tailSlot;
return { matches: !this.drift, activeBlocks: this.activeBlocks, tailGaps: this.tailGaps, diagnostics };
}
}

function missingSlot(): DriftSlotFingerprint {
return { id: "-", role: "-", length: 0, sha8: "-" };
}

function messageSignature(value: unknown): { digest: string; slot: DriftSlotFingerprint } {
const json = JSON.stringify(canonicalize(value)) ?? "null";
const digest = sha256(json);
const message = value as { id?: unknown; role?: unknown } | null;
return {
digest,
slot: !message || typeof message !== "object" ? missingSlot() : {
id: typeof message.id === "string" ? message.id : "-",
role: typeof message.role === "string" ? message.role : "-",
length: json.length,
sha8: digest.slice(0, 8),
},
};
}

// #3919: which dimensions disagree when the sampled verify detects drift —
// the digest is a single hash and leaves classification to guesswork. Pure;
// runs only on detected drift (rare). Fingerprints carry id/role/length/
Expand Down
Loading
Loading