Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,11 @@ export class UnifiedShadow {
let store: UnifiedStore;
try {
store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit });
// #3285: watermark trust does not cross process boundaries — a
// restart-handoff overlap leaves the previous writer's rows AND
// watermark behind; drop them all so this process's first save
// per session re-establishes trust from its own view.
store.resetAllIngestWatermarks();
} catch (err) {
// #2771: a corrupt/unreadable index.db must degrade to legacy-only
// (same shape as the encryption opt-out), never brick SessionStore
Expand Down Expand Up @@ -158,7 +163,9 @@ export class UnifiedShadow {
// a full ingest, never a corrupt one.
if (!this.ingestDelta(legacy, hasRawTail)) {
const plan = planIngest(legacy);
this.store.ingestSession(plan);
// #3285: only a raw-tail plan is the complete log — that is
// what lets the full path sweep foreign rows as stale.
this.store.ingestSession(plan, undefined, { completeLog: hasRawTail });
}
this.stats.ingests++;
// #3133 sampled verification: ingest integrity is already guarded
Expand Down
62 changes: 62 additions & 0 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,7 @@ export class UnifiedStore {
ingestSession(
input: IngestSessionInput,
delta?: { trustedSeq: number; boundaryMsgId: string; boundaryHash: string; trustedTailSeqs: number[] },
opts?: { completeLog?: boolean },
): boolean {
const s = input.session;
const ok = this.db.transaction((): boolean => {
Expand Down Expand Up @@ -412,6 +413,43 @@ export class UnifiedStore {
const full = this.remapPlanToAbsolute(input);
messages = full.messages;
watermark = full.watermark;
// #3285: authoritative-set discipline for messages — same as
// blocks / refs / dead_refs / ccr_entries below, but ONLY when
// completeLog says the plan IS the whole log (the raw tail).
// Envelope-only plans are pruned renders (bounded folded view,
// #2960 ⑤; folded-resident re-renders, persist ②a caveat), and
// sweeping on those would delete raw rows that earlier saves
// interned beyond the render window. With a complete plan,
// rows whose identity is absent are stale — foreign-writer
// leftovers from a restart-handoff overlap (#3285/#2671 shared
// directory) or below-boundary rewrites the sampled digest
// caught (#3166). INSERT OR IGNORE alone can never remove
// them, so without this sweep they ride every unified replay
// forever (permanent DIGEST DRIFT even after the #3166
// watermark-clear heal). Release their CAS refs and drop them.
//
// LANE EXCLUSION (#3248): the incremental lane consumes this
// table POSITIONALLY — its baseSeq epoch is an absolute row
// count recorded at resync, and rejected-branch rows must stay
// in the table as an audit floor the served history slices
// past. Sweeping them would invalidate the epoch (cold rebuild
// serves an empty canonical → 409 loop). Lane-owned sessions
// therefore opt out; they keep #3166-class orphans (bounded —
// the boot watermark reset still re-establishes delta trust).
if (opts?.completeLog === true && !this.hasLaneExt(s.sessionId)) {
const incomingIds = new Set<string>();
for (const m of messages) {
incomingIds.add(`${m.msgId}\u0000${m.contentHash ?? sha256Hex(m.content)}`);
}
const staleRows = this.db
.prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?")
.all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[];
for (const row of staleRows) {
if (incomingIds.has(`${row.msg_id}\u0000${row.content_hash}`)) continue;
this.releaseContent(row.content_hash);
this.db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq);
}
}
}
this.db
.prepare(
Expand Down Expand Up @@ -709,6 +747,30 @@ export class UnifiedStore {
this.db.prepare("UPDATE sessions SET last_ingested_seq = NULL WHERE session_id = ?").run(sessionId);
}

/** #3285: drop EVERY session's ingest watermark at once. A watermark
* vouches "the writer that set it verified this prefix" but carries no
* writer identity, so the trust must not survive a process handoff:
* under the multi-process shared-directory contract (#2671) a
* restart-handoff overlap (old instance still draining while the new
* one boots, both sharing index.db) leaves a foreign process's rows
* AND its watermark behind, and inheriting that trust lets the new
* process take unverified empty deltas against rows it never wrote.
* Called once at writer init (UnifiedShadow.maybeOpen); pure readers
* (export) must not call it. After the reset each session's first save
* takes the exact full path — an authoritative replace, see
* ingestSession — re-establishing trust from this process's own view.
* Cost: one full ingest per saved session per boot. */
resetAllIngestWatermarks(): void {
this.db.prepare("UPDATE sessions SET last_ingested_seq = NULL").run();
}

/** #3248 lane-ownership probe for the message sweep (#3285): every ext
* key the incremental lane persists is "inc:"-prefixed, so any such key
* marks a session whose messages table the lane consumes positionally. */
private hasLaneExt(sessionId: string): boolean {
return this.db.prepare("SELECT 1 FROM session_ext WHERE session_id = ? AND key LIKE 'inc:%' LIMIT 1").get(sessionId) !== undefined;
}

/** #3133 boundary probe: the identity of the messages row at
* (session_id, seq) plus the session's current max seq — both lookups
* ride the PRIMARY KEY (session_id, seq) / idx_messages_session (no
Expand Down
72 changes: 72 additions & 0 deletions tests/storage-ingest-watermark.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -603,3 +603,75 @@ test("watermark: v4 database migrates to v5 additively; newer user_version no-op
rmBestEffort(dir);
}
});

test("watermark: restart handoff — the previous writer's rows and watermark do not survive (#3285)", () => {
const dir = tmpdir("bili-wm-handoff-");
try {
// Live saves carry the raw tail (persist.ts guardedBuild) — that is
// what makes a plan the complete log. Both processes save through
// that production shape.
const withRaw = (id: string, lastMessages: CoreMessage[]): [LegacySessionLike, { lastMessages: CoreMessage[] }] =>
[mkLegacy(id, lastMessages), { lastMessages }];
// Process A: ingests 10 messages, then exits mid-drain (a host
// restart while the old instance still flushes — the #3285 shape).
const a = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike);
assert.ok(a, "first process must open");
try {
const msgsA = Array.from({ length: 10 }, (_, i) => msg(i));
a.record(...withRaw("handoff", msgsA));
} finally {
a.close();
}
// Process B boots on the same directory. Its view is what the host
// resent AFTER the restart: two tail turns are gone (the desktop
// rebuilt history from its own store), so B's truth is shorter than
// what A ingested.
const b = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike);
assert.ok(b, "second process must open");
const storeB = UnifiedStore.open(path.join(dir, "index.db"));
try {
assert.equal(storeB.ingestWatermark("handoff"), null, "a foreign writer's watermark must not be inherited");
const view = [msg(0), msg(1), msg(2)];
b.record(...withRaw("handoff", view));
const s = stats(b);
assert.equal(s.deltaIngests, 0, "first save after the handoff takes the full path");
assert.equal(s.verifyDrift, 0, "no DIGEST DRIFT: the full path replaced A's rows");
assert.equal(s.verifyOk, 1, "sampled verification passes on the first ingest");
const rows = storeB.getMessages("handoff");
assert.equal(rows.length, 3, "A's orphaned rows are gone");
assert.deepEqual(rows.map((r) => r.msgId), ["h_msg0", "h_msg1", "h_msg2"]);
// Steady state resumes against B's own watermark.
b.record(...withRaw("handoff", [...view, msg(10)]));
assert.equal(stats(b).deltaIngests, 1, "post-handoff saves delta again");
assert.equal(stats(b).verifyDrift, 0);
} finally {
storeB.close();
b.close();
}
} finally {
rmBestEffort(dir);
}
});

test("watermark: completeLog sweep skips lane-owned sessions (#3248/#3285)", () => {
const dir = tmpdir("bili-wm-lane-skip-");
try {
const store = UnifiedStore.open(path.join(dir, "index.db"));
try {
// Lane-owned: any "inc:" ext key marks positional consumption of the
// messages table — its rows must survive even when absent from the plan.
store.ingestSession(planIngest(mkLegacy("lane-a", [msg(0), msg(1)])));
store.setSessionExt("lane-a", "inc:base-seq", 2);
store.ingestSession(planIngest(mkLegacy("lane-a", [msg(2)])), undefined, { completeLog: true });
assert.deepEqual(store.getMessages("lane-a").map((r) => r.msgId), ["h_msg0", "h_msg1", "h_msg2"], "lane rows survive the sweep");
// Control: without a lane marker the #3285 sweep still applies.
store.ingestSession(planIngest(mkLegacy("plain-b", [msg(0), msg(1)])));
store.ingestSession(planIngest(mkLegacy("plain-b", [msg(2)])), undefined, { completeLog: true });
assert.deepEqual(store.getMessages("plain-b").map((r) => r.msgId), ["h_msg2"], "non-lane foreign rows swept");
} finally {
store.close();
}
} finally {
rmBestEffort(dir);
}
});
Loading