From 5aeef338dd32d7535046ed972dd0fe9e7f369734 Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 15 Oct 2026 21:29:38 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20unified=20store=20=E2=80=94=20waterm?= =?UTF-8?q?ark=20trust=20does=20not=20cross=20process=20handoffs,=20full?= =?UTF-8?q?=20path=20sweeps=20foreign=20message=20rows=20(#3285)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restart handoff with two processes sharing the state dir let a previous writer's index.db rows and last_ingested_seq watermark survive into the new process: the new writer inherited foreign delta trust (empty unverified deltas) and its full re-ingest kept foreign orphan rows via INSERT OR IGNORE + identity remap, so every sampled digest check flagged DIGEST DRIFT forever. The b9e82f641 heal (clear watermark on drift) was insufficient because the subsequent full re-ingest still kept orphans. Fix: resetAllIngestWatermarks() at writer init (once per boot; readers/export untouched), and the ingestSession full path sweeps message rows absent from the plan when the plan is a complete raw-tail log (completeLog gate — envelope-only plans are pruned folded renders and must never trigger sweeps). Regression test: tests/storage-ingest-watermark.test.ts restart-handoff case (fails without the fix). --- src/storage/shadow.ts | 9 ++++- src/storage/store.ts | 46 ++++++++++++++++++++++++ tests/storage-ingest-watermark.test.ts | 49 ++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d3e21a7f60..5aef41ed1a 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -85,6 +85,11 @@ export class UnifiedShadow { let store: UnifiedStore; try { store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit }); + // #3285: watermark trust does not cross process boundaries — a + // restart-handoff overlap leaves the previous writer's rows AND + // watermark behind; drop them all so this process's first save + // per session re-establishes trust from its own view. + store.resetAllIngestWatermarks(); } catch (err) { // #2771: a corrupt/unreadable index.db must degrade to legacy-only // (same shape as the encryption opt-out), never brick SessionStore @@ -158,7 +163,9 @@ export class UnifiedShadow { // a full ingest, never a corrupt one. if (!this.ingestDelta(legacy, hasRawTail)) { const plan = planIngest(legacy); - this.store.ingestSession(plan); + // #3285: only a raw-tail plan is the complete log — that is + // what lets the full path sweep foreign rows as stale. + this.store.ingestSession(plan, undefined, { completeLog: hasRawTail }); } this.stats.ingests++; // #3133 sampled verification: ingest integrity is already guarded diff --git a/src/storage/store.ts b/src/storage/store.ts index 1612c6674e..cb79c87fae 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -340,6 +340,7 @@ export class UnifiedStore { ingestSession( input: IngestSessionInput, delta?: { trustedSeq: number; boundaryMsgId: string; boundaryHash: string; trustedTailSeqs: number[] }, + opts?: { completeLog?: boolean }, ): boolean { const s = input.session; const ok = this.db.transaction((): boolean => { @@ -389,6 +390,34 @@ export class UnifiedStore { const full = this.remapPlanToAbsolute(input); messages = full.messages; watermark = full.watermark; + // #3285: authoritative-set discipline for messages — same as + // blocks / refs / dead_refs / ccr_entries below, but ONLY when + // completeLog says the plan IS the whole log (the raw tail). + // Envelope-only plans are pruned renders (bounded folded view, + // #2960 ⑤; folded-resident re-renders, persist ②a caveat), and + // sweeping on those would delete raw rows that earlier saves + // interned beyond the render window. With a complete plan, + // rows whose identity is absent are stale — foreign-writer + // leftovers from a restart-handoff overlap (#3285/#2671 shared + // directory) or below-boundary rewrites the sampled digest + // caught (#3166). INSERT OR IGNORE alone can never remove + // them, so without this sweep they ride every unified replay + // forever (permanent DIGEST DRIFT even after the #3166 + // watermark-clear heal). Release their CAS refs and drop them. + if (opts?.completeLog === true) { + const incomingIds = new Set(); + for (const m of messages) { + incomingIds.add(`${m.msgId}\u0000${m.contentHash ?? sha256Hex(m.content)}`); + } + const staleRows = this.db + .prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; + for (const row of staleRows) { + if (incomingIds.has(`${row.msg_id}\u0000${row.content_hash}`)) continue; + this.releaseContent(row.content_hash); + this.db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); + } + } } this.db .prepare( @@ -686,6 +715,23 @@ export class UnifiedStore { this.db.prepare("UPDATE sessions SET last_ingested_seq = NULL WHERE session_id = ?").run(sessionId); } + /** #3285: drop EVERY session's ingest watermark at once. A watermark + * vouches "the writer that set it verified this prefix" but carries no + * writer identity, so the trust must not survive a process handoff: + * under the multi-process shared-directory contract (#2671) a + * restart-handoff overlap (old instance still draining while the new + * one boots, both sharing index.db) leaves a foreign process's rows + * AND its watermark behind, and inheriting that trust lets the new + * process take unverified empty deltas against rows it never wrote. + * Called once at writer init (UnifiedShadow.maybeOpen); pure readers + * (export) must not call it. After the reset each session's first save + * takes the exact full path — an authoritative replace, see + * ingestSession — re-establishing trust from this process's own view. + * Cost: one full ingest per saved session per boot. */ + resetAllIngestWatermarks(): void { + this.db.prepare("UPDATE sessions SET last_ingested_seq = NULL").run(); + } + /** #3133 boundary probe: the identity of the messages row at * (session_id, seq) plus the session's current max seq — both lookups * ride the PRIMARY KEY (session_id, seq) / idx_messages_session (no diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 151d3c937c..f29d8e10b7 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -603,3 +603,52 @@ test("watermark: v4 database migrates to v5 additively; newer user_version no-op rmBestEffort(dir); } }); + +test("watermark: restart handoff — the previous writer's rows and watermark do not survive (#3285)", () => { + const dir = tmpdir("bili-wm-handoff-"); + try { + // Live saves carry the raw tail (persist.ts guardedBuild) — that is + // what makes a plan the complete log. Both processes save through + // that production shape. + const withRaw = (id: string, lastMessages: CoreMessage[]): [LegacySessionLike, { lastMessages: CoreMessage[] }] => + [mkLegacy(id, lastMessages), { lastMessages }]; + // Process A: ingests 10 messages, then exits mid-drain (a host + // restart while the old instance still flushes — the #3285 shape). + const a = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(a, "first process must open"); + try { + const msgsA = Array.from({ length: 10 }, (_, i) => msg(i)); + a.record(...withRaw("handoff", msgsA)); + } finally { + a.close(); + } + // Process B boots on the same directory. Its view is what the host + // resent AFTER the restart: two tail turns are gone (the desktop + // rebuilt history from its own store), so B's truth is shorter than + // what A ingested. + const b = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(b, "second process must open"); + const storeB = UnifiedStore.open(path.join(dir, "index.db")); + try { + assert.equal(storeB.ingestWatermark("handoff"), null, "a foreign writer's watermark must not be inherited"); + const view = [msg(0), msg(1), msg(2)]; + b.record(...withRaw("handoff", view)); + const s = stats(b); + assert.equal(s.deltaIngests, 0, "first save after the handoff takes the full path"); + assert.equal(s.verifyDrift, 0, "no DIGEST DRIFT: the full path replaced A's rows"); + assert.equal(s.verifyOk, 1, "sampled verification passes on the first ingest"); + const rows = storeB.getMessages("handoff"); + assert.equal(rows.length, 3, "A's orphaned rows are gone"); + assert.deepEqual(rows.map((r) => r.msgId), ["h_msg0", "h_msg1", "h_msg2"]); + // Steady state resumes against B's own watermark. + b.record(...withRaw("handoff", [...view, msg(10)])); + assert.equal(stats(b).deltaIngests, 1, "post-handoff saves delta again"); + assert.equal(stats(b).verifyDrift, 0); + } finally { + storeB.close(); + b.close(); + } + } finally { + rmBestEffort(dir); + } +}); From b85e5d6d483f5c1cbe40a3f9b3bf03c97891d2a9 Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 15 Oct 2026 22:32:05 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20message=20sweep=20skips=20lane-owned?= =?UTF-8?q?=20sessions=20=E2=80=94=20the=20incremental=20lane's=20baseSeq?= =?UTF-8?q?=20epoch=20consumes=20the=20messages=20table=20positionally=20(?= =?UTF-8?q?#3248,=20CI=20regression=20of=20the=20#3285=20sweep)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/storage/store.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/storage/store.ts b/src/storage/store.ts index cb79c87fae..488e2ea6e1 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -404,7 +404,16 @@ export class UnifiedStore { // them, so without this sweep they ride every unified replay // forever (permanent DIGEST DRIFT even after the #3166 // watermark-clear heal). Release their CAS refs and drop them. - if (opts?.completeLog === true) { + // + // LANE EXCLUSION (#3248): the incremental lane consumes this + // table POSITIONALLY — its baseSeq epoch is an absolute row + // count recorded at resync, and rejected-branch rows must stay + // in the table as an audit floor the served history slices + // past. Sweeping them would invalidate the epoch (cold rebuild + // serves an empty canonical → 409 loop). Lane-owned sessions + // therefore opt out; they keep #3166-class orphans (bounded — + // the boot watermark reset still re-establishes delta trust). + if (opts?.completeLog === true && !this.hasLaneExt(s.sessionId)) { const incomingIds = new Set(); for (const m of messages) { incomingIds.add(`${m.msgId}\u0000${m.contentHash ?? sha256Hex(m.content)}`); @@ -732,6 +741,13 @@ export class UnifiedStore { this.db.prepare("UPDATE sessions SET last_ingested_seq = NULL").run(); } + /** #3248 lane-ownership probe for the message sweep (#3285): every ext + * key the incremental lane persists is "inc:"-prefixed, so any such key + * marks a session whose messages table the lane consumes positionally. */ + private hasLaneExt(sessionId: string): boolean { + return this.db.prepare("SELECT 1 FROM session_ext WHERE session_id = ? AND key LIKE 'inc:%' LIMIT 1").get(sessionId) !== undefined; + } + /** #3133 boundary probe: the identity of the messages row at * (session_id, seq) plus the session's current max seq — both lookups * ride the PRIMARY KEY (session_id, seq) / idx_messages_session (no From d6e703a2d989b9d3b6e52c5d5843f90a6100ace1 Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 15 Oct 2026 23:02:43 +0800 Subject: [PATCH 3/3] test: pin the lane-owned sweep exclusion at the storage layer (#3248/#3285) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e incremental-lane case covers the user-visible 409 loop; this unit case pins the mechanism itself — a session carrying any "inc:" ext key survives a completeLog ingest untouched, while a marker-less control session still gets its foreign rows swept. --- tests/storage-ingest-watermark.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index f29d8e10b7..7f717187fe 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -652,3 +652,26 @@ test("watermark: restart handoff — the previous writer's rows and watermark do rmBestEffort(dir); } }); + +test("watermark: completeLog sweep skips lane-owned sessions (#3248/#3285)", () => { + const dir = tmpdir("bili-wm-lane-skip-"); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + // Lane-owned: any "inc:" ext key marks positional consumption of the + // messages table — its rows must survive even when absent from the plan. + store.ingestSession(planIngest(mkLegacy("lane-a", [msg(0), msg(1)]))); + store.setSessionExt("lane-a", "inc:base-seq", 2); + store.ingestSession(planIngest(mkLegacy("lane-a", [msg(2)])), undefined, { completeLog: true }); + assert.deepEqual(store.getMessages("lane-a").map((r) => r.msgId), ["h_msg0", "h_msg1", "h_msg2"], "lane rows survive the sweep"); + // Control: without a lane marker the #3285 sweep still applies. + store.ingestSession(planIngest(mkLegacy("plain-b", [msg(0), msg(1)]))); + store.ingestSession(planIngest(mkLegacy("plain-b", [msg(2)])), undefined, { completeLog: true }); + assert.deepEqual(store.getMessages("plain-b").map((r) => r.msgId), ["h_msg2"], "non-lane foreign rows swept"); + } finally { + store.close(); + } + } finally { + rmBestEffort(dir); + } +});