From c74a23c2a2b40a79da21a57c53149e24cd27c270 Mon Sep 17 00:00:00 2001 From: awork Date: Fri, 23 Oct 2026 17:21:19 +0800 Subject: [PATCH 1/2] docs(storage): state planIngestDelta's strict-append precondition and pin the #4182 duplicate-identity corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit planIngestDelta's doc claimed 'resulting store state equals a full ingest's' unconditionally, but its raw plan diverges from planIngest when the new tail portion carries an already-stored identity (resent/ replayed message): full dedups via seqByIdentity, delta appends a fresh placeholder slot. The divergence is by design — the delta planner cannot see stored prefix identities — and is neutralized at the store layer by the strictly-append guard in UnifiedStore.ingestSession's delta branch (#3133, SQLite + MySQL backends), which refuses such plans in-transaction so the exact full path runs instead. State that contract explicitly in planIngestDelta/finishPlan docs, and pin it with two regression tests: a row-driven matrix over the #4182 five-shape corpus (rows/watermark/tailSeqs/served-view equal a pure full-ingest oracle; no-duplicate control still rides the delta) and a planner-level pin of the intentional [0,1,2] vs [0,1,0] divergence. No runtime behavior change: comment + test diff only. Closes #4182 --- src/storage/ingest.ts | 27 +++++++++--- tests/storage-ingest-watermark.test.ts | 59 ++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index d9470406b5..3ef7f72442 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -223,8 +223,23 @@ export function planIngest(session: LegacySessionLike): IngestPlan { * written by that save), and the probe pins slot L-1 to row `watermark` * with nothing above it — so this save only has to append tail[L..] at * watermark+1, watermark+2, … Only messages PAST the trusted prefix pay - * stringify+hash; every other row family is byte-identical to - * planIngest's, so the resulting store state equals a full ingest's. + * stringify+hash; given the same log, finishPlan() emits every row family + * byte-identical to planIngest's. The store-state equivalence holds under + * ONE strict-append precondition (#4182): no message in tail[L..] may + * carry an ALREADY-STORED identity (msg_id + content_hash). The planner + * cannot see stored prefix identities — that is the point of skipping + * them — so a resent/replayed message (#3661-class: content-derived ids + * mean a resend IS the same identity) is appended as a fresh placeholder + * slot, and the RAW PLAN diverges from planIngest, which dedups it onto + * its existing seq via seqByIdentity. That divergence is by design, + * neutralized at the store layer: UnifiedStore.ingestSession's delta + * branch probes every new message's identity against the messages table + * inside its transaction (the strictly-append guard behind + * UNIQUE(session_id, msg_id, content_hash), SQLite + MySQL) and refuses + * the whole plan on any hit; the caller then runs the exact full path, + * whose dedup makes the COMMITTED store state equal to a pure full + * ingest's anyway. Cost of a violation: one refused delta attempt + one + * redundant full write — never wrong rows. * * Returns null (→ full ingest) when: a pluginSnapshot is present (its * entries may dedup against the trusted prefix and renumber the log), @@ -325,9 +340,11 @@ export function planIngestDelta( return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); } -/** Shared plan tail (everything except the messages log): byte-identical - * output for both planners — the watermark fast path must not change what - * a full ingest would write for the same session. */ +/** Shared plan tail (everything except the messages log): given the same + * log + session, byte-identical output for both planners — the watermark + * fast path must not change what a full ingest would write. (#4182: the + * duplicate-identity case feeds this function DIFFERENT logs by design; + * see planIngestDelta's strict-append precondition.) */ function finishPlan( session: LegacySessionLike, byRawRef: Record, diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 1d2f523bb1..761be7eb5a 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -20,6 +20,7 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; +import { createHash } from "node:crypto"; import { UnifiedStore } from "../src/storage/store.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; import { openSqliteDatabase } from "../src/storage/driver.ts"; @@ -447,6 +448,64 @@ test("watermark: rewound/re-derived tail falls back to full (no silent drops)", } }); +test("watermark: every duplicate-identity tail shape falls back to full — corpus from #4182", () => { + const dir = tmpdir("bili-wm-resend-corpus-"); + try { + const { shadow, store } = openShadow(dir); + try { + // The #4182 corpus: every shape whose NEW tail portion carries an + // already-stored identity (the delta planner cannot see stored + // identities, so its raw plan diverges from planIngest) must be + // refused by the store's strictly-append guard and fall back to + // the exact full path — final state equal to a pure full ingest, + // no extra rows. The no-duplicate control must still ride the + // delta (the guard must not over-refuse). + const shapes: { name: string; first: CoreMessage[]; second: CoreMessage[]; ridesDelta: boolean }[] = [ + { name: "adjacent duplicate at the tip", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(2)], ridesDelta: false }, + { name: "duplicate in the middle of the new tail", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(0), msg(2)], ridesDelta: false }, + { name: "double duplicate (both prefix messages resent)", first: [msg(0), msg(1)], second: [msg(0), msg(1), msg(0), msg(1)], ridesDelta: false }, + { name: "duplicate of an interior prefix message", first: [msg(0), msg(1), msg(2), msg(3)], second: [msg(0), msg(1), msg(2), msg(3), msg(1)], ridesDelta: false }, + { name: "control: no duplicate", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(4)], ridesDelta: true }, + ]; + shapes.forEach((shape, k) => { + const id = `wm-corpus-${k}`; + const before = stats(shadow).deltaIngests; + shadow.record(mkLegacy(id, shape.first)); + shadow.record(mkLegacy(id, shape.second)); + assert.equal(stats(shadow).deltaIngests - before, shape.ridesDelta ? 1 : 0, `${shape.name}: ${shape.ridesDelta ? "must ride" : "must NOT ride"} the delta`); + // Oracle: what a pure full ingest of the same view writes. + const fullPlan = planIngest(mkLegacy(id, shape.second)); + const expTail = (JSON.parse(fullPlan.session.tailJson) as { tailSeqs: number[] }).tailSeqs; + assert.equal(store.getMessages(id).length, fullPlan.messages.length, `${shape.name}: no extra rows for resent identities`); + assert.deepEqual(store.ingestWatermark(id), { seq: expTail[expTail.length - 1], tailSeqs: expTail }, `${shape.name}: watermark/tailSeqs equal the full ingest's`); + assert.deepEqual(view(store, id).lastMessages, shape.second, `${shape.name}: served view keeps every tail slot`); + }); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("planner contract (#4182): duplicate-identity tail plans diverge BY DESIGN; the store guard compensates", () => { + const A = msg(0); + const B = msg(1); + const hB = createHash("sha256").update(JSON.stringify(B)).digest("hex"); + const full = planIngest(mkLegacy("p-full", [A, B, A])); + assert.equal(full.messages.length, 2, "full plan dedups the resent identity onto its existing seq"); + assert.deepEqual((JSON.parse(full.session.tailJson) as { tailSeqs: number[] }).tailSeqs, [0, 1, 0]); + const delta = planIngestDelta(mkLegacy("p-delta", [A, B, A]), 1, [0, 1], { msgId: B.id, contentHash: hB }, true); + assert.ok(delta, "the delta planner accepts the shape — it cannot see stored prefix identities"); + assert.equal(delta!.messages.length, 1, "only the resent tail slot is emitted as a new row"); + assert.equal(delta!.messages[0].msgId, A.id); + // THE #4182 divergence, pinned: the resend gets a FRESH seq here where + // planIngest above pointed the same slot at A's existing seq (…0 vs …2). + // Compensated at the store layer by the strictly-append guard. + assert.deepEqual((JSON.parse(delta!.session.tailJson) as { tailSeqs: number[] }).tailSeqs, [0, 1, 2]); +}); + test("watermark: snapshot-bearing sessions stay on the full path", () => { const dir = tmpdir("bili-wm-snapshot-"); try { From 61bbcd81e6aa688e6092d36eb402d234952ecd95 Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 24 Oct 2026 01:36:54 +0800 Subject: [PATCH 2/2] docs(storage): state the intra-plan duplicate facet beside the strict-append precondition (#4182) --- src/storage/ingest.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 3ef7f72442..6c3fe79511 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -225,7 +225,7 @@ export function planIngest(session: LegacySessionLike): IngestPlan { * watermark+1, watermark+2, … Only messages PAST the trusted prefix pay * stringify+hash; given the same log, finishPlan() emits every row family * byte-identical to planIngest's. The store-state equivalence holds under - * ONE strict-append precondition (#4182): no message in tail[L..] may + * a strict-append precondition (#4182): no message in tail[L..] may * carry an ALREADY-STORED identity (msg_id + content_hash). The planner * cannot see stored prefix identities — that is the point of skipping * them — so a resent/replayed message (#3661-class: content-derived ids @@ -241,6 +241,16 @@ export function planIngest(session: LegacySessionLike): IngestPlan { * ingest's anyway. Cost of a violation: one refused delta attempt + one * redundant full write — never wrong rows. * + * A second, intra-plan facet of the #4182 divergence stays open: two NEW + * byte-identical messages inside the same tail share ONE content-derived + * identity, so both pass the stored-identity probe above; the UNIQUE + * constraint then silently skips the second insert, leaving its tailSeqs + * slot pointing at nothing — the read view drops that entry for one save + * cycle. The next save self-heals: the boundary probe finds no row at the + * new watermark, refuses the delta, and the exact full path re-dedups. + * Bounded and unusual-input-only; closing it (an intra-plan seen-set in + * the store guard) changes fallback behavior and is a separate decision. + * * Returns null (→ full ingest) when: a pluginSnapshot is present (its * entries may dedup against the trusted prefix and renumber the log), * the tail is absent or not longer than the trusted prefix (shrunk /