diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index d9470406b5..6c3fe79511 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -223,8 +223,33 @@ export function planIngest(session: LegacySessionLike): IngestPlan { * written by that save), and the probe pins slot L-1 to row `watermark` * with nothing above it — so this save only has to append tail[L..] at * watermark+1, watermark+2, … Only messages PAST the trusted prefix pay - * stringify+hash; every other row family is byte-identical to - * planIngest's, so the resulting store state equals a full ingest's. + * stringify+hash; given the same log, finishPlan() emits every row family + * byte-identical to planIngest's. The store-state equivalence holds under + * a strict-append precondition (#4182): no message in tail[L..] may + * carry an ALREADY-STORED identity (msg_id + content_hash). The planner + * cannot see stored prefix identities — that is the point of skipping + * them — so a resent/replayed message (#3661-class: content-derived ids + * mean a resend IS the same identity) is appended as a fresh placeholder + * slot, and the RAW PLAN diverges from planIngest, which dedups it onto + * its existing seq via seqByIdentity. That divergence is by design, + * neutralized at the store layer: UnifiedStore.ingestSession's delta + * branch probes every new message's identity against the messages table + * inside its transaction (the strictly-append guard behind + * UNIQUE(session_id, msg_id, content_hash), SQLite + MySQL) and refuses + * the whole plan on any hit; the caller then runs the exact full path, + * whose dedup makes the COMMITTED store state equal to a pure full + * ingest's anyway. Cost of a violation: one refused delta attempt + one + * redundant full write — never wrong rows. + * + * A second, intra-plan facet of the #4182 divergence stays open: two NEW + * byte-identical messages inside the same tail share ONE content-derived + * identity, so both pass the stored-identity probe above; the UNIQUE + * constraint then silently skips the second insert, leaving its tailSeqs + * slot pointing at nothing — the read view drops that entry for one save + * cycle. The next save self-heals: the boundary probe finds no row at the + * new watermark, refuses the delta, and the exact full path re-dedups. + * Bounded and unusual-input-only; closing it (an intra-plan seen-set in + * the store guard) changes fallback behavior and is a separate decision. * * Returns null (→ full ingest) when: a pluginSnapshot is present (its * entries may dedup against the trusted prefix and renumber the log), @@ -325,9 +350,11 @@ export function planIngestDelta( return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); } -/** Shared plan tail (everything except the messages log): byte-identical - * output for both planners — the watermark fast path must not change what - * a full ingest would write for the same session. */ +/** Shared plan tail (everything except the messages log): given the same + * log + session, byte-identical output for both planners — the watermark + * fast path must not change what a full ingest would write. (#4182: the + * duplicate-identity case feeds this function DIFFERENT logs by design; + * see planIngestDelta's strict-append precondition.) */ function finishPlan( session: LegacySessionLike, byRawRef: Record, diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 1d2f523bb1..761be7eb5a 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -20,6 +20,7 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; +import { createHash } from "node:crypto"; import { UnifiedStore } from "../src/storage/store.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; import { openSqliteDatabase } from "../src/storage/driver.ts"; @@ -447,6 +448,64 @@ test("watermark: rewound/re-derived tail falls back to full (no silent drops)", } }); +test("watermark: every duplicate-identity tail shape falls back to full — corpus from #4182", () => { + const dir = tmpdir("bili-wm-resend-corpus-"); + try { + const { shadow, store } = openShadow(dir); + try { + // The #4182 corpus: every shape whose NEW tail portion carries an + // already-stored identity (the delta planner cannot see stored + // identities, so its raw plan diverges from planIngest) must be + // refused by the store's strictly-append guard and fall back to + // the exact full path — final state equal to a pure full ingest, + // no extra rows. The no-duplicate control must still ride the + // delta (the guard must not over-refuse). + const shapes: { name: string; first: CoreMessage[]; second: CoreMessage[]; ridesDelta: boolean }[] = [ + { name: "adjacent duplicate at the tip", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(2)], ridesDelta: false }, + { name: "duplicate in the middle of the new tail", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(0), msg(2)], ridesDelta: false }, + { name: "double duplicate (both prefix messages resent)", first: [msg(0), msg(1)], second: [msg(0), msg(1), msg(0), msg(1)], ridesDelta: false }, + { name: "duplicate of an interior prefix message", first: [msg(0), msg(1), msg(2), msg(3)], second: [msg(0), msg(1), msg(2), msg(3), msg(1)], ridesDelta: false }, + { name: "control: no duplicate", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(4)], ridesDelta: true }, + ]; + shapes.forEach((shape, k) => { + const id = `wm-corpus-${k}`; + const before = stats(shadow).deltaIngests; + shadow.record(mkLegacy(id, shape.first)); + shadow.record(mkLegacy(id, shape.second)); + assert.equal(stats(shadow).deltaIngests - before, shape.ridesDelta ? 1 : 0, `${shape.name}: ${shape.ridesDelta ? "must ride" : "must NOT ride"} the delta`); + // Oracle: what a pure full ingest of the same view writes. + const fullPlan = planIngest(mkLegacy(id, shape.second)); + const expTail = (JSON.parse(fullPlan.session.tailJson) as { tailSeqs: number[] }).tailSeqs; + assert.equal(store.getMessages(id).length, fullPlan.messages.length, `${shape.name}: no extra rows for resent identities`); + assert.deepEqual(store.ingestWatermark(id), { seq: expTail[expTail.length - 1], tailSeqs: expTail }, `${shape.name}: watermark/tailSeqs equal the full ingest's`); + assert.deepEqual(view(store, id).lastMessages, shape.second, `${shape.name}: served view keeps every tail slot`); + }); + } finally { + shadow.close(); + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("planner contract (#4182): duplicate-identity tail plans diverge BY DESIGN; the store guard compensates", () => { + const A = msg(0); + const B = msg(1); + const hB = createHash("sha256").update(JSON.stringify(B)).digest("hex"); + const full = planIngest(mkLegacy("p-full", [A, B, A])); + assert.equal(full.messages.length, 2, "full plan dedups the resent identity onto its existing seq"); + assert.deepEqual((JSON.parse(full.session.tailJson) as { tailSeqs: number[] }).tailSeqs, [0, 1, 0]); + const delta = planIngestDelta(mkLegacy("p-delta", [A, B, A]), 1, [0, 1], { msgId: B.id, contentHash: hB }, true); + assert.ok(delta, "the delta planner accepts the shape — it cannot see stored prefix identities"); + assert.equal(delta!.messages.length, 1, "only the resent tail slot is emitted as a new row"); + assert.equal(delta!.messages[0].msgId, A.id); + // THE #4182 divergence, pinned: the resend gets a FRESH seq here where + // planIngest above pointed the same slot at A's existing seq (…0 vs …2). + // Compensated at the store layer by the strictly-append guard. + assert.deepEqual((JSON.parse(delta!.session.tailJson) as { tailSeqs: number[] }).tailSeqs, [0, 1, 2]); +}); + test("watermark: snapshot-bearing sessions stay on the full path", () => { const dir = tmpdir("bili-wm-snapshot-"); try {