Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 32 additions & 5 deletions src/storage/ingest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,33 @@ export function planIngest(session: LegacySessionLike): IngestPlan {
* written by that save), and the probe pins slot L-1 to row `watermark`
* with nothing above it — so this save only has to append tail[L..] at
* watermark+1, watermark+2, … Only messages PAST the trusted prefix pay
* stringify+hash; every other row family is byte-identical to
* planIngest's, so the resulting store state equals a full ingest's.
* stringify+hash; given the same log, finishPlan() emits every row family
* byte-identical to planIngest's. The store-state equivalence holds under
* a strict-append precondition (#4182): no message in tail[L..] may
* carry an ALREADY-STORED identity (msg_id + content_hash). The planner
* cannot see stored prefix identities — that is the point of skipping
* them — so a resent/replayed message (#3661-class: content-derived ids
* mean a resend IS the same identity) is appended as a fresh placeholder
* slot, and the RAW PLAN diverges from planIngest, which dedups it onto
* its existing seq via seqByIdentity. That divergence is by design,
* neutralized at the store layer: UnifiedStore.ingestSession's delta
* branch probes every new message's identity against the messages table
* inside its transaction (the strictly-append guard behind
* UNIQUE(session_id, msg_id, content_hash), SQLite + MySQL) and refuses
* the whole plan on any hit; the caller then runs the exact full path,
* whose dedup makes the COMMITTED store state equal to a pure full
* ingest's anyway. Cost of a violation: one refused delta attempt + one
* redundant full write — never wrong rows.
*
* A second, intra-plan facet of the #4182 divergence stays open: two NEW
* byte-identical messages inside the same tail share ONE content-derived
* identity, so both pass the stored-identity probe above; the UNIQUE
* constraint then silently skips the second insert, leaving its tailSeqs
* slot pointing at nothing — the read view drops that entry for one save
* cycle. The next save self-heals: the boundary probe finds no row at the
* new watermark, refuses the delta, and the exact full path re-dedups.
* Bounded and unusual-input-only; closing it (an intra-plan seen-set in
* the store guard) changes fallback behavior and is a separate decision.
*
* Returns null (→ full ingest) when: a pluginSnapshot is present (its
* entries may dedup against the trusted prefix and renumber the log),
Expand Down Expand Up @@ -325,9 +350,11 @@ export function planIngestDelta(
return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs);
}

/** Shared plan tail (everything except the messages log): byte-identical
* output for both planners — the watermark fast path must not change what
* a full ingest would write for the same session. */
/** Shared plan tail (everything except the messages log): given the same
* log + session, byte-identical output for both planners — the watermark
* fast path must not change what a full ingest would write. (#4182: the
* duplicate-identity case feeds this function DIFFERENT logs by design;
* see planIngestDelta's strict-append precondition.) */
function finishPlan(
session: LegacySessionLike,
byRawRef: Record<string, string>,
Expand Down
59 changes: 59 additions & 0 deletions tests/storage-ingest-watermark.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import assert from "node:assert/strict";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { createHash } from "node:crypto";
import { UnifiedStore } from "../src/storage/store.ts";
import { UnifiedShadow } from "../src/storage/shadow.ts";
import { openSqliteDatabase } from "../src/storage/driver.ts";
Expand Down Expand Up @@ -447,6 +448,64 @@ test("watermark: rewound/re-derived tail falls back to full (no silent drops)",
}
});

test("watermark: every duplicate-identity tail shape falls back to full — corpus from #4182", () => {
const dir = tmpdir("bili-wm-resend-corpus-");
try {
const { shadow, store } = openShadow(dir);
try {
// The #4182 corpus: every shape whose NEW tail portion carries an
// already-stored identity (the delta planner cannot see stored
// identities, so its raw plan diverges from planIngest) must be
// refused by the store's strictly-append guard and fall back to
// the exact full path — final state equal to a pure full ingest,
// no extra rows. The no-duplicate control must still ride the
// delta (the guard must not over-refuse).
const shapes: { name: string; first: CoreMessage[]; second: CoreMessage[]; ridesDelta: boolean }[] = [
{ name: "adjacent duplicate at the tip", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(2)], ridesDelta: false },
{ name: "duplicate in the middle of the new tail", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(0), msg(2)], ridesDelta: false },
{ name: "double duplicate (both prefix messages resent)", first: [msg(0), msg(1)], second: [msg(0), msg(1), msg(0), msg(1)], ridesDelta: false },
{ name: "duplicate of an interior prefix message", first: [msg(0), msg(1), msg(2), msg(3)], second: [msg(0), msg(1), msg(2), msg(3), msg(1)], ridesDelta: false },
{ name: "control: no duplicate", first: [msg(0), msg(1), msg(2)], second: [msg(0), msg(1), msg(2), msg(4)], ridesDelta: true },
];
shapes.forEach((shape, k) => {
const id = `wm-corpus-${k}`;
const before = stats(shadow).deltaIngests;
shadow.record(mkLegacy(id, shape.first));
shadow.record(mkLegacy(id, shape.second));
assert.equal(stats(shadow).deltaIngests - before, shape.ridesDelta ? 1 : 0, `${shape.name}: ${shape.ridesDelta ? "must ride" : "must NOT ride"} the delta`);
// Oracle: what a pure full ingest of the same view writes.
const fullPlan = planIngest(mkLegacy(id, shape.second));
const expTail = (JSON.parse(fullPlan.session.tailJson) as { tailSeqs: number[] }).tailSeqs;
assert.equal(store.getMessages(id).length, fullPlan.messages.length, `${shape.name}: no extra rows for resent identities`);
assert.deepEqual(store.ingestWatermark(id), { seq: expTail[expTail.length - 1], tailSeqs: expTail }, `${shape.name}: watermark/tailSeqs equal the full ingest's`);
assert.deepEqual(view(store, id).lastMessages, shape.second, `${shape.name}: served view keeps every tail slot`);
});
} finally {
shadow.close();
store.close();
}
} finally {
rmBestEffort(dir);
}
});

test("planner contract (#4182): duplicate-identity tail plans diverge BY DESIGN; the store guard compensates", () => {
const A = msg(0);
const B = msg(1);
const hB = createHash("sha256").update(JSON.stringify(B)).digest("hex");
const full = planIngest(mkLegacy("p-full", [A, B, A]));
assert.equal(full.messages.length, 2, "full plan dedups the resent identity onto its existing seq");
assert.deepEqual((JSON.parse(full.session.tailJson) as { tailSeqs: number[] }).tailSeqs, [0, 1, 0]);
const delta = planIngestDelta(mkLegacy("p-delta", [A, B, A]), 1, [0, 1], { msgId: B.id, contentHash: hB }, true);
assert.ok(delta, "the delta planner accepts the shape — it cannot see stored prefix identities");
assert.equal(delta!.messages.length, 1, "only the resent tail slot is emitted as a new row");
assert.equal(delta!.messages[0].msgId, A.id);
// THE #4182 divergence, pinned: the resend gets a FRESH seq here where
// planIngest above pointed the same slot at A's existing seq (…0 vs …2).
// Compensated at the store layer by the strictly-append guard.
assert.deepEqual((JSON.parse(delta!.session.tailJson) as { tailSeqs: number[] }).tailSeqs, [0, 1, 2]);
});

test("watermark: snapshot-bearing sessions stay on the full path", () => {
const dir = tmpdir("bili-wm-snapshot-");
try {
Expand Down
Loading