diff --git a/src/persist.ts b/src/persist.ts index 4b16a3ebb9..ed0762b211 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -556,6 +556,17 @@ export class SessionStore { * synchronously on EVERY call; a microtask-batched run per turn keeps * the immediate-ish semantics while coalescing same-tick calls. */ private readonly writerOffPending = new Map void>(); + /** #6780 P0: armed proof that a pre-forward durability ingest (#6326, + * ingestRowsNow) already made THIS session's current state durable in + * this tick. The writer-off coalesced build (armed by prepare's + * markDirty microseconds later) checks it at run time and, when the + * state is still byte-identical, skips its duplicate full rebuild + + * full delta-ingest — a no-op that still pays the three-table full + * reads, the whole-record serialization and a second commit round-trip + * (~1.3s of #6780's pre-forward cost on a 23k-message session). Any + * mismatch falls through to the full build; the arm is a hint, never a + * commitment. */ + private readonly durabilityArms = new Map; metadataFp: string }>(); /** #5408: per-session cooldown for the frozen-skip ERROR log (epoch ms of * the last notice) — a live session saves every turn, so without it the * line would spam; one notice per cooldown still makes the freeze loud. */ @@ -1605,6 +1616,7 @@ export class SessionStore { * proxy runs and nothing is resident in memory — that is exactly the * point. */ async removeSession(id: string): Promise<{ envelopeFound: boolean; unifiedHadRows: boolean }> { + this.durabilityArms.delete(id); // #6780 P0 (review F5): no arm for a dead session // Resolve identity from whichever side knows it: the unified row // carries protocol+origin even when the JSON tree is already gone, // and the envelope carries them when only the tree survives. @@ -2098,6 +2110,17 @@ export class SessionStore { private consumePendingLedgerAlign(session: Session): void { if (!session.pendingLedgerAlign) return; delete session.pendingLedgerAlign; + // #6780 P0 (review F1): the restamp below mutates state that this tick's + // pre-forward durability ingest (ingestRowsNow) already made durable — + // it renumbers refs and re-keys the CCR store AFTER that ingest. Consuming + // the arm here forces the restamp turn's coalesced build to run in full + // (once per session lifetime — maybeRestampToLedger no-ops once aligned), + // keeping the #4733 invariant: aligned state and re-keyed CCR land in the + // SAME serialized record. Without this, pendingLedgerAlign is already + // deleted by the time the writer-off gate reads it, so the gate's own + // align check would be a dead signal and the restamped bytes would ride + // one ingestRowsNow late. + this.durabilityArms.delete(session.id); this.maybeRestampToLedger(session); } @@ -3231,6 +3254,7 @@ export class SessionStore { this.convergeToDisk(session, disk); this.shadow.record(disk); this.noteOwnCommit(session.id, disk); + this.armDurabilitySkip(session, disk); // #6780 P0: the disk view is what is durable now return; } const payload = session.decompositionLog; @@ -3241,11 +3265,30 @@ export class SessionStore { ? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded } : undefined, session.contentStore ?? undefined); this.noteOwnCommit(session.id, record); + this.armDurabilitySkip(session, record); // #6780 P0: arm the same-tick dedup for prepare's markDirty } catch (err) { this.log("warn", `[persist] ingestRowsNow failed for ${session.id}: ${String(err)} — the debounced save retries the full path`); } } + /** #6780 P0: record that `record` (just ingested by ingestRowsNow) is + * exactly the session's current state, so the writer-off coalesced + * build armed by this tick's prepare markDirty can skip its duplicate + * heavy legs. Validated against LIVE state at run time — see + * scheduleSave's writerOff branch for the gate. */ + private armDurabilitySkip(session: Session, record: PersistedSession): void { + this.durabilityArms.set(session.id, { + epoch: session.revisionEpoch ?? 0, + messagesRef: session.lastMessages, + metadata: record.metadata ?? {}, + // buildRecordForSave already ran above (its + // publicSnapshotStoredRefs mutation is idempotent), so this + // fingerprint is stable as long as nothing else touches the + // metadata before the coalesced build runs. + metadataFp: JSON.stringify(session.metadata), + }); + } + scheduleSave(session: Session): void { // #5608-D: a committed fold IS the session's next legitimate // cache-breaking boundary — run the deferred #4316 citation-space @@ -3271,11 +3314,40 @@ export class SessionStore { // can only ride along, never be missing, so the spill gate's // equality check against the CURRENT epoch stays conservative. const proofRevision = session.revisionEpoch; + // #6780 P0: the pre-forward durability ingest (#6326) already ran + // earlier this tick from the same prepare and ingested this exact + // state. If nothing changed since — the epoch moved by exactly + // prepare's own markDirty (markDirty is the mutation chokepoint), + // the same lastMessages array is live (a re-snapshot does not + // bump the epoch), no new decomposition payload, no CCR re-key, + // no pending ledger align, and the metadata bytes are untouched — + // the full rebuild + full delta-ingest below is a pure no-op + // that still pays the three-table full reads, the whole-record + // serialization and a second commit round-trip. Skip the heavy + // legs; the settle protocol below runs unchanged on the armed + // fingerprint. Any mismatch takes the full build (the arm is a + // hint, never a commitment). + const armed = this.durabilityArms.get(session.id); + const durabilitySkip = armed !== undefined && + (session.revisionEpoch ?? 0) === armed.epoch + 1 && + session.lastMessages === armed.messagesRef && + session.decompositionLog === undefined && + !session.contentStoreDirty && + session.pendingLedgerAlign === undefined && + JSON.stringify(session.metadata) === armed.metadataFp; + // #6780 P0 (review F2): single-shot arm — this run made the + // decision; the arm must not survive to justify a skip for a + // LATER save (a stale arm could match any later epoch exactly + // one beyond the one it was armed at). The next ingestRowsNow + // re-arms if its state is durable. + this.durabilityArms.delete(session.id); let settledOk = false; try { - const rec = this.guardedBuild(session)(); + const recMeta = durabilitySkip && armed + ? armed.metadata + : this.guardedBuild(session)().metadata ?? {}; this.maybeCutForkBase(session); - this.noteMetaFp(session.id, rec.metadata ?? {}); // #5506: index-only build landed + this.noteMetaFp(session.id, recMeta); // #5506: index-only build landed // #4788 D2: the index-only build landed synchronously — // run the same settle protocol as a kernel-write land // (invariant #5: this routing path gets equivalent diff --git a/src/storage/store.ts b/src/storage/store.ts index 1e71740687..9b6f6a4525 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1119,6 +1119,11 @@ export class UnifiedStore { tail_json = excluded.tail_json, ccr_json = CASE WHEN ccr_json IS excluded.ccr_json THEN sessions.ccr_json ELSE excluded.ccr_json END, last_ingested_seq = excluded.last_ingested_seq, + -- #6780: no same-value skip here (or for the #6325 columns): an unchanged + -- OVERFLOW blob (a JSON column big enough to spill onto overflow pages) is + -- copied onto fresh pages whenever ANY column of this row changes, so a + -- CASE-when skip only saves inline-size values. Moving the big blobs out of + -- the row (table-ization) is the real byte-cost fix. orphans_json = excluded.orphans_json`, ) .run({ diff --git a/tests/issue6780-pre-forward-dedup.test.ts b/tests/issue6780-pre-forward-dedup.test.ts new file mode 100644 index 0000000000..606ffdeb98 --- /dev/null +++ b/tests/issue6780-pre-forward-dedup.test.ts @@ -0,0 +1,208 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore } from "../src/persist.ts"; +import { snapshotMessages, type Session } from "../src/session.ts"; +import type { CoreMessage } from "acp-kernel"; + +/** + * #6780 P0: with the pure-SQLite default (#6480, JSON writer off) the prepare + * phase runs TWO full unified-index ingests per turn BEFORE the model call — + * the synchronous durability ingest (#6326, ingestRowsNow) and the coalesced + * writer-off save armed by prepare's own markDirty (5a80e2b8 removed the + * debounce on this lane). The second one is a no-op delta (the watermark was + * already advanced) that still pays the three-table full reads, the + * whole-record re-serialization and a second commit round-trip — ~1.3s of + * #6780's pre-forward cost on the reporter's 23k-message session. + * + * Pinned here (writer-off mode, exactly what the prepare phase does): + * 1. When nothing changed between ingestRowsNow and the coalesced build, + * the build SKIPS the heavy legs — the sidecar record() is not called a + * second time — but the settle protocol still lands (lastSaveLanded). + * 2. When the state DID change after the arm (re-snapshot + markDirty), + * the full build still runs — the arm is a hint, never a commitment. + * 3. (review F6) A lastMessages re-snapshot ALONE (no kernel mutation, one + * markDirty) defeats the skip — isolating the ref-swap gate condition. + * 4. (review F1) A fold-turn ledger align consumed during scheduleSave + * restamps refs AFTER the arm; the coalesced build must take the full + * path (the #4733 same-record invariant), i.e. the align path consumes + * the arm. + * 5. (review F2) The arm is single-shot: once the coalesced build has made + * its decision, a LATER standalone markDirty must not be able to skip on + * the stale arm. + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: pin BOTH tiers so every +// leg controls its own mode explicitly (absent now defaults to full, #3818). +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-6780-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +const CORPUS_ROOT = path.resolve(import.meta.dirname, "golden/storage-corpus"); + +function loadCorpusCase(dir: string, name: string, id: string, protocol: string) { + fs.cpSync(path.join(CORPUS_ROOT, name), path.join(dir, name), { recursive: true }); + const loader = new SessionStore({ dir: path.join(dir, name), debounceMs: 1, enabled: true }); + const loaded = loader.loadSync(id, { protocol, upstreamOrigin: "http://127.0.0.1:8199" }); + loader.cancelAll(); + loader.closeQuietly(); + assert.ok(loaded, `${name}: corpus loadSync returned null`); + return loaded as Session; +} + +/** Windows: an ephemeral stray handle must not fail CI — best-effort delete + * with bounded retries, swallowed on persistent EBUSY/EPERM. */ +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +type Count = () => number; + +interface Harness { + session: Session; + /** The synchronous pre-forward durability ingest (prepare phase step). */ + ingestRowsNow: () => void; + /** Arm the sidecar record() spy (call right after the first + * ingestRowsNow). Returns the ingest counter. */ + spy: () => Count; + /** Bump the epoch exactly like session.ts markDirty does and schedule the + * save (calling session.markDirty would hit the module-singleton store, + * not the harness store). */ + markDirty: () => void; + /** The writer-off lane's scheduleSave without an epoch bump (e.g. the + * incremental-lane close-out shape). */ + scheduleSave: () => void; + lastSaveLanded: Map; + /** Let the coalesced microtask run. */ + settle: () => Promise; +} + +async function withHarness(run: (h: Harness) => Promise): Promise { + const prevUnified = process.env.BILI_STORAGE_UNIFIED; + const prevWriter = process.env.BILI_STORAGE_JSON_WRITER; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "off"; // no envelope — the writer-off coalesced lane + const seed = fs.mkdtempSync(path.join(os.tmpdir(), "bili-6780-seed-")); + const session = loadCorpusCase(seed, "folded", "corpus-folded", "anthropic"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-6780-dedup-")); + try { + const store = new SessionStore({ dir, debounceMs: 60_000, enabled: true }); + const anyStore = store as unknown as { + shadow: { record: (...args: unknown[]) => unknown }; + scheduleSave: (s: Session) => void; + lastSaveLanded: Map; + ingestRowsNow: (s: Session) => void; + }; + let ingests = 0; + const harness: Harness = { + session, + ingestRowsNow: () => anyStore.ingestRowsNow(session), + spy: () => { + const real = anyStore.shadow.record.bind(anyStore.shadow); + anyStore.shadow.record = (...args: unknown[]) => { ingests++; return real(...args); }; + return () => ingests; + }, + markDirty: () => { + session.revisionEpoch = (session.revisionEpoch ?? 0) + 1; + anyStore.scheduleSave(session); + }, + scheduleSave: () => anyStore.scheduleSave(session), + lastSaveLanded: anyStore.lastSaveLanded, + settle: () => new Promise((r) => setTimeout(r, 50)), + }; + await run(harness); + store.cancelAll(); + store.closeQuietly(); + } finally { + rmBestEffort(dir); + rmBestEffort(seed); + process.env.BILI_STORAGE_UNIFIED = prevUnified; + process.env.BILI_STORAGE_JSON_WRITER = prevWriter; + } +} + +test("#6780 P0: the writer-off coalesced save skips the duplicate ingest when ingestRowsNow already covered this tick's state", async () => { + await withHarness(async ({ session, ingestRowsNow, spy, markDirty, lastSaveLanded, settle }) => { + // Exactly what the prepare phase does before the model call: the + // synchronous durability ingest, then the plain markDirty. + ingestRowsNow(); + const ingests = spy(); + + markDirty(); + await settle(); + assert.equal(ingests(), 0, "state unchanged since ingestRowsNow — the coalesced save must NOT re-run the full rebuild + full delta-ingest"); + assert.equal(lastSaveLanded.get(session.id), true, "the settle protocol must still land on the skip path"); + + // Counterfactual: the state changes AFTER the arm (a re-snapshot + // swaps lastMessages without bumping the epoch; the follow-up + // markDirty bumps it) → the full build must still run. + snapshotMessages(session, [...(session.lastMessages ?? []), { id: "m-dedup-probe", role: "user", contentType: "text", text: "probe" } as unknown as CoreMessage]); + markDirty(); + await settle(); + assert.equal(ingests(), 1, "a changed state must still take the full build (the arm never commits)"); + }); +}); + +test("#6780 P0 (F6): a lastMessages re-snapshot alone (single markDirty) defeats the skip", async () => { + await withHarness(async ({ session, ingestRowsNow, spy, markDirty, settle }) => { + ingestRowsNow(); + const ingests = spy(); + + // A fresh client request re-snapshots the raw messages: a NEW array + // reference, no kernel mutation, no epoch bump — then only + // prepare's own markDirty. The ref-swap condition must alone force + // the full build (epoch is exactly armed.epoch + 1, so every other + // gate condition passes). + snapshotMessages(session, [...(session.lastMessages ?? []), { id: "m-ref-swap", role: "user", contentType: "text", text: "ref swap" } as unknown as CoreMessage]); + markDirty(); + await settle(); + assert.equal(ingests(), 1, "a re-snapshotted lastMessages (same epoch delta) must take the full build"); + }); +}); + +test("#6780 P0 (F1): a fold-turn ledger align consumed during scheduleSave forces the full build", async () => { + await withHarness(async ({ session, ingestRowsNow, spy, markDirty, lastSaveLanded, settle }) => { + ingestRowsNow(); + const ingests = spy(); + + // A fold commit between the previous save and this prepare set the + // align flag. ingestRowsNow does NOT consume it — scheduleSave's + // entry does, and the restamp mutates state (ref renumbering + CCR + // re-key) AFTER the arm was set. + session.pendingLedgerAlign = true; + markDirty(); + await settle(); + assert.equal(ingests(), 1, "the restamped state must take the full build — the align path consumes the arm (the #4733 same-record invariant)"); + assert.equal(lastSaveLanded.get(session.id), true, "the settle protocol must land on the forced full build"); + assert.equal(session.pendingLedgerAlign, undefined, "the align flag is consumed exactly once"); + }); +}); + +test("#6780 P0 (F2): the arm is single-shot — a later standalone markDirty cannot skip on the consumed arm", async () => { + await withHarness(async ({ session, ingestRowsNow, spy, markDirty, scheduleSave, settle }) => { + ingestRowsNow(); // arms at epoch E + const ingests = spy(); + + // A scheduleSave WITHOUT the epoch bump (the incremental-lane + // close-out shape): the gate fails (E !== E + 1) → full build, and + // the arm is consumed by this run's decision. + scheduleSave(); + await settle(); + assert.equal(ingests(), 1, "epoch unchanged since the arm → full build"); + + // Now a single non-metadata mutation + markDirty reaches E+1 — the + // OLD (never-consumed) arm would match the whole gate and wrongly + // skip this one. The single-shot arm must not allow that. + session.stats.outputTokens += 1; + markDirty(); + await settle(); + assert.equal(ingests(), 2, "a stale arm must not justify skipping a save it never covered"); + }); +});