From 58a53c79dfed7333d6764c063296be5b90c75274 Mon Sep 17 00:00:00 2001 From: qiangu2020 <57162279+qiangu2020@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:03:19 +0800 Subject: [PATCH 1/4] kernel: preserve issued refs and quarantine known historical collisions (#5196) --- kernel/package-lock.json | 4 +- kernel/package.json | 2 +- kernel/src/compress.ts | 1 + kernel/src/rebuild.ts | 1 + kernel/src/refs.ts | 69 ++++- kernel/tests/refs.test.ts | 89 +++++- package-lock.json | 2 +- src/absorb.ts | 5 +- src/external-summary-compress.ts | 4 +- src/fork-adoption.ts | 4 +- src/persist.ts | 144 +++------ src/plugin.ts | 11 +- src/session.ts | 23 ++ src/storage/shadow.ts | 16 +- src/storage/store.ts | 30 +- src/stream.ts | 23 +- tests/absorb.test.ts | 19 ++ tests/corrupt-index-child.ts | 24 +- tests/external-summary-runtime.test.ts | 20 ++ tests/issue1875-plugin-tool-outcome.test.ts | 18 ++ tests/plugin-fork.test.ts | 38 ++- tests/seq-ledger-pr3.test.ts | 321 ++++++++++++++++++-- tests/storage-engine-proofs.test.ts | 30 +- tests/storage-orphan-recovery.test.ts | 15 +- tests/storage-unified.test.ts | 30 +- 25 files changed, 736 insertions(+), 207 deletions(-) diff --git a/kernel/package-lock.json b/kernel/package-lock.json index bbe88cba18..9a50787c32 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.170", + "version": "0.0.171", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.170", + "version": "0.0.171", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 28b2aac68f..37f51d6aef 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.170", + "version": "0.0.171", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 5be1706e1d..5d73ce4fb9 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -1048,6 +1048,7 @@ const assignRefsNode: PipelineNode = { isProtected: protectedFn, // Ephemeral retrieval injections never consume a ref slot. shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), + deadRefs: io.state.deadRefs, }); // #3939 ②/#3805: a BLOCKED→numeric migration mints the ref NOW but the // carrier's tag render is DEFERRED to the next fold boundary — diff --git a/kernel/src/rebuild.ts b/kernel/src/rebuild.ts index e1c88ca443..ccbbe27c2e 100644 --- a/kernel/src/rebuild.ts +++ b/kernel/src/rebuild.ts @@ -37,6 +37,7 @@ export function rebuildCompressionState( existing: state.messageRefs, nextIndex: highestUsedIndex(state.messageRefs) + 1, shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), + deadRefs: state.deadRefs, }); let working: CompressionState = { ...state, messageRefs: refResult.map }; diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index 639d11a1a6..b3281b1c59 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -73,20 +73,57 @@ export interface AssignRefsOptions { nextIndex: number; isProtected?: (message: CoreMessage) => boolean; shouldSkip?: (message: CoreMessage) => boolean; + /** Refs that belonged to pruned messages. They are never handed out again. */ + deadRefs?: readonly string[]; +} + +function reservedRefIndices(deadRefs: readonly string[] | undefined): Set { + const reserved = new Set(); + for (const ref of deadRefs ?? []) { + const index = refToIndex(ref); + if (index !== null) reserved.add(index); + } + return reserved; +} + +/** Highest numeric number present on either side of the map. byRef can hold + * orphans whose byRaw owner is gone; those numbers are still issued. */ +function highestIssuedIndex(map: MessageRefMap): number { + let highest = highestUsedIndex(map); + for (const ref of Object.keys(map.byRef)) { + const index = refToIndex(ref); + if (index !== null && index > highest) highest = index; + } + return highest; } export function assignRefs( messages: CoreMessage[], options: AssignRefsOptions, ): AssignRefsResult { + const existingByRaw = options.existing?.byRaw ?? {}; + const existingByRef = options.existing?.byRef ?? {}; const map: MessageRefMap = { - byRaw: { ...options.existing.byRaw }, - byRef: { ...options.existing.byRef }, + byRaw: { ...existingByRaw }, + byRef: { ...existingByRef }, }; - let cursor = + const reserved = reservedRefIndices(options.deadRefs); + // The high-water mark is the highest ref this map or its tombstones have + // ever carried. A caller-supplied cursor below it must not reopen that + // range: a low ledgerSeq or a counter restart would otherwise reissue a + // number whose previous owner is gone from byRef. + let historicalHighWater = highestIssuedIndex({ byRaw: existingByRaw, byRef: existingByRef }); + for (const index of reserved) { + if (index > historicalHighWater) historicalHighWater = index; + } + const requestedCursor = Number.isInteger(options.nextIndex) && options.nextIndex >= MIN_INDEX ? options.nextIndex : MIN_INDEX; + if (requestedCursor - 1 > historicalHighWater) { + historicalHighWater = requestedCursor - 1; + } + let cursor = Math.max(requestedCursor, historicalHighWater + 1); let newlyAssigned = 0; const migratedIds: string[] = []; @@ -140,7 +177,15 @@ export function assignRefs( ledgerSeq <= MAX_INDEX ) { const text = indexToRef(ledgerSeq); - if (!map.byRef[text]) { + // A ledger stamp only proves identity when it lands above every ref + // already issued for this state and outside the dead-ref tombstones. + // Below the high-water mark the sequence was reused or reset; trusting + // seq+1 there would mint a collision with a historical owner. + if ( + ledgerSeq > historicalHighWater && + !map.byRef[text] && + !reserved.has(ledgerSeq) + ) { map.byRaw[message.id] = text; map.byRef[text] = message.id; newlyAssigned++; @@ -149,7 +194,7 @@ export function assignRefs( } } - const ref = allocateFreeRef(map, cursor); + const ref = allocateFreeRef(map, cursor, reserved); cursor = ref.index + 1; map.byRaw[message.id] = ref.text; map.byRef[ref.text] = message.id; @@ -162,11 +207,12 @@ export function assignRefs( function allocateFreeRef( map: MessageRefMap, start: number, + reserved: ReadonlySet = new Set(), ): { text: string; index: number } { let candidate = Math.max(start, MIN_INDEX); while (candidate <= MAX_INDEX) { const text = indexToRef(candidate); - if (!map.byRef[text]) { + if (!map.byRef[text] && !reserved.has(candidate)) { return { text, index: candidate }; } candidate++; @@ -235,10 +281,15 @@ export function restampRefsToLedger( const byRef: Record = { ...refs.byRef }; const scratch: MessageRefMap = { byRaw, byRef }; const displaced: string[] = []; + const reserved = reservedRefIndices(state.deadRefs); // MessageRefMap carries no cursor field — the caller recomputes it from the // highest used index (compress.ts:1047 pattern). We still track a local - // cursor for free-number allocation during this pass. - let nextIndex = highestUsedIndex(refs) + 1; + // cursor for free-number allocation during this pass. Tombstoned refs are + // part of the historical floor even though no byRef key survives for them. + let nextIndex = highestIssuedIndex(refs) + 1; + for (const index of reserved) { + if (index + 1 > nextIndex) nextIndex = index + 1; + } for (const [rawId, seq] of seqByRawId) { const current = byRaw[rawId]; @@ -273,7 +324,7 @@ export function restampRefsToLedger( const held = byRaw[rawId]; if (held === undefined) continue; if (byRef[held] === rawId) continue; - const free = allocateFreeRef(scratch, nextIndex); + const free = allocateFreeRef(scratch, nextIndex, reserved); out.superseded.set(held, rawId); byRaw[rawId] = free.text; byRef[free.text] = rawId; diff --git a/kernel/tests/refs.test.ts b/kernel/tests/refs.test.ts index feba055bef..ebef682e6e 100644 --- a/kernel/tests/refs.test.ts +++ b/kernel/tests/refs.test.ts @@ -11,7 +11,8 @@ import { refToIndex, rebuildRefIndex, } from "../src/refs.js"; -import type { CoreMessage } from "../src/types.js"; +import { createInitialState } from "../src/state.js"; +import type { CompressionState, CoreMessage } from "../src/types.js"; function msg(id: string, role: CoreMessage["role"] = "user"): CoreMessage { return { id, role, contentType: "text", text: id }; @@ -121,14 +122,94 @@ test("assignRefs skips messages per shouldSkip", () => { assert.equal(refForRaw(result.map, "c"), "m00002"); }); -test("assignRefs skips free indices already taken (no collision)", () => { +test("assignRefs starts above an orphan byRef high-water", () => { const existing = emptyRefMap(); existing.byRaw["old"] = "m00002"; existing.byRef["m00002"] = "old"; + // An orphan reservation can keep a byRef key without a byRaw owner. Its + // number is still issued, so the new allocation starts above it. + existing.byRef["m00003"] = "ghost"; const result = assignRefs([msg("x")], { existing, nextIndex: 1 }); - assert.equal(refForRaw(result.map, "x"), "m00001"); - assert.equal(result.nextIndex, 2); + assert.equal(refForRaw(result.map, "x"), "m00004"); + assert.equal(result.nextIndex, 5); +}); + +test("assignRefs counter never fills below an orphan byRef high-water", () => { + const existing = emptyRefMap(); + existing.byRef["m00100"] = "ghost"; + + const result = assignRefs([msg("new")], { existing, nextIndex: 1 }); + + assert.equal(refForRaw(result.map, "new"), "m00101"); + assert.equal(result.nextIndex, 102); +}); + +test("assignRefs refuses a reused ledger stamp below the historical floor", () => { + const existing = emptyRefMap(); + existing.byRaw["floor"] = "m00010"; + existing.byRef["m00010"] = "floor"; + const newMessage: CoreMessage = { ...msg("new"), ledgerSeq: 7 }; + + const result = assignRefs([newMessage], { + existing, + nextIndex: 1, + deadRefs: ["m00007"], + }); + + assert.notEqual(refForRaw(result.map, "new"), "m00007", "dead ref never reissued"); + assert.equal(refForRaw(result.map, "new"), "m00011", "counter allocation stays above the floor"); + assert.equal(result.nextIndex, 12); +}); + +test("assignRefs counter path never drops below the historical floor", () => { + const existing = emptyRefMap(); + existing.byRaw["floor"] = "m00010"; + existing.byRef["m00010"] = "floor"; + + const result = assignRefs([msg("new")], { + existing, + nextIndex: 1, + deadRefs: ["m00007"], + }); + + assert.equal(refForRaw(result.map, "new"), "m00011"); + assert.equal(result.nextIndex, 12); +}); + +test("assignRefs still accepts a ledger stamp above the historical floor", () => { + const existing = emptyRefMap(); + existing.byRaw["floor"] = "m00010"; + existing.byRef["m00010"] = "floor"; + const newMessage: CoreMessage = { ...msg("new"), ledgerSeq: 11 }; + + const result = assignRefs([newMessage], { + existing, + nextIndex: 1, + deadRefs: ["m00007"], + }); + + assert.equal(refForRaw(result.map, "new"), "m00011"); + assert.equal(result.nextIndex, 12); +}); + +test("assignRefs fresh allocation still starts at seq+1", () => { + const stamped: CoreMessage = { ...msg("first"), ledgerSeq: 1 }; + const result = assignRefs([stamped], { existing: emptyRefMap(), nextIndex: 1 }); + assert.equal(refForRaw(result.map, "first"), "m00001"); +}); + +test("restampRefsToLedger re-homes displaced refs above dead tombstones", async () => { + const { restampRefsToLedger } = await import("../src/refs.js"); + const state: CompressionState = createInitialState(); + state.messageRefs.byRaw.a = "m00001"; state.messageRefs.byRef["m00001"] = "a"; + state.messageRefs.byRaw.b = "m00003"; state.messageRefs.byRef["m00003"] = "b"; + state.messageRefs.byRaw.d = "m00002"; state.messageRefs.byRef["m00002"] = "d"; + state.deadRefs = ["m00004"]; + + const outcome = restampRefsToLedger(state, new Map([["a", 0], ["b", 1]])); + assert.equal(state.messageRefs.byRaw.d, "m00005", "dead m00004 is not a rehome target"); + assert.equal(outcome.rehomed, 1); }); test("rebuildRefIndex drops stale byRef entries and ignores BLOCKED", () => { diff --git a/package-lock.json b/package-lock.json index eced96ec98..bfc642e9f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.170", + "version": "0.0.171", "dev": true, "license": "MIT", "devDependencies": { diff --git a/src/absorb.ts b/src/absorb.ts index ff1cae483f..245ff0d3a3 100644 --- a/src/absorb.ts +++ b/src/absorb.ts @@ -15,7 +15,7 @@ import { PROXY_TOOL_NAMES, READ_ORIGINAL_IMAGE_TOOL_NAME, RULE_TOOL_NAME } from import { effectiveRulesEnabled } from "./rules-feature.js"; import { ccrEnabled, retrieveToolName } from "./store.js"; import { log as loggerLog } from "./logger.js"; -import type { Session } from "./session.js"; +import { hasRefLedgerConflict, type Session } from "./session.js"; import { toolFail, toolOk, type ProxyToolResult } from "./proxy-tool-result.js"; const EFFECTIVE_ABSORB_KEY = "effectiveAbsorb"; @@ -84,6 +84,9 @@ type AbsorbExecCtx = { // repeat-failure guard reasons about them uniformly with compress failures. export function executeAbsorb(args: unknown, callId: string | undefined, absorb: AbsorbConfig, ctx: AbsorbExecCtx): ProxyToolResult { const log = ctx.log ?? ((msg: string) => loggerLog("info", msg)); + if (hasRefLedgerConflict(ctx.session)) { + return { ...toolFail("[absorb FAILED: this session's retained ref history is ambiguous. Absorbing could hide the wrong message, so nothing was changed. Raw messages, blocks, and the ref ledger are preserved.]"), reason: "ref-ledger-conflict" }; + } const parsed = parseAbsorbInput(args, callId, (w) => log(`[acp-absorb] ${w}`)); if (!parsed) { return toolFail('[absorb FAILED: invalid input — expected { ref: "mNNNNN", summary: "..." }]'); diff --git a/src/external-summary-compress.ts b/src/external-summary-compress.ts index 24c0d217e1..29185747cd 100644 --- a/src/external-summary-compress.ts +++ b/src/external-summary-compress.ts @@ -2,7 +2,7 @@ import { collectBlockContent } from "acp-kernel"; import { buildCompressSystemPrompt, parseCompressInput, type ParsedRange } from "./compress-tool.js"; import { configuredSummaryPlan } from "./external-summary-runtime.js"; import type { ResolvedKernelConfig } from "./compress-settings.js"; -import { applyRanges, noVisibleContextRefusal, normalizeRangeOrder, recoverEvictedView, type RewriteCtx } from "./stream.js"; +import { applyRanges, refLedgerConflictRefusal, noVisibleContextRefusal, normalizeRangeOrder, recoverEvictedView, type RewriteCtx } from "./stream.js"; import { compressResult, type ProxyToolResult } from "./proxy-tool-result.js"; import { imagePlaceholders } from "./image-note.js"; import { describeExternalSummaryFailure, summaryLengthBudgetClause, type SummaryWork } from "./external-summary.js"; @@ -34,6 +34,8 @@ function withOptionalSummaries(input: unknown): unknown { /** Called under the host session lock; generation is separate from kernel commit. */ export async function applyConfiguredCompression(input: unknown, ctx: RewriteCtx, callId?: string, signal?: AbortSignal): Promise { + const conflict = refLedgerConflictRefusal(ctx); + if (conflict) return conflict; if (signal?.aborted) return compressResult("[Compression FAILED: cancelled. Nothing compressed.]", "refused", 0); // The plan rides the request Config rail (#833: ctx.config is the // session's effective Config — same cascade that produced the wire). diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 8bc295b144..c31c2b2a6c 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -4,7 +4,7 @@ import { stripRawStage } from "./marker/engine.js"; import { normalizeResponsesMessageItems, sanitizeResponsesInputIds, dropWhitespaceResponsesMessages } from "./loop/adapter-responses.js"; import { responsesToCoreWithToolImages } from "./responses-tool-output.js"; import { replaceBiliCompactionItems } from "./codex-compact.js"; -import { peekSession, markDirty, reserveRefsThrough, type Session } from "./session.js"; +import { peekSession, markDirty, reserveRefsThrough, hasRefLedgerConflict, type Session } from "./session.js"; import { getStore } from "./persist.js"; import { adoptContentStore, cloneStoreForRefs, contentStoreOf } from "./store.js"; import type { WireProtocol } from "./util.js"; @@ -241,6 +241,8 @@ export function planForkAdoption( /** Seed a fresh fork session from the plan. Copy-on-fork: every value is a * clone; the parent session is never touched. */ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, parent: Session): void { + // Filtered adoption does not copy the parent's historical ref rows. + if (hasRefLedgerConflict(parent)) session.metadata.refLedgerConflict = true; for (const b of plan.blocks) { session.state.blocks.push(b); const content = parent.blockContents.get(b.blockId); diff --git a/src/persist.ts b/src/persist.ts index 1586370479..4a2134846f 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -29,16 +29,14 @@ import { VERSION } from "./version.js"; import { createStorageCodec, parseEncryptionKey } from "./encrypt.js"; import { PersistEpermAlert } from "./persist-eperm.js"; import { createInitialState, defaultCountTokens, prune, type CompressionState, type CoreMessage, type MessageContentStore } from "acp-kernel"; -import { markDirty } from "./session.js"; +import { hasRefLedgerConflict, markDirty } from "./session.js"; import { mergeState } from "./state-hydration.js"; -import { restampRefsToLedger, restampContentStore } from "acp-kernel"; import { LazyBlockContents, isDroppedMetadata, lazyMetadata, peekSession, sessionWeight } from "./session.js"; import type { Session, BlockContent, BlockView } from "./session.js"; import { orphanContents, resolveStorageMode, UnifiedShadow, type StorageMode } from "./storage/shadow.js"; import { resolveBackendName } from "./storage/backend.js"; import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js"; import { sha256Hex, type CheckpointRow, type MessageRefContentRow } from "./storage/store.js"; -import { contentStoreOf } from "./store.js"; import type { WireProtocol } from "./util.js"; import { currentContextObservation, CALIBRATION_CLAMP_MAX, CALIBRATION_CLAMP_MIN, CALIBRATION_SAMPLE_MAX, CALIBRATION_SAMPLE_MIN, CALIBRATION_SAMPLE_WINDOW } from "./cache-ledger.js"; @@ -1083,9 +1081,15 @@ export class SessionStore { this.shadow.importRecord(payload); } const view = this.shadow.viewFor(id); - if (view) return buildSession(viewToRecord(view)); + if (view) { + const session = buildSession(viewToRecord(view)); + this.markRefLedgerConflicts(session); + return session; + } } - return buildSession(payload); + const session = buildSession(payload); + this.markRefLedgerConflicts(session); + return session; } /** #708/#1080 review: boot NEVER rewrites session file CONTENT. The first @@ -1508,6 +1512,7 @@ export class SessionStore { } } const drifted = buildSession(driftedEnvelope.payload); + this.markRefLedgerConflicts(drifted); // #4323: "drifted" only proves the two copies DIVERGED — // not which side is ahead. With multiple writers sharing a // sessions directory (the 2026-10-22 incident) the INDEX @@ -1587,6 +1592,7 @@ export class SessionStore { // Legacy fallback — same single-build semantics as plain legacy // mode (see the note below on why no adapter round-trip here). const session = buildSession(envelope.payload); + this.markRefLedgerConflicts(session); if (hasNegativePersistedTokens(envelope.payload)) { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); @@ -1604,6 +1610,7 @@ export class SessionStore { // normalization round-trip and reshape persisted-state bytes (the // storage-render goldens catch exactly that). const session = buildSession(envelope.payload); + this.markRefLedgerConflicts(session); if (hasNegativePersistedTokens(envelope.payload)) { // #408: sync context — debounce the stale-file rewrite // (buildSession already clamped the in-memory value). @@ -1623,93 +1630,36 @@ export class SessionStore { private afterLoad(id: string, view: LegacySessionLike): Session { const record = viewToRecord(view); const session = buildSession(record); + this.markRefLedgerConflicts(session); // #4788 D2: a cold load IS a disk proof of this live revision. this.noteDurable(id, session.revisionEpoch); if (hasNegativePersistedTokens(record)) { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); } - // #4316 layer 3 PR3/3: align the citation space with the unified - // ledger exactly once per state, at the ONLY safe moment — cold - // load. Mid-conversation alignment would shift refs the client has - // already seen (byte-stability contract, snapshot-render D-suite); - // a restart boundary has no in-flight compress call, so nothing can - // observe a half-renumbered map. Post-load arrivals keep minting - // counter refs (fresh numbers, no collision). Envelope-only loads - // (no rows) skip honestly. - this.maybeRestampToLedger(session); + // Issued refs are sealed at cold load. Renumbering them here moved a + // number from one raw message to another while old process-lifetime + // citations still resolved live-first; a stale compress then folded + // the wrong raw. New arrivals are numbered above the historical floor + // by the kernel instead. this.logColdLoad(id, "unified store", session, record.savedAt); return session; } - /** #4316 PR3/3: one-time ref/ledger alignment at cold load. Guards: - * state already aligned, no ref map, unified store unavailable, or no - * rows for this session → no-op. Multi-generation ids (edits) are - * excluded by seqByRaw itself — their legacy refs stay put. */ - private maybeRestampToLedger(session: Session): void { - const state = session.state as (CompressionState & { refLedgerAligned?: boolean }); - if (state.refLedgerAligned === true) return; - if (!state.messageRefs || Object.keys(state.messageRefs.byRaw).length === 0) return; - try { - const seqByRawId = this.unifiedSeqByRaw(session.id); - if (seqByRawId === null || seqByRawId.size === 0) return; - const outcome = restampRefsToLedger(state, seqByRawId); - // #4733: the ref restamp alone leaves the CCR content store keyed by - // store-time refs — a stored original would sit under a number the - // ref map now vends to a DIFFERENT message (misattribution on - // retrieve) and consistency gates (plugin fork snapshot) reject the - // session. Re-key the store onto the same map, same pass. - // #4599: guarded — contentStoreOf MATERIALIZES an empty envelope - // for storeless sessions; touching it speculatively here changed - // the persisted shape of every full-mode cold load (legacy keeps - // undefined) and broke the storage-unified parity pin. Storeless - // sessions have nothing to re-key. - if (session.contentStore) { - const store = contentStoreOf(session); - const restampedStore = restampContentStore(store, state.messageRefs.byRaw); - if (restampedStore !== store) session.contentStore = restampedStore; - } - if (outcome.superseded.size > 0) { - session.restampSuperseded = outcome.superseded; - } else { - // #4797: a converged pass (everything already aligned) must - // CLEAR any stale map — the guard keeps consulting the - // process-lifetime map, and an empty outcome that leaves an - // old map in place keeps refusing live refs forever. - delete session.restampSuperseded; - } - // The token snapshot is keyed by ref (#3488 render-refs): renumbered - // refs would otherwise inherit the stale count of whichever message - // held the new number pre-alignment — the wire tag then disagrees - // with the legacy-restart leg beyond citation renumbering (storage - // engine-proofs wire-parity, #4599). Two-pass moves: stage all - // (oldRef → newRef → count) first, then delete, then write — a - // single-pass swap would lose entries whose newRef is another - // move's oldRef. - const snap = (state as { tokenSnapshot?: Record }).tokenSnapshot; - if (snap && outcome.superseded.size > 0) { - const byRawNow = state.messageRefs.byRaw; - const moves: Array<[oldRef: string, newRef: string, count: number]> = []; - for (const [oldRef, rawId] of outcome.superseded) { - const count = snap[oldRef]; - const newRef = byRawNow[rawId]; - if (count === undefined || newRef === undefined || newRef === oldRef) continue; - moves.push([oldRef, newRef, count]); - } - if (moves.length > 0) { - for (const [oldRef] of moves) delete snap[oldRef]; - for (const [, newRef, count] of moves) snap[newRef] = count; - this.log("info", `[persist] remapped ${moves.length} token-snapshot entries onto ledger-aligned refs for ${session.id} (#4599)`); - } - } - state.refLedgerAligned = true; - this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`); - this.scheduleSave(session); - } catch (err) { - // Alignment is an optimization of the citation space, never a - // load blocker — a failed stamp leaves the legacy refs intact. - this.log("warn", `[persist] ref/ledger alignment skipped for ${session.id}: ${err instanceof Error ? err.message : String(err)}`); - } + /** Detects retained old-vs-current ref collisions before this process can + * resolve a ref into a mutation. Read-only: it neither renumbers nor + * deletes anything. An unavailable query cannot clear an earlier + * conflict; missing historical rows cannot establish past uniqueness. */ + private markRefLedgerConflicts(session: Session): void { + const collisions = this.shadow?.refHistoryCollisions(session.id) ?? null; + if (collisions === null || collisions.length === 0) return; + session.refLedgerConflict = collisions; + session.metadata.refLedgerConflict = true; + const sample = collisions.slice(0, 5).map((c) => c.ref).join(", "); + this.log( + "warn", + `[persist] ${session.id}: ${collisions.length} retained message ref(s) collide with the current refs ledger (${sample}); ref-resolving mutations are blocked for this session`, + ); } /** Unified-first view load (full mode). Null on miss. */ @@ -1828,19 +1778,6 @@ export class SessionStore { return this.enabled && this.shadow?.readsUnified === true; } - /** #4316 PR3/3: raw id → canonical seq map for the one-time ledger - * re-stamp (single-row ids only — multi-generation ids are excluded by - * the store). Null when the unified store is unavailable; the re-stamp - * then degrades to legacy refs (honest skip, never a guess). */ - unifiedSeqByRaw(id: string): Map | null { - if (!this.shadow || this.shadow.stats.disabled) return null; - try { - return this.shadow.seqByRaw(id); - } catch { - return null; - } - } - /** #4730/#4911: highest live (non-retired) seq — the fork endpoint's * inclusive throughSeq, awaited from the (async) fork lane. NOT count - 1 * from unifiedLiveRowCount: tombstones keep their seq, so a retired @@ -2375,6 +2312,10 @@ export class SessionStore { const envelope = this.loadEnvelope(incoming.id, incoming.meta); if (!envelope) return null; const disk = envelope.payload; + // Quarantine is monotonic even when the incoming state wins arbitration. + if (disk.metadata?.refLedgerConflict === true) { + (incoming.metadata ??= {}).refLedgerConflict = true; + } const diskRequests = disk.stats?.requests ?? disk.requests ?? 0; const incRequests = incoming.stats?.requests ?? incoming.requests ?? 0; if (incRequests < diskRequests) return disk; @@ -2492,8 +2433,18 @@ export class SessionStore { // the JSON envelope, and the stamp must say legacy for that copy. const stamp = this.liveStorageStamp; return () => { + // The unified record can be newer than its envelope, including + // index-only writes. Counter arbitration cannot clear this fact. + if (hasRefLedgerConflict(session) || this.shadow?.getSessionMetadata(session.id)?.refLedgerConflict === true) { + session.metadata.refLedgerConflict = true; + } const record = this.buildRecordForSave(session, stamp); const disk = this.staleDiskPayload(record); + if (record.metadata?.refLedgerConflict === true || disk?.metadata?.refLedgerConflict === true) { + session.metadata.refLedgerConflict = true; + (record.metadata ??= {}).refLedgerConflict = true; + if (disk) (disk.metadata ??= {}).refLedgerConflict = true; + } if (disk === null) { // #3301: external-converter import payload rides exactly ONE // save into the unified index (first-open imports take the @@ -2540,6 +2491,7 @@ export class SessionStore { // arbitration the write side just used. this.convergeToDisk(session, disk); this.shadow?.record(disk); + this.markRefLedgerConflicts(session); // Rewrite the disk's own payload: content-identical no-op that // preserves the newer state while satisfying the write chain. this.noteOwnCommit(session.id, disk); @@ -3102,8 +3054,10 @@ export class SessionStore { const cloned = await this.shadow?.createForkAsync(parentId, toLegacyLike(session)) ?? false; if (!cloned) { if (!this.flushSync(session)) throw new Error("fork persistence failed"); + this.markRefLedgerConflicts(session); return; } + this.markRefLedgerConflicts(session); await this.maybeCutForkBaseAsync(session); // A failure here leaves a COMPLETE durable fork receipt in the DB. // The exact retry repairs its envelope; never delete an uncertain commit. diff --git a/src/plugin.ts b/src/plugin.ts index be2b50aa28..1b3cf242de 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { createHash } from "node:crypto"; import { getStore, storageStatusLabel } from "./persist.js"; import { cloneStoreForRefs } from "./store.js"; -import { acquireInFlight, createSession, getSession, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, listSessions, markCompactionBoundary, markDirty, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; +import { acquireInFlight, createSession, getSession, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, hasRefLedgerConflict, listSessions, markCompactionBoundary, markDirty, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; import { clientConversationHeader } from "./session-id.js"; import { buildContextView } from "./context-view.js"; import { ABSORB_TOOL_NAME, BILI_ACP_TOOLS_ANTHROPIC, BILI_ACP_TOOLS_ANTHROPIC_NO_RANGE, BILI_ACP_TOOLS_OPENAI, BILI_ACP_TOOLS_OPENAI_NO_RANGE, BILI_ACP_TOOLS_RESPONSES_NO_RANGE, PROXY_TOOL_NAMES, READ_ORIGINAL_IMAGE_TOOL, READ_ORIGINAL_IMAGE_TOOL_NAME, READ_ORIGINAL_IMAGE_TOOL_OPENAI, READ_ORIGINAL_IMAGE_TOOL_RESPONSES, RETRIEVE_TOOL_NAME, RULE_TOOL, RULE_TOOL_NAME, RULE_TOOL_OPENAI, RULE_TOOL_RESPONSES, SEARCH_CONTEXT_TOOL_NAME, absorbToolsFor, retrieveToolsFor, withAcpNotes, withAcpNotesOpenAI } from "./compress-tool.js"; @@ -1562,6 +1562,7 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr if (entry && (entry.rawId !== rawId || typeof parentStore.byHash[entry.hash] !== "string")) return forkReply(res, 409, { ok: false, status: "unavailable", code: "PARENT_STATE_INCOMPLETE", error: "CCR original unavailable or alias inconsistent" }); } child.contentStore = cloneStoreForRefs(parentStore, new Set(prefix.filter((m) => m.ref !== null).map((m) => m.ref!))) ?? undefined; + if (hasRefLedgerConflict(parent)) child.metadata.refLedgerConflict = true; child.metadata.publicSnapshotTextComparable = true; child.metadata.publicSnapshotStoredRefs = Object.keys(child.contentStore?.byRef ?? {}); for (const key of ["pluginAgent", "lastModel", "effectiveConfig", "effectiveConfigEpoch", "effectiveCcr", "effectiveContextLimit", "reservedContextLimit", "lastWindowSource", "systemPromptTokens"]) if (parent.metadata[key] !== undefined) child.metadata[key] = structuredClone(parent.metadata[key]); @@ -2222,10 +2223,12 @@ export async function handlePluginTool( // (sticky) — the next model request stops injecting the duplicate ephemeral wire // tools. Guarded: only flips a session with NO existing agent binding, never // overriding a pi/omp/opencode plugin or a launcher-registered agent. - if (typeof session.metadata.pluginAgent !== "string" && tool !== retrieveToolName(session)) { - session.metadata.pluginAgent = "mcp"; + if (result.reason !== "ref-ledger-conflict") { + if (typeof session.metadata.pluginAgent !== "string" && tool !== retrieveToolName(session)) { + session.metadata.pluginAgent = "mcp"; + } + markDirty(session); } - markDirty(session); deps.log("info", `[${session.id}] [plugin] tool ${tool} executed via plugin (routed by ${routedBy}, #1685) (${result.text.length} chars, outcome=${result.outcome ?? "n/a"}${result.reason ? `, reason=${result.reason}` : ""})`); // Same deep link on the /acp-cache display surfaces: clients wrap this text in // [acp-cache]/[/acp-cache] markers and strip it from model context by marker diff --git a/src/session.ts b/src/session.ts index 870bc484da..f203aae7e4 100644 --- a/src/session.ts +++ b/src/session.ts @@ -236,6 +236,19 @@ export type PendingRetrieval = { ccr?: boolean; }; +/** A persisted message row was issued `ref` at some point, but the current + * refs ledger binds that same number to a different raw message. This is the + * residue of the retired cold-load re-stamp: old `messages.ref` values and + * the current `refs` table disagree. Any compress that resolves refs would + * be ambiguous, so the session-level guard refuses ref-resolving mutations + * while keeping raw rows, blocks, and the ledger intact. */ +export interface RefHistoryCollision { + ref: string; + refNum: number; + oldRawId: string; + currentRawId: string; +} + export type Session = { /** 派生缓存随会话释放,供驻留字节计量遍历;不进入持久化投影或 wire。 */ transient?: { @@ -544,6 +557,12 @@ export type Session = { * the message that now owns the number). A fresh process that never * re-stamped has no map and every ref behaves normally. */ restampSuperseded?: Map; + /** Detectable old-vs-current ref collisions found at cold load. Diagnostic + * rows stay transient; metadata.refLedgerConflict preserves known + * ambiguity when adoption cannot carry the parent's historical rows. + * Detection cannot be universal: pruned, null, BLOCKED, or orphaned + * historical rows may leave no trace. */ + refLedgerConflict?: RefHistoryCollision[]; createdAt: number; lastSeen: number; /** Original content of compressed blocks, captured at compress time when @@ -668,6 +687,10 @@ type AnonymousPrefixAffinityStamp = { producer?: AnonymousProvenance; }; +export function hasRefLedgerConflict(session: Session): boolean { + return session.metadata.refLedgerConflict === true || (session.refLedgerConflict?.length ?? 0) > 0; +} + // #833: wire paths resolve the kernel Config per request (global → provider → // model compress settings + self-heal + output headroom), while the plugin // status/tool API reads sessions with no request context and was falling back diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index b9dd9f19dc..dfa7889b1d 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -23,7 +23,7 @@ import { log as loggerLog } from "../logger.js"; import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; -import type { BlockView } from "../session.js"; +import type { BlockView, RefHistoryCollision } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import { renameSync } from "node:fs"; @@ -927,6 +927,20 @@ export class UnifiedShadow { return this.sync.seqByRaw(sessionId); } + /** Retained ref-history collisions (see UnifiedStore.findRefHistoryCollisions). + * Full-trust only: a stale index could invent or miss a conflict, so a + * degraded/shadow read returns null and the caller must not treat that as + * a clean bill of health. */ + refHistoryCollisions(sessionId: string): RefHistoryCollision[] | null { + if (this.sync === null) return null; + if (this.stats.disabled) return null; + try { + return this.sync.findRefHistoryCollisions(sessionId); + } catch { + return null; + } + } + /** #4316 PR3/3: live (non-retired) message row count. */ liveRowCount(sessionId: string): number { this.assertLive(); diff --git a/src/storage/store.ts b/src/storage/store.ts index f57b897c83..8fe3749470 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -27,7 +27,7 @@ import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } import { integrityModeFor, writeIntegrityCache, scheduleBackgroundIntegrityCheck, backgroundIntegrityInflightForTests, noteIntegrityCheckRun } from "./integrity-check.js"; import { listInstances, type RegistryEntry } from "../instance.js"; import type { MessageContentStore } from "acp-kernel"; -import type { BlockView } from "../session.js"; +import type { BlockView, RefHistoryCollision } from "../session.js"; /** #3250: run a PASSIVE wal_checkpoint every N successful ingests. The * built-in autocheckpoint alone cannot guarantee reclamation in a long-lived @@ -2979,6 +2979,34 @@ export class UnifiedStore { })); } + /** Historical message rows whose issued ref now belongs to another raw + * message in the current refs ledger. `messages.ref` is first-seen + * (INSERT OR IGNORE), so it preserves the number a row was originally + * issued; comparing it with the live ledger exposes a retired re-stamp + * without hydrating message content. Retired rows are included because + * they are historical evidence; a non-empty result means ref resolution + * is ambiguous and destructive ref work must fail closed. `limit` bounds + * the read; one hit is enough for the guard, and the sample drives the + * operator-facing message. */ + findRefHistoryCollisions(sessionId: string, limit = 100): RefHistoryCollision[] { + const rows = this.db + .prepare( + `SELECT DISTINCT m.ref AS ref, m.ref_num AS ref_num, m.msg_id AS old_raw_id, r.raw_id AS current_raw_id + FROM messages m + JOIN refs r ON r.session_id = m.session_id AND r.ref_num = m.ref_num + WHERE m.session_id = ? AND m.ref_num IS NOT NULL AND m.msg_id <> r.raw_id + ORDER BY m.ref_num, old_raw_id, current_raw_id + LIMIT ?`, + ) + .all(sessionId, limit) as Record[]; + return rows.map((r) => ({ + ref: r.ref as string, + refNum: r.ref_num as number, + oldRawId: r.old_raw_id as string, + currentRawId: r.current_raw_id as string, + })); + } + getCcr(sessionId: string): MessageContentStore | null { const row = this.db .prepare("SELECT ccr_json FROM sessions WHERE session_id = ?") diff --git a/src/stream.ts b/src/stream.ts index 2a18273036..3f88ec5fad 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -2,7 +2,7 @@ import { createHash, randomBytes } from "node:crypto"; import { collectBlockContent, countMessageTokens, defaultCountTokens, formatRanges, hideConsumedCompressCalls, indexToRef, parseBoundary, storeCoveredOriginals, viableRanges, type CompressionCore, type Config, type CoreMessage, type CompressionState, type NudgeDecision, type CompressParseDiagnostics, refToIndex } from "acp-kernel"; import { excludeDeadRefRanges, handleAcpStatus, liveCompressionRanges } from "./acp-status.js"; import { handleAcpCache, prefixTokensBeforeRef, recordCacheFoldsFromBlocks } from "./cache-ledger.js"; -import { type Session, cacheBlockContent, markDirty, residentPressure, scrubTurnSeparatorIds, statusInputBaseline, withoutTurnSeparators } from "./session.js"; +import { type Session, hasRefLedgerConflict, cacheBlockContent, markDirty, residentPressure, scrubTurnSeparatorIds, statusInputBaseline, withoutTurnSeparators } from "./session.js"; import { COMPRESS_TOOL_NAME, parseCompressInput, ABSORB_TOOL_NAME, type ParsedRange } from "./compress-tool.js"; import { effectiveAbsorbConfig, executeAbsorb, isProxyToolFor } from "./absorb.js"; import { executeSearchContextTarget, resolveDecompress } from "./decompress-shared.js"; @@ -909,11 +909,32 @@ export function supersededRefRejection c.ref))); + const listed = refs.length > 0 ? ` for ref(s) ${refs.slice(0, 5).join(", ")}` : " inherited from an ambiguous ref ledger"; + const more = refs.length > 5 ? ` (+${refs.length - 5} more)` : ""; + ctx.log( + `[acp-proxy: compress REFUSED for ${ctx.session.id} — known retained or inherited ref-history ambiguity; raw messages, blocks, and the ref ledger preserved]`, + ); + return compressResult( + `[Compression REFUSED: this session's retained ref history is ambiguous${listed}${more}. Retained or inherited references have conflicting raw-message owners. Compressing could fold the wrong messages, so nothing was changed. Raw messages, blocks, and the ref ledger are preserved. Read-only status, decompress, and export still work; no refs were translated, renumbered, or deleted automatically. Re-read acp_status, or export and continue in a new session if the labels remain inconsistent.]`, + "refused", + 0, + "ref-ledger-conflict", + ); + } + return undefined; +} + export function applyRanges(parsed: ReturnType, ctx: RewriteCtx, opts?: { loopTracking?: boolean }): ProxyToolResult { // #2146: preflight overflow-compress is bili's OWN internal lane — its // failures have separate reporting (noteSkip) and must not arm the // model-facing circuit breaker on attempts the model never made. const trackLoop = opts?.loopTracking !== false; + const conflict = refLedgerConflictRefusal(ctx); + if (conflict) return conflict; let { ranges, diagnostics } = parsed; if (ranges.length === 0) { ctx.log("[acp-proxy: compress call had no valid ranges; nothing compressed.]"); diff --git a/tests/absorb.test.ts b/tests/absorb.test.ts index 5ff7be1c0d..9f1eba2948 100644 --- a/tests/absorb.test.ts +++ b/tests/absorb.test.ts @@ -271,3 +271,22 @@ test("persist round-trip: absorbed records and absorbedTokens survive save/load" assert.deepEqual(loaded!.state.absorbed, [record], "absorbed records must survive restart"); assert.equal(loaded!.state.stats.absorbedTokens, 4321); }); + + +test("executeAbsorb: retained ref-history conflicts refuse before any mutation", () => { + const { turn, config, refFor } = makeTurn(); + const ref = refFor("t-res"); + assert.ok(ref); + const session = makeSession(); + session.state = turn.state; + session.refLedgerConflict = [{ ref, refNum: Number(ref.slice(1)), oldRawId: "previous-owner", currentRawId: "t-res" }]; + const stateBefore = structuredClone(session.state); + const statsBefore = structuredClone(session.stats); + const result = executeAbsorb({ ref, summary: "must not absorb a reused ref" }, "conflict-call", config.absorb!, { + config, messages: turn.messages, session, + }); + assert.match(result.text, /FAILED/); + assert.match(result.text, /ambiguous/); + assert.deepEqual(session.state, stateBefore, "ambiguous ref must not alter absorptions or refs"); + assert.deepEqual(session.stats, statsBefore, "ambiguous ref must not credit tokens"); +}); diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 76eb4d5335..b3880a8be9 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -81,17 +81,19 @@ export function viewStateDigest(view: ReturnType): string }))); } -/** #4316 layer 3 PR3/3 (380ef4dac): a unified full-mode cold load performs a - * ONE-TIME ledger alignment — citation numbers are re-homed onto ledger - * sequence (kernel restampRefsToLedger) and state.refLedgerAligned is stamped. - * Cross-mode / cross-read-path "identical state" proofs must therefore compare - * the state modulo that one-time renumbering: this strips the alignment stamp - * and re-homes every ref number to a canonical order (k-th smallest raw id -> - * m0000k), so two views differing ONLY in which numbering scheme produced - * their refs hash identically under viewDigest/viewStateDigest. Object key - * order is normalized too: different hydration paths build identical content - * under different insertion orders, and canonicalize preserves them. Pure - * function — returns a new object, input untouched. */ +/** #4316 layer 3 PR3/3 (380ef4dac): the historical unified full-mode cold load + * performed a ONE-TIME ledger alignment — citation numbers were re-homed onto + * ledger sequence (kernel restampRefsToLedger) and state.refLedgerAligned was + * stamped. Current cold loads seal refs and do not apply that rewrite, but + * committed goldens and older fixtures still carry its numbering. Cross-mode / + * cross-read-path "identical state" proofs therefore compare state modulo the + * historical renumbering: this strips any alignment stamp and re-homes every + * ref number to a canonical order (k-th smallest raw id -> m0000k), so two + * views differing ONLY in which numbering scheme produced their refs hash + * identically under viewDigest/viewStateDigest. Object key order is normalized + * too: different hydration paths build identical content under different + * insertion orders, and canonicalize preserves them. Pure function — returns a + * new object, input untouched. */ export function stripOneTimeAlignment(view: T): T { const v = view as { state?: unknown; blockContents?: unknown }; if (!v.state || typeof v.state !== "object") return view; diff --git a/tests/external-summary-runtime.test.ts b/tests/external-summary-runtime.test.ts index fee423ea6d..e11ecf12b9 100644 --- a/tests/external-summary-runtime.test.ts +++ b/tests/external-summary-runtime.test.ts @@ -204,6 +204,26 @@ test("missing primary key skips its network call and tries the configured backup }, (req, res) => { paths.push(req.url ?? ""); success(res); }); }); +test("known ref ambiguity refuses configured summaries before work or no-view bookkeeping", async () => { + let calls = 0; + await fixture(async (_base, externalSummary) => { + for (const emptyView of [false, true]) { + const ctx = context(externalSummary); + const input = args(ctx, SUMMARY); + ctx.session.refLedgerConflict = [{ ref: ctx.session.state.messageRefs.byRaw.history, refNum: 1, oldRawId: "old-history", currentRawId: "history" }]; + if (emptyView) ctx.messages = []; + const before = JSON.stringify({ state: ctx.session.state, stats: ctx.session.stats, metadata: ctx.session.metadata, + revision: ctx.session.revisionEpoch }); + const result = await executeProxyToolAsync("compress", input, ctx); + assert.equal(result.reason, "ref-ledger-conflict"); + assert.equal(result.outcome, "refused"); + assert.equal(calls, 0, "ambiguous source must not reach the configured summary server"); + assert.equal(JSON.stringify({ state: ctx.session.state, stats: ctx.session.stats, metadata: ctx.session.metadata, + revision: ctx.session.revisionEpoch }), before); + } + }, (_req, res) => { calls++; success(res); }); +}); + test("protected ranges cause no paid call and no fold", async () => { let calls = 0; await fixture(async (_base, externalSummary) => { diff --git a/tests/issue1875-plugin-tool-outcome.test.ts b/tests/issue1875-plugin-tool-outcome.test.ts index aa02621896..98642f6bad 100644 --- a/tests/issue1875-plugin-tool-outcome.test.ts +++ b/tests/issue1875-plugin-tool-outcome.test.ts @@ -16,6 +16,7 @@ import { afterEach, beforeEach, describe, it } from "node:test"; import { createCore, defaultConfig, type Config, type CoreMessage } from "acp-kernel"; import { type PluginToolDeps, _resetPluginStateForTest, handlePluginTool } from "../src/plugin.ts"; import { getSession } from "../src/session.ts"; +import { getStore } from "../src/persist.ts"; import { applyRanges } from "../src/stream.ts"; import { parseCompressInput } from "../src/compress-tool.ts"; import { forwardTool } from "../src/mcp.ts"; @@ -55,6 +56,23 @@ describe("#1875: plugin tool endpoint reports business outcome, not just ok:true assert.ok(String(r.json.result).startsWith("[Compression FAILED"), `receipt: ${r.json.result}`); }); + it("known ambiguity refusal leaves plugin ownership, revision and storage untouched", async () => { + const session = getSession(sessionId); + session.refLedgerConflict = [{ ref: "m00001", refNum: 1, oldRawId: "old", currentRawId: "current" }]; + const before = JSON.stringify({ metadata: session.metadata, revision: session.revisionEpoch, state: session.state, stats: session.stats }); + const store = getStore(); + const scheduleSave = store.scheduleSave; + let saves = 0; + store.scheduleSave = function (value) { saves++; return scheduleSave.call(this, value); }; + try { + const result = await call("compress", { content: [{ startId: "m00001", endId: "m00001", summary: "Historical source." }] }); + assert.equal(result.status, 200); + assert.equal(result.json.outcome, "refused"); + assert.equal(JSON.stringify({ metadata: session.metadata, revision: session.revisionEpoch, state: session.state, stats: session.stats }), before); + assert.equal(saves, 0); + } finally { store.scheduleSave = scheduleSave; } + }); + it("decompress of unknown block → 200 ok:true + outcome:failure, no blocksCreated", async () => { const r = await call("decompress", { blockId: "b999" }); assert.equal(r.status, 200); diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 5e77c14886..c1dbcf726a 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -63,7 +63,7 @@ interface StatusResponse { type PluginResponse = Path extends "/__bili/plugin/manifest" ? { tools: { openai: { type: "function"; function: { name: string; parameters: Record } }[] }; capabilities: { fork: { protocolVersion: number; endpoint: string; snapshotEndpoint: string } } } : Path extends "/__bili/plugin/fork" ? ForkResponse - : Path extends "/__bili/plugin/tool" ? { result: string; code?: string; conversationId?: string } + : Path extends "/__bili/plugin/tool" ? { result: string; code?: string; conversationId?: string; outcome?: string } : Path extends `/__bili/plugin/status${string}` ? StatusResponse : SnapshotResponse; @@ -129,6 +129,36 @@ test("plain proxy panel snapshots do not change raw-history persistence", { time } finally { await h.close(); } }); +for (const inheritedConflict of ["marker", "diagnostics"] as const) { + test(`HTTP fork preserves known ref ambiguity through cold restore (${inheritedConflict})`, { timeout: 60_000 }, async () => { + const h = await harness(true); + try { + const parent = resolveConversation("parent").session!; + if (inheritedConflict === "marker") parent.metadata.refLedgerConflict = true; + else parent.refLedgerConflict = [{ ref: "m00001", refNum: 1, oldRawId: "retired-owner", currentRawId: parent.state.messageRefs.byRef.m00001 }]; + const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; + assert.equal((await h.request("/__bili/plugin/fork", forkRequest(snapshot))).status, 201); + for (const cold of [false, true]) { + if (cold) { + assert(h.store.flushSync(resolveConversation("child").session!)); + _resetPluginStateForTest(); + _resetSessionsForTest(); + } + const childSnapshot = await h.request("/__bili/plugin/snapshot?conversationId=child"); + assert.equal(childSnapshot.status, 200); + const child = resolveConversation("child").session!; + const before = JSON.stringify({ state: child.state, stats: child.stats, metadata: child.metadata, revision: child.revisionEpoch }); + const ref = childSnapshot.body.orderedMessages[0]!.ref; + const result = await h.request("/__bili/plugin/tool", { conversationId: "child", tool: "compress", args: { content: [{ startId: ref, endId: ref, summary }] } }); + assert.equal(result.status, 200); + assert.equal(result.body.outcome, "refused"); + assert.equal(child.metadata.refLedgerConflict, true); + assert.equal(JSON.stringify({ state: child.state, stats: child.stats, metadata: child.metadata, revision: child.revisionEpoch }), before); + } + } finally { await h.close(); } + }); +} + test("HTTP fork preserves sparse parent ledger before the child's first save", { timeout: 60_000 }, async () => { const oldMode = process.env.BILI_STORAGE_UNIFIED; process.env.BILI_STORAGE_UNIFIED = "full"; @@ -906,10 +936,10 @@ for (const storageMode of ["legacy", "full"] as const) { parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); parent.contentStoreDirty = true; assert(h.store.flushSync(parent)); - // #4316 PR3/3: only a unified cold load runs the one-time ledger - // alignment; the CI legacy baseline retains the original state. + // A cold load must not re-stamp or renumber issued refs: the state + // it serves is byte-for-byte the state that was flushed, in every + // storage mode (no one-time ledger alignment marker). const persistedState = JSON.parse(JSON.stringify(parent.state)); - if (h.store.replicasAvailable()) persistedState.refLedgerAligned = true; assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); diff --git a/tests/seq-ledger-pr3.test.ts b/tests/seq-ledger-pr3.test.ts index 3b63c44166..f863002d7d 100644 --- a/tests/seq-ledger-pr3.test.ts +++ b/tests/seq-ledger-pr3.test.ts @@ -13,8 +13,8 @@ import { join } from "node:path"; * child appends from the parent's max seq + 1. * S14: a compress call citing a ref superseded by the one-time ledger re-stamp * is refused with renumber guidance instead of folding the wrong range. - * S15: cold load re-stamps legacy counter refs onto seq+1 exactly once and - * records the superseded map on the session. + * S15: cold load leaves issued refs untouched (no re-stamp) and a second + * load sees the exact same refs and no superseded map. */ process.env.NODE_ENV = "test"; process.env.BILI_STORAGE_UNIFIED = "full"; @@ -25,7 +25,7 @@ for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CON import { UnifiedStore } from "../src/storage/store.ts"; import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; import type { CoreMessage } from "acp-kernel"; -import { supersededRefRejection } from "../src/stream.ts"; +import { applyRanges, supersededRefRejection } from "../src/stream.ts"; import type { Session } from "../src/session.ts"; function msg(id: string, role: "user" | "assistant" = "user", text = `content for ${id} ${"x".repeat(120)}`): CoreMessage { @@ -150,11 +150,12 @@ test("S14: superseded-ref compress resolution — old refs translate (or refuse assert.equal(supersededRefRejection([{ startRef: "m00003", endRef: "m00004" }], {} as Session).rejection, null); }); -test("S15: cold load re-stamps legacy counter refs onto seq+1 exactly once", async () => { +test("S15: cold load seals legacy counter refs without re-stamping them", async () => { // A persisted session whose kernel refs came from the counter path (a // BLOCKED protected message skipped a number, so refs drift ahead of - // seq+1). Cold load must align refs = seq+1, record the superseded map, - // and a SECOND cold load must be a no-op (kept only, map preserved). + // seq+1). Cold load must preserve every issued number exactly as stored: + // no re-stamp, no superseded map, and a SECOND cold load must see the + // same refs. const dir = join(testRoot, `s15-${Math.random().toString(36).slice(2)}`); const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); @@ -192,33 +193,29 @@ test("S15: cold load re-stamps legacy counter refs onto seq+1 exactly once", asy _setStoreForTest(store2); const reloaded = store2.loadSync("pr3-restamp"); assert.ok(reloaded, "session loads"); - // Cold load fires the one-time alignment (restart boundary — no - // in-flight compress call can observe the renumbering). + // Cold load must not rewrite the issued numbers. const refs = reloaded.state.messageRefs; assert.equal(refs.byRaw.a, "m00001"); assert.equal(refs.byRaw.b, "m00002"); assert.equal(refs.byRaw.c, "BLOCKED", "protected keeps taking no number"); - assert.equal(refs.byRaw.d, "m00004", "d re-stamped to seq3+1"); - assert.equal(refs.byRaw.e, "m00005", "e re-stamped to seq4+1"); - assert.equal((reloaded.state as { refLedgerAligned?: boolean }).refLedgerAligned, true); - assert.ok(reloaded.restampSuperseded instanceof Map); - assert.equal(reloaded.restampSuperseded!.get("m00003"), "d"); - assert.equal(reloaded.restampSuperseded!.get("m00004"), "e"); - // Persist the aligned state deterministically before closing (the - // restamp scheduled a debounced save; closeQuietly should flush it, - // but the second reload below must not race that decision). - assert.ok(store2.flushSync(reloaded), "aligned state committed"); - - // Second cold load: aligned state is a no-op (kept only), and the - // superseded map rides the Session object for this process only. + assert.equal(refs.byRaw.d, "m00003", "d keeps its issued counter ref"); + assert.equal(refs.byRaw.e, "m00004", "e keeps its issued counter ref"); + assert.equal(refs.byRef.m00003, "d"); + assert.equal(refs.byRef.m00004, "e"); + assert.equal((reloaded.state as { refLedgerAligned?: boolean }).refLedgerAligned, undefined); + assert.equal(reloaded.restampSuperseded, undefined); + assert.ok(store2.flushSync(reloaded), "sealed state committed"); + + // Second cold load: identical refs, still no re-stamp. store2.closeQuietly(); const store3 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); _setStoreForTest(store3); const again = store3.loadSync("pr3-restamp"); assert.ok(again); - assert.equal(again.state.messageRefs.byRaw.d, "m00004", "aligned refs stable across reloads"); - assert.equal(again.state.messageRefs.byRaw.e, "m00005"); - // Fresh process: no superseded map — refs behave normally. + assert.equal(again.state.messageRefs.byRaw.d, "m00003", "refs stable across reloads"); + assert.equal(again.state.messageRefs.byRaw.e, "m00004"); + assert.equal(again.state.messageRefs.byRef.m00003, "d"); + assert.equal(again.state.messageRefs.byRef.m00004, "e"); assert.equal(again.restampSuperseded, undefined); store3.closeQuietly(); } finally { @@ -227,3 +224,279 @@ test("S15: cold load re-stamps legacy counter refs onto seq+1 exactly once", asy } rmSync(dir, { recursive: true, force: true }); }); + +test("S16: retained old-vs-current ref collision fails closed without changing bytes", async () => { + const dir = join(testRoot, `s16-${Math.random().toString(36).slice(2)}`); + const dbPath = join(dir, "index.db"); + // First ingest issues m00001/m00002 to a/b. A second ingest keeps those + // first-seen message-row refs (INSERT OR IGNORE) while the refs table + // moves m00001 to c — the exact residue a retired cold-load re-stamp + // leaves behind. + const st = UnifiedStore.open(dbPath, undefined as never) as unknown as UnifiedStore; + const first = legacyLike("pr3-collision", [msg("a"), msg("b")]); + first.state.messageRefs = { byRaw: { a: "m00001", b: "m00002" }, byRef: { m00001: "a", m00002: "b" } }; + assert.equal(st.ingestSession(planIngest(first)), true); + const second = legacyLike("pr3-collision", [msg("a"), msg("b"), msg("c")]); + second.state.messageRefs = { + byRaw: { a: "m00002", b: "m00003", c: "m00001" }, + byRef: { m00001: "c", m00002: "a", m00003: "b" }, + }; + assert.equal(st.ingestSession(planIngest(second)), true); + const refsBefore = st.getRefs("pr3-collision"); + const rowsBefore = st.getMessages("pr3-collision", { includeRetired: true }); + const blocksBefore = st.getBlocks("pr3-collision"); + const countsBefore = st.counts(); + assert.ok(st.findRefHistoryCollisions("pr3-collision").length > 0, "fixture must leave a detectable collision"); + st.close(); + + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store); + try { + const session = store.loadSync("pr3-collision"); + assert.ok(session, "collision fixture must still load"); + assert.ok(session.refLedgerConflict && session.refLedgerConflict.length > 0, "cold load must flag the collision"); + const refused = applyRanges( + { ranges: [{ startRef: "m00001", endRef: "m00001", summary: "fold" }], diagnostics: { kind: "ok", invalidItems: 0 } } as never, + { session, log: () => {} } as never, + { loopTracking: false }, + ); + assert.equal(refused.outcome, "refused"); + assert.equal(refused.reason, "ref-ledger-conflict"); + assert.match(refused.text, /ambiguous/); + assert.match(refused.text, /Raw messages, blocks, and the ref ledger are preserved/); + assert.equal(refused.blocksCreated, 0); + store.closeQuietly(); + + const after = UnifiedStore.open(dbPath, undefined as never) as unknown as UnifiedStore; + try { + assert.deepEqual(after.getRefs("pr3-collision"), refsBefore, "refs ledger unchanged"); + assert.deepEqual(after.getMessages("pr3-collision", { includeRetired: true }), rowsBefore, "raw rows unchanged"); + assert.deepEqual(after.getBlocks("pr3-collision"), blocksBefore, "blocks unchanged"); + assert.deepEqual(after.counts(), countsBefore, "no rows added or removed"); + } finally { + after.close(); + } + + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store2); + const again = store2.loadSync("pr3-collision"); + assert.ok(again); + assert.deepEqual(again.refLedgerConflict, session.refLedgerConflict, "detection is stable across cold loads"); + store2.closeQuietly(); + } finally { + _setStoreForTest({} as never); + } +}); + + +test("S17: warm startup retains known ref-history collision refusal", async () => { + const dir = join(testRoot, `s17-${Math.random().toString(36).slice(2)}`); + const id = "pr3-warm-collision"; + const st = UnifiedStore.open(join(dir, "index.db"), undefined as never) as unknown as UnifiedStore; + const first = legacyLike(id, [msg("a"), msg("b")]); + first.state.messageRefs = { byRaw: { a: "m00001", b: "m00002" }, byRef: { m00001: "a", m00002: "b" } }; + assert.equal(st.ingestSession(planIngest(first)), true); + const second = legacyLike(id, [msg("a"), msg("b"), msg("c")]); + second.state.messageRefs = { + byRaw: { a: "m00002", b: "m00003", c: "m00001" }, + byRef: { m00001: "c", m00002: "a", m00003: "b" }, + }; + assert.equal(st.ingestSession(planIngest(second)), true); + st.close(); + const { SessionStore } = await import("../src/persist.ts"); + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const cold = store.loadSync(id); + assert.ok(cold?.refLedgerConflict?.length, "cold fixture must detect retained conflict"); + assert.equal(store.flushSync(cold), true, "write the normal legacy startup envelope"); + for (const method of ["loadAll", "boot"] as const) { + const warm = (await store[method]()).get(id); + assert.ok(warm?.refLedgerConflict?.length, `${method} must retain the same conflict`); + const before = structuredClone(warm.state); + const refused = applyRanges( + { ranges: [{ startRef: "m00001", endRef: "m00001", summary: "fold" }], diagnostics: { kind: "ok", invalidItems: 0 } } as never, + { session: warm, log: () => {} } as never, + { loopTracking: false }, + ); + assert.equal(refused.outcome, "refused"); + assert.equal(refused.reason, "ref-ledger-conflict"); + assert.deepEqual(warm.state, before, "refusal must not mutate loaded state"); + } + const child: Session = { ...cold, id: `${id}-child`, state: structuredClone(cold.state), refLedgerConflict: undefined }; + await store.persistForkSession(id, child); + assert.ok(child.refLedgerConflict?.length, "an in-memory fork must detect the cloned ref history"); + } finally { + store.closeQuietly(); + } + const mode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "shadow"; + const legacyStore = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const legacy = legacyStore.loadSync(id); + assert.ok(legacy?.refLedgerConflict?.length, "legacy fallback must retain a detectable collision"); + } finally { + legacyStore.closeQuietly(); + if (mode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = mode; + } +}); + +for (const mode of ["legacy", "full"] as const) { + test(`S18: ambiguous resume inheritance survives ${mode} restart and another adoption`, async () => { + const { createCore, defaultConfig, DEFAULT_ABSORB_CONFIG } = await import("acp-kernel"); + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const { createSession } = await import("../src/session.ts"); + const { incomingCoreMessages, maybeAdoptResume } = await import("../src/fork-adoption.ts"); + const { executeProxyToolAsync } = await import("../src/loop/core.ts"); + const { executeAbsorb } = await import("../src/absorb.ts"); + const dir = mkdtempSync(join(testRoot, `resume-${mode}-`)); + const previousMode = process.env.BILI_STORAGE_UNIFIED; + const previousWriter = process.env.BILI_STORAGE_JSON_WRITER; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "on"; + _setStoreForTest(new SessionStore({ enabled: false })); + const parsed = { model: "synthetic", messages: [ + { role: "user", content: "Original request. " + "Retained source context. ".repeat(200) }, + { role: "assistant", content: "First reply. " + "Retained first answer. ".repeat(200) }, + { role: "assistant", content: "Second reply. " + "Retained second answer. ".repeat(200) }, + { role: "user", content: "Keep the original references." }, + ] }; + const messages = incomingCoreMessages("openai", parsed)!; + const [a, b, c, d] = messages.map((m) => m.id); + const fixture = legacyLike(`parent-${mode}`, messages); + fixture.meta = { protocol: "openai" }; + fixture.state.messageRefs = { byRaw: { [a]: "m00001", [b]: "m00002", [c]: "m00003", [d]: "m00004" }, byRef: { m00001: a, m00002: b, m00003: c, m00004: d } }; + const db = UnifiedStore.open(join(dir, "index.db")); + assert.equal(db.ingestSession(planIngest(fixture)), true); + fixture.state.messageRefs = { byRaw: { [a]: "m00002", [b]: "m00003", [c]: "m00001", [d]: "m00004" }, byRef: { m00001: c, m00002: a, m00003: b, m00004: d } }; + assert.equal(db.ingestSession(planIngest(fixture)), true); + const parentRows = db.getMessages(fixture.id, { includeRetired: true }); + db.close(); + let store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const parent = store.loadSync(fixture.id)!; + assert.ok(parent.refLedgerConflict?.length, "fixture hydration must find real retained ambiguity"); + const core = createCore(); + const config = defaultConfig(400_000); + config.preserveRecentMessages = 0; + config.preserveRecentTokens = 0; + config.compress.minCompressRange = 100; + const child = createSession(`child-${mode}`, { protocol: "openai" }); + const resumed = { ...parsed, messages: [...parsed.messages, { role: "user", content: "Resume with the old m00001 citation." }] }; + const childMessages = incomingCoreMessages("openai", resumed)!; + const adopt = (target: Session, source: Session) => maybeAdoptResume({ + session: target, parent: source, sharedDepth: messages.length, protocol: "openai", parsed: resumed, + upstreamOrigin: "https://synthetic.invalid", blocksEnabled: true, log: () => {}, + }); + assert.equal(adopt(child, parent).seededRefs, 4); + child.state = core.processTurn({ messages: childMessages, state: child.state, config, tokenCount: 30000, renderTags: "text-only" }).state; + child.lastMessages = childMessages; + const refuse = async (session: Session) => { + const before = JSON.stringify({ state: session.state, stats: session.stats, metadata: session.metadata, revision: session.revisionEpoch }); + const result = await executeProxyToolAsync("compress", { content: [{ startId: "m00001", endId: "m00001", summary: "Retain the original source context and exact reference ownership without changing any source message or stored block." }] }, { + core, config, session, messages: childMessages, log: () => {}, + }); + assert.equal(result.outcome, "refused", "known inherited ambiguity must refuse, never fold the current wrong owner"); + assert.equal(result.reason, "ref-ledger-conflict"); + const absorbed = executeAbsorb({ ref: "m00001", summary: "must not hide ambiguous source" }, "test-call", DEFAULT_ABSORB_CONFIG, { config, session, messages: childMessages }); + assert.equal(absorbed.reason, "ref-ledger-conflict"); + assert.equal(JSON.stringify({ state: session.state, stats: session.stats, metadata: session.metadata, revision: session.revisionEpoch }), before); + }; + await refuse(child); + assert.equal(child.refLedgerConflict, undefined, "adoption need not invent historical diagnostic rows"); + assert.equal(child.metadata.refLedgerConflict, true); + store.closeQuietly(); + process.env.BILI_STORAGE_UNIFIED = mode; + store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + assert.equal(store.flushSync(child), true); + store.closeQuietly(); + store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + const loaded = store.loadSync(child.id, { protocol: "openai" })!; + assert.ok(loaded); + assert.equal(loaded.refLedgerConflict, undefined, "child has no retained old-owner rows to rediscover"); + assert.equal(loaded.metadata.refLedgerConflict, true, "existing metadata must keep quarantine across restart"); + await refuse(loaded); + const grandchild = createSession(`grandchild-${mode}`, { protocol: "openai" }); + assert.ok(adopt(grandchild, loaded).seededRefs > 0); + await refuse(grandchild); + const healthy = createSession(`healthy-${mode}`, { protocol: "openai" }); + healthy.state = structuredClone(loaded.state); + const healthyChild = createSession(`healthy-child-${mode}`, { protocol: "openai" }); + adopt(healthyChild, healthy); + assert.equal(healthyChild.metadata.refLedgerConflict, undefined, "healthy adoption must not acquire quarantine"); + const checker = UnifiedStore.open(join(dir, "index.db")); + try { + assert.deepEqual(checker.getMessages(fixture.id, { includeRetired: true }), parentRows); + assert.deepEqual(checker.findRefHistoryCollisions(child.id), [], "the marker, not a new SQL collision, must enforce refusal"); + } finally { checker.close(); } + } finally { + store.closeQuietly(); + _setStoreForTest(new SessionStore({ enabled: false })); + if (previousMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previousMode; + if (previousWriter === undefined) delete process.env.BILI_STORAGE_JSON_WRITER; + else process.env.BILI_STORAGE_JSON_WRITER = previousWriter; + } + }); +} + + +for (const mode of ["legacy", "full"] as const) { + const scenarios = ["disk-newer", "local-newer", "local-known", "healthy", ...(mode === "full" ? ["index-only"] : [])]; + for (const scenario of scenarios) { + test(`S19: ${mode} save keeps known quarantine when ${scenario}`, async () => { + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const { createSession, acquireInFlight, releaseInFlight, hasRefLedgerConflict } = await import("../src/session.ts"); + const previousMode = process.env.BILI_STORAGE_UNIFIED; + const previousWriter = process.env.BILI_STORAGE_JSON_WRITER; + process.env.BILI_STORAGE_UNIFIED = mode; + process.env.BILI_STORAGE_JSON_WRITER = "on"; + const dir = mkdtempSync(join(testRoot, `convergence-${mode}-`)); + const localStore = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + if (scenario === "index-only") process.env.BILI_STORAGE_JSON_WRITER = "off"; + const foreignStore = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + process.env.BILI_STORAGE_JSON_WRITER = "on"; + _setStoreForTest(new SessionStore({ enabled: false })); + const local = createSession(`convergence-${scenario}`, { protocol: "openai" }); + acquireInFlight(local); + let foreign: Session | undefined; + try { + local.metadata.localOnly = "keep resident metadata"; + await localStore.writeNow(local); + foreign = foreignStore.loadSync(local.id, { protocol: "openai" })!; + assert.ok(foreign); + acquireInFlight(foreign); + foreign.stats.requests = 2; + foreign.metadata.foreignOnly = "do not copy unrelated metadata"; + if (scenario === "disk-newer" || scenario === "local-newer" || scenario === "index-only") foreign.metadata.refLedgerConflict = true; + await foreignStore.writeNow(foreign); + if (scenario === "local-known") local.metadata.refLedgerConflict = true; + const localWins = scenario === "local-newer" || scenario === "index-only"; + if (localWins) local.stats.requests = 3; + await localStore.writeNow(local); + const expected = scenario !== "healthy"; + assert.equal(hasRefLedgerConflict(local), expected, "active resident must retain known quarantine whichever state wins"); + assert.equal(local.stats.requests, localWins ? 3 : 2); + assert.equal(local.metadata.localOnly, "keep resident metadata"); + assert.equal(local.metadata.foreignOnly, undefined, "do not replace unrelated resident metadata"); + const reader = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const loaded = reader.loadSync(local.id, { protocol: "openai" })!; + assert.ok(loaded); + assert.equal(hasRefLedgerConflict(loaded), expected, "the selected persisted payload must retain the same fact"); + } finally { reader.closeQuietly(); } + } finally { + releaseInFlight(local); + if (foreign) releaseInFlight(foreign); + localStore.closeQuietly(); + foreignStore.closeQuietly(); + _setStoreForTest(new SessionStore({ enabled: false })); + if (previousMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previousMode; + if (previousWriter === undefined) delete process.env.BILI_STORAGE_JSON_WRITER; + else process.env.BILI_STORAGE_JSON_WRITER = previousWriter; + } + }); + } +} diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 9bbe15e75e..e7bec2cf71 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -11,11 +11,11 @@ * render goldens, both with a mid-conversation restart * and with the legacy file DELETED at the restart (the * index alone carries the session — the migration's - * end-state promise). #4316 PR3/3 (380ef4dac): a - * full-mode cold load additionally re-homes citation - * numbers once (one-time ledger alignment) — comparisons - * run modulo that renumbering (citation-normalized - * digests), which is the ONLY permitted delta. + * end-state promise). #4316 PR3/3 (380ef4dac) used to + * re-home citation numbers on a full-mode cold load + * (one-time ledger alignment); that rewrite is now + * removed, but committed goldens still carry its + * numbering, so comparisons stay citation-normalized. * STALE REJECT — a multi-instance stale write that the guarded build * rejects must never reach index.db; the converged disk * record is what gets ingested. @@ -149,7 +149,7 @@ function unifiedRows(store: UnifiedStore, id: string): ReturnType { + test(`storage engine: full-mode restart is invisible on the ${wire} wire (== legacy restart / committed goldens, modulo historical #4316 citation numbering in the goldens)`, { timeout: 240_000 }, async () => { const golden = JSON.parse(fs.readFileSync(path.join(RENDER_ROOT, `${wire}.json`), "utf8")) as { turns: Array<{ i: number; sha: string; shaCiteNorm: string }>; foldAt: number; editedAt: number; summaryFrom: number; @@ -168,9 +168,9 @@ for (const wire of WIRES) { assert.equal(record.turns.length, legacy.turns.length, `${wire}/${leg}: body count drifted vs legacy restart`); for (let i = 0; i < legacy.turns.length; i++) { if (i <= record.editedAt) assert.equal(record.turns[i]!.sha, golden.turns[i]!.sha, `${wire}/${leg}: pre-migration body #${i} digest drift vs committed goldens`); - // #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation - // numbers once (one-time ledger alignment) — the ONLY permitted - // wire delta. Citation-normalized digests must match turn-for-turn. + // #4316 PR3/3 (380ef4dac) goldens were captured with the old + // one-time citation renumbering. Current cold loads seal refs; + // the remaining normalization targets those historical goldens. assert.equal(record.turns[i]!.shaCiteNorm, legacy.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs legacy restart`); assert.equal(record.turns[i]!.shaCiteNorm, golden.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs committed goldens (restart-inherent bs-field loss aside)`); } @@ -307,7 +307,7 @@ test("storage engine: rollback contract — explicit legacy mode serves the iden const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(viaFull, "full-mode reload returned null"); - // Persist the one-time cold-load alignment before exercising rollback. + // Persist the full-mode load result before exercising rollback. await storeFull.writeNow(viaFull); storeFull.closeQuietly(); const digestFull = viewDigest(toLegacyLike(viaFull)); @@ -319,8 +319,8 @@ test("storage engine: rollback contract — explicit legacy mode serves the iden assert.ok(legacy, "legacy-only reload returned null after rollback"); // #2960 ⑤: cross-mode equality is on the STATE proper — full mode // serves the raw table tail, legacy the envelope's folded view. - // #4316 PR3/3: modulo the one-time ledger alignment (full-side cold - // load re-homes citation numbers; legacy keeps counter numbers). + // #4316 PR3/3: committed fixtures may still carry the historical + // cold-load numbering, so these cross-mode digests stay normalized. assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(legacy))), viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))), "rollback: legacy-only load state digest drift vs full-mode load"); assert.ok(viaFull.lastMessages && viaFull.lastMessages.length > 0, "⑤: full-mode load serves the table tail"); assert.ok(legacy.lastMessages && legacy.lastMessages.length > 0, "⑤: rollback keeps the envelope tail"); @@ -398,9 +398,9 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki // executed in the child, verified in the parent). The child also runs // the #2781 mid-flow rename guard plus a post-close guard, so future // handle-leak drift trips there by name instead of at teardown. - // #4316 PR3/3: expected digest computed modulo the one-time ledger - // alignment — the clean read above already carried it in memory, and - // the child's fresh load applies it again; both sides must normalize. + // #4316 PR3/3: expected digest stays normalized for historical + // cold-load numbering. The clean read above and the child's fresh load + // must agree on state regardless of the retained fixture numbering. const stateDigestClean = viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))); const childEnv: NodeJS.ProcessEnv = { ...process.env, diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index ae8bdd7b97..dc6535f5d3 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -69,15 +69,12 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", assert.equal(recovered.stats.requests, 876); assert.equal(recovered.state.nextBlockId, 4); assert.deepEqual(recovered.state.blocks.map(b => b.blockId), ["b3"]); - // #4316 PR3/3: a full-mode cold load re-homes citation numbers once - // (ledger alignment) — pin the content-addressable round-trip, not the - // counter-era number, so the assertion holds under any numbering scheme. - const alignedRef = recovered.state.messageRefs.byRaw["raw-new"]; - assert.ok(alignedRef, "raw-new must keep a citation number after recovery"); - assert.equal(recovered.state.messageRefs.byRef[alignedRef], "raw-new"); - // The pre-alignment number is recorded as superseded, not silently dropped. - assert.equal(alignedRef, "m00001", "the first stored message aligns to seq 0 + 1"); - assert.equal(recovered.restampSuperseded?.get("m00003"), "raw-new"); + // Cold load must not renumber issued refs: the recovered raw keeps the + // exact number the live session handed out, and no superseded translation + // is synthesized. + assert.equal(recovered.state.messageRefs.byRaw["raw-new"], "m00003"); + assert.equal(recovered.state.messageRefs.byRef["m00003"], "raw-new"); + assert.equal(recovered.restampSuperseded, undefined); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 3bc5342c5a..74f7869283 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -9,7 +9,6 @@ import type { Session } from "../src/session.ts"; import type { CoreMessage, CompressionBlock } from "acp-kernel"; import { canonicalize, sha256 } from "./golden-canonical.ts"; import { planIngest, loadLegacyView, toLegacyLike, type LegacySessionLike } from "../src/storage/ingest.ts"; -import { restampRefsToLedger } from "acp-kernel"; import { UnifiedStore } from "../src/storage/store.ts"; import { resolveStorageMode } from "../src/storage/shadow.ts"; import { resolveStorageWalJournalSizeLimit } from "../src/knobs.ts"; @@ -406,12 +405,11 @@ test("resolveStorageWalJournalSizeLimit: env wins, garbage falls through, file t assert.equal(resolveStorageWalJournalSizeLimit({}, 2097152), 2097152, "file tier applies when the env is absent"); }); -/** #4316 PR3/3 parity oracle: the expected digest for a FULL-mode load is - * the LEGACY-mode view of the same files plus the deterministic one-time - * ledger alignment (same seqByRaw the full-mode load aligned against). The - * corpus manifest keeps pinning the PURE view (storage-corpus suite); this - * oracle pins the equivalence "full serve = legacy serve + alignment". */ -function alignedLegacyDigest(caseDir: string, meta: { protocol: string; id: string }, seqByRawId: Map): string { +/** Full mode now seals issued refs at cold load, so the expected digest for + * a FULL-mode load is the PURE legacy view of the same files. The corpus + * manifest pins the same digest (storage-corpus suite); this oracle keeps + * the full-vs-legacy equivalence explicit. */ +function legacyDigest(caseDir: string, meta: { protocol: string; id: string }): string { const legacyDir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-legacy-oracle-")); try { fs.cpSync(caseDir, path.join(legacyDir, "case"), { recursive: true }); @@ -426,11 +424,7 @@ function alignedLegacyDigest(caseDir: string, meta: { protocol: string; id: stri store = new SessionStore({ dir: path.join(legacyDir, "case"), debounceMs: 1, enabled: true }); const pure = store.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(pure, "legacy oracle load returned null"); - const view = toLegacyLike(pure); - const state = structuredClone(view.state as Record); - restampRefsToLedger(state as never, seqByRawId); - delete state.refLedgerAligned; - return sessionDigest({ ...view, state: state as never }); + return sessionDigest(toLegacyLike(pure)); } finally { store?.closeQuietly(); if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; @@ -460,9 +454,7 @@ test("storage unified FULL mode: read path serves from the index with organic im let expectedDigest = ""; try { assert.ok(probe.getSessionRow(meta.id), "organic import did not populate index.db"); - // #4316 PR3/3: expected = legacy view of the same files + the - // deterministic alignment against these very rows. - expectedDigest = alignedLegacyDigest(caseDir, meta, probe.seqByRaw(meta.id)); + expectedDigest = legacyDigest(caseDir, meta); } finally { probe.close(); } @@ -557,13 +549,7 @@ test("storage unified: file-config storage.unified selects the mode; env tier st let session = store.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(session, "file-tier full loadSync returned null"); store.closeQuietly(); - const probeStore = UnifiedStore.open(path.join(caseDir, "index.db")); - let fileTierDigest = ""; - try { - fileTierDigest = alignedLegacyDigest(caseDir, meta, probeStore.seqByRaw(meta.id)); - } finally { - probeStore.close(); - } + const fileTierDigest = legacyDigest(caseDir, meta); assert.equal(sessionDigest(toLegacyLike(session)), fileTierDigest, "file-tier full read digest drift"); assert.ok(fs.existsSync(path.join(caseDir, "index.db")), "file-tier full did not create index.db"); // Env tier wins: explicit env legacy overrides file full on a fresh From 5f7218c6cb2329232f27b3b6210988433e1f93a7 Mon Sep 17 00:00:00 2001 From: qiangu2020 <57162279+qiangu2020@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:51:13 +0800 Subject: [PATCH 2/4] fix(ci): restore baseline config docs and MySQL stamp expectation --- CONFIGURATION.md | 2 +- CONFIGURATION.zh-CN.md | 2 ++ LIFECYCLE.md | 1 + tests/issue4666-migration-dedup.test.ts | 3 ++- website/config-reference/process.yaml | 8 ++++++++ 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 6aefadce0b..0df8c5d4d5 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -1885,8 +1885,8 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_SESSION_GC_INTERVAL_MS` | `sessions.gc.intervalMs` | 3600000 | | `BILI_SESSION_GC_MAX_AGE_DAYS` | `sessions.gc.maxAgeDays` | 7 | | `BILI_SESSION_GC_MAX_TOKENS` | `sessions.gc.maxTokens` | 1000000 | +| `BILI_SESSION_RESIDENT_HARD_MAX_BYTES` | `sessions.hardMaxBytes` | (same as `sessions.residentMaxBytes`) | | `BILI_SESSION_RESIDENT_MAX_BYTES` | `sessions.residentMaxBytes` | 536870912 | -| `BILI_SESSION_RESIDENT_HARD_MAX_BYTES` | `sessions.hardMaxBytes` | (same as `BILI_SESSION_RESIDENT_MAX_BYTES`) | | `BILI_SESSION_WARM_MAX` | `sessions.warmMax` | 0 | | `BILI_SESSION_WARM_MAX_BYTES` | `sessions.warmMaxBytes` | 67108864 | | `BILI_SQLITE_AUTO_DOWNLOAD` | `storage.sqliteAutoDownload` | true | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index ac054a66e2..763b5d34f6 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -203,6 +203,7 @@ | `sessions.warmMax` | number | 0 | BILI_SESSION_WARM_MAX | 有界启动(#2803/#3050):启动监听前预热加载的最近保存会话数上限,更早的状态按需重新加载。默认 0(不预热——首次访问按需加载一次;设 8 恢复预热行为)。受 sessions.max 封顶。 | | `sessions.warmMaxBytes` | number | 67108864 | BILI_SESSION_WARM_MAX_BYTES | 启动预热集的解码字节预算(#2803)。按解码后大小计而非磁盘压缩字节,zstd 归档无法绕过内存上限;单文件超预算则永不预热。 | | `sessions.residentMaxBytes` | number | 536870912 | BILI_SESSION_RESIDENT_MAX_BYTES | 驻留会话估算字节的软目标(#2803)。活跃/在途状态始终固定保留;超过目标时,空闲状态随请求完成回写磁盘释放。 | +| `sessions.hardMaxBytes` | number | (同 `sessions.residentMaxBytes`) | BILI_SESSION_RESIDENT_HARD_MAX_BYTES | 单会话驻留字节估算值的硬上限(#4788,决策 #4792 D1-A)。与只能逐出其他冷会话、从不触及唯一活跃会话的会话池软上限不同,它限制单个会话:请求结束边界处超过上限时,即使该会话免于会话池逐出,也会将可丢弃字段(折叠块原文、元数据台账、有界视图之外的原始尾部)强制释放回磁盘;下一轮通过窄范围磁盘读取透明地恢复所需内容。只要能证明最近一次保存已落盘,每个请求边界都会执行安全释放;此上限额外触发单会话警告(按 60 秒节流)。`0` 禁用上限检查。 | | `sessions.gc.enabled` | boolean | false | BILI_SESSION_GC | 过期持久化会话的可选垃圾回收(未开启时用户数据绝不被悄悄清理)。 | | `sessions.gc.maxAgeDays` | number | 7 | BILI_SESSION_GC_MAX_AGE_DAYS | GC 年龄阈值(天)。 | | `sessions.gc.maxTokens` | number | 1000000 | BILI_SESSION_GC_MAX_TOKENS | GC 单会话记录的 token 大小阈值。 | @@ -1867,6 +1868,7 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `BILI_SESSION_GC_INTERVAL_MS` | `sessions.gc.intervalMs` | 3600000 | | `BILI_SESSION_GC_MAX_AGE_DAYS` | `sessions.gc.maxAgeDays` | 7 | | `BILI_SESSION_GC_MAX_TOKENS` | `sessions.gc.maxTokens` | 1000000 | +| `BILI_SESSION_RESIDENT_HARD_MAX_BYTES` | `sessions.hardMaxBytes` | (same as `sessions.residentMaxBytes`) | | `BILI_SESSION_RESIDENT_MAX_BYTES` | `sessions.residentMaxBytes` | 536870912 | | `BILI_SESSION_WARM_MAX` | `sessions.warmMax` | 0 | | `BILI_SESSION_WARM_MAX_BYTES` | `sessions.warmMaxBytes` | 67108864 | diff --git a/LIFECYCLE.md b/LIFECYCLE.md index 9564227eb1..5e5186319b 100644 --- a/LIFECYCLE.md +++ b/LIFECYCLE.md @@ -121,6 +121,7 @@ | `BILI_SESSION_GC` | 会话 GC 总开关(`=1/true/on` 启用,默认 off;启用后下面三个参数生效) | | `BILI_SESSION_GC_INTERVAL_MS` / `BILI_SESSION_GC_MAX_AGE_DAYS` / `BILI_SESSION_GC_MAX_TOKENS` | 会话 GC 参数(默认 1h / 7d / 1M tokens) | | `BILI_SESSION_RESIDENT_MAX_BYTES` | sessions.residentMaxBytes,512 MiB 软上限(#2803) | +| `BILI_SESSION_RESIDENT_HARD_MAX_BYTES` | sessions.hardMaxBytes,单会话驻留字节硬上限;默认同 sessions.residentMaxBytes,0 禁用检查(#4788 / #4792 D1-A) | | `BILI_SESSION_WARM_MAX_BYTES` | warm 解码字节上限,64 MiB(#2803) | | `BILI_PERSIST` | `=0` 关闭持久化(ephemeral/test) | | `BILI_PERSIST_DEBOUNCE_MS` | persist.debounceMs,500 | diff --git a/tests/issue4666-migration-dedup.test.ts b/tests/issue4666-migration-dedup.test.ts index bf7f9e62bf..3696c91097 100644 --- a/tests/issue4666-migration-dedup.test.ts +++ b/tests/issue4666-migration-dedup.test.ts @@ -503,6 +503,7 @@ test("#4666 mysql: concurrent boots serialize on the migration lock", try { be = await openBackend("mysql", H.dsn); await be.init?.(); + const initializedSchemaVersion = await H.stamp(); await H.rewindToV7(); await seedDirtyFixtureMysql(H, "s1"); await be.close(); @@ -537,7 +538,7 @@ test("#4666 mysql: concurrent boots serialize on the migration lock", assert.equal(Number(released[0].ok), 1, "lock released exactly once"); locker.release(); await migrating; - assert.equal(await H.stamp(), "8", "migration completed after the lock freed"); + assert.equal(await H.stamp(), initializedSchemaVersion, "migration completed after the lock freed"); const s1 = await H.q("SELECT seq FROM messages WHERE session_id = 's1' ORDER BY seq"); assert.deepEqual(s1.map((r) => Number(r.seq)), [1, 2, 3, 5], "fixture collapsed exactly once — no double-delete"); const sLock = await H.q("SELECT seq, identity FROM messages WHERE session_id = 's_lock' ORDER BY seq"); diff --git a/website/config-reference/process.yaml b/website/config-reference/process.yaml index 1d4aaebc3b..38cfa7c057 100644 --- a/website/config-reference/process.yaml +++ b/website/config-reference/process.yaml @@ -367,6 +367,14 @@ entries: group: process desc_en: "Soft target for estimated resident session bytes (#2803). Active/in-flight state stays pinned above it; when over target, idle state discharges back to disk as requests complete." desc_zh: "驻留会话估算字节的软目标(#2803)。活跃/在途状态始终固定保留;超过目标时,空闲状态随请求完成回写磁盘释放。" + - key: sessions.hardMaxBytes + type: number + default: "(same as `sessions.residentMaxBytes`)" + default_zh: "(同 `sessions.residentMaxBytes`)" + env: BILI_SESSION_RESIDENT_HARD_MAX_BYTES + group: process + desc_en: "Per-SESSION hard cap on estimated resident bytes (#4788, decision #4792 D1-A). Unlike the pool soft cap — which can only evict OTHER cold sessions and never touches a sole-active session — this bounds ONE session: over the cap at a request-end boundary, its droppable fields (folded-block originals, metadata ledger, the raw tail beyond the bounded view) are forced back to disk even for a pool-exempt session, and the next turn transparently re-reads what it needs via narrow disk reads. Safe drops run at every request boundary as long as the last save has provably landed; the cap adds the per-session call-out (warn, throttled to 60 s). `0` disables the cap check." + desc_zh: "单会话驻留字节估算值的硬上限(#4788,决策 #4792 D1-A)。与只能逐出其他冷会话、从不触及唯一活跃会话的会话池软上限不同,它限制单个会话:请求结束边界处超过上限时,即使该会话免于会话池逐出,也会将可丢弃字段(折叠块原文、元数据台账、有界视图之外的原始尾部)强制释放回磁盘;下一轮通过窄范围磁盘读取透明地恢复所需内容。只要能证明最近一次保存已落盘,每个请求边界都会执行安全释放;此上限额外触发单会话警告(按 60 秒节流)。`0` 禁用上限检查。" - key: sessions.gc.enabled type: boolean default: "false" From 1ffd4e755e348febb36794a4f6a63a3983fbafe9 Mon Sep 17 00:00:00 2001 From: qiangu2020 <57162279+qiangu2020@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:09:44 +0800 Subject: [PATCH 3/4] test: preserve ref-safety assertions across CI backends --- tests/cache-theorem.test.ts | 6 +-- tests/fork-adoption-plan.test.ts | 2 + tests/issue4739-bounded-session-load.test.ts | 4 +- tests/issue4788-resident-minimal.test.ts | 38 ++++++++++------ .../issue4986-digest-drift-ref-order.test.ts | 18 ++++---- ...ue5103-startup-recovery-visibility.test.ts | 5 ++- tests/plugin-fork.test.ts | 45 +++++++++++-------- 7 files changed, 70 insertions(+), 48 deletions(-) diff --git a/tests/cache-theorem.test.ts b/tests/cache-theorem.test.ts index 67980203a2..95458e4066 100644 --- a/tests/cache-theorem.test.ts +++ b/tests/cache-theorem.test.ts @@ -278,7 +278,7 @@ test("[grounding:g1] ref cursor mechanics pinned to assignRefs/allocateFreeRef", assert.equal(res.map.byRaw["q1"], "m00006", "smallest free >= nextIndex"); assert.equal(highestUsedIndex(res.map), 6); } - // pre-existing binding untouched; gap below the cursor is claimable + // pre-existing binding untouched; a stale cursor cannot reopen issued ref space { const existing = emptyRefMap(); existing.byRaw["a"] = "m00005"; @@ -286,8 +286,8 @@ test("[grounding:g1] ref cursor mechanics pinned to assignRefs/allocateFreeRef", const res = assignRefs([msg("a"), msg("q")], { existing, nextIndex: 4 }); assert.equal(res.map.byRaw["a"], "m00005", "existing untouched"); assert.equal(res.newlyAssigned, 1, "existing message consumes no number"); - assert.equal(res.map.byRaw["q"], "m00004", "gap below cursor is the first free slot"); - assert.equal(res.nextIndex, 5); + assert.equal(res.map.byRaw["q"], "m00006", "fresh ref stays above every issued number despite a stale cursor"); + assert.equal(res.nextIndex, 7); } // BLOCKED consumes no number { diff --git a/tests/fork-adoption-plan.test.ts b/tests/fork-adoption-plan.test.ts index b7143ad88d..cc0f1f6ee8 100644 --- a/tests/fork-adoption-plan.test.ts +++ b/tests/fork-adoption-plan.test.ts @@ -62,6 +62,7 @@ test("planForkAdoption default (no seedAllRefs) seeds only block coverage — do test("resume adoption never writes the kernel BLOCKED sentinel into byRef (#2620)", () => { _setStoreForTest(new SessionStore({ enabled: false })); const parent = { + metadata: {}, state: { blocks: [] as unknown[], tokenSnapshot: {} as Record, @@ -71,6 +72,7 @@ test("resume adoption never writes the kernel BLOCKED sentinel into byRef (#2620 }, } as unknown as Session; const child = { + metadata: {}, state: { blocks: [] as unknown[], tokenSnapshot: {} as Record, diff --git a/tests/issue4739-bounded-session-load.test.ts b/tests/issue4739-bounded-session-load.test.ts index bc9906b5f0..bf91303252 100644 --- a/tests/issue4739-bounded-session-load.test.ts +++ b/tests/issue4739-bounded-session-load.test.ts @@ -139,7 +139,7 @@ test("full mode: viewFor decodes only pointer rows and matches the full-log deco assert.deepEqual(via, reference, "bounded decode byte-equal to the full-log decode"); assert.equal(delta, expectedPointers, `viewFor read ${delta} message rows for ${expectedPointers} pointer seqs (full log = ${liveSeqs.size})`); } finally { - store.cancelAll(); + store.closeQuietly(); if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prev; rmrf(dir); @@ -178,7 +178,7 @@ test("full mode: dropped metadata restores from the index alone (narrow read, no assert.ok(again, "envelope rewritten by later save"); assert.deepEqual(Object.keys(JSON.parse(fs.readFileSync(again, "utf8")).payload.metadata as object).sort(), persistedKeys); } finally { - store.cancelAll(); + store.closeQuietly(); if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prev; rmrf(dir); diff --git a/tests/issue4788-resident-minimal.test.ts b/tests/issue4788-resident-minimal.test.ts index 5f20c83aa8..57641f942a 100644 --- a/tests/issue4788-resident-minimal.test.ts +++ b/tests/issue4788-resident-minimal.test.ts @@ -14,7 +14,7 @@ // Invariants exercised here: #1 frozen routing byte-identical; #5 index-only // paths get equivalent drops; #6 CompressionState untouched by any drop. -import test from "node:test"; +import test, { type TestContext } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, readFileSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -50,6 +50,8 @@ process.env.BILI_PERSIST_TAIL_TOKENS = "600"; // tests (a)-(d) assert behaviorally. process.env.BILI_SESSION_RESIDENT_HARD_MAX_BYTES = "4096"; +const DEBOUNCE_MS = 50; + function rawTail(n: number): CoreMessage[] { const out: CoreMessage[] = []; for (let i = 0; i < n; i++) { @@ -82,11 +84,13 @@ function makeHeavy(id: string, blocks: number): Session { /** One simulated request against a store: mutate (what markDirty would do), * arm the debounced save, wait for it to settle on the kernel chain. */ -async function turn(store: SessionStore, s: Session, midFlight?: boolean): Promise { +async function turn(t: TestContext, store: SessionStore, s: Session, midFlight?: boolean): Promise { if (midFlight) s.inFlight = 1; s.revisionEpoch = (s.revisionEpoch ?? 0) + 1; store.scheduleSave(s); - const landed = await store.afterNextFlush(s.id); + const pendingFlush = store.afterNextFlush(s.id); + t.mock.timers.tick(DEBOUNCE_MS); + const landed = await pendingFlush; if (midFlight) s.inFlight = 0; return landed; } @@ -102,16 +106,17 @@ function findEnvelope(dir: string, protocol: string): string | null { return files.length > 0 ? path.join(protoDir, files[0]) : null; } -test("(a) between-turns drop: every landed request-end boundary discharges droppable fields", async () => { +test("(a) between-turns drop: every landed request-end boundary discharges droppable fields", async (ctx) => { + ctx.mock.timers.enable({ apis: ["setTimeout"] }); const dir = mkdtempSync(path.join(tmpdir(), "bili-bc-4788-a-")); - const store = new SessionStore({ dir, debounceMs: 50, enabled: true }); + const store = new SessionStore({ dir, debounceMs: DEBOUNCE_MS, enabled: true }); try { const s = makeHeavy("rt-a", 40); const beforeTotal = sessionWeight(s); assert.ok(beforeTotal > 8_000, `seeded session should be heavy, got ${beforeTotal}`); // Turn 1 ends: save settles, then the boundary spill runs. - assert.equal(await turn(store, s), true, "debounced save must land with proof"); + assert.equal(await turn(ctx, store, s), true, "debounced save must land with proof"); await store.spillBetweenTurns(s); assert.ok(s.blockContents instanceof LazyBlockContents && (s.blockContents as LazyBlockContents).isDropped, "block contents folded to lazy shell"); assert.ok(isDroppedMetadata(s.metadata), "metadata folded to lazy proxy"); @@ -124,26 +129,28 @@ test("(a) between-turns drop: every landed request-end boundary discharges dropp assert.equal(afterA.state > 0, true, "state stays resident"); // Turn 2: the same boundary works repeatedly (idempotent drops). - assert.equal(await turn(store, s), true); + assert.equal(await turn(ctx, store, s), true); await store.spillBetweenTurns(s); assert.ok((s.blockContents as LazyBlockContents).isDropped, "still folded after the second boundary"); assert.ok(isDroppedMetadata(s.metadata)); } finally { store.cancelAll(); + ctx.mock.timers.reset(); rmrf(dir); } }); -test("(b) narrow-read reconstruction deepEquals the pre-drop reference", async () => { +test("(b) narrow-read reconstruction deepEquals the pre-drop reference", async (ctx) => { + ctx.mock.timers.enable({ apis: ["setTimeout"] }); const dir = mkdtempSync(path.join(tmpdir(), "bili-bc-4788-b-")); - const store = new SessionStore({ dir, debounceMs: 50, enabled: true }); + const store = new SessionStore({ dir, debounceMs: DEBOUNCE_MS, enabled: true }); try { const s = makeHeavy("rt-b", 30); const refBlocks: Record = {}; for (const [k, v] of s.blockContents) refBlocks[k] = structuredClone(v); const refMeta: Record = structuredClone({ ...s.metadata }); - assert.equal(await turn(store, s), true); + assert.equal(await turn(ctx, store, s), true); await store.spillBetweenTurns(s); assert.ok((s.blockContents as LazyBlockContents).isDropped); @@ -159,27 +166,29 @@ test("(b) narrow-read reconstruction deepEquals the pre-drop reference", async ( assert.deepEqual({ ...s.metadata }, refMeta, "metadata rebuilt identically from the narrow read"); // A later save persists the SAME full set again (no thinned erase). - await turn(store, s); + await turn(ctx, store, s); const parsed = JSON.parse(readFileSync(findEnvelope(dir, "openai")!, "utf8")) as { payload?: Record }; const rec = (parsed.payload ?? parsed) as Record; assert.equal(Object.keys((rec.blockContents ?? {}) as object).length, Object.keys(refBlocks).length, "full set round-trips through the envelope"); } finally { store.cancelAll(); + ctx.mock.timers.reset(); rmrf(dir); } }); -test("(c) hard cap under sustained activity: boundaries drop, in-flight skips, warn throttled", async () => { +test("(c) hard cap under sustained activity: boundaries drop, in-flight skips, warn throttled", async (ctx) => { + ctx.mock.timers.enable({ apis: ["setTimeout"] }); const dir = mkdtempSync(path.join(tmpdir(), "bili-bc-4788-c-")); const lines: string[] = []; - const store = new SessionStore({ dir, debounceMs: 50, enabled: true, log: (_l, m) => lines.push(m) }); + const store = new SessionStore({ dir, debounceMs: DEBOUNCE_MS, enabled: true, log: (_l, m) => lines.push(m) }); try { const s = makeHeavy("rt-c", 40); assert.ok(sessionWeight(s) > Number(process.env.BILI_SESSION_RESIDENT_HARD_MAX_BYTES), "session must exceed the (tiny) cap"); // Three consecutive busy turns: mid-flight the spill must NOT run… for (let t = 1; t <= 3; t++) { - assert.equal(await turn(store, s, true), true, `turn ${t} save lands`); + assert.equal(await turn(ctx, store, s, true), true, `turn ${t} save lands`); // Mid-response (still streaming): the spill must decline. s.inFlight = 1; await store.spillBetweenTurns(s); @@ -204,6 +213,7 @@ test("(c) hard cap under sustained activity: boundaries drop, in-flight skips, w } } finally { store.cancelAll(); + ctx.mock.timers.reset(); rmrf(dir); } }); diff --git a/tests/issue4986-digest-drift-ref-order.test.ts b/tests/issue4986-digest-drift-ref-order.test.ts index d427f387b1..239f5dfbfc 100644 --- a/tests/issue4986-digest-drift-ref-order.test.ts +++ b/tests/issue4986-digest-drift-ref-order.test.ts @@ -8,11 +8,10 @@ import { SessionStore } from "../src/persist.ts"; /** * #4986: the sampled ingest read-back verify must not report DIGEST DRIFT * when the in-memory messageRefs key order diverges from the stored - * refs-row order. The golden corpora are exactly that shape: byRef sits in - * ref-number order while byRaw — which finishPlan freezes into row ord at - * full ingest — sits in a different key order, so the row-rebuilt leg's - * byRef starts at m00041 while the live leg's starts at m00001. Identical - * content, different key insertion order. Pre-fix, every such save logged + * refs-row order. The fixture reorders byRef without changing any binding; + * byRaw — which finishPlan freezes into row ord at full ingest — keeps its + * original order. Identical content, different key insertion order. + * Pre-fix, every such save logged * an error-level DIGEST DRIFT and dropped the ingest watermark (full-tail * re-ingest next save); post-fix the parity digest compares contents, not * key insertion order (#3407/#3113 philosophy). @@ -53,10 +52,11 @@ test("ingest verify ignores messageRefs key-order divergence (#4986)", async () try { const session = loadCorpusCase(seed, "folded", "corpus-folded", "anthropic"); - // Precondition: the corpus exhibits the divergent shape — the - // row order (byRaw key order, what finishPlan stores as ord) - // projects byRef to a DIFFERENT key sequence than the live map's - // own key order. Without this the test would pass vacuously. + const refsBefore = structuredClone(session.state.messageRefs); + session.state.messageRefs.byRef = Object.fromEntries(Object.entries(refsBefore.byRef).reverse()); + assert.deepEqual(session.state.messageRefs, refsBefore, "order-only fixture preserves every raw/ref binding"); + + // Without different key orders this regression would pass vacuously. const byRaw = session.state.messageRefs.byRaw; const byRef = session.state.messageRefs.byRef; const rowOrderProjectedRefs = Object.keys(byRaw) diff --git a/tests/issue5103-startup-recovery-visibility.test.ts b/tests/issue5103-startup-recovery-visibility.test.ts index 07c20f48b7..0e89c35ddd 100644 --- a/tests/issue5103-startup-recovery-visibility.test.ts +++ b/tests/issue5103-startup-recovery-visibility.test.ts @@ -183,6 +183,7 @@ function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]): test("#5103: first ingest failure warns immediately with the db path; disable warning keeps naming it", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5103-shadow-")); + const dbPath = path.join(dir, "index.db").replaceAll("\\", "/"); try { const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); assert.ok(shadow, "shadow mode sidecar must open"); @@ -203,7 +204,7 @@ test("#5103: first ingest failure warns immediately with the db path; disable wa shadow.record(mkLegacy("s1", [msg(1), msg(2)])); const firstWarn = captured.find((c) => c.level === "warn" && c.msg.includes("failed to ingest")); assert.ok(firstWarn, "the FIRST failure must warn (before #5103 strikes one and two were silent)"); - assert.ok(firstWarn!.msg.includes(path.join(dir, "index.db")), "warning must name the database file"); + assert.ok(firstWarn!.msg.replaceAll("\\", "/").includes(dbPath), "warning must name the database file"); if (process.platform === "win32") { assert.ok(firstWarn!.msg.includes("(win32:"), "win32 runner carries the OS-level hint"); } @@ -214,7 +215,7 @@ test("#5103: first ingest failure warns immediately with the db path; disable wa shadow.record(mkLegacy("s1", [msg(1), msg(2), msg(3), msg(4)])); const disableWarn = captured.find((c) => c.level === "warn" && c.msg.includes("disabled after 3 consecutive failures")); assert.ok(disableWarn, "third consecutive failure disables with a loud warning"); - assert.ok(disableWarn!.msg.includes(path.join(dir, "index.db")), "disable warning must keep naming the file"); + assert.ok(disableWarn!.msg.replaceAll("\\", "/").includes(dbPath), "disable warning must keep naming the file"); assert.equal(shadow.stats.disabled, true); } finally { setLogCapture(null); diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index c1dbcf726a..2d240d77c5 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -1492,25 +1492,34 @@ test("HTTP a mid-history rewrite replaces the snapshot wholesale with an exact b }); test("HTTP a cold-restored non-retained parent serves its intact raw tail immediately (#5140)", { timeout: 60_000 }, async () => { - const h = await harness(true); + const prevMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; try { - const parent = resolveConversation("parent").session!; - assert(h.store.flushSync(parent)); - h.store.cancelAll(); - _resetSessionsForTest(); - _resetPluginStateForTest(); - const restored = getSession("parent"); - assert.equal(restored.pluginSnapshot, undefined, "a non-forked session restores without its raw snapshot"); - // #5140: the intact raw tail is reconstructable from the unified index, so the cold - // parent is servable immediately — no model call needed to recover stored source. - // The 409 body names the exact forkSnapshot refusal branch — a bare status assert - // swallowed it and left the CI flake undiagnosable (#4599 sentinel 2026-10-30). - const coldSnap = await h.request("/__bili/plugin/snapshot?conversationId=parent"); - assert.equal(coldSnap.status, 200, JSON.stringify(coldSnap.body)); - await sendModel(h, "parent", h.messages); - assert.equal((await h.request("/__bili/plugin/snapshot?conversationId=parent")).status, 200); - assert.equal(resolveConversation("parent").session!.pluginSnapshot!.length, 3); - } finally { await h.close(); } + const h = await harness(true); + try { + const parent = resolveConversation("parent").session!; + assert(h.store.flushSync(parent)); + h.store.cancelAll(); + _resetSessionsForTest(); + _resetPluginStateForTest(); + const restored = getSession("parent"); + assert.equal(restored.pluginSnapshot, undefined, "a non-forked session restores without its raw snapshot"); + // #5140: the intact raw tail is reconstructable from the unified index, so the cold + // parent is servable immediately — no model call needed to recover stored source. + // The 409 body names the exact forkSnapshot refusal branch — a bare status assert + // swallowed it and left the CI flake undiagnosable (#4599 sentinel 2026-10-30). + const coldSnap = await h.request("/__bili/plugin/snapshot?conversationId=parent"); + assert.equal(coldSnap.status, 200, JSON.stringify(coldSnap.body)); + assert.deepEqual(coldSnap.body.messages.map((m) => m.text), h.messages.map((m) => m.content), "cold snapshot preserves the exact raw text and order"); + assert.deepEqual(coldSnap.body.orderedMessages.map((m) => m.rawId), restored.state.lastPassIds, "cold snapshot preserves the exact raw message identities and order"); + await sendModel(h, "parent", h.messages); + assert.equal((await h.request("/__bili/plugin/snapshot?conversationId=parent")).status, 200); + assert.equal(resolveConversation("parent").session!.pluginSnapshot!.length, 3); + } finally { await h.close(); } + } finally { + if (prevMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prevMode; + } }); test("fork protocol v2: snapshot serves the light revision, a stale v1 whole-store revision is still accepted (#2671 Phase 2b)", { timeout: 60_000 }, async () => { From 3e137c4ac8eeeb21df9a462b0e2d04c199c8d353 Mon Sep 17 00:00:00 2001 From: qiangu2020 <57162279+qiangu2020@users.noreply.github.com> Date: Mon, 2 Nov 2026 03:00:48 +0800 Subject: [PATCH 4/4] fix: fence async session loads across fork publication --- src/session.ts | 19 +- tests/session-fork-publication.test.ts | 355 +++++++++++++++++++++++++ 2 files changed, 371 insertions(+), 3 deletions(-) create mode 100644 tests/session-fork-publication.test.ts diff --git a/src/session.ts b/src/session.ts index c018ac4c3a..d57de18751 100644 --- a/src/session.ts +++ b/src/session.ts @@ -1399,8 +1399,20 @@ export async function getSessionAsync(id: string, meta?: SessionMeta): Promise adoptOrCreate(id, meta, await store.loadAsync(id, meta)))() - .finally(() => { pendingSessionLoads.delete(id); }); + pending = (async () => { + const reloaded = await store.loadAsync(id, meta); + if (publishingForks.has(id)) throw new Error("child fork publication in progress"); + const winner = sessions.get(id); + if (winner) { + stampExisting(winner, meta); + return winner; + } + // A published fork may already be evicted or committed despite a persistence error. + if (pendingSessionLoads.get(id) !== pending) throw new Error("session load superseded by fork publication"); + return adoptOrCreate(id, meta, reloaded); + })().finally(() => { + if (pendingSessionLoads.get(id) === pending) pendingSessionLoads.delete(id); + }); pendingSessionLoads.set(id, pending); } return pending; @@ -1430,9 +1442,10 @@ const publishingForks = new Set(); // backends, or a fork could silently clobber a DB-only child row. export async function publishForkSession(session: Session, parentId: string): Promise { if (publishingForks.has(session.id)) throw new Error("child fork publication in progress"); - if (sessions.has(session.id) || (await getStore().loadAsync(session.id))) throw new Error("child session already exists"); publishingForks.add(session.id); try { + pendingSessionLoads.delete(session.id); + if (sessions.has(session.id) || (await getStore().loadAsync(session.id))) throw new Error("child session already exists"); makeResidentRoom(sessionWeight(session)); await getStore().persistForkSession(parentId, session); session.persisted = getStore().enabled; diff --git a/tests/session-fork-publication.test.ts b/tests/session-fork-publication.test.ts new file mode 100644 index 0000000000..c88849e458 --- /dev/null +++ b/tests/session-fork-publication.test.ts @@ -0,0 +1,355 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { type TestContext } from "node:test"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetSessionsForTest, createSession, getSession, getSessionAsync, hasRefLedgerConflict, memoryReport, peekSession, publishForkSession, type Session } from "../src/session.ts"; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +async function fixture(t: TestContext, full = false): Promise { + const previousEnv = Object.fromEntries(["BILI_STORAGE_UNIFIED", "BILI_STORAGE_JSON_WRITER", "BILI_PERSIST_ZSTD"].map((key) => [key, process.env[key]])); + if (full) Object.assign(process.env, { BILI_STORAGE_UNIFIED: "full", BILI_STORAGE_JSON_WRITER: "on", BILI_PERSIST_ZSTD: "0" }); + const dir = mkdtempSync(join(tmpdir(), "bili-fork-publication-")); + const store = new SessionStore({ dir, enabled: true, debounceMs: 60000 }); + _resetSessionsForTest(); + _setStoreForTest(store); + t.after(async () => { + store.cancelAll(); + await store.closeQuietlyAsync(); + _resetSessionsForTest(); + rmSync(dir, { recursive: true, force: true }); + if (full) for (const [key, value] of Object.entries(previousEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + await store.ready(); + if (full) assert.ok(existsSync(join(dir, "index.db")), "full mode must use a real SQLite database"); + for (const id of ["parent-a", "parent-b"]) assert.equal(store.flushSync(createSession(id)), true); + return store; +} + +test("fork publication reserves a child before its existence probe settles", async (t) => { + const store = await fixture(t); + const probe = deferred(); + const load = store.loadAsync.bind(store); + const persist = store.persistForkSession.bind(store); + const writes: string[] = []; + let probes = 0; + store.loadAsync = async (id, meta) => ++probes === 1 ? probe.promise : load(id, meta); + store.persistForkSession = async (parentId, session) => { + await persist(parentId, session); + writes.push(parentId); + }; + const winner = createSession("child", { label: "first publisher" }); + const loser = createSession("child", { label: "second publisher" }); + const first = publishForkSession(winner, "parent-a"); + const [second] = await Promise.allSettled([publishForkSession(loser, "parent-b")]); + const [reader] = await Promise.allSettled([getSessionAsync("child")]); + let syncReadError: unknown; + try { getSession("child"); } catch (error) { syncReadError = error; } + probe.resolve(null); + const [published] = await Promise.allSettled([first]); + t.diagnostic(JSON.stringify({ probes, writes, first: published.status, second: second.status, durableLabel: store.loadSync("child")?.meta.label })); + + assert.equal(second.status, "rejected", `different-parent contender must be refused; writes=${writes.join(",")}`); + if (second.status === "rejected") assert.match(String(second.reason), /child fork publication in progress/); + assert.equal(published.status, "fulfilled"); + assert.equal(reader.status, "rejected", "new async readers must be refused during the existence probe"); + if (reader.status === "rejected") assert.match(String(reader.reason), /child fork publication in progress/); + assert.match(String(syncReadError), /child fork publication in progress/); + assert.equal(probes, 1, "the losing publisher must not even enter the existence probe"); + assert.deepEqual(writes, ["parent-a"]); + assert.equal(peekSession("child"), winner); + assert.equal(store.loadSync("child")?.meta.label, "first publisher"); +}); + +test("an in-flight load cannot adopt a child during fork persistence", async (t) => { + const store = await fixture(t); + const loaded = deferred(); + const persisting = deferred(); + const release = deferred(); + const load = store.loadAsync.bind(store); + const persist = store.persistForkSession.bind(store); + let probes = 0; + store.loadAsync = async (id, meta) => ++probes === 1 ? loaded.promise : load(id, meta); + store.persistForkSession = async (parentId, session) => { + persisting.resolve(); + await release.promise; + await persist(parentId, session); + }; + const reading = Promise.allSettled([getSessionAsync("child")]); + const child = createSession("child", { label: "published child" }); + const publishing = publishForkSession(child, "parent-a"); + await persisting.promise; + loaded.resolve(null); + const [read] = await reading; + const midPublication = peekSession("child"); + release.resolve(); + await publishing; + + t.diagnostic(JSON.stringify({ read: read.status, residentDuringPublication: midPublication?.id ?? null })); + assert.equal(read.status, "rejected", "a load started before publication must recheck exclusion after its await"); + if (read.status === "rejected") assert.match(String(read.reason), /child fork publication in progress/); + assert.equal(midPublication, undefined, "no ghost may become resident while the fork is publishing"); + assert.equal(await getSessionAsync("child"), child); +}); + +for (const result of ["missing", "stale"] as const) { + test(`an in-flight ${result} load returns the completed fork winner`, async (t) => { + const store = await fixture(t); + const loaded = deferred(); + const load = store.loadAsync.bind(store); + let probes = 0; + store.loadAsync = async (id, meta) => ++probes === 1 ? loaded.promise : load(id, meta); + const reading = getSessionAsync("child"); + const child = createSession("child", { label: "published child" }); + child.metadata.refLedgerConflict = true; + await publishForkSession(child, "parent-a"); + loaded.resolve(result === "missing" ? null : createSession("child", { label: "stale load" })); + + const winner = await reading; + assert.equal(winner, child, "a completed publication must win over the earlier load result"); + assert.equal(hasRefLedgerConflict(winner), true, "a stale load must not clear the published quarantine"); + assert.equal(peekSession("child"), child); + const durable = store.loadSync("child"); + assert.equal(durable?.meta.label, "published child"); + assert.ok(durable && hasRefLedgerConflict(durable)); + }); +} + +test("missing-child loads remain single-flight without publication", async (t) => { + const store = await fixture(t); + const loaded = deferred(); + let probes = 0; + store.loadAsync = async () => { probes++; return loaded.promise; }; + const first = getSessionAsync("missing"); + const second = getSessionAsync("missing"); + loaded.resolve(null); + const [a, b] = await Promise.all([first, second]); + + assert.equal(probes, 1); + assert.equal(a, b); + assert.equal(getSession("missing"), a); +}); + +test("serialized publication refuses resident and durable children without leaving exclusion set", async (t) => { + const store = await fixture(t); + const child = createSession("child", { label: "published child" }); + await publishForkSession(child, "parent-a"); + await assert.rejects(publishForkSession(createSession("child"), "parent-b"), /child session already exists/); + assert.equal(getSession("child"), child); + _resetSessionsForTest(); + await assert.rejects(publishForkSession(createSession("child"), "parent-b"), /child session already exists/); + + const loaded = await getSessionAsync("child"); + assert.equal(loaded.meta.label, "published child"); + assert.equal(getSession("child"), loaded); + assert.equal(store.loadSync("child")?.meta.label, "published child"); +}); + +for (const failure of ["probe", "persist"] as const) { + test(`fork ${failure} failure releases child exclusion for a clean retry`, async (t) => { + const store = await fixture(t); + const load = store.loadAsync.bind(store); + const persist = store.persistForkSession.bind(store); + if (failure === "probe") store.loadAsync = async () => { throw new Error("synthetic probe failure"); }; + else store.persistForkSession = async () => { throw new Error("synthetic persist failure"); }; + const child = createSession("child"); + await assert.rejects(publishForkSession(child, "parent-a"), new RegExp(`synthetic ${failure} failure`)); + assert.equal(peekSession("child"), undefined); + store.loadAsync = load; + store.persistForkSession = persist; + + await publishForkSession(child, "parent-a"); + assert.equal(await getSessionAsync("child"), child); + assert.equal(store.loadSync("child")?.id, "child"); + }); +} + +async function inheritedFixture(t: TestContext) { + const store = await fixture(t, true); + const parent = createSession("parent-a"); + parent.metadata.refLedgerConflict = true; + parent.state.messageRefs = { byRaw: { "issued-raw": "m00090" }, byRef: { m00090: "issued-raw" } }; + parent.state.deadRefs = ["m00080"]; + parent.lastMessages = [{ id: "issued-raw", role: "user", contentType: "text", text: "Synthetic inherited ref fixture." }]; + assert.equal(store.flushSync(parent), true); + const child = createSession("child", { label: "inherited quarantined fork" }); + child.metadata.refLedgerConflict = true; + child.state = structuredClone(parent.state); + child.lastMessages = structuredClone(parent.lastMessages); + return { store, child }; +} + +function assertInherited(session: Session | null | undefined) { + assert.ok(session); + assert.equal(hasRefLedgerConflict(session), true); + assert.equal(session.state.messageRefs.byRaw["issued-raw"], "m00090"); + assert.ok(session.state.deadRefs?.includes("m00080")); +} + +function holdEarlierRead(store: SessionStore, result: "missing" | "stale") { + const started = deferred(); + const release = deferred(); + const load = store.loadAsync.bind(store); + let held = false; + store.loadAsync = async (id, meta) => { + const hold = id === "child" && !held; + if (hold) held = true; + const snapshot = await load(id, meta); + if (!hold) return snapshot; + assert.equal(snapshot, null, "the first read must actually miss before publication"); + started.resolve(); + await release.promise; + return result === "missing" ? snapshot : createSession(id, { label: "earlier snapshot" }); + }; + const reading = Promise.allSettled([getSessionAsync("child")]); + return { started, release, reading, load }; +} + +function failEnvelope(t: TestContext, store: SessionStore) { + const disk = (store as unknown as { store: { flushSync(id: string, build: () => unknown): boolean } }).store; + const flush = disk.flushSync.bind(disk); + return t.mock.method(disk, "flushSync", (id: string, build: () => unknown) => id === "child" ? false : flush(id, build)); +} + +for (const result of ["missing", "stale"] as const) { + test(`an earlier ${result} load cannot replace a fork after ordinary LRU eviction`, async (t) => { + const { store, child } = await inheritedFixture(t); + const pool = memoryReport().pool; + _resetSessionsForTest(1, pool.budgetBytes); + const held = holdEarlierRead(store, result); + try { + await held.started.promise; + await publishForkSession(child, "parent-a"); + assert.equal(peekSession("child"), child); + getSession("eviction-pressure"); + assert.equal(peekSession("child"), undefined, "real one-slot LRU must evict the completed child"); + assertInherited(await held.load("child")); + held.release.resolve(); + const [read] = await held.reading; + const durable = await held.load("child"); + assertInherited(durable); + t.diagnostic(JSON.stringify({ schedule: "publish-evict-release", result, reader: read.status, returnedRefs: read.status === "fulfilled" ? read.value.state.messageRefs : null, returnedQuarantine: read.status === "fulfilled" ? hasRefLedgerConflict(read.value) : null, durableRefs: durable?.state.messageRefs, durableQuarantine: durable && hasRefLedgerConflict(durable) })); + assert.equal(read.status, "rejected", "the old claim must not adopt after the resident winner is evicted"); + if (read.status === "rejected") assert.match(String(read.reason), /session load superseded/); + assert.equal(peekSession("child"), undefined); + assertInherited(await getSessionAsync("child")); + } finally { + held.release.resolve(); + await held.reading; + _resetSessionsForTest(pool.maxSessions, pool.budgetBytes); + } + }); + + test(`an earlier ${result} load cannot replace a committed fork after envelope failure`, async (t) => { + const { store, child } = await inheritedFixture(t); + const held = holdEarlierRead(store, result); + try { + await held.started.promise; + const fault = failEnvelope(t, store); + try { + await assert.rejects(publishForkSession(child, "parent-a"), /fork envelope persistence failed; retry the same request/); + } finally { fault.mock.restore(); } + assert.equal(peekSession("child"), undefined); + assert.equal((await store.findSessionFiles("child")).length, 0); + assertInherited(await held.load("child")); + held.release.resolve(); + const [read] = await held.reading; + const durable = await held.load("child"); + assertInherited(durable); + t.diagnostic(JSON.stringify({ schedule: "committed-envelope-error-release", result, reader: read.status, returnedRefs: read.status === "fulfilled" ? read.value.state.messageRefs : null, returnedQuarantine: read.status === "fulfilled" ? hasRefLedgerConflict(read.value) : null, durableRefs: durable?.state.messageRefs, durableQuarantine: durable && hasRefLedgerConflict(durable) })); + assert.equal(read.status, "rejected", "a committed-then-error publication must invalidate the earlier claim too"); + if (read.status === "rejected") assert.match(String(read.reason), /session load superseded/); + assert.equal(peekSession("child"), undefined); + assertInherited(await getSessionAsync("child")); + } finally { + held.release.resolve(); + await held.reading; + } + }); +} + +test("a fresh reader recovers the actual committed fork after envelope failure", async (t) => { + const { store, child } = await inheritedFixture(t); + const fault = failEnvelope(t, store); + try { + await assert.rejects(publishForkSession(child, "parent-a"), /fork envelope persistence failed; retry the same request/); + } finally { fault.mock.restore(); } + assert.equal(peekSession("child"), undefined); + assert.equal((await store.findSessionFiles("child")).length, 0); + assertInherited(await store.loadAsync("child")); + assertInherited(await getSessionAsync("child")); +}); + +test("a failed load releases its own claim for an ordinary missing-child retry", async (t) => { + const store = await fixture(t); + const load = store.loadAsync.bind(store); + let probes = 0; + store.loadAsync = async (id, meta) => { + if (++probes === 1) throw new Error("synthetic read failure"); + return load(id, meta); + }; + await assert.rejects(getSessionAsync("missing"), /synthetic read failure/); + const missing = await getSessionAsync("missing"); + assert.equal(probes, 2); + assert.equal(await getSessionAsync("missing"), missing); +}); + + +test("an invalidated load's finally cannot clear a newer reader's single-flight claim", async (t) => { + const { store, child } = await inheritedFixture(t); + const pool = memoryReport().pool; + _resetSessionsForTest(1, pool.budgetBytes); + const load = store.loadAsync.bind(store); + const started = deferred(); + const releaseOld = deferred(); + const releaseNew = deferred(); + let reads = 0; + store.loadAsync = async (id, meta) => { + const probe = id === "child" ? ++reads : 0; + const snapshot = await load(id, meta); + if (probe === 1) { + assert.equal(snapshot, null); + started.resolve(); + await releaseOld.promise; + } else if (probe > 2) await releaseNew.promise; + return snapshot; + }; + const older = Promise.allSettled([getSessionAsync("child")]); + const settling = [older]; + try { + await started.promise; + await publishForkSession(child, "parent-a"); + getSession("eviction-pressure"); + assert.equal(peekSession("child"), undefined); + const newer = Promise.allSettled([getSessionAsync("child")]); + settling.push(newer); + releaseOld.resolve(); + const [old] = await older; + const joining = Promise.allSettled([getSessionAsync("child")]); + settling.push(joining); + const issuedReads = reads; + releaseNew.resolve(); + const [[next], [joined]] = await Promise.all([newer, joining]); + assertInherited(await load("child")); + t.diagnostic(JSON.stringify({ schedule: "old-finally-new-claim", issuedReads, old: old.status, newer: next.status, joining: joined.status })); + assert.equal(issuedReads, 3, "two newer readers must share one fresh load after the old claim settles"); + assert.equal(old.status, "rejected"); + assert.ok(next.status === "fulfilled" && joined.status === "fulfilled"); + assert.equal(next.value, joined.value); + assertInherited(next.value); + } finally { + releaseOld.resolve(); + releaseNew.resolve(); + await Promise.all(settling); + _resetSessionsForTest(pool.maxSessions, pool.budgetBytes); + } +});