diff --git a/kernel/package-lock.json b/kernel/package-lock.json index bad3402fdd..3d36764818 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index e7b5b52f29..bb32450b06 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 906c7a8cdd..d099ed90e6 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -1186,6 +1186,7 @@ const assignRefsNode: PipelineNode = { isProtected: protectedFn, // Ephemeral retrieval injections never consume a ref slot. shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), + deadRefs: io.state.deadRefs, }); // #3939 ②/#3805: a BLOCKED→numeric migration mints the ref NOW but the // carrier's tag render is DEFERRED to the next fold boundary — diff --git a/kernel/src/rebuild.ts b/kernel/src/rebuild.ts index e1c88ca443..ccbbe27c2e 100644 --- a/kernel/src/rebuild.ts +++ b/kernel/src/rebuild.ts @@ -37,6 +37,7 @@ export function rebuildCompressionState( existing: state.messageRefs, nextIndex: highestUsedIndex(state.messageRefs) + 1, shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), + deadRefs: state.deadRefs, }); let working: CompressionState = { ...state, messageRefs: refResult.map }; diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index 054fedd624..5b27d1bf30 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -73,20 +73,57 @@ export interface AssignRefsOptions { nextIndex: number; isProtected?: (message: CoreMessage) => boolean; shouldSkip?: (message: CoreMessage) => boolean; + /** Refs that belonged to pruned messages. They are never handed out again. */ + deadRefs?: readonly string[]; +} + +function reservedRefIndices(deadRefs: readonly string[] | undefined): Set { + const reserved = new Set(); + for (const ref of deadRefs ?? []) { + const index = refToIndex(ref); + if (index !== null) reserved.add(index); + } + return reserved; +} + +/** Highest numeric number present on either side of the map. byRef can hold + * orphans whose byRaw owner is gone; those numbers are still issued. */ +function highestIssuedIndex(map: MessageRefMap): number { + let highest = highestUsedIndex(map); + for (const ref of Object.keys(map.byRef)) { + const index = refToIndex(ref); + if (index !== null && index > highest) highest = index; + } + return highest; } export function assignRefs( messages: CoreMessage[], options: AssignRefsOptions, ): AssignRefsResult { + const existingByRaw = options.existing?.byRaw ?? {}; + const existingByRef = options.existing?.byRef ?? {}; const map: MessageRefMap = { - byRaw: { ...options.existing.byRaw }, - byRef: { ...options.existing.byRef }, + byRaw: { ...existingByRaw }, + byRef: { ...existingByRef }, }; - let cursor = + const reserved = reservedRefIndices(options.deadRefs); + // The high-water mark is the highest ref this map or its tombstones have + // ever carried. A caller-supplied cursor below it must not reopen that + // range: a low ledgerSeq or a counter restart would otherwise reissue a + // number whose previous owner is gone from byRef. + let historicalHighWater = highestIssuedIndex({ byRaw: existingByRaw, byRef: existingByRef }); + for (const index of reserved) { + if (index > historicalHighWater) historicalHighWater = index; + } + const requestedCursor = Number.isInteger(options.nextIndex) && options.nextIndex >= MIN_INDEX ? options.nextIndex : MIN_INDEX; + if (requestedCursor - 1 > historicalHighWater) { + historicalHighWater = requestedCursor - 1; + } + let cursor = Math.max(requestedCursor, historicalHighWater + 1); let newlyAssigned = 0; const migratedIds: string[] = []; @@ -140,7 +177,15 @@ export function assignRefs( ledgerSeq <= MAX_INDEX ) { const text = indexToRef(ledgerSeq); - if (!map.byRef[text]) { + // A ledger stamp only proves identity when it lands above every ref + // already issued for this state and outside the dead-ref tombstones. + // Below the high-water mark the sequence was reused or reset; trusting + // seq+1 there would mint a collision with a historical owner. + if ( + ledgerSeq > historicalHighWater && + !map.byRef[text] && + !reserved.has(ledgerSeq) + ) { map.byRaw[message.id] = text; map.byRef[text] = message.id; newlyAssigned++; @@ -149,7 +194,7 @@ export function assignRefs( } } - const ref = allocateFreeRef(map, cursor); + const ref = allocateFreeRef(map, cursor, reserved); cursor = ref.index + 1; map.byRaw[message.id] = ref.text; map.byRef[ref.text] = message.id; @@ -162,11 +207,12 @@ export function assignRefs( function allocateFreeRef( map: MessageRefMap, start: number, + reserved: ReadonlySet = new Set(), ): { text: string; index: number } { let candidate = Math.max(start, MIN_INDEX); while (candidate <= MAX_INDEX) { const text = indexToRef(candidate); - if (!map.byRef[text]) { + if (!map.byRef[text] && !reserved.has(candidate)) { return { text, index: candidate }; } candidate++; @@ -250,12 +296,17 @@ export function restampRefsToLedger( const byRef: Record = { ...refs.byRef }; const scratch: MessageRefMap = { byRaw, byRef }; const displaced: string[] = []; + const reserved = reservedRefIndices(state.deadRefs); const retiredThisPass = new Set(); const leftovers: string[] = []; // MessageRefMap carries no cursor field — the caller recomputes it from the // highest used index (compress.ts:1047 pattern). We still track a local - // cursor for free-number allocation during this pass. - let nextIndex = highestUsedIndex(refs) + 1; + // cursor for free-number allocation during this pass. Tombstoned refs are + // part of the historical floor even though no byRef key survives for them. + let nextIndex = highestIssuedIndex(refs) + 1; + for (const index of reserved) { + if (index + 1 > nextIndex) nextIndex = index + 1; + } for (const [rawId, seq] of seqByRawId) { const current = byRaw[rawId]; @@ -309,8 +360,8 @@ export function restampRefsToLedger( for (const rawId of displaced) { const held = byRaw[rawId]; if (held === undefined) continue; - if (byRef[held] === rawId) continue; // back where we found it (no-op guard) - const free = allocateFreeRef(scratch, nextIndex); + if (byRef[held] === rawId) continue; + const free = allocateFreeRef(scratch, nextIndex, reserved); out.superseded.set(held, rawId); byRaw[rawId] = free.text; byRef[free.text] = rawId; @@ -323,7 +374,7 @@ export function restampRefsToLedger( const held = byRaw[rawId]; if (held === undefined) continue; if (byRef[held] === rawId) delete byRef[held]; - const free = allocateFreeRef(scratch, nextIndex); + const free = allocateFreeRef(scratch, nextIndex, reserved); out.superseded.set(held, rawId); byRaw[rawId] = free.text; byRef[free.text] = rawId; diff --git a/kernel/tests/refs.test.ts b/kernel/tests/refs.test.ts index feba055bef..ebef682e6e 100644 --- a/kernel/tests/refs.test.ts +++ b/kernel/tests/refs.test.ts @@ -11,7 +11,8 @@ import { refToIndex, rebuildRefIndex, } from "../src/refs.js"; -import type { CoreMessage } from "../src/types.js"; +import { createInitialState } from "../src/state.js"; +import type { CompressionState, CoreMessage } from "../src/types.js"; function msg(id: string, role: CoreMessage["role"] = "user"): CoreMessage { return { id, role, contentType: "text", text: id }; @@ -121,14 +122,94 @@ test("assignRefs skips messages per shouldSkip", () => { assert.equal(refForRaw(result.map, "c"), "m00002"); }); -test("assignRefs skips free indices already taken (no collision)", () => { +test("assignRefs starts above an orphan byRef high-water", () => { const existing = emptyRefMap(); existing.byRaw["old"] = "m00002"; existing.byRef["m00002"] = "old"; + // An orphan reservation can keep a byRef key without a byRaw owner. Its + // number is still issued, so the new allocation starts above it. + existing.byRef["m00003"] = "ghost"; const result = assignRefs([msg("x")], { existing, nextIndex: 1 }); - assert.equal(refForRaw(result.map, "x"), "m00001"); - assert.equal(result.nextIndex, 2); + assert.equal(refForRaw(result.map, "x"), "m00004"); + assert.equal(result.nextIndex, 5); +}); + +test("assignRefs counter never fills below an orphan byRef high-water", () => { + const existing = emptyRefMap(); + existing.byRef["m00100"] = "ghost"; + + const result = assignRefs([msg("new")], { existing, nextIndex: 1 }); + + assert.equal(refForRaw(result.map, "new"), "m00101"); + assert.equal(result.nextIndex, 102); +}); + +test("assignRefs refuses a reused ledger stamp below the historical floor", () => { + const existing = emptyRefMap(); + existing.byRaw["floor"] = "m00010"; + existing.byRef["m00010"] = "floor"; + const newMessage: CoreMessage = { ...msg("new"), ledgerSeq: 7 }; + + const result = assignRefs([newMessage], { + existing, + nextIndex: 1, + deadRefs: ["m00007"], + }); + + assert.notEqual(refForRaw(result.map, "new"), "m00007", "dead ref never reissued"); + assert.equal(refForRaw(result.map, "new"), "m00011", "counter allocation stays above the floor"); + assert.equal(result.nextIndex, 12); +}); + +test("assignRefs counter path never drops below the historical floor", () => { + const existing = emptyRefMap(); + existing.byRaw["floor"] = "m00010"; + existing.byRef["m00010"] = "floor"; + + const result = assignRefs([msg("new")], { + existing, + nextIndex: 1, + deadRefs: ["m00007"], + }); + + assert.equal(refForRaw(result.map, "new"), "m00011"); + assert.equal(result.nextIndex, 12); +}); + +test("assignRefs still accepts a ledger stamp above the historical floor", () => { + const existing = emptyRefMap(); + existing.byRaw["floor"] = "m00010"; + existing.byRef["m00010"] = "floor"; + const newMessage: CoreMessage = { ...msg("new"), ledgerSeq: 11 }; + + const result = assignRefs([newMessage], { + existing, + nextIndex: 1, + deadRefs: ["m00007"], + }); + + assert.equal(refForRaw(result.map, "new"), "m00011"); + assert.equal(result.nextIndex, 12); +}); + +test("assignRefs fresh allocation still starts at seq+1", () => { + const stamped: CoreMessage = { ...msg("first"), ledgerSeq: 1 }; + const result = assignRefs([stamped], { existing: emptyRefMap(), nextIndex: 1 }); + assert.equal(refForRaw(result.map, "first"), "m00001"); +}); + +test("restampRefsToLedger re-homes displaced refs above dead tombstones", async () => { + const { restampRefsToLedger } = await import("../src/refs.js"); + const state: CompressionState = createInitialState(); + state.messageRefs.byRaw.a = "m00001"; state.messageRefs.byRef["m00001"] = "a"; + state.messageRefs.byRaw.b = "m00003"; state.messageRefs.byRef["m00003"] = "b"; + state.messageRefs.byRaw.d = "m00002"; state.messageRefs.byRef["m00002"] = "d"; + state.deadRefs = ["m00004"]; + + const outcome = restampRefsToLedger(state, new Map([["a", 0], ["b", 1]])); + assert.equal(state.messageRefs.byRaw.d, "m00005", "dead m00004 is not a rehome target"); + assert.equal(outcome.rehomed, 1); }); test("rebuildRefIndex drops stale byRef entries and ignores BLOCKED", () => { diff --git a/kernel/tests/seq-restamp.test.ts b/kernel/tests/seq-restamp.test.ts index 569f74b6c6..c7a8308c02 100644 --- a/kernel/tests/seq-restamp.test.ts +++ b/kernel/tests/seq-restamp.test.ts @@ -218,3 +218,46 @@ test("#5196: the parked layout survives the next cold load untouched", () => { assert.equal(state.messageRefs.byRaw.z, "m00003"); }); +for (const fixture of [ + { name: "retired-seat leftover", byRaw: { a: "m00007", b: "m00009" }, seqs: [["a", 4], ["b", 6]], parked: "b", mover: "a", target: "m00005" }, + { name: "displaced unbacked owner", byRaw: { a: "m00004", b: "m00005" }, seqs: [["a", 4]], parked: "b", mover: "a", target: "m00005" }, + { name: "displaced and leftover overlap", byRaw: { a: "m00002", b: "m00001" }, seqs: [["a", 0], ["b", 1]], parked: "b", mover: "a", target: "m00001" }, +] satisfies Array<{ name: string; byRaw: Record; seqs: Array<[string, number]>; parked: string; mover: string; target: string }>) { + test(`#5196 composition: ${fixture.name} parks above orphan and tombstone history`, () => { + const state = stateWith({ ...fixture.byRaw, protected: "BLOCKED" }, { deadRefs: ["m00090"] }); + state.messageRefs.byRef.m00070 = "historical-orphan"; + const seqs = new Map([...fixture.seqs, ["protected", 99] as [string, number]]); + const out = restampRefsToLedger(state, seqs); + + assert.equal(state.messageRefs.byRaw[fixture.parked], "m00091"); + assert.equal(state.messageRefs.byRef.m00091, fixture.parked); + assert.equal(state.messageRefs.byRaw[fixture.mover], fixture.target); + assert.equal(state.messageRefs.byRef[fixture.target], fixture.mover); + assert.equal(state.messageRefs.byRef.m00070, "historical-orphan"); + assert.equal(state.messageRefs.byRef.m00090, undefined); + assert.equal(state.messageRefs.byRaw.protected, "BLOCKED"); + assert.equal(out.rehomed, 1, "an owner in both settlement lists parks only once"); + assert.equal(out.blocked, 1); + assert.deepEqual(state.deadRefs, ["m00090"]); + if (fixture.name === "retired-seat leftover") { + assert.equal(state.messageRefs.byRef.m00007, undefined); + assert.equal(out.superseded.get("m00007"), "a"); + } + const after = structuredClone(state); + const again = restampRefsToLedger(state, seqs); + assert.equal(again.kept + again.restamped + again.rehomed, 0); + assert.deepEqual(state, after, "the parked layout and history survive a second pass"); + }); +} + +for (const seqs of [ + new Map([["a", 4], ["b", 6]]), + new Map([["a", 8]]), +]) { + test(`#5196 composition: exhausted ${seqs.size === 2 ? "leftover" : "displaced"} parking is atomic`, () => { + const state = stateWith({ a: "m00007", b: "m00009" }, { deadRefs: ["m9999999"] }); + const before = structuredClone(state); + assert.throws(() => restampRefsToLedger(state, seqs), /ref capacity exhausted/); + assert.deepEqual(state, before, "no partial ref swap or aligned marker escapes a failed pass"); + }); +} diff --git a/package-lock.json b/package-lock.json index 561cacb77b..73e28ff4ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.177", + "version": "0.0.178", "dev": true, "license": "MIT", "devDependencies": { diff --git a/src/absorb.ts b/src/absorb.ts index ff1cae483f..245ff0d3a3 100644 --- a/src/absorb.ts +++ b/src/absorb.ts @@ -15,7 +15,7 @@ import { PROXY_TOOL_NAMES, READ_ORIGINAL_IMAGE_TOOL_NAME, RULE_TOOL_NAME } from import { effectiveRulesEnabled } from "./rules-feature.js"; import { ccrEnabled, retrieveToolName } from "./store.js"; import { log as loggerLog } from "./logger.js"; -import type { Session } from "./session.js"; +import { hasRefLedgerConflict, type Session } from "./session.js"; import { toolFail, toolOk, type ProxyToolResult } from "./proxy-tool-result.js"; const EFFECTIVE_ABSORB_KEY = "effectiveAbsorb"; @@ -84,6 +84,9 @@ type AbsorbExecCtx = { // repeat-failure guard reasons about them uniformly with compress failures. export function executeAbsorb(args: unknown, callId: string | undefined, absorb: AbsorbConfig, ctx: AbsorbExecCtx): ProxyToolResult { const log = ctx.log ?? ((msg: string) => loggerLog("info", msg)); + if (hasRefLedgerConflict(ctx.session)) { + return { ...toolFail("[absorb FAILED: this session's retained ref history is ambiguous. Absorbing could hide the wrong message, so nothing was changed. Raw messages, blocks, and the ref ledger are preserved.]"), reason: "ref-ledger-conflict" }; + } const parsed = parseAbsorbInput(args, callId, (w) => log(`[acp-absorb] ${w}`)); if (!parsed) { return toolFail('[absorb FAILED: invalid input — expected { ref: "mNNNNN", summary: "..." }]'); diff --git a/src/external-summary-compress.ts b/src/external-summary-compress.ts index 6df0e557b6..dda645a29f 100644 --- a/src/external-summary-compress.ts +++ b/src/external-summary-compress.ts @@ -3,7 +3,7 @@ import { buildCompressSystemPrompt, parseCompressInput, type ParsedRange } from import { messageTimestampSections } from "./external-summary-surface.js"; import { configuredSummaryPlan } from "./external-summary-runtime.js"; import type { ResolvedKernelConfig } from "./compress-settings.js"; -import { applyRanges, noVisibleContextRefusal, normalizeRangeOrder, recoverEvictedView, type RewriteCtx } from "./stream.js"; +import { applyRanges, refLedgerConflictRefusal, noVisibleContextRefusal, normalizeRangeOrder, recoverEvictedView, type RewriteCtx } from "./stream.js"; import { compressResult, type ProxyToolResult } from "./proxy-tool-result.js"; import { imagePlaceholders } from "./image-note.js"; import { describeExternalSummaryFailure, summaryLengthBudgetClause, type SummaryWork } from "./external-summary.js"; @@ -35,6 +35,8 @@ function withOptionalSummaries(input: unknown): unknown { /** Called under the host session lock; generation is separate from kernel commit. */ export async function applyConfiguredCompression(input: unknown, ctx: RewriteCtx, callId?: string, signal?: AbortSignal): Promise { + const conflict = refLedgerConflictRefusal(ctx); + if (conflict) return conflict; if (signal?.aborted) return compressResult("[Compression FAILED: cancelled. Nothing compressed.]", "refused", 0); // The plan rides the request Config rail (#833: ctx.config is the // session's effective Config — same cascade that produced the wire). diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 3e005817a3..3781508bfc 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -4,7 +4,7 @@ import { stripRawStage } from "./marker/engine.js"; import { normalizeResponsesMessageItems, sanitizeResponsesInputIds, dropWhitespaceResponsesMessages } from "./loop/adapter-responses.js"; import { responsesToCoreWithToolImages } from "./responses-tool-output.js"; import { replaceBiliCompactionItems } from "./codex-compact.js"; -import { peekSession, markDirty, reserveRefsThrough, type Session } from "./session.js"; +import { peekSession, markDirty, reserveRefsThrough, hasRefLedgerConflict, type Session } from "./session.js"; import { getStore } from "./persist.js"; import { adoptContentStore, cloneStoreForRefs, contentStoreOf } from "./store.js"; import type { WireProtocol } from "./util.js"; @@ -241,6 +241,8 @@ export function planForkAdoption( /** Seed a fresh fork session from the plan. Copy-on-fork: every value is a * clone; the parent session is never touched. */ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, parent: Session): void { + // Filtered adoption does not copy the parent's historical ref rows. + if (hasRefLedgerConflict(parent)) session.metadata.refLedgerConflict = true; for (const b of plan.blocks) { session.state.blocks.push(b); const content = parent.blockContents.get(b.blockId); diff --git a/src/persist.ts b/src/persist.ts index 2f0e15ff67..af415c909f 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -30,10 +30,9 @@ import { VERSION } from "./version.js"; import { createStorageCodec, parseEncryptionKey } from "./encrypt.js"; import { PersistEpermAlert } from "./persist-eperm.js"; import { createInitialState, defaultCountTokens, prune, type CompressionState, type CoreMessage, type MessageContentStore } from "acp-kernel"; -import { markDirty } from "./session.js"; +import { hasRefLedgerConflict, markDirty } from "./session.js"; import { validCostUsd } from "./billing.js"; import { mergeState } from "./state-hydration.js"; -import { restampRefsToLedger, restampContentStore } from "acp-kernel"; import { LazyBlockContents, isDroppedContentStore, isDroppedMetadata, lazyContentStore, lazyMetadata, peekSession, sessionWeight } from "./session.js"; import type { Session, BlockContent, BlockView } from "./session.js"; import { orphanContents, resolveStorageMode, UnifiedShadow, hadIntegrityQuickCheckFailure, type StorageMode } from "./storage/shadow.js"; @@ -41,7 +40,6 @@ import { noteColdLoad } from "./attach-heartbeat.js"; import { resolveBackendName } from "./storage/backend.js"; import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js"; import { sha256Hex, type CheckpointRow, type MessageRefContentRow } from "./storage/store.js"; -import { contentStoreOf } from "./store.js"; import type { WireProtocol } from "./util.js"; import { currentContextObservation, CALIBRATION_CLAMP_MAX, CALIBRATION_CLAMP_MIN, CALIBRATION_SAMPLE_MAX, CALIBRATION_SAMPLE_MIN, CALIBRATION_SAMPLE_WINDOW } from "./cache-ledger.js"; @@ -1202,9 +1200,15 @@ export class SessionStore { this.shadow.importRecord(payload); } const view = this.shadow.viewFor(id); - if (view) return buildSession(viewToRecord(view)); + if (view) { + const session = buildSession(viewToRecord(view)); + this.markRefLedgerConflicts(session); + return session; + } } - return buildSession(payload); + const session = buildSession(payload); + this.markRefLedgerConflicts(session); + return session; } /** #708/#1080 review: boot NEVER rewrites session file CONTENT. The first @@ -1639,6 +1643,7 @@ export class SessionStore { } } const drifted = buildSession(driftedEnvelope.payload); + this.markRefLedgerConflicts(drifted); // #4323: "drifted" only proves the two copies DIVERGED — // not which side is ahead. With multiple writers sharing a // sessions directory (the 2026-10-22 incident) the INDEX @@ -1718,6 +1723,7 @@ export class SessionStore { // Legacy fallback — same single-build semantics as plain legacy // mode (see the note below on why no adapter round-trip here). const session = buildSession(envelope.payload); + this.markRefLedgerConflicts(session); if (hasNegativePersistedTokens(envelope.payload)) { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); @@ -1735,6 +1741,7 @@ export class SessionStore { // normalization round-trip and reshape persisted-state bytes (the // storage-render goldens catch exactly that). const session = buildSession(envelope.payload); + this.markRefLedgerConflicts(session); if (hasNegativePersistedTokens(envelope.payload)) { // #408: sync context — debounce the stale-file rewrite // (buildSession already clamped the in-memory value). @@ -1776,93 +1783,36 @@ export class SessionStore { private afterLoad(id: string, view: LegacySessionLike): Session { const record = viewToRecord(view); const session = buildSession(record); + this.markRefLedgerConflicts(session); // #4788 D2: a cold load IS a disk proof of this live revision. this.noteDurable(id, session.revisionEpoch); if (hasNegativePersistedTokens(record)) { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); } - // #4316 layer 3 PR3/3: align the citation space with the unified - // ledger exactly once per state, at the ONLY safe moment — cold - // load. Mid-conversation alignment would shift refs the client has - // already seen (byte-stability contract, snapshot-render D-suite); - // a restart boundary has no in-flight compress call, so nothing can - // observe a half-renumbered map. Post-load arrivals keep minting - // counter refs (fresh numbers, no collision). Envelope-only loads - // (no rows) skip honestly. - this.maybeRestampToLedger(session); + // Issued refs are sealed at cold load. Renumbering them here moved a + // number from one raw message to another while old process-lifetime + // citations still resolved live-first; a stale compress then folded + // the wrong raw. New arrivals are numbered above the historical floor + // by the kernel instead. this.logColdLoad(id, "unified store", session, record.savedAt); return session; } - /** #4316 PR3/3: one-time ref/ledger alignment at cold load. Guards: - * state already aligned, no ref map, unified store unavailable, or no - * rows for this session → no-op. Multi-generation ids (edits) are - * excluded by seqByRaw itself — their legacy refs stay put. */ - private maybeRestampToLedger(session: Session): void { - const state = session.state as (CompressionState & { refLedgerAligned?: boolean }); - if (state.refLedgerAligned === true) return; - if (!state.messageRefs || Object.keys(state.messageRefs.byRaw).length === 0) return; - try { - const seqByRawId = this.unifiedSeqByRaw(session.id); - if (seqByRawId === null || seqByRawId.size === 0) return; - const outcome = restampRefsToLedger(state, seqByRawId); - // #4733: the ref restamp alone leaves the CCR content store keyed by - // store-time refs — a stored original would sit under a number the - // ref map now vends to a DIFFERENT message (misattribution on - // retrieve) and consistency gates (plugin fork snapshot) reject the - // session. Re-key the store onto the same map, same pass. - // #4599: guarded — contentStoreOf MATERIALIZES an empty envelope - // for storeless sessions; touching it speculatively here changed - // the persisted shape of every full-mode cold load (legacy keeps - // undefined) and broke the storage-unified parity pin. Storeless - // sessions have nothing to re-key. - if (session.contentStore) { - const store = contentStoreOf(session); - const restampedStore = restampContentStore(store, state.messageRefs.byRaw); - if (restampedStore !== store) session.contentStore = restampedStore; - } - if (outcome.superseded.size > 0) { - session.restampSuperseded = outcome.superseded; - } else { - // #4797: a converged pass (everything already aligned) must - // CLEAR any stale map — the guard keeps consulting the - // process-lifetime map, and an empty outcome that leaves an - // old map in place keeps refusing live refs forever. - delete session.restampSuperseded; - } - // The token snapshot is keyed by ref (#3488 render-refs): renumbered - // refs would otherwise inherit the stale count of whichever message - // held the new number pre-alignment — the wire tag then disagrees - // with the legacy-restart leg beyond citation renumbering (storage - // engine-proofs wire-parity, #4599). Two-pass moves: stage all - // (oldRef → newRef → count) first, then delete, then write — a - // single-pass swap would lose entries whose newRef is another - // move's oldRef. - const snap = (state as { tokenSnapshot?: Record }).tokenSnapshot; - if (snap && outcome.superseded.size > 0) { - const byRawNow = state.messageRefs.byRaw; - const moves: Array<[oldRef: string, newRef: string, count: number]> = []; - for (const [oldRef, rawId] of outcome.superseded) { - const count = snap[oldRef]; - const newRef = byRawNow[rawId]; - if (count === undefined || newRef === undefined || newRef === oldRef) continue; - moves.push([oldRef, newRef, count]); - } - if (moves.length > 0) { - for (const [oldRef] of moves) delete snap[oldRef]; - for (const [, newRef, count] of moves) snap[newRef] = count; - this.log("info", `[persist] remapped ${moves.length} token-snapshot entries onto ledger-aligned refs for ${session.id} (#4599)`); - } - } - state.refLedgerAligned = true; - this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`); - this.scheduleSave(session); - } catch (err) { - // Alignment is an optimization of the citation space, never a - // load blocker — a failed stamp leaves the legacy refs intact. - this.log("warn", `[persist] ref/ledger alignment skipped for ${session.id}: ${err instanceof Error ? err.message : String(err)}`); - } + /** Detects retained old-vs-current ref collisions before this process can + * resolve a ref into a mutation. Read-only: it neither renumbers nor + * deletes anything. An unavailable query cannot clear an earlier + * conflict; missing historical rows cannot establish past uniqueness. */ + private markRefLedgerConflicts(session: Session): void { + const collisions = this.shadow?.refHistoryCollisions(session.id) ?? null; + if (collisions === null || collisions.length === 0) return; + session.refLedgerConflict = collisions; + session.metadata.refLedgerConflict = true; + const sample = collisions.slice(0, 5).map((c) => c.ref).join(", "); + this.log( + "warn", + `[persist] ${session.id}: ${collisions.length} retained message ref(s) collide with the current refs ledger (${sample}); ref-resolving mutations are blocked for this session`, + ); } /** Unified-first view load (full mode). Null on miss. */ @@ -2020,19 +1970,6 @@ export class SessionStore { return this.enabled && this.shadow?.readsUnified === true; } - /** #4316 PR3/3: raw id → canonical seq map for the one-time ledger - * re-stamp (single-row ids only — multi-generation ids are excluded by - * the store). Null when the unified store is unavailable; the re-stamp - * then degrades to legacy refs (honest skip, never a guess). */ - unifiedSeqByRaw(id: string): Map | null { - if (!this.shadow || this.shadow.stats.disabled) return null; - try { - return this.shadow.seqByRaw(id); - } catch { - return null; - } - } - /** #4730/#4911: highest live (non-retired) seq — the fork endpoint's * inclusive throughSeq, awaited from the (async) fork lane. NOT count - 1 * from unifiedLiveRowCount: tombstones keep their seq, so a retired @@ -2618,6 +2555,10 @@ export class SessionStore { const envelope = this.loadEnvelope(incoming.id, incoming.meta); if (!envelope) return null; const disk = envelope.payload; + // Quarantine is monotonic even when the incoming state wins arbitration. + if (disk.metadata?.refLedgerConflict === true) { + (incoming.metadata ??= {}).refLedgerConflict = true; + } const diskRequests = disk.stats?.requests ?? disk.requests ?? 0; const incRequests = incoming.stats?.requests ?? incoming.requests ?? 0; if (incRequests < diskRequests) return disk; @@ -2735,8 +2676,18 @@ export class SessionStore { // the JSON envelope, and the stamp must say legacy for that copy. const stamp = this.liveStorageStamp; return () => { + // The unified record can be newer than its envelope, including + // index-only writes. Counter arbitration cannot clear this fact. + if (hasRefLedgerConflict(session) || this.shadow?.getSessionMetadata(session.id)?.refLedgerConflict === true) { + session.metadata.refLedgerConflict = true; + } const record = this.buildRecordForSave(session, stamp); const disk = this.staleDiskPayload(record); + if (record.metadata?.refLedgerConflict === true || disk?.metadata?.refLedgerConflict === true) { + session.metadata.refLedgerConflict = true; + (record.metadata ??= {}).refLedgerConflict = true; + if (disk) (disk.metadata ??= {}).refLedgerConflict = true; + } if (disk === null) { // #3301: external-converter import payload rides exactly ONE // save into the unified index (first-open imports take the @@ -2783,6 +2734,7 @@ export class SessionStore { // arbitration the write side just used. this.convergeToDisk(session, disk); this.shadow?.record(disk); + this.markRefLedgerConflicts(session); // Rewrite the disk's own payload: content-identical no-op that // preserves the newer state while satisfying the write chain. this.noteOwnCommit(session.id, disk); @@ -3441,8 +3393,10 @@ export class SessionStore { const cloned = await this.shadow?.createForkAsync(parentId, toLegacyLike(session)) ?? false; if (!cloned) { if (!this.flushSync(session)) throw new Error("fork persistence failed"); + this.markRefLedgerConflicts(session); return; } + this.markRefLedgerConflicts(session); await this.maybeCutForkBaseAsync(session); // A failure here leaves a COMPLETE durable fork receipt in the DB. // The exact retry repairs its envelope; never delete an uncertain commit. diff --git a/src/plugin.ts b/src/plugin.ts index 5b6578e7c5..d9a81a9de8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { createHash } from "node:crypto"; import { getStore, storageStatusLabel } from "./persist.js"; import { cloneStoreForRefs } from "./store.js"; -import { acquireInFlight, createSession, getSessionAsync, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, listSessions, markCompactionBoundary, markDirty, noteAbortedTurn, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; +import { acquireInFlight, createSession, getSessionAsync, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, hasRefLedgerConflict, listSessions, markCompactionBoundary, markDirty, noteAbortedTurn, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; import { clientConversationHeader } from "./session-id.js"; import { buildContextView } from "./context-view.js"; import { ABSORB_TOOL_NAME, BILI_ACP_TOOLS_ANTHROPIC, BILI_ACP_TOOLS_ANTHROPIC_NO_RANGE, BILI_ACP_TOOLS_OPENAI, BILI_ACP_TOOLS_OPENAI_NO_RANGE, BILI_ACP_TOOLS_RESPONSES_NO_RANGE, PROXY_TOOL_NAMES, READ_ORIGINAL_IMAGE_TOOL, READ_ORIGINAL_IMAGE_TOOL_NAME, READ_ORIGINAL_IMAGE_TOOL_OPENAI, READ_ORIGINAL_IMAGE_TOOL_RESPONSES, RETRIEVE_TOOL_NAME, RULE_TOOL, RULE_TOOL_NAME, RULE_TOOL_OPENAI, RULE_TOOL_RESPONSES, SEARCH_CONTEXT_TOOL_NAME, absorbToolsFor, retrieveToolsFor, withAcpNotes, withAcpNotesOpenAI } from "./compress-tool.js"; @@ -1619,6 +1619,7 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr if (entry && (entry.rawId !== rawId || typeof parentStore.byHash[entry.hash] !== "string")) return forkReply(res, 409, { ok: false, status: "unavailable", code: "PARENT_STATE_INCOMPLETE", error: "CCR original unavailable or alias inconsistent" }); } child.contentStore = cloneStoreForRefs(parentStore, new Set(prefix.filter((m) => m.ref !== null).map((m) => m.ref!))) ?? undefined; + if (hasRefLedgerConflict(parent)) child.metadata.refLedgerConflict = true; child.metadata.publicSnapshotTextComparable = true; child.metadata.publicSnapshotStoredRefs = Object.keys(child.contentStore?.byRef ?? {}); for (const key of ["pluginAgent", "lastModel", "effectiveConfig", "effectiveConfigEpoch", "effectiveCcr", "effectiveContextLimit", "reservedContextLimit", "lastWindowSource", "systemPromptTokens"]) if (parent.metadata[key] !== undefined) child.metadata[key] = structuredClone(parent.metadata[key]); @@ -2281,10 +2282,12 @@ export async function handlePluginTool( // (sticky) — the next model request stops injecting the duplicate ephemeral wire // tools. Guarded: only flips a session with NO existing agent binding, never // overriding a pi/omp/opencode plugin or a launcher-registered agent. - if (typeof session.metadata.pluginAgent !== "string" && tool !== retrieveToolName(session)) { - session.metadata.pluginAgent = "mcp"; + if (result.reason !== "ref-ledger-conflict") { + if (typeof session.metadata.pluginAgent !== "string" && tool !== retrieveToolName(session)) { + session.metadata.pluginAgent = "mcp"; + } + markDirty(session); } - markDirty(session); deps.log("info", `[${session.id}] [plugin] tool ${tool} executed via plugin (routed by ${routedBy}, #1685) (${result.text.length} chars, outcome=${result.outcome ?? "n/a"}${result.reason ? `, reason=${result.reason}` : ""})`); // Same deep link on the /acp-cache display surfaces: clients wrap this text in // [acp-cache]/[/acp-cache] markers and strip it from model context by marker diff --git a/src/session.ts b/src/session.ts index dc5f83ac88..d57de18751 100644 --- a/src/session.ts +++ b/src/session.ts @@ -303,6 +303,19 @@ export type PendingRetrieval = { ccr?: boolean; }; +/** A persisted message row was issued `ref` at some point, but the current + * refs ledger binds that same number to a different raw message. This is the + * residue of the retired cold-load re-stamp: old `messages.ref` values and + * the current `refs` table disagree. Any compress that resolves refs would + * be ambiguous, so the session-level guard refuses ref-resolving mutations + * while keeping raw rows, blocks, and the ledger intact. */ +export interface RefHistoryCollision { + ref: string; + refNum: number; + oldRawId: string; + currentRawId: string; +} + export type Session = { /** 派生缓存随会话释放,供驻留字节计量遍历;不进入持久化投影或 wire。 */ transient?: { @@ -621,6 +634,12 @@ export type Session = { * the message that now owns the number). A fresh process that never * re-stamped has no map and every ref behaves normally. */ restampSuperseded?: Map; + /** Detectable old-vs-current ref collisions found at cold load. Diagnostic + * rows stay transient; metadata.refLedgerConflict preserves known + * ambiguity when adoption cannot carry the parent's historical rows. + * Detection cannot be universal: pruned, null, BLOCKED, or orphaned + * historical rows may leave no trace. */ + refLedgerConflict?: RefHistoryCollision[]; createdAt: number; lastSeen: number; /** Original content of compressed blocks, captured at compress time when @@ -745,6 +764,10 @@ type AnonymousPrefixAffinityStamp = { producer?: AnonymousProvenance; }; +export function hasRefLedgerConflict(session: Session): boolean { + return session.metadata.refLedgerConflict === true || (session.refLedgerConflict?.length ?? 0) > 0; +} + // #833: wire paths resolve the kernel Config per request (global → provider → // model compress settings + self-heal + output headroom), while the plugin // status/tool API reads sessions with no request context and was falling back @@ -1376,8 +1399,20 @@ export async function getSessionAsync(id: string, meta?: SessionMeta): Promise adoptOrCreate(id, meta, await store.loadAsync(id, meta)))() - .finally(() => { pendingSessionLoads.delete(id); }); + pending = (async () => { + const reloaded = await store.loadAsync(id, meta); + if (publishingForks.has(id)) throw new Error("child fork publication in progress"); + const winner = sessions.get(id); + if (winner) { + stampExisting(winner, meta); + return winner; + } + // A published fork may already be evicted or committed despite a persistence error. + if (pendingSessionLoads.get(id) !== pending) throw new Error("session load superseded by fork publication"); + return adoptOrCreate(id, meta, reloaded); + })().finally(() => { + if (pendingSessionLoads.get(id) === pending) pendingSessionLoads.delete(id); + }); pendingSessionLoads.set(id, pending); } return pending; @@ -1407,9 +1442,10 @@ const publishingForks = new Set(); // backends, or a fork could silently clobber a DB-only child row. export async function publishForkSession(session: Session, parentId: string): Promise { if (publishingForks.has(session.id)) throw new Error("child fork publication in progress"); - if (sessions.has(session.id) || (await getStore().loadAsync(session.id))) throw new Error("child session already exists"); publishingForks.add(session.id); try { + pendingSessionLoads.delete(session.id); + if (sessions.has(session.id) || (await getStore().loadAsync(session.id))) throw new Error("child session already exists"); makeResidentRoom(sessionWeight(session)); await getStore().persistForkSession(parentId, session); session.persisted = getStore().enabled; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index fddaaf272e..dc0f22b034 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -26,7 +26,7 @@ import { log as loggerLog } from "../logger.js"; import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; -import type { BlockView } from "../session.js"; +import type { BlockView, RefHistoryCollision } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import path from "node:path"; @@ -979,6 +979,20 @@ export class UnifiedShadow { return this.sync.seqByRaw(sessionId); } + /** Retained ref-history collisions (see UnifiedStore.findRefHistoryCollisions). + * Full-trust only: a stale index could invent or miss a conflict, so a + * degraded/shadow read returns null and the caller must not treat that as + * a clean bill of health. */ + refHistoryCollisions(sessionId: string): RefHistoryCollision[] | null { + if (this.sync === null) return null; + if (this.stats.disabled) return null; + try { + return this.sync.findRefHistoryCollisions(sessionId); + } catch { + return null; + } + } + /** #4316 PR3/3: live (non-retired) message row count. */ liveRowCount(sessionId: string): number { if (this.sync !== null) return this.sync.liveRowCount(sessionId); diff --git a/src/storage/store.ts b/src/storage/store.ts index ba6aed922f..ff80843b10 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -28,7 +28,7 @@ import { integrityModeFor, writeIntegrityCache, scheduleBackgroundIntegrityCheck import { sqliteCorruptionDetail } from "./sqlite-errors.js"; import { listInstances, type RegistryEntry } from "../instance.js"; import type { MessageContentStore } from "acp-kernel"; -import type { BlockView } from "../session.js"; +import type { BlockView, RefHistoryCollision } from "../session.js"; /** #3250: run a PASSIVE wal_checkpoint every N successful ingests. The * built-in autocheckpoint alone cannot guarantee reclamation in a long-lived @@ -3185,6 +3185,34 @@ export class UnifiedStore { })); } + /** Historical message rows whose issued ref now belongs to another raw + * message in the current refs ledger. `messages.ref` is first-seen + * (INSERT OR IGNORE), so it preserves the number a row was originally + * issued; comparing it with the live ledger exposes a retired re-stamp + * without hydrating message content. Retired rows are included because + * they are historical evidence; a non-empty result means ref resolution + * is ambiguous and destructive ref work must fail closed. `limit` bounds + * the read; one hit is enough for the guard, and the sample drives the + * operator-facing message. */ + findRefHistoryCollisions(sessionId: string, limit = 100): RefHistoryCollision[] { + const rows = this.db + .prepare( + `SELECT DISTINCT m.ref AS ref, m.ref_num AS ref_num, m.msg_id AS old_raw_id, r.raw_id AS current_raw_id + FROM messages m + JOIN refs r ON r.session_id = m.session_id AND r.ref_num = m.ref_num + WHERE m.session_id = ? AND m.ref_num IS NOT NULL AND m.msg_id <> r.raw_id + ORDER BY m.ref_num, old_raw_id, current_raw_id + LIMIT ?`, + ) + .all(sessionId, limit) as Record[]; + return rows.map((r) => ({ + ref: r.ref as string, + refNum: r.ref_num as number, + oldRawId: r.old_raw_id as string, + currentRawId: r.current_raw_id as string, + })); + } + getCcr(sessionId: string): MessageContentStore | null { const row = this.db .prepare("SELECT ccr_json FROM sessions WHERE session_id = ?") diff --git a/src/stream.ts b/src/stream.ts index 2979f8401d..92be2bc66b 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -3,7 +3,7 @@ import { collectBlockContent, countMessageTokens, defaultCountTokens, formatRang import { loadConfigFile, resolveMessageTimestamps } from "./config.js"; import { excludeDeadRefRanges, handleAcpStatus, liveCompressionRanges } from "./acp-status.js"; import { handleAcpCache, prefixTokensBeforeRef, recordCacheFoldsFromBlocks } from "./cache-ledger.js"; -import { type Session, cacheBlockContent, markDirty, residentPressure, scrubTurnSeparatorIds, statusInputBaseline, withoutTurnSeparators } from "./session.js"; +import { type Session, hasRefLedgerConflict, cacheBlockContent, markDirty, residentPressure, scrubTurnSeparatorIds, statusInputBaseline, withoutTurnSeparators } from "./session.js"; import { COMPRESS_TOOL_NAME, parseCompressInput, ABSORB_TOOL_NAME, type ParsedRange } from "./compress-tool.js"; import { effectiveAbsorbConfig, executeAbsorb, isProxyToolFor } from "./absorb.js"; import { executeSearchContextTarget, resolveDecompress } from "./decompress-shared.js"; @@ -947,11 +947,32 @@ export function supersededRefRejection c.ref))); + const listed = refs.length > 0 ? ` for ref(s) ${refs.slice(0, 5).join(", ")}` : " inherited from an ambiguous ref ledger"; + const more = refs.length > 5 ? ` (+${refs.length - 5} more)` : ""; + ctx.log( + `[acp-proxy: compress REFUSED for ${ctx.session.id} — known retained or inherited ref-history ambiguity; raw messages, blocks, and the ref ledger preserved]`, + ); + return compressResult( + `[Compression REFUSED: this session's retained ref history is ambiguous${listed}${more}. Retained or inherited references have conflicting raw-message owners. Compressing could fold the wrong messages, so nothing was changed. Raw messages, blocks, and the ref ledger are preserved. Read-only status, decompress, and export still work; no refs were translated, renumbered, or deleted automatically. Re-read acp_status, or export and continue in a new session if the labels remain inconsistent.]`, + "refused", + 0, + "ref-ledger-conflict", + ); + } + return undefined; +} + export function applyRanges(parsed: ReturnType, ctx: RewriteCtx, opts?: { loopTracking?: boolean }): ProxyToolResult { // #2146: preflight overflow-compress is bili's OWN internal lane — its // failures have separate reporting (noteSkip) and must not arm the // model-facing circuit breaker on attempts the model never made. const trackLoop = opts?.loopTracking !== false; + const conflict = refLedgerConflictRefusal(ctx); + if (conflict) return conflict; let { ranges, diagnostics } = parsed; if (ranges.length === 0) { ctx.log("[acp-proxy: compress call had no valid ranges; nothing compressed.]"); diff --git a/tests/absorb.test.ts b/tests/absorb.test.ts index 5ff7be1c0d..9f1eba2948 100644 --- a/tests/absorb.test.ts +++ b/tests/absorb.test.ts @@ -271,3 +271,22 @@ test("persist round-trip: absorbed records and absorbedTokens survive save/load" assert.deepEqual(loaded!.state.absorbed, [record], "absorbed records must survive restart"); assert.equal(loaded!.state.stats.absorbedTokens, 4321); }); + + +test("executeAbsorb: retained ref-history conflicts refuse before any mutation", () => { + const { turn, config, refFor } = makeTurn(); + const ref = refFor("t-res"); + assert.ok(ref); + const session = makeSession(); + session.state = turn.state; + session.refLedgerConflict = [{ ref, refNum: Number(ref.slice(1)), oldRawId: "previous-owner", currentRawId: "t-res" }]; + const stateBefore = structuredClone(session.state); + const statsBefore = structuredClone(session.stats); + const result = executeAbsorb({ ref, summary: "must not absorb a reused ref" }, "conflict-call", config.absorb!, { + config, messages: turn.messages, session, + }); + assert.match(result.text, /FAILED/); + assert.match(result.text, /ambiguous/); + assert.deepEqual(session.state, stateBefore, "ambiguous ref must not alter absorptions or refs"); + assert.deepEqual(session.stats, statsBefore, "ambiguous ref must not credit tokens"); +}); diff --git a/tests/cache-theorem.test.ts b/tests/cache-theorem.test.ts index 67980203a2..95458e4066 100644 --- a/tests/cache-theorem.test.ts +++ b/tests/cache-theorem.test.ts @@ -278,7 +278,7 @@ test("[grounding:g1] ref cursor mechanics pinned to assignRefs/allocateFreeRef", assert.equal(res.map.byRaw["q1"], "m00006", "smallest free >= nextIndex"); assert.equal(highestUsedIndex(res.map), 6); } - // pre-existing binding untouched; gap below the cursor is claimable + // pre-existing binding untouched; a stale cursor cannot reopen issued ref space { const existing = emptyRefMap(); existing.byRaw["a"] = "m00005"; @@ -286,8 +286,8 @@ test("[grounding:g1] ref cursor mechanics pinned to assignRefs/allocateFreeRef", const res = assignRefs([msg("a"), msg("q")], { existing, nextIndex: 4 }); assert.equal(res.map.byRaw["a"], "m00005", "existing untouched"); assert.equal(res.newlyAssigned, 1, "existing message consumes no number"); - assert.equal(res.map.byRaw["q"], "m00004", "gap below cursor is the first free slot"); - assert.equal(res.nextIndex, 5); + assert.equal(res.map.byRaw["q"], "m00006", "fresh ref stays above every issued number despite a stale cursor"); + assert.equal(res.nextIndex, 7); } // BLOCKED consumes no number { diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 41df7f2065..8be1e88bc5 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -85,17 +85,19 @@ export function viewStateDigest(view: ReturnType): string }))); } -/** #4316 layer 3 PR3/3 (380ef4dac): a unified full-mode cold load performs a - * ONE-TIME ledger alignment — citation numbers are re-homed onto ledger - * sequence (kernel restampRefsToLedger) and state.refLedgerAligned is stamped. - * Cross-mode / cross-read-path "identical state" proofs must therefore compare - * the state modulo that one-time renumbering: this strips the alignment stamp - * and re-homes every ref number to a canonical order (k-th smallest raw id -> - * m0000k), so two views differing ONLY in which numbering scheme produced - * their refs hash identically under viewDigest/viewStateDigest. Object key - * order is normalized too: different hydration paths build identical content - * under different insertion orders, and canonicalize preserves them. Pure - * function — returns a new object, input untouched. */ +/** #4316 layer 3 PR3/3 (380ef4dac): the historical unified full-mode cold load + * performed a ONE-TIME ledger alignment — citation numbers were re-homed onto + * ledger sequence (kernel restampRefsToLedger) and state.refLedgerAligned was + * stamped. Current cold loads seal refs and do not apply that rewrite, but + * committed goldens and older fixtures still carry its numbering. Cross-mode / + * cross-read-path "identical state" proofs therefore compare state modulo the + * historical renumbering: this strips any alignment stamp and re-homes every + * ref number to a canonical order (k-th smallest raw id -> m0000k), so two + * views differing ONLY in which numbering scheme produced their refs hash + * identically under viewDigest/viewStateDigest. Object key order is normalized + * too: different hydration paths build identical content under different + * insertion orders, and canonicalize preserves them. Pure function — returns a + * new object, input untouched. */ export function stripOneTimeAlignment(view: T): T { const v = view as { state?: unknown; blockContents?: unknown }; if (!v.state || typeof v.state !== "object") return view; diff --git a/tests/external-summary-runtime.test.ts b/tests/external-summary-runtime.test.ts index fee423ea6d..e11ecf12b9 100644 --- a/tests/external-summary-runtime.test.ts +++ b/tests/external-summary-runtime.test.ts @@ -204,6 +204,26 @@ test("missing primary key skips its network call and tries the configured backup }, (req, res) => { paths.push(req.url ?? ""); success(res); }); }); +test("known ref ambiguity refuses configured summaries before work or no-view bookkeeping", async () => { + let calls = 0; + await fixture(async (_base, externalSummary) => { + for (const emptyView of [false, true]) { + const ctx = context(externalSummary); + const input = args(ctx, SUMMARY); + ctx.session.refLedgerConflict = [{ ref: ctx.session.state.messageRefs.byRaw.history, refNum: 1, oldRawId: "old-history", currentRawId: "history" }]; + if (emptyView) ctx.messages = []; + const before = JSON.stringify({ state: ctx.session.state, stats: ctx.session.stats, metadata: ctx.session.metadata, + revision: ctx.session.revisionEpoch }); + const result = await executeProxyToolAsync("compress", input, ctx); + assert.equal(result.reason, "ref-ledger-conflict"); + assert.equal(result.outcome, "refused"); + assert.equal(calls, 0, "ambiguous source must not reach the configured summary server"); + assert.equal(JSON.stringify({ state: ctx.session.state, stats: ctx.session.stats, metadata: ctx.session.metadata, + revision: ctx.session.revisionEpoch }), before); + } + }, (_req, res) => { calls++; success(res); }); +}); + test("protected ranges cause no paid call and no fold", async () => { let calls = 0; await fixture(async (_base, externalSummary) => { diff --git a/tests/fork-adoption-plan.test.ts b/tests/fork-adoption-plan.test.ts index b7143ad88d..cc0f1f6ee8 100644 --- a/tests/fork-adoption-plan.test.ts +++ b/tests/fork-adoption-plan.test.ts @@ -62,6 +62,7 @@ test("planForkAdoption default (no seedAllRefs) seeds only block coverage — do test("resume adoption never writes the kernel BLOCKED sentinel into byRef (#2620)", () => { _setStoreForTest(new SessionStore({ enabled: false })); const parent = { + metadata: {}, state: { blocks: [] as unknown[], tokenSnapshot: {} as Record, @@ -71,6 +72,7 @@ test("resume adoption never writes the kernel BLOCKED sentinel into byRef (#2620 }, } as unknown as Session; const child = { + metadata: {}, state: { blocks: [] as unknown[], tokenSnapshot: {} as Record, diff --git a/tests/issue1875-plugin-tool-outcome.test.ts b/tests/issue1875-plugin-tool-outcome.test.ts index aa02621896..98642f6bad 100644 --- a/tests/issue1875-plugin-tool-outcome.test.ts +++ b/tests/issue1875-plugin-tool-outcome.test.ts @@ -16,6 +16,7 @@ import { afterEach, beforeEach, describe, it } from "node:test"; import { createCore, defaultConfig, type Config, type CoreMessage } from "acp-kernel"; import { type PluginToolDeps, _resetPluginStateForTest, handlePluginTool } from "../src/plugin.ts"; import { getSession } from "../src/session.ts"; +import { getStore } from "../src/persist.ts"; import { applyRanges } from "../src/stream.ts"; import { parseCompressInput } from "../src/compress-tool.ts"; import { forwardTool } from "../src/mcp.ts"; @@ -55,6 +56,23 @@ describe("#1875: plugin tool endpoint reports business outcome, not just ok:true assert.ok(String(r.json.result).startsWith("[Compression FAILED"), `receipt: ${r.json.result}`); }); + it("known ambiguity refusal leaves plugin ownership, revision and storage untouched", async () => { + const session = getSession(sessionId); + session.refLedgerConflict = [{ ref: "m00001", refNum: 1, oldRawId: "old", currentRawId: "current" }]; + const before = JSON.stringify({ metadata: session.metadata, revision: session.revisionEpoch, state: session.state, stats: session.stats }); + const store = getStore(); + const scheduleSave = store.scheduleSave; + let saves = 0; + store.scheduleSave = function (value) { saves++; return scheduleSave.call(this, value); }; + try { + const result = await call("compress", { content: [{ startId: "m00001", endId: "m00001", summary: "Historical source." }] }); + assert.equal(result.status, 200); + assert.equal(result.json.outcome, "refused"); + assert.equal(JSON.stringify({ metadata: session.metadata, revision: session.revisionEpoch, state: session.state, stats: session.stats }), before); + assert.equal(saves, 0); + } finally { store.scheduleSave = scheduleSave; } + }); + it("decompress of unknown block → 200 ok:true + outcome:failure, no blocksCreated", async () => { const r = await call("decompress", { blockId: "b999" }); assert.equal(r.status, 200); diff --git a/tests/issue4986-digest-drift-ref-order.test.ts b/tests/issue4986-digest-drift-ref-order.test.ts index d427f387b1..239f5dfbfc 100644 --- a/tests/issue4986-digest-drift-ref-order.test.ts +++ b/tests/issue4986-digest-drift-ref-order.test.ts @@ -8,11 +8,10 @@ import { SessionStore } from "../src/persist.ts"; /** * #4986: the sampled ingest read-back verify must not report DIGEST DRIFT * when the in-memory messageRefs key order diverges from the stored - * refs-row order. The golden corpora are exactly that shape: byRef sits in - * ref-number order while byRaw — which finishPlan freezes into row ord at - * full ingest — sits in a different key order, so the row-rebuilt leg's - * byRef starts at m00041 while the live leg's starts at m00001. Identical - * content, different key insertion order. Pre-fix, every such save logged + * refs-row order. The fixture reorders byRef without changing any binding; + * byRaw — which finishPlan freezes into row ord at full ingest — keeps its + * original order. Identical content, different key insertion order. + * Pre-fix, every such save logged * an error-level DIGEST DRIFT and dropped the ingest watermark (full-tail * re-ingest next save); post-fix the parity digest compares contents, not * key insertion order (#3407/#3113 philosophy). @@ -53,10 +52,11 @@ test("ingest verify ignores messageRefs key-order divergence (#4986)", async () try { const session = loadCorpusCase(seed, "folded", "corpus-folded", "anthropic"); - // Precondition: the corpus exhibits the divergent shape — the - // row order (byRaw key order, what finishPlan stores as ord) - // projects byRef to a DIFFERENT key sequence than the live map's - // own key order. Without this the test would pass vacuously. + const refsBefore = structuredClone(session.state.messageRefs); + session.state.messageRefs.byRef = Object.fromEntries(Object.entries(refsBefore.byRef).reverse()); + assert.deepEqual(session.state.messageRefs, refsBefore, "order-only fixture preserves every raw/ref binding"); + + // Without different key orders this regression would pass vacuously. const byRaw = session.state.messageRefs.byRaw; const byRef = session.state.messageRefs.byRef; const rowOrderProjectedRefs = Object.keys(byRaw) diff --git a/tests/issue5103-startup-recovery-visibility.test.ts b/tests/issue5103-startup-recovery-visibility.test.ts index 3fce3ce791..ac620ae12d 100644 --- a/tests/issue5103-startup-recovery-visibility.test.ts +++ b/tests/issue5103-startup-recovery-visibility.test.ts @@ -188,6 +188,7 @@ function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]): test("#5103: first ingest failure warns immediately with the db path; failures NEVER self-disable", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5103-shadow-")); + const dbPath = path.join(dir, "index.db").replaceAll("\\", "/"); try { const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); assert.ok(shadow, "shadow mode sidecar must open"); @@ -208,7 +209,7 @@ test("#5103: first ingest failure warns immediately with the db path; failures N shadow.record(mkLegacy("s1", [msg(1), msg(2)])); const firstWarn = captured.find((c) => c.level === "warn" && c.msg.includes("failed to ingest")); assert.ok(firstWarn, "the FIRST failure must warn (#5103) — no silent strikes any more"); - assert.ok(firstWarn!.msg.includes(path.join(dir, "index.db")), "warning must name the database file"); + assert.ok(firstWarn!.msg.replaceAll("\\", "/").includes(dbPath), "warning must name the database file"); if (process.platform === "win32") { assert.ok(firstWarn!.msg.includes("(win32:"), "win32 runner carries the OS-level hint"); } diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index 88720715af..1940384478 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -63,7 +63,7 @@ interface StatusResponse { type PluginResponse = Path extends "/__bili/plugin/manifest" ? { tools: { openai: { type: "function"; function: { name: string; parameters: Record } }[] }; capabilities: { fork: { protocolVersion: number; endpoint: string; snapshotEndpoint: string } } } : Path extends "/__bili/plugin/fork" ? ForkResponse - : Path extends "/__bili/plugin/tool" ? { result: string; code?: string; conversationId?: string } + : Path extends "/__bili/plugin/tool" ? { result: string; code?: string; conversationId?: string; outcome?: string } : Path extends `/__bili/plugin/status${string}` ? StatusResponse : SnapshotResponse; @@ -129,6 +129,36 @@ test("plain proxy panel snapshots do not change raw-history persistence", { time } finally { await h.close(); } }); +for (const inheritedConflict of ["marker", "diagnostics"] as const) { + test(`HTTP fork preserves known ref ambiguity through cold restore (${inheritedConflict})`, { timeout: 60_000 }, async () => { + const h = await harness(true); + try { + const parent = resolveConversation("parent").session!; + if (inheritedConflict === "marker") parent.metadata.refLedgerConflict = true; + else parent.refLedgerConflict = [{ ref: "m00001", refNum: 1, oldRawId: "retired-owner", currentRawId: parent.state.messageRefs.byRef.m00001 }]; + const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; + assert.equal((await h.request("/__bili/plugin/fork", forkRequest(snapshot))).status, 201); + for (const cold of [false, true]) { + if (cold) { + assert(h.store.flushSync(resolveConversation("child").session!)); + _resetPluginStateForTest(); + _resetSessionsForTest(); + } + const childSnapshot = await h.request("/__bili/plugin/snapshot?conversationId=child"); + assert.equal(childSnapshot.status, 200); + const child = resolveConversation("child").session!; + const before = JSON.stringify({ state: child.state, stats: child.stats, metadata: child.metadata, revision: child.revisionEpoch }); + const ref = childSnapshot.body.orderedMessages[0]!.ref; + const result = await h.request("/__bili/plugin/tool", { conversationId: "child", tool: "compress", args: { content: [{ startId: ref, endId: ref, summary }] } }); + assert.equal(result.status, 200); + assert.equal(result.body.outcome, "refused"); + assert.equal(child.metadata.refLedgerConflict, true); + assert.equal(JSON.stringify({ state: child.state, stats: child.stats, metadata: child.metadata, revision: child.revisionEpoch }), before); + } + } finally { await h.close(); } + }); +} + test("HTTP fork preserves sparse parent ledger before the child's first save", { timeout: 60_000 }, async () => { const oldMode = process.env.BILI_STORAGE_UNIFIED; process.env.BILI_STORAGE_UNIFIED = "full"; @@ -958,10 +988,10 @@ for (const storageMode of ["legacy", "full"] as const) { parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); parent.contentStoreDirty = true; assert(h.store.flushSync(parent)); - // #4316 PR3/3: only a unified cold load runs the one-time ledger - // alignment; the CI legacy baseline retains the original state. + // A cold load must not re-stamp or renumber issued refs: the state + // it serves is byte-for-byte the state that was flushed, in every + // storage mode (no one-time ledger alignment marker). const persistedState = JSON.parse(JSON.stringify(parent.state)); - if (h.store.replicasAvailable()) persistedState.refLedgerAligned = true; assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); @@ -1535,33 +1565,33 @@ test("HTTP a mid-history rewrite replaces the snapshot wholesale with an exact b }); test("HTTP a cold-restored non-retained parent serves its intact raw tail immediately (#5140)", { timeout: 60_000 }, async () => { - // The reconstruction path reads the unified index, so this test must run - // under unified-full — the global test baseline pins BILI_STORAGE_UNIFIED= - // "legacy" for non-storage suites, under which the sidecar never opens and - // replicasAvailable() is structurally false (a bare sidecar-missing 409). - const oldMode = process.env.BILI_STORAGE_UNIFIED; + const prevMode = process.env.BILI_STORAGE_UNIFIED; process.env.BILI_STORAGE_UNIFIED = "full"; - const h = await harness(true); try { - const parent = resolveConversation("parent").session!; - assert(h.store.flushSync(parent)); - h.store.cancelAll(); - _resetSessionsForTest(); - _resetPluginStateForTest(); - const restored = getSession("parent"); - assert.equal(restored.pluginSnapshot, undefined, "a non-forked session restores without its raw snapshot"); - // #5140: the intact raw tail is reconstructable from the unified index, so the cold - // parent is servable immediately — no model call needed to recover stored source. - // The 409 body names the exact forkSnapshot refusal branch — a bare status assert - // swallowed it and left the CI flake undiagnosable (#4599 sentinel 2026-10-30). - const coldSnap = await h.request("/__bili/plugin/snapshot?conversationId=parent"); - assert.equal(coldSnap.status, 200, JSON.stringify(coldSnap.body)); - await sendModel(h, "parent", h.messages); - assert.equal((await h.request("/__bili/plugin/snapshot?conversationId=parent")).status, 200); - assert.equal(resolveConversation("parent").session!.pluginSnapshot!.length, 3); + const h = await harness(true); + try { + const parent = resolveConversation("parent").session!; + assert(h.store.flushSync(parent)); + h.store.cancelAll(); + _resetSessionsForTest(); + _resetPluginStateForTest(); + const restored = getSession("parent"); + assert.equal(restored.pluginSnapshot, undefined, "a non-forked session restores without its raw snapshot"); + // #5140: the intact raw tail is reconstructable from the unified index, so the cold + // parent is servable immediately — no model call needed to recover stored source. + // The 409 body names the exact forkSnapshot refusal branch — a bare status assert + // swallowed it and left the CI flake undiagnosable (#4599 sentinel 2026-10-30). + const coldSnap = await h.request("/__bili/plugin/snapshot?conversationId=parent"); + assert.equal(coldSnap.status, 200, JSON.stringify(coldSnap.body)); + assert.deepEqual(coldSnap.body.messages.map((m) => m.text), h.messages.map((m) => m.content), "cold snapshot preserves the exact raw text and order"); + assert.deepEqual(coldSnap.body.orderedMessages.map((m) => m.rawId), restored.state.lastPassIds, "cold snapshot preserves the exact raw message identities and order"); + await sendModel(h, "parent", h.messages); + assert.equal((await h.request("/__bili/plugin/snapshot?conversationId=parent")).status, 200); + assert.equal(resolveConversation("parent").session!.pluginSnapshot!.length, 3); + } finally { await h.close(); } } finally { - await h.close(); - if (oldMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = oldMode; + if (prevMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prevMode; } }); diff --git a/tests/seq-ledger-pr3.test.ts b/tests/seq-ledger-pr3.test.ts index 3b63c44166..f863002d7d 100644 --- a/tests/seq-ledger-pr3.test.ts +++ b/tests/seq-ledger-pr3.test.ts @@ -13,8 +13,8 @@ import { join } from "node:path"; * child appends from the parent's max seq + 1. * S14: a compress call citing a ref superseded by the one-time ledger re-stamp * is refused with renumber guidance instead of folding the wrong range. - * S15: cold load re-stamps legacy counter refs onto seq+1 exactly once and - * records the superseded map on the session. + * S15: cold load leaves issued refs untouched (no re-stamp) and a second + * load sees the exact same refs and no superseded map. */ process.env.NODE_ENV = "test"; process.env.BILI_STORAGE_UNIFIED = "full"; @@ -25,7 +25,7 @@ for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CON import { UnifiedStore } from "../src/storage/store.ts"; import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; import type { CoreMessage } from "acp-kernel"; -import { supersededRefRejection } from "../src/stream.ts"; +import { applyRanges, supersededRefRejection } from "../src/stream.ts"; import type { Session } from "../src/session.ts"; function msg(id: string, role: "user" | "assistant" = "user", text = `content for ${id} ${"x".repeat(120)}`): CoreMessage { @@ -150,11 +150,12 @@ test("S14: superseded-ref compress resolution — old refs translate (or refuse assert.equal(supersededRefRejection([{ startRef: "m00003", endRef: "m00004" }], {} as Session).rejection, null); }); -test("S15: cold load re-stamps legacy counter refs onto seq+1 exactly once", async () => { +test("S15: cold load seals legacy counter refs without re-stamping them", async () => { // A persisted session whose kernel refs came from the counter path (a // BLOCKED protected message skipped a number, so refs drift ahead of - // seq+1). Cold load must align refs = seq+1, record the superseded map, - // and a SECOND cold load must be a no-op (kept only, map preserved). + // seq+1). Cold load must preserve every issued number exactly as stored: + // no re-stamp, no superseded map, and a SECOND cold load must see the + // same refs. const dir = join(testRoot, `s15-${Math.random().toString(36).slice(2)}`); const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); @@ -192,33 +193,29 @@ test("S15: cold load re-stamps legacy counter refs onto seq+1 exactly once", asy _setStoreForTest(store2); const reloaded = store2.loadSync("pr3-restamp"); assert.ok(reloaded, "session loads"); - // Cold load fires the one-time alignment (restart boundary — no - // in-flight compress call can observe the renumbering). + // Cold load must not rewrite the issued numbers. const refs = reloaded.state.messageRefs; assert.equal(refs.byRaw.a, "m00001"); assert.equal(refs.byRaw.b, "m00002"); assert.equal(refs.byRaw.c, "BLOCKED", "protected keeps taking no number"); - assert.equal(refs.byRaw.d, "m00004", "d re-stamped to seq3+1"); - assert.equal(refs.byRaw.e, "m00005", "e re-stamped to seq4+1"); - assert.equal((reloaded.state as { refLedgerAligned?: boolean }).refLedgerAligned, true); - assert.ok(reloaded.restampSuperseded instanceof Map); - assert.equal(reloaded.restampSuperseded!.get("m00003"), "d"); - assert.equal(reloaded.restampSuperseded!.get("m00004"), "e"); - // Persist the aligned state deterministically before closing (the - // restamp scheduled a debounced save; closeQuietly should flush it, - // but the second reload below must not race that decision). - assert.ok(store2.flushSync(reloaded), "aligned state committed"); - - // Second cold load: aligned state is a no-op (kept only), and the - // superseded map rides the Session object for this process only. + assert.equal(refs.byRaw.d, "m00003", "d keeps its issued counter ref"); + assert.equal(refs.byRaw.e, "m00004", "e keeps its issued counter ref"); + assert.equal(refs.byRef.m00003, "d"); + assert.equal(refs.byRef.m00004, "e"); + assert.equal((reloaded.state as { refLedgerAligned?: boolean }).refLedgerAligned, undefined); + assert.equal(reloaded.restampSuperseded, undefined); + assert.ok(store2.flushSync(reloaded), "sealed state committed"); + + // Second cold load: identical refs, still no re-stamp. store2.closeQuietly(); const store3 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); _setStoreForTest(store3); const again = store3.loadSync("pr3-restamp"); assert.ok(again); - assert.equal(again.state.messageRefs.byRaw.d, "m00004", "aligned refs stable across reloads"); - assert.equal(again.state.messageRefs.byRaw.e, "m00005"); - // Fresh process: no superseded map — refs behave normally. + assert.equal(again.state.messageRefs.byRaw.d, "m00003", "refs stable across reloads"); + assert.equal(again.state.messageRefs.byRaw.e, "m00004"); + assert.equal(again.state.messageRefs.byRef.m00003, "d"); + assert.equal(again.state.messageRefs.byRef.m00004, "e"); assert.equal(again.restampSuperseded, undefined); store3.closeQuietly(); } finally { @@ -227,3 +224,279 @@ test("S15: cold load re-stamps legacy counter refs onto seq+1 exactly once", asy } rmSync(dir, { recursive: true, force: true }); }); + +test("S16: retained old-vs-current ref collision fails closed without changing bytes", async () => { + const dir = join(testRoot, `s16-${Math.random().toString(36).slice(2)}`); + const dbPath = join(dir, "index.db"); + // First ingest issues m00001/m00002 to a/b. A second ingest keeps those + // first-seen message-row refs (INSERT OR IGNORE) while the refs table + // moves m00001 to c — the exact residue a retired cold-load re-stamp + // leaves behind. + const st = UnifiedStore.open(dbPath, undefined as never) as unknown as UnifiedStore; + const first = legacyLike("pr3-collision", [msg("a"), msg("b")]); + first.state.messageRefs = { byRaw: { a: "m00001", b: "m00002" }, byRef: { m00001: "a", m00002: "b" } }; + assert.equal(st.ingestSession(planIngest(first)), true); + const second = legacyLike("pr3-collision", [msg("a"), msg("b"), msg("c")]); + second.state.messageRefs = { + byRaw: { a: "m00002", b: "m00003", c: "m00001" }, + byRef: { m00001: "c", m00002: "a", m00003: "b" }, + }; + assert.equal(st.ingestSession(planIngest(second)), true); + const refsBefore = st.getRefs("pr3-collision"); + const rowsBefore = st.getMessages("pr3-collision", { includeRetired: true }); + const blocksBefore = st.getBlocks("pr3-collision"); + const countsBefore = st.counts(); + assert.ok(st.findRefHistoryCollisions("pr3-collision").length > 0, "fixture must leave a detectable collision"); + st.close(); + + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store); + try { + const session = store.loadSync("pr3-collision"); + assert.ok(session, "collision fixture must still load"); + assert.ok(session.refLedgerConflict && session.refLedgerConflict.length > 0, "cold load must flag the collision"); + const refused = applyRanges( + { ranges: [{ startRef: "m00001", endRef: "m00001", summary: "fold" }], diagnostics: { kind: "ok", invalidItems: 0 } } as never, + { session, log: () => {} } as never, + { loopTracking: false }, + ); + assert.equal(refused.outcome, "refused"); + assert.equal(refused.reason, "ref-ledger-conflict"); + assert.match(refused.text, /ambiguous/); + assert.match(refused.text, /Raw messages, blocks, and the ref ledger are preserved/); + assert.equal(refused.blocksCreated, 0); + store.closeQuietly(); + + const after = UnifiedStore.open(dbPath, undefined as never) as unknown as UnifiedStore; + try { + assert.deepEqual(after.getRefs("pr3-collision"), refsBefore, "refs ledger unchanged"); + assert.deepEqual(after.getMessages("pr3-collision", { includeRetired: true }), rowsBefore, "raw rows unchanged"); + assert.deepEqual(after.getBlocks("pr3-collision"), blocksBefore, "blocks unchanged"); + assert.deepEqual(after.counts(), countsBefore, "no rows added or removed"); + } finally { + after.close(); + } + + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store2); + const again = store2.loadSync("pr3-collision"); + assert.ok(again); + assert.deepEqual(again.refLedgerConflict, session.refLedgerConflict, "detection is stable across cold loads"); + store2.closeQuietly(); + } finally { + _setStoreForTest({} as never); + } +}); + + +test("S17: warm startup retains known ref-history collision refusal", async () => { + const dir = join(testRoot, `s17-${Math.random().toString(36).slice(2)}`); + const id = "pr3-warm-collision"; + const st = UnifiedStore.open(join(dir, "index.db"), undefined as never) as unknown as UnifiedStore; + const first = legacyLike(id, [msg("a"), msg("b")]); + first.state.messageRefs = { byRaw: { a: "m00001", b: "m00002" }, byRef: { m00001: "a", m00002: "b" } }; + assert.equal(st.ingestSession(planIngest(first)), true); + const second = legacyLike(id, [msg("a"), msg("b"), msg("c")]); + second.state.messageRefs = { + byRaw: { a: "m00002", b: "m00003", c: "m00001" }, + byRef: { m00001: "c", m00002: "a", m00003: "b" }, + }; + assert.equal(st.ingestSession(planIngest(second)), true); + st.close(); + const { SessionStore } = await import("../src/persist.ts"); + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const cold = store.loadSync(id); + assert.ok(cold?.refLedgerConflict?.length, "cold fixture must detect retained conflict"); + assert.equal(store.flushSync(cold), true, "write the normal legacy startup envelope"); + for (const method of ["loadAll", "boot"] as const) { + const warm = (await store[method]()).get(id); + assert.ok(warm?.refLedgerConflict?.length, `${method} must retain the same conflict`); + const before = structuredClone(warm.state); + const refused = applyRanges( + { ranges: [{ startRef: "m00001", endRef: "m00001", summary: "fold" }], diagnostics: { kind: "ok", invalidItems: 0 } } as never, + { session: warm, log: () => {} } as never, + { loopTracking: false }, + ); + assert.equal(refused.outcome, "refused"); + assert.equal(refused.reason, "ref-ledger-conflict"); + assert.deepEqual(warm.state, before, "refusal must not mutate loaded state"); + } + const child: Session = { ...cold, id: `${id}-child`, state: structuredClone(cold.state), refLedgerConflict: undefined }; + await store.persistForkSession(id, child); + assert.ok(child.refLedgerConflict?.length, "an in-memory fork must detect the cloned ref history"); + } finally { + store.closeQuietly(); + } + const mode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "shadow"; + const legacyStore = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const legacy = legacyStore.loadSync(id); + assert.ok(legacy?.refLedgerConflict?.length, "legacy fallback must retain a detectable collision"); + } finally { + legacyStore.closeQuietly(); + if (mode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = mode; + } +}); + +for (const mode of ["legacy", "full"] as const) { + test(`S18: ambiguous resume inheritance survives ${mode} restart and another adoption`, async () => { + const { createCore, defaultConfig, DEFAULT_ABSORB_CONFIG } = await import("acp-kernel"); + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const { createSession } = await import("../src/session.ts"); + const { incomingCoreMessages, maybeAdoptResume } = await import("../src/fork-adoption.ts"); + const { executeProxyToolAsync } = await import("../src/loop/core.ts"); + const { executeAbsorb } = await import("../src/absorb.ts"); + const dir = mkdtempSync(join(testRoot, `resume-${mode}-`)); + const previousMode = process.env.BILI_STORAGE_UNIFIED; + const previousWriter = process.env.BILI_STORAGE_JSON_WRITER; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "on"; + _setStoreForTest(new SessionStore({ enabled: false })); + const parsed = { model: "synthetic", messages: [ + { role: "user", content: "Original request. " + "Retained source context. ".repeat(200) }, + { role: "assistant", content: "First reply. " + "Retained first answer. ".repeat(200) }, + { role: "assistant", content: "Second reply. " + "Retained second answer. ".repeat(200) }, + { role: "user", content: "Keep the original references." }, + ] }; + const messages = incomingCoreMessages("openai", parsed)!; + const [a, b, c, d] = messages.map((m) => m.id); + const fixture = legacyLike(`parent-${mode}`, messages); + fixture.meta = { protocol: "openai" }; + fixture.state.messageRefs = { byRaw: { [a]: "m00001", [b]: "m00002", [c]: "m00003", [d]: "m00004" }, byRef: { m00001: a, m00002: b, m00003: c, m00004: d } }; + const db = UnifiedStore.open(join(dir, "index.db")); + assert.equal(db.ingestSession(planIngest(fixture)), true); + fixture.state.messageRefs = { byRaw: { [a]: "m00002", [b]: "m00003", [c]: "m00001", [d]: "m00004" }, byRef: { m00001: c, m00002: a, m00003: b, m00004: d } }; + assert.equal(db.ingestSession(planIngest(fixture)), true); + const parentRows = db.getMessages(fixture.id, { includeRetired: true }); + db.close(); + let store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const parent = store.loadSync(fixture.id)!; + assert.ok(parent.refLedgerConflict?.length, "fixture hydration must find real retained ambiguity"); + const core = createCore(); + const config = defaultConfig(400_000); + config.preserveRecentMessages = 0; + config.preserveRecentTokens = 0; + config.compress.minCompressRange = 100; + const child = createSession(`child-${mode}`, { protocol: "openai" }); + const resumed = { ...parsed, messages: [...parsed.messages, { role: "user", content: "Resume with the old m00001 citation." }] }; + const childMessages = incomingCoreMessages("openai", resumed)!; + const adopt = (target: Session, source: Session) => maybeAdoptResume({ + session: target, parent: source, sharedDepth: messages.length, protocol: "openai", parsed: resumed, + upstreamOrigin: "https://synthetic.invalid", blocksEnabled: true, log: () => {}, + }); + assert.equal(adopt(child, parent).seededRefs, 4); + child.state = core.processTurn({ messages: childMessages, state: child.state, config, tokenCount: 30000, renderTags: "text-only" }).state; + child.lastMessages = childMessages; + const refuse = async (session: Session) => { + const before = JSON.stringify({ state: session.state, stats: session.stats, metadata: session.metadata, revision: session.revisionEpoch }); + const result = await executeProxyToolAsync("compress", { content: [{ startId: "m00001", endId: "m00001", summary: "Retain the original source context and exact reference ownership without changing any source message or stored block." }] }, { + core, config, session, messages: childMessages, log: () => {}, + }); + assert.equal(result.outcome, "refused", "known inherited ambiguity must refuse, never fold the current wrong owner"); + assert.equal(result.reason, "ref-ledger-conflict"); + const absorbed = executeAbsorb({ ref: "m00001", summary: "must not hide ambiguous source" }, "test-call", DEFAULT_ABSORB_CONFIG, { config, session, messages: childMessages }); + assert.equal(absorbed.reason, "ref-ledger-conflict"); + assert.equal(JSON.stringify({ state: session.state, stats: session.stats, metadata: session.metadata, revision: session.revisionEpoch }), before); + }; + await refuse(child); + assert.equal(child.refLedgerConflict, undefined, "adoption need not invent historical diagnostic rows"); + assert.equal(child.metadata.refLedgerConflict, true); + store.closeQuietly(); + process.env.BILI_STORAGE_UNIFIED = mode; + store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + assert.equal(store.flushSync(child), true); + store.closeQuietly(); + store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + const loaded = store.loadSync(child.id, { protocol: "openai" })!; + assert.ok(loaded); + assert.equal(loaded.refLedgerConflict, undefined, "child has no retained old-owner rows to rediscover"); + assert.equal(loaded.metadata.refLedgerConflict, true, "existing metadata must keep quarantine across restart"); + await refuse(loaded); + const grandchild = createSession(`grandchild-${mode}`, { protocol: "openai" }); + assert.ok(adopt(grandchild, loaded).seededRefs > 0); + await refuse(grandchild); + const healthy = createSession(`healthy-${mode}`, { protocol: "openai" }); + healthy.state = structuredClone(loaded.state); + const healthyChild = createSession(`healthy-child-${mode}`, { protocol: "openai" }); + adopt(healthyChild, healthy); + assert.equal(healthyChild.metadata.refLedgerConflict, undefined, "healthy adoption must not acquire quarantine"); + const checker = UnifiedStore.open(join(dir, "index.db")); + try { + assert.deepEqual(checker.getMessages(fixture.id, { includeRetired: true }), parentRows); + assert.deepEqual(checker.findRefHistoryCollisions(child.id), [], "the marker, not a new SQL collision, must enforce refusal"); + } finally { checker.close(); } + } finally { + store.closeQuietly(); + _setStoreForTest(new SessionStore({ enabled: false })); + if (previousMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previousMode; + if (previousWriter === undefined) delete process.env.BILI_STORAGE_JSON_WRITER; + else process.env.BILI_STORAGE_JSON_WRITER = previousWriter; + } + }); +} + + +for (const mode of ["legacy", "full"] as const) { + const scenarios = ["disk-newer", "local-newer", "local-known", "healthy", ...(mode === "full" ? ["index-only"] : [])]; + for (const scenario of scenarios) { + test(`S19: ${mode} save keeps known quarantine when ${scenario}`, async () => { + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const { createSession, acquireInFlight, releaseInFlight, hasRefLedgerConflict } = await import("../src/session.ts"); + const previousMode = process.env.BILI_STORAGE_UNIFIED; + const previousWriter = process.env.BILI_STORAGE_JSON_WRITER; + process.env.BILI_STORAGE_UNIFIED = mode; + process.env.BILI_STORAGE_JSON_WRITER = "on"; + const dir = mkdtempSync(join(testRoot, `convergence-${mode}-`)); + const localStore = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + if (scenario === "index-only") process.env.BILI_STORAGE_JSON_WRITER = "off"; + const foreignStore = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + process.env.BILI_STORAGE_JSON_WRITER = "on"; + _setStoreForTest(new SessionStore({ enabled: false })); + const local = createSession(`convergence-${scenario}`, { protocol: "openai" }); + acquireInFlight(local); + let foreign: Session | undefined; + try { + local.metadata.localOnly = "keep resident metadata"; + await localStore.writeNow(local); + foreign = foreignStore.loadSync(local.id, { protocol: "openai" })!; + assert.ok(foreign); + acquireInFlight(foreign); + foreign.stats.requests = 2; + foreign.metadata.foreignOnly = "do not copy unrelated metadata"; + if (scenario === "disk-newer" || scenario === "local-newer" || scenario === "index-only") foreign.metadata.refLedgerConflict = true; + await foreignStore.writeNow(foreign); + if (scenario === "local-known") local.metadata.refLedgerConflict = true; + const localWins = scenario === "local-newer" || scenario === "index-only"; + if (localWins) local.stats.requests = 3; + await localStore.writeNow(local); + const expected = scenario !== "healthy"; + assert.equal(hasRefLedgerConflict(local), expected, "active resident must retain known quarantine whichever state wins"); + assert.equal(local.stats.requests, localWins ? 3 : 2); + assert.equal(local.metadata.localOnly, "keep resident metadata"); + assert.equal(local.metadata.foreignOnly, undefined, "do not replace unrelated resident metadata"); + const reader = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + const loaded = reader.loadSync(local.id, { protocol: "openai" })!; + assert.ok(loaded); + assert.equal(hasRefLedgerConflict(loaded), expected, "the selected persisted payload must retain the same fact"); + } finally { reader.closeQuietly(); } + } finally { + releaseInFlight(local); + if (foreign) releaseInFlight(foreign); + localStore.closeQuietly(); + foreignStore.closeQuietly(); + _setStoreForTest(new SessionStore({ enabled: false })); + if (previousMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previousMode; + if (previousWriter === undefined) delete process.env.BILI_STORAGE_JSON_WRITER; + else process.env.BILI_STORAGE_JSON_WRITER = previousWriter; + } + }); + } +} diff --git a/tests/session-fork-publication.test.ts b/tests/session-fork-publication.test.ts new file mode 100644 index 0000000000..c88849e458 --- /dev/null +++ b/tests/session-fork-publication.test.ts @@ -0,0 +1,355 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { type TestContext } from "node:test"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetSessionsForTest, createSession, getSession, getSessionAsync, hasRefLedgerConflict, memoryReport, peekSession, publishForkSession, type Session } from "../src/session.ts"; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} + +async function fixture(t: TestContext, full = false): Promise { + const previousEnv = Object.fromEntries(["BILI_STORAGE_UNIFIED", "BILI_STORAGE_JSON_WRITER", "BILI_PERSIST_ZSTD"].map((key) => [key, process.env[key]])); + if (full) Object.assign(process.env, { BILI_STORAGE_UNIFIED: "full", BILI_STORAGE_JSON_WRITER: "on", BILI_PERSIST_ZSTD: "0" }); + const dir = mkdtempSync(join(tmpdir(), "bili-fork-publication-")); + const store = new SessionStore({ dir, enabled: true, debounceMs: 60000 }); + _resetSessionsForTest(); + _setStoreForTest(store); + t.after(async () => { + store.cancelAll(); + await store.closeQuietlyAsync(); + _resetSessionsForTest(); + rmSync(dir, { recursive: true, force: true }); + if (full) for (const [key, value] of Object.entries(previousEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + await store.ready(); + if (full) assert.ok(existsSync(join(dir, "index.db")), "full mode must use a real SQLite database"); + for (const id of ["parent-a", "parent-b"]) assert.equal(store.flushSync(createSession(id)), true); + return store; +} + +test("fork publication reserves a child before its existence probe settles", async (t) => { + const store = await fixture(t); + const probe = deferred(); + const load = store.loadAsync.bind(store); + const persist = store.persistForkSession.bind(store); + const writes: string[] = []; + let probes = 0; + store.loadAsync = async (id, meta) => ++probes === 1 ? probe.promise : load(id, meta); + store.persistForkSession = async (parentId, session) => { + await persist(parentId, session); + writes.push(parentId); + }; + const winner = createSession("child", { label: "first publisher" }); + const loser = createSession("child", { label: "second publisher" }); + const first = publishForkSession(winner, "parent-a"); + const [second] = await Promise.allSettled([publishForkSession(loser, "parent-b")]); + const [reader] = await Promise.allSettled([getSessionAsync("child")]); + let syncReadError: unknown; + try { getSession("child"); } catch (error) { syncReadError = error; } + probe.resolve(null); + const [published] = await Promise.allSettled([first]); + t.diagnostic(JSON.stringify({ probes, writes, first: published.status, second: second.status, durableLabel: store.loadSync("child")?.meta.label })); + + assert.equal(second.status, "rejected", `different-parent contender must be refused; writes=${writes.join(",")}`); + if (second.status === "rejected") assert.match(String(second.reason), /child fork publication in progress/); + assert.equal(published.status, "fulfilled"); + assert.equal(reader.status, "rejected", "new async readers must be refused during the existence probe"); + if (reader.status === "rejected") assert.match(String(reader.reason), /child fork publication in progress/); + assert.match(String(syncReadError), /child fork publication in progress/); + assert.equal(probes, 1, "the losing publisher must not even enter the existence probe"); + assert.deepEqual(writes, ["parent-a"]); + assert.equal(peekSession("child"), winner); + assert.equal(store.loadSync("child")?.meta.label, "first publisher"); +}); + +test("an in-flight load cannot adopt a child during fork persistence", async (t) => { + const store = await fixture(t); + const loaded = deferred(); + const persisting = deferred(); + const release = deferred(); + const load = store.loadAsync.bind(store); + const persist = store.persistForkSession.bind(store); + let probes = 0; + store.loadAsync = async (id, meta) => ++probes === 1 ? loaded.promise : load(id, meta); + store.persistForkSession = async (parentId, session) => { + persisting.resolve(); + await release.promise; + await persist(parentId, session); + }; + const reading = Promise.allSettled([getSessionAsync("child")]); + const child = createSession("child", { label: "published child" }); + const publishing = publishForkSession(child, "parent-a"); + await persisting.promise; + loaded.resolve(null); + const [read] = await reading; + const midPublication = peekSession("child"); + release.resolve(); + await publishing; + + t.diagnostic(JSON.stringify({ read: read.status, residentDuringPublication: midPublication?.id ?? null })); + assert.equal(read.status, "rejected", "a load started before publication must recheck exclusion after its await"); + if (read.status === "rejected") assert.match(String(read.reason), /child fork publication in progress/); + assert.equal(midPublication, undefined, "no ghost may become resident while the fork is publishing"); + assert.equal(await getSessionAsync("child"), child); +}); + +for (const result of ["missing", "stale"] as const) { + test(`an in-flight ${result} load returns the completed fork winner`, async (t) => { + const store = await fixture(t); + const loaded = deferred(); + const load = store.loadAsync.bind(store); + let probes = 0; + store.loadAsync = async (id, meta) => ++probes === 1 ? loaded.promise : load(id, meta); + const reading = getSessionAsync("child"); + const child = createSession("child", { label: "published child" }); + child.metadata.refLedgerConflict = true; + await publishForkSession(child, "parent-a"); + loaded.resolve(result === "missing" ? null : createSession("child", { label: "stale load" })); + + const winner = await reading; + assert.equal(winner, child, "a completed publication must win over the earlier load result"); + assert.equal(hasRefLedgerConflict(winner), true, "a stale load must not clear the published quarantine"); + assert.equal(peekSession("child"), child); + const durable = store.loadSync("child"); + assert.equal(durable?.meta.label, "published child"); + assert.ok(durable && hasRefLedgerConflict(durable)); + }); +} + +test("missing-child loads remain single-flight without publication", async (t) => { + const store = await fixture(t); + const loaded = deferred(); + let probes = 0; + store.loadAsync = async () => { probes++; return loaded.promise; }; + const first = getSessionAsync("missing"); + const second = getSessionAsync("missing"); + loaded.resolve(null); + const [a, b] = await Promise.all([first, second]); + + assert.equal(probes, 1); + assert.equal(a, b); + assert.equal(getSession("missing"), a); +}); + +test("serialized publication refuses resident and durable children without leaving exclusion set", async (t) => { + const store = await fixture(t); + const child = createSession("child", { label: "published child" }); + await publishForkSession(child, "parent-a"); + await assert.rejects(publishForkSession(createSession("child"), "parent-b"), /child session already exists/); + assert.equal(getSession("child"), child); + _resetSessionsForTest(); + await assert.rejects(publishForkSession(createSession("child"), "parent-b"), /child session already exists/); + + const loaded = await getSessionAsync("child"); + assert.equal(loaded.meta.label, "published child"); + assert.equal(getSession("child"), loaded); + assert.equal(store.loadSync("child")?.meta.label, "published child"); +}); + +for (const failure of ["probe", "persist"] as const) { + test(`fork ${failure} failure releases child exclusion for a clean retry`, async (t) => { + const store = await fixture(t); + const load = store.loadAsync.bind(store); + const persist = store.persistForkSession.bind(store); + if (failure === "probe") store.loadAsync = async () => { throw new Error("synthetic probe failure"); }; + else store.persistForkSession = async () => { throw new Error("synthetic persist failure"); }; + const child = createSession("child"); + await assert.rejects(publishForkSession(child, "parent-a"), new RegExp(`synthetic ${failure} failure`)); + assert.equal(peekSession("child"), undefined); + store.loadAsync = load; + store.persistForkSession = persist; + + await publishForkSession(child, "parent-a"); + assert.equal(await getSessionAsync("child"), child); + assert.equal(store.loadSync("child")?.id, "child"); + }); +} + +async function inheritedFixture(t: TestContext) { + const store = await fixture(t, true); + const parent = createSession("parent-a"); + parent.metadata.refLedgerConflict = true; + parent.state.messageRefs = { byRaw: { "issued-raw": "m00090" }, byRef: { m00090: "issued-raw" } }; + parent.state.deadRefs = ["m00080"]; + parent.lastMessages = [{ id: "issued-raw", role: "user", contentType: "text", text: "Synthetic inherited ref fixture." }]; + assert.equal(store.flushSync(parent), true); + const child = createSession("child", { label: "inherited quarantined fork" }); + child.metadata.refLedgerConflict = true; + child.state = structuredClone(parent.state); + child.lastMessages = structuredClone(parent.lastMessages); + return { store, child }; +} + +function assertInherited(session: Session | null | undefined) { + assert.ok(session); + assert.equal(hasRefLedgerConflict(session), true); + assert.equal(session.state.messageRefs.byRaw["issued-raw"], "m00090"); + assert.ok(session.state.deadRefs?.includes("m00080")); +} + +function holdEarlierRead(store: SessionStore, result: "missing" | "stale") { + const started = deferred(); + const release = deferred(); + const load = store.loadAsync.bind(store); + let held = false; + store.loadAsync = async (id, meta) => { + const hold = id === "child" && !held; + if (hold) held = true; + const snapshot = await load(id, meta); + if (!hold) return snapshot; + assert.equal(snapshot, null, "the first read must actually miss before publication"); + started.resolve(); + await release.promise; + return result === "missing" ? snapshot : createSession(id, { label: "earlier snapshot" }); + }; + const reading = Promise.allSettled([getSessionAsync("child")]); + return { started, release, reading, load }; +} + +function failEnvelope(t: TestContext, store: SessionStore) { + const disk = (store as unknown as { store: { flushSync(id: string, build: () => unknown): boolean } }).store; + const flush = disk.flushSync.bind(disk); + return t.mock.method(disk, "flushSync", (id: string, build: () => unknown) => id === "child" ? false : flush(id, build)); +} + +for (const result of ["missing", "stale"] as const) { + test(`an earlier ${result} load cannot replace a fork after ordinary LRU eviction`, async (t) => { + const { store, child } = await inheritedFixture(t); + const pool = memoryReport().pool; + _resetSessionsForTest(1, pool.budgetBytes); + const held = holdEarlierRead(store, result); + try { + await held.started.promise; + await publishForkSession(child, "parent-a"); + assert.equal(peekSession("child"), child); + getSession("eviction-pressure"); + assert.equal(peekSession("child"), undefined, "real one-slot LRU must evict the completed child"); + assertInherited(await held.load("child")); + held.release.resolve(); + const [read] = await held.reading; + const durable = await held.load("child"); + assertInherited(durable); + t.diagnostic(JSON.stringify({ schedule: "publish-evict-release", result, reader: read.status, returnedRefs: read.status === "fulfilled" ? read.value.state.messageRefs : null, returnedQuarantine: read.status === "fulfilled" ? hasRefLedgerConflict(read.value) : null, durableRefs: durable?.state.messageRefs, durableQuarantine: durable && hasRefLedgerConflict(durable) })); + assert.equal(read.status, "rejected", "the old claim must not adopt after the resident winner is evicted"); + if (read.status === "rejected") assert.match(String(read.reason), /session load superseded/); + assert.equal(peekSession("child"), undefined); + assertInherited(await getSessionAsync("child")); + } finally { + held.release.resolve(); + await held.reading; + _resetSessionsForTest(pool.maxSessions, pool.budgetBytes); + } + }); + + test(`an earlier ${result} load cannot replace a committed fork after envelope failure`, async (t) => { + const { store, child } = await inheritedFixture(t); + const held = holdEarlierRead(store, result); + try { + await held.started.promise; + const fault = failEnvelope(t, store); + try { + await assert.rejects(publishForkSession(child, "parent-a"), /fork envelope persistence failed; retry the same request/); + } finally { fault.mock.restore(); } + assert.equal(peekSession("child"), undefined); + assert.equal((await store.findSessionFiles("child")).length, 0); + assertInherited(await held.load("child")); + held.release.resolve(); + const [read] = await held.reading; + const durable = await held.load("child"); + assertInherited(durable); + t.diagnostic(JSON.stringify({ schedule: "committed-envelope-error-release", result, reader: read.status, returnedRefs: read.status === "fulfilled" ? read.value.state.messageRefs : null, returnedQuarantine: read.status === "fulfilled" ? hasRefLedgerConflict(read.value) : null, durableRefs: durable?.state.messageRefs, durableQuarantine: durable && hasRefLedgerConflict(durable) })); + assert.equal(read.status, "rejected", "a committed-then-error publication must invalidate the earlier claim too"); + if (read.status === "rejected") assert.match(String(read.reason), /session load superseded/); + assert.equal(peekSession("child"), undefined); + assertInherited(await getSessionAsync("child")); + } finally { + held.release.resolve(); + await held.reading; + } + }); +} + +test("a fresh reader recovers the actual committed fork after envelope failure", async (t) => { + const { store, child } = await inheritedFixture(t); + const fault = failEnvelope(t, store); + try { + await assert.rejects(publishForkSession(child, "parent-a"), /fork envelope persistence failed; retry the same request/); + } finally { fault.mock.restore(); } + assert.equal(peekSession("child"), undefined); + assert.equal((await store.findSessionFiles("child")).length, 0); + assertInherited(await store.loadAsync("child")); + assertInherited(await getSessionAsync("child")); +}); + +test("a failed load releases its own claim for an ordinary missing-child retry", async (t) => { + const store = await fixture(t); + const load = store.loadAsync.bind(store); + let probes = 0; + store.loadAsync = async (id, meta) => { + if (++probes === 1) throw new Error("synthetic read failure"); + return load(id, meta); + }; + await assert.rejects(getSessionAsync("missing"), /synthetic read failure/); + const missing = await getSessionAsync("missing"); + assert.equal(probes, 2); + assert.equal(await getSessionAsync("missing"), missing); +}); + + +test("an invalidated load's finally cannot clear a newer reader's single-flight claim", async (t) => { + const { store, child } = await inheritedFixture(t); + const pool = memoryReport().pool; + _resetSessionsForTest(1, pool.budgetBytes); + const load = store.loadAsync.bind(store); + const started = deferred(); + const releaseOld = deferred(); + const releaseNew = deferred(); + let reads = 0; + store.loadAsync = async (id, meta) => { + const probe = id === "child" ? ++reads : 0; + const snapshot = await load(id, meta); + if (probe === 1) { + assert.equal(snapshot, null); + started.resolve(); + await releaseOld.promise; + } else if (probe > 2) await releaseNew.promise; + return snapshot; + }; + const older = Promise.allSettled([getSessionAsync("child")]); + const settling = [older]; + try { + await started.promise; + await publishForkSession(child, "parent-a"); + getSession("eviction-pressure"); + assert.equal(peekSession("child"), undefined); + const newer = Promise.allSettled([getSessionAsync("child")]); + settling.push(newer); + releaseOld.resolve(); + const [old] = await older; + const joining = Promise.allSettled([getSessionAsync("child")]); + settling.push(joining); + const issuedReads = reads; + releaseNew.resolve(); + const [[next], [joined]] = await Promise.all([newer, joining]); + assertInherited(await load("child")); + t.diagnostic(JSON.stringify({ schedule: "old-finally-new-claim", issuedReads, old: old.status, newer: next.status, joining: joined.status })); + assert.equal(issuedReads, 3, "two newer readers must share one fresh load after the old claim settles"); + assert.equal(old.status, "rejected"); + assert.ok(next.status === "fulfilled" && joined.status === "fulfilled"); + assert.equal(next.value, joined.value); + assertInherited(next.value); + } finally { + releaseOld.resolve(); + releaseNew.resolve(); + await Promise.all(settling); + _resetSessionsForTest(pool.maxSessions, pool.budgetBytes); + } +}); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 9e39c8ff7d..769d052492 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -11,11 +11,11 @@ * render goldens, both with a mid-conversation restart * and with the legacy file DELETED at the restart (the * index alone carries the session — the migration's - * end-state promise). #4316 PR3/3 (380ef4dac): a - * full-mode cold load additionally re-homes citation - * numbers once (one-time ledger alignment) — comparisons - * run modulo that renumbering (citation-normalized - * digests), which is the ONLY permitted delta. + * end-state promise). #4316 PR3/3 (380ef4dac) used to + * re-home citation numbers on a full-mode cold load + * (one-time ledger alignment); that rewrite is now + * removed, but committed goldens still carry its + * numbering, so comparisons stay citation-normalized. * STALE REJECT — a multi-instance stale write that the guarded build * rejects must never reach index.db; the converged disk * record is what gets ingested. @@ -159,7 +159,7 @@ function unifiedRows(store: UnifiedStore, id: string): ReturnType { + test(`storage engine: full-mode restart is invisible on the ${wire} wire (== legacy restart / committed goldens, modulo historical #4316 citation numbering in the goldens)`, { timeout: 240_000 }, async () => { const golden = JSON.parse(fs.readFileSync(path.join(RENDER_ROOT, `${wire}.json`), "utf8")) as { turns: Array<{ i: number; sha: string; shaCiteNorm: string }>; foldAt: number; editedAt: number; summaryFrom: number; @@ -178,9 +178,9 @@ for (const wire of WIRES) { assert.equal(record.turns.length, legacy.turns.length, `${wire}/${leg}: body count drifted vs legacy restart`); for (let i = 0; i < legacy.turns.length; i++) { if (i <= record.editedAt) assert.equal(record.turns[i]!.sha, golden.turns[i]!.sha, `${wire}/${leg}: pre-migration body #${i} digest drift vs committed goldens`); - // #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation - // numbers once (one-time ledger alignment) — the ONLY permitted - // wire delta. Citation-normalized digests must match turn-for-turn. + // #4316 PR3/3 (380ef4dac) goldens were captured with the old + // one-time citation renumbering. Current cold loads seal refs; + // the remaining normalization targets those historical goldens. assert.equal(record.turns[i]!.shaCiteNorm, legacy.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs legacy restart`); assert.equal(record.turns[i]!.shaCiteNorm, golden.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs committed goldens (restart-inherent bs-field loss aside)`); } @@ -317,7 +317,7 @@ test("storage engine: rollback contract — explicit legacy mode serves the iden const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(viaFull, "full-mode reload returned null"); - // Persist the one-time cold-load alignment before exercising rollback. + // Persist the full-mode load result before exercising rollback. await storeFull.writeNow(viaFull); storeFull.closeQuietly(); const digestFull = viewDigest(toLegacyLike(viaFull)); @@ -329,8 +329,8 @@ test("storage engine: rollback contract — explicit legacy mode serves the iden assert.ok(legacy, "legacy-only reload returned null after rollback"); // #2960 ⑤: cross-mode equality is on the STATE proper — full mode // serves the raw table tail, legacy the envelope's folded view. - // #4316 PR3/3: modulo the one-time ledger alignment (full-side cold - // load re-homes citation numbers; legacy keeps counter numbers). + // #4316 PR3/3: committed fixtures may still carry the historical + // cold-load numbering, so these cross-mode digests stay normalized. assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(legacy))), viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))), "rollback: legacy-only load state digest drift vs full-mode load"); assert.ok(viaFull.lastMessages && viaFull.lastMessages.length > 0, "⑤: full-mode load serves the table tail"); assert.ok(legacy.lastMessages && legacy.lastMessages.length > 0, "⑤: rollback keeps the envelope tail"); diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index ae8bdd7b97..dc6535f5d3 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -69,15 +69,12 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", assert.equal(recovered.stats.requests, 876); assert.equal(recovered.state.nextBlockId, 4); assert.deepEqual(recovered.state.blocks.map(b => b.blockId), ["b3"]); - // #4316 PR3/3: a full-mode cold load re-homes citation numbers once - // (ledger alignment) — pin the content-addressable round-trip, not the - // counter-era number, so the assertion holds under any numbering scheme. - const alignedRef = recovered.state.messageRefs.byRaw["raw-new"]; - assert.ok(alignedRef, "raw-new must keep a citation number after recovery"); - assert.equal(recovered.state.messageRefs.byRef[alignedRef], "raw-new"); - // The pre-alignment number is recorded as superseded, not silently dropped. - assert.equal(alignedRef, "m00001", "the first stored message aligns to seq 0 + 1"); - assert.equal(recovered.restampSuperseded?.get("m00003"), "raw-new"); + // Cold load must not renumber issued refs: the recovered raw keeps the + // exact number the live session handed out, and no superseded translation + // is synthesized. + assert.equal(recovered.state.messageRefs.byRaw["raw-new"], "m00003"); + assert.equal(recovered.state.messageRefs.byRef["m00003"], "raw-new"); + assert.equal(recovered.restampSuperseded, undefined); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 3c5908f29a..379b323010 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -9,7 +9,6 @@ import type { Session } from "../src/session.ts"; import type { CoreMessage, CompressionBlock } from "acp-kernel"; import { canonicalize, sha256 } from "./golden-canonical.ts"; import { planIngest, loadLegacyView, toLegacyLike, type LegacySessionLike } from "../src/storage/ingest.ts"; -import { restampRefsToLedger } from "acp-kernel"; import { UnifiedStore } from "../src/storage/store.ts"; import { resolveStorageMode } from "../src/storage/shadow.ts"; import { resolveStorageWalJournalSizeLimit } from "../src/knobs.ts"; @@ -406,12 +405,11 @@ test("resolveStorageWalJournalSizeLimit: env wins, garbage falls through, file t assert.equal(resolveStorageWalJournalSizeLimit({}, 2097152), 2097152, "file tier applies when the env is absent"); }); -/** #4316 PR3/3 parity oracle: the expected digest for a FULL-mode load is - * the LEGACY-mode view of the same files plus the deterministic one-time - * ledger alignment (same seqByRaw the full-mode load aligned against). The - * corpus manifest keeps pinning the PURE view (storage-corpus suite); this - * oracle pins the equivalence "full serve = legacy serve + alignment". */ -function alignedLegacyDigest(caseDir: string, meta: { protocol: string; id: string }, seqByRawId: Map): string { +/** Full mode now seals issued refs at cold load, so the expected digest for + * a FULL-mode load is the PURE legacy view of the same files. The corpus + * manifest pins the same digest (storage-corpus suite); this oracle keeps + * the full-vs-legacy equivalence explicit. */ +function legacyDigest(caseDir: string, meta: { protocol: string; id: string }): string { const legacyDir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-legacy-oracle-")); try { fs.cpSync(caseDir, path.join(legacyDir, "case"), { recursive: true }); @@ -426,11 +424,7 @@ function alignedLegacyDigest(caseDir: string, meta: { protocol: string; id: stri store = new SessionStore({ dir: path.join(legacyDir, "case"), debounceMs: 1, enabled: true }); const pure = store.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(pure, "legacy oracle load returned null"); - const view = toLegacyLike(pure); - const state = structuredClone(view.state as Record); - restampRefsToLedger(state as never, seqByRawId); - delete state.refLedgerAligned; - return sessionDigest({ ...view, state: state as never }); + return sessionDigest(toLegacyLike(pure)); } finally { store?.closeQuietly(); if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; @@ -460,9 +454,7 @@ test("storage unified FULL mode: read path serves from the index with organic im let expectedDigest = ""; try { assert.ok(probe.getSessionRow(meta.id), "organic import did not populate index.db"); - // #4316 PR3/3: expected = legacy view of the same files + the - // deterministic alignment against these very rows. - expectedDigest = alignedLegacyDigest(caseDir, meta, probe.seqByRaw(meta.id)); + expectedDigest = legacyDigest(caseDir, meta); } finally { probe.close(); } @@ -557,13 +549,7 @@ test("storage unified: file-config storage.unified selects the mode; env tier st let session = store.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(session, "file-tier full loadSync returned null"); store.closeQuietly(); - const probeStore = UnifiedStore.open(path.join(caseDir, "index.db")); - let fileTierDigest = ""; - try { - fileTierDigest = alignedLegacyDigest(caseDir, meta, probeStore.seqByRaw(meta.id)); - } finally { - probeStore.close(); - } + const fileTierDigest = legacyDigest(caseDir, meta); assert.equal(sessionDigest(toLegacyLike(session)), fileTierDigest, "file-tier full read digest drift"); assert.ok(fs.existsSync(path.join(caseDir, "index.db")), "file-tier full did not create index.db"); // Env tier wins: explicit env legacy overrides file full on a fresh