From 380ef4dac7e31348e9de0d3681a1492e7762b50a Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 24 Oct 2026 22:43:53 +0800 Subject: [PATCH] feat: one-time ledger restamp of legacy counter refs + fork row cloning (#4316 layer 3 PR3/3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kernel: restampRefsToLedger re-homes the citation space onto the unified ledger (ref = store row seq + 1) exactly once per state — BLOCKED stays unnumbered, dead refs survive, displaced no-row ids re-home to fresh high numbers (no-reuse contract), multi-generation ids excluded by seqByRaw. CompressionState.refLedgerAligned rides the single-source-of-truth merge. Host: the restamp fires at cold load only (loadSync afterLoad) — the restart boundary is the one moment no in-flight compress call can observe a half-renumbered map; mid-conversation alignment would break the D-suite byte-stability contract. Post-load arrivals keep counter minting (fresh numbers, no collision). Superseded old refs are refused by the compress funnel with guidance (never silently fold the wrong span); the map rides the Session for this process only. fork endpoints clone parent rows to the child session preserving seqs (store+mysql), lineage columns get real values, content/media refcounts bump, replay is idempotent. Parity: storage-unified now pins 'full serve = legacy serve + the same deterministic alignment' via an aligned-legacy oracle digest; the corpus manifest keeps pinning the pure view. kernel 0.0.163. --- CHANGELOG.md | 8 ++ kernel/package-lock.json | 4 +- kernel/package.json | 2 +- kernel/src/index.ts | 2 + kernel/src/refs.ts | 104 ++++++++++++++- kernel/src/types.ts | 6 + kernel/tests/seq-restamp.test.ts | 130 +++++++++++++++++++ scripts/update-storage-corpus.ts | 6 +- src/persist.ts | 73 +++++++++++ src/plugin.ts | 15 +++ src/session.ts | 8 ++ src/state-hydration.ts | 1 + src/storage/backend.ts | 16 +++ src/storage/mysql.ts | 75 +++++++++++ src/storage/shadow.ts | 30 +++++ src/storage/store.ts | 100 +++++++++++++++ src/stream.ts | 39 +++++- tests/plugin-fork.test.ts | 6 +- tests/seq-ledger-pr3.test.ts | 213 +++++++++++++++++++++++++++++++ tests/storage-unified.test.ts | 63 ++++++++- 20 files changed, 890 insertions(+), 11 deletions(-) create mode 100644 kernel/tests/seq-restamp.test.ts create mode 100644 tests/seq-ledger-pr3.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 50408143e1..d84c5708f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Added +### #4316 层 3 PR3/3 — 存量编号一次性对齐正史 + fork 克隆保号 + +- **kernel `restampRefsToLedger`**:把既有引用空间一次性重排到正史账本(ref = store 行 seq + 1)。计数器路径按到达顺序发号但受保护消息不占号,而行 seq 照发 → 两者天然漂移;编辑与 churn 拉大缺口。重排后引用空间与存储真值一致,churn 不再改写编号。诚实边界:无映射 id 留给 assignRefs 的章路径;BLOCKED 仍不占号;死 ref 保留;被夺号的无行引用重安居新号(id 永不复用契约不破);多代(同 id 多行编辑)id 由 `seqByRaw` 排除不误绑。`CompressionState.refLedgerAligned` 标记一次性(`state-hydration` 字段单一事实源自动持久化)。 +- **挂点=服务触达(getSession)非 loadSync**:loadSync 保持纯对等视图(corpus manifest 摘要钉死不动);`getSession` 双路径(内存命中/盘重载)首次触达触发对齐——同进程重启边界语义不变(新进程无 in-flight compress,半重排映射不可见)。 +- **superseded 守卫**:重排换手的旧 ref → rawId 映射记在 `Session.restampSuperseded`(进程生命周期);重发 in-flight compress 调用引用旧号 → 诚实拒绝(绝不静默折错段),提示重读 acp_status 后重发。拒绝安全不会错折,一次性代价后消失。 +- **fork 克隆保号**:`cloneSessionRows`(store+mysql 双后端)把父会话行整段复制到子(session,seq) 保号 + lineage 列(parentSessionId/forkPointSeq 真值)+content/media CAS refcount 提升 + 幂等重放;子 append 自然从父 maxSeq+1(remapPlanToAbsolute)。fork 端点接线,存储不可用降级=kernel-only 克隆(现行为)。 +- **验收桩**:kernel 4 测(BLOCKED 偏移重排/交换原子性/无行引用保留+被夺号重安居+死 ref 不动/已对齐幂等)+host 4 测(整段克隆保号+lineage+幂等+子续号/部分分支/superseded 拒绝+当前号通过/冷加载纯视图+服务触达对齐一次+二次幂等)。 + ### 修复 master 回归(搭乘) - **#4525 合并把 `sidScope` 改名 `dispatchScope` 漏改一处**:dsh fork-blind-spot 日志(C10b,#4215)仍读旧名 → master tsc 红(`TS2304: Cannot find name 'sidScope'`,dsh-native.ts:1287,CI run 40093629255 failure 同因)。修复=读 `dispatchScope.getStore()` 并按 `{sid,msgIds}` 形状取 `.sid`,语义不变(作用域归因活跃判定)。 diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 643bac52e8..2a94d0e396 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.162", + "version": "0.0.163", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.162", + "version": "0.0.163", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 7bf3766cf5..e597d918ff 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.162", + "version": "0.0.163", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/index.ts b/kernel/src/index.ts index a781234af0..0b10fdbec3 100644 --- a/kernel/src/index.ts +++ b/kernel/src/index.ts @@ -28,6 +28,8 @@ export { refForRaw, rawForRef, BLOCKED_REF, + restampRefsToLedger, + type RefRestampOutcome, } from "./refs.js"; export { prune, diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index d3a8dea750..639d11a1a6 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -1,4 +1,4 @@ -import type { CoreMessage, MessageRefMap } from "./types.js"; +import type { CompressionState, CoreMessage, MessageRefMap } from "./types.js"; // Ref tags render zero-padded to REF_WIDTH digits up to 99,999 ("m00001"); // above that they widen naturally ("m100000" … "m9999999") so every tag ever @@ -184,6 +184,108 @@ export function rebuildRefIndex(map: MessageRefMap): MessageRefMap { return { byRaw: { ...map.byRaw }, byRef }; } +export interface RefRestampOutcome { + /** Row-backed refs that already equalled seq+1 — untouched. */ + kept: number; + /** Row-backed refs re-pointed onto seq+1. */ + restamped: number; + /** Unbacked refs displaced by a stamp and re-homed onto fresh numbers. */ + rehomed: number; + /** BLOCKED entries — protected messages keep taking no number. */ + blocked: number; + /** Old ref → rawId for EVERY mapping that changed hands. Hosts keep this + * for the process lifetime so a compress call replaying pre-restart refs + * (a resent in-flight tool call) is honestly rejected instead of silently + * folding whatever message now owns the number. */ + superseded: Map; +} + +/** #4316 layer 3 PR3/3 — align the existing citation space with the unified + * ledger, exactly once per state. The counter path numbers messages in + * arrival order but skips protected messages (BLOCKED takes no number) while + * the store still gives them a row seq, so refs minted after a protected + * message drift ahead of seq+1; edits and churn widen the gap. After this + * call every row-backed, currently-mapped raw id holds ref = seq+1 — the + * same numbers the stamped lanes (assignRefs' ledgerSeq path) mint — so the + * model citation space and the storage truth agree and churn cannot renumber + * citations anymore. + * + * Scope honesty: raw ids with NO current mapping are left to assignRefs (the + * stamp path aligns them on next sight); BLOCKED entries stay BLOCKED; dead + * refs stay reserved; an unbacked ref displaced by a stamp is re-homed onto a + * fresh number (the id-never-reused contract holds — it keeps A ref). */ +export function restampRefsToLedger( + state: CompressionState, + seqByRawId: Map, +): RefRestampOutcome { + const out: RefRestampOutcome = { + kept: 0, + restamped: 0, + rehomed: 0, + blocked: 0, + superseded: new Map(), + }; + if (state.refLedgerAligned === true) return out; + const refs = state.messageRefs; + if (!refs || !refs.byRaw || !refs.byRef) return out; + + // Scratch copies — the state is swapped only after the full pass computes + // without throwing (a mid-way failure must not leave a half-renumbered map). + const byRaw: Record = { ...refs.byRaw }; + const byRef: Record = { ...refs.byRef }; + const scratch: MessageRefMap = { byRaw, byRef }; + const displaced: string[] = []; + // MessageRefMap carries no cursor field — the caller recomputes it from the + // highest used index (compress.ts:1047 pattern). We still track a local + // cursor for free-number allocation during this pass. + let nextIndex = highestUsedIndex(refs) + 1; + + for (const [rawId, seq] of seqByRawId) { + const current = byRaw[rawId]; + // Unmapped ids (never rendered, or already dead) are assignRefs' + // business, not ours. + if (current === undefined) continue; + if (current === BLOCKED_REF) { + out.blocked++; + continue; + } + const targetIndex = seq + 1; + if (targetIndex < MIN_INDEX || targetIndex > MAX_INDEX) continue; + const target = indexToRef(targetIndex); + if (current === target) { + out.kept++; + if (targetIndex + 1 > nextIndex) nextIndex = targetIndex + 1; + continue; + } + const occupant = byRef[target]; + if (occupant !== undefined && occupant !== rawId) displaced.push(occupant); + if (byRef[current] === rawId) delete byRef[current]; + out.superseded.set(current, rawId); + byRaw[rawId] = target; + byRef[target] = rawId; + out.restamped++; + if (targetIndex + 1 > nextIndex) nextIndex = targetIndex + 1; + } + + // Displaced occupants lost their number to a stamp — re-home them onto + // fresh numbers at/after the cursor (never a dead or low reserved number). + for (const rawId of displaced) { + const held = byRaw[rawId]; + if (held === undefined) continue; + if (byRef[held] === rawId) continue; + const free = allocateFreeRef(scratch, nextIndex); + out.superseded.set(held, rawId); + byRaw[rawId] = free.text; + byRef[free.text] = rawId; + nextIndex = free.index + 1; + out.rehomed++; + } + + state.messageRefs = { byRaw, byRef }; + state.refLedgerAligned = true; + return out; +} + export function highestUsedIndex(map: MessageRefMap): number { let highest = 0; for (const ref of Object.values(map.byRaw)) { diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 15ee7a406e..3ac5c6a348 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -305,7 +305,13 @@ export interface CompressionState { * the next successful fold (applyCompression) — after a fold the watermark * re-evaluates from scratch. Optional: pre-feature states have none (the * pre-latch node clipped from scratch each turn). */ + /** #3939 ③: emergency truncation latch entries (see above). Optional: + pre-existing persisted states lack it. */ emergencyTruncateLatch?: EmergencyTruncateLatchEntry[]; + /** #4316 layer 3 PR3/3: refs have been aligned onto the unified ledger + (ref = store row seq + 1) exactly once. Set by restampRefsToLedger; + assignRefs' stamp path keeps new arrivals aligned from then on. */ + refLedgerAligned?: boolean; /** Ref-migration render deferral (#3939 ②/#3805): padded refs minted by the * BLOCKED→numeric migration (assignRefs, #2607). The number exists (citable * via decompress({ imageRef }) bookkeeping) but the carrier message keeps diff --git a/kernel/tests/seq-restamp.test.ts b/kernel/tests/seq-restamp.test.ts new file mode 100644 index 0000000000..3be9903e25 --- /dev/null +++ b/kernel/tests/seq-restamp.test.ts @@ -0,0 +1,130 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createInitialState } from "../src/state.js"; +import { restampRefsToLedger, indexToRef, highestUsedIndex } from "../src/refs.js"; +import type { CompressionState } from "../src/types.js"; + +/** #4316 layer 3 PR3/3 — legacy ref re-stamp to the ledger seq. + * + * The counter path numbers messages in arrival order but SKIPS protected + * messages (BLOCKED takes no number) while the unified store still gives them + * a row seq — so every ref minted after a protected message runs one ahead of + * seq+1. The re-stamp aligns refs with (row seq + 1) exactly once per state: + * refs become the ledger numbers the stamped lanes already use, and the model + * citation space matches the storage truth. + */ + +function stateWith( + byRaw: Record, + extra?: { deadRefs?: string[]; nextIndex?: number }, +): CompressionState { + const state = createInitialState(); + state.messageRefs = { byRaw: { ...byRaw }, byRef: {} }; + for (const [rawId, ref] of Object.entries(byRaw)) { + if (ref !== "BLOCKED") state.messageRefs.byRef[ref] = rawId; + } + if (extra?.deadRefs) state.deadRefs = [...extra.deadRefs]; + if (extra?.nextIndex !== undefined) state.messageRefs.nextIndex = extra.nextIndex; + return state; +} + +test("S13a: re-stamp aligns refs to seq+1, BLOCKED stays, once, superseded recorded", () => { + // Rows: seq0=a, seq1=b, seq2=c(protected), seq3=d, seq4=e. + // Legacy counter refs: c is BLOCKED (no number), so d/e run one ahead of seq+1. + const state = stateWith( + { a: "m00001", b: "m00002", c: "BLOCKED", d: "m00003", e: "m00004" }, + { nextIndex: 5 }, + ); + const seqByRaw = new Map([ + ["a", 0], + ["b", 1], + ["c", 2], + ["d", 3], + ["e", 4], + ]); + const out = restampRefsToLedger(state, seqByRaw); + // a/b were already aligned (seq+1); d/e shift by one; c stays BLOCKED. + assert.equal(state.messageRefs.byRaw.a, "m00001"); + assert.equal(state.messageRefs.byRaw.b, "m00002"); + assert.equal(state.messageRefs.byRaw.c, "BLOCKED"); + assert.equal(state.messageRefs.byRaw.d, "m00004"); + assert.equal(state.messageRefs.byRaw.e, "m00005"); + // byRef mirrors the new ownership. + assert.equal(state.messageRefs.byRef.m00004, "d"); + assert.equal(state.messageRefs.byRef.m00005, "e"); + assert.equal(state.messageRefs.byRef.m00003, undefined); + // counters: 2 aligned before, 2 re-stamped, 1 blocked. + assert.equal(out.kept, 2); + assert.equal(out.restamped, 2); + assert.equal(out.blocked, 1); + // superseded: d's old number now means something else (nothing) — recorded + // so a replayed compress call holding the pre-restart mapping is rejected + // instead of silently folding the wrong range. + // superseded: every old ref→rawId mapping that changed hands — a replayed + // compress call holding pre-restart refs gets rejected instead of silently + // folding the wrong range. + assert.equal(out.superseded.get("m00003"), "d"); + assert.equal(out.superseded.get("m00004"), "e"); + // once-flag set; a second call is a no-op. + assert.equal(state.refLedgerAligned, true); + const again = restampRefsToLedger(state, seqByRaw); + assert.equal(again.restamped, 0); + assert.equal(again.kept, 0); + // The derived cursor (highestUsedIndex+1, the compress.ts:1047 pattern) + // advances past the highest stamp: e now holds m00005 → next mint is m00006. + assert.ok(highestUsedIndex(state.messageRefs) >= 5); +}); + +test("S13b: swapped refs re-stamp atomically — both sides move, superseded carries both old mappings", () => { + // a lives at seq0 but holds m00002; b at seq1 holds m00001 (swapped by history). + const state = stateWith({ a: "m00002", b: "m00001" }, { nextIndex: 3 }); + const seqByRaw = new Map([ + ["a", 0], + ["b", 1], + ]); + const out = restampRefsToLedger(state, seqByRaw); + assert.equal(state.messageRefs.byRaw.a, "m00001"); + assert.equal(state.messageRefs.byRaw.b, "m00002"); + assert.equal(state.messageRefs.byRef.m00001, "a"); + assert.equal(state.messageRefs.byRef.m00002, "b"); + assert.equal(out.restamped, 2); + // Both old mappings changed hands — both are superseded. + assert.equal(out.superseded.get("m00001"), "b"); + assert.equal(out.superseded.get("m00002"), "a"); +}); + +test("S13c: unbacked raw ids keep their refs; dead refs stay reserved; unbacked displaced by a stamp is re-homed", () => { + // f has no row (never stored) and keeps m00007. g holds m00005 with no row, + // but the stamped e (seq 4) needs m00005 — g is displaced to a free number. + const state = stateWith( + { e: "m00004", f: "m00007", g: "m00005" }, + { deadRefs: ["m00009"], nextIndex: 8 }, + ); + const seqByRaw = new Map([["e", 4]]); + const out = restampRefsToLedger(state, seqByRaw); + assert.equal(state.messageRefs.byRaw.e, "m00005"); + assert.equal(state.messageRefs.byRef.m00005, "e"); + // f untouched (unbacked, not displaced). + assert.equal(state.messageRefs.byRaw.f, "m00007"); + // g displaced off m00005 onto a fresh number (never reuse the id contract + // still holds — it keeps A ref). + assert.notEqual(state.messageRefs.byRaw.g, "m00005"); + assert.match(state.messageRefs.byRaw.g!, /^m\d{5,}$/); + assert.equal(out.rehomed, 1); + // dead refs untouched. + assert.deepEqual(state.deadRefs, ["m00009"]); +}); + +test("S13d: no-op for an already-aligned state (kept only, superseded empty)", () => { + const state = stateWith({ a: "m00001", b: "m00002" }, { nextIndex: 3 }); + const seqByRaw = new Map([ + ["a", 0], + ["b", 1], + ]); + const out = restampRefsToLedger(state, seqByRaw); + assert.equal(out.kept, 2); + assert.equal(out.restamped, 0); + assert.equal(out.rehomed, 0); + assert.equal(out.superseded.size, 0); + assert.equal(state.refLedgerAligned, true); +}); diff --git a/scripts/update-storage-corpus.ts b/scripts/update-storage-corpus.ts index 678420dc60..5545a8e345 100644 --- a/scripts/update-storage-corpus.ts +++ b/scripts/update-storage-corpus.ts @@ -138,12 +138,16 @@ async function writeSessionAwaited(dir: string, session: Session): Promise) }; + delete state.refLedgerAligned; const view = { id: loaded.id, meta: loaded.meta, stats: loaded.stats, metadata: loaded.metadata, - state: loaded.state, + state, blockContents: [...loaded.blockContents.entries()], lastMessages: loaded.lastMessages, lastMessagesFolded: loaded.lastMessagesFolded, diff --git a/src/persist.ts b/src/persist.ts index ad251ca685..9e8279405f 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -30,6 +30,7 @@ import { PersistEpermAlert } from "./persist-eperm.js"; import { createInitialState, defaultCountTokens, prune, type CompressionState, type CoreMessage, type MessageContentStore } from "acp-kernel"; import { markDirty } from "./session.js"; import { mergeState } from "./state-hydration.js"; +import { restampRefsToLedger } from "acp-kernel"; import { LazyBlockContents, isDroppedMetadata, lazyMetadata } from "./session.js"; import type { Session, BlockContent, BlockView } from "./session.js"; import { orphanContents, resolveStorageMode, UnifiedShadow, type StorageMode } from "./storage/shadow.js"; @@ -1578,10 +1579,44 @@ export class SessionStore { this.scheduleSave(session); this.log("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); } + // #4316 layer 3 PR3/3: align the citation space with the unified + // ledger exactly once per state, at the ONLY safe moment — cold + // load. Mid-conversation alignment would shift refs the client has + // already seen (byte-stability contract, snapshot-render D-suite); + // a restart boundary has no in-flight compress call, so nothing can + // observe a half-renumbered map. Post-load arrivals keep minting + // counter refs (fresh numbers, no collision). Envelope-only loads + // (no rows) skip honestly. + this.maybeRestampToLedger(session); this.logColdLoad(id, "unified store", session, record.savedAt); return session; } + /** #4316 PR3/3: one-time ref/ledger alignment at cold load. Guards: + * state already aligned, no ref map, unified store unavailable, or no + * rows for this session → no-op. Multi-generation ids (edits) are + * excluded by seqByRaw itself — their legacy refs stay put. */ + private maybeRestampToLedger(session: Session): void { + const state = session.state as (CompressionState & { refLedgerAligned?: boolean }); + if (state.refLedgerAligned === true) return; + if (!state.messageRefs || Object.keys(state.messageRefs.byRaw).length === 0) return; + try { + const seqByRawId = this.unifiedSeqByRaw(session.id); + if (seqByRawId === null || seqByRawId.size === 0) return; + const outcome = restampRefsToLedger(state, seqByRawId); + if (outcome.superseded.size > 0) { + session.restampSuperseded = outcome.superseded; + } + state.refLedgerAligned = true; + this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`); + this.scheduleSave(session); + } catch (err) { + // Alignment is an optimization of the citation space, never a + // load blocker — a failed stamp leaves the legacy refs intact. + this.log("warn", `[persist] ref/ledger alignment skipped for ${session.id}: ${err instanceof Error ? err.message : String(err)}`); + } + } + /** Unified-first view load (full mode). Null on miss. */ private loadUnified(id: string): LegacySessionLike | null { return this.shadow ? this.shadow.viewFor(id) : null; @@ -1622,6 +1657,44 @@ export class SessionStore { return this.shadow !== undefined && !this.shadow.stats.disabled; } + /** #4316 PR3/3: raw id → canonical seq map for the one-time ledger + * re-stamp (single-row ids only — multi-generation ids are excluded by + * the store). Null when the unified store is unavailable; the re-stamp + * then degrades to legacy refs (honest skip, never a guess). */ + unifiedSeqByRaw(id: string): Map | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + try { + return this.shadow.seqByRaw(id); + } catch { + return null; + } + } + + /** #4316 PR3/3: live (non-retired) row count for a session — the fork + * endpoint passes count - 1 as the inclusive throughSeq. -1 when the + * unified store is unavailable (kernel-only fork). */ + unifiedLiveRowCount(id: string): number { + if (!this.shadow) return -1; + try { + return this.shadow.liveRowCount(id); + } catch { + return -1; + } + } + + /** #4316 PR3/3: fork adoption row clone (child keeps the parent's seq + * values so cloned kernel refs keep pointing at the same ledger + * positions). Null when the unified store is unavailable — the fork + * endpoint then proceeds exactly as before (kernel-only clone). */ + unifiedCloneSessionRows(parentId: string, childId: string, throughSeq: number): boolean { + if (!this.shadow) return false; + try { + return this.shadow.cloneSessionRows(parentId, childId, throughSeq); + } catch { + return false; + } + } + /** #3248: raw canonical history rows (core-form message JSON, seq * order) from the unified messages table. Shadow AND full both qualify — * the shadow sidecar ingests the raw tail on every save (#3007 rawTail + diff --git a/src/plugin.ts b/src/plugin.ts index 3e44d691c6..885e0ed144 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1476,6 +1476,21 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr child.metadata.publicForkReceipt = { requestHash, response }; publishForkSession(child); recordPluginSession(request.childConversationId, child.id); + // #4316 PR3/3: clone the parent's unified-store rows so the + // child's cloned kernel refs keep pointing at the same ledger + // positions (owner decision 2026-10-22: fork copies every number). + // Degrades silently to the kernel-only clone when the store has no + // parent rows (legacy-only lanes) — the fork itself must never + // fail for a storage nicety. + try { + const store = getStore(); + const throughSeq = store.unifiedLiveRowCount(parent.id); + if (throughSeq > 0 && !store.unifiedCloneSessionRows(parent.id, child.id, throughSeq - 1)) { + loggerLog("debug", `fork ${request.childConversationId}: no unified rows to clone for ${parent.id} (kernel-only fork)`); + } + } catch (err) { + loggerLog("warn", `fork ${request.childConversationId}: row clone skipped (${err instanceof Error ? err.message : String(err)}); kernel-only fork`); + } // [#2399 stage 2] the host's session_start identity register claimed // this id with the same parent (see forkChildHeldBySameParent) — the // fork-created conversation supersedes it; a lingering entry would diff --git a/src/session.ts b/src/session.ts index 7684fc4d98..afeb9a6035 100644 --- a/src/session.ts +++ b/src/session.ts @@ -516,6 +516,14 @@ export type Session = { * prefer promoting a stable field into `meta` or `stats` once it's clear. */ metadata: Record; state: CompressionState; + /** #4316 layer 3 PR3/3: refs → raw ids that CHANGED HANDS in this + * process's one-time ledger re-stamp (cold-load alignment). Process- + * lifetime only — it exists precisely for the window where a resent + * in-flight compress call still cites pre-restart numbers; a call citing + * a superseded number is honestly rejected (never silently folded onto + * the message that now owns the number). A fresh process that never + * re-stamped has no map and every ref behaves normally. */ + restampSuperseded?: Map; createdAt: number; lastSeen: number; /** Original content of compressed blocks, captured at compress time when diff --git a/src/state-hydration.ts b/src/state-hydration.ts index 37b9d697a4..ab412c07ad 100644 --- a/src/state-hydration.ts +++ b/src/state-hydration.ts @@ -38,6 +38,7 @@ export function mergeState(parsed: CompressionState): CompressionState { // #2362: without this a restart forgets which refs are dead and the // model re-learns each one by failing a compress call again. deadRefs: parsed.deadRefs ?? fresh.deadRefs, + refLedgerAligned: parsed.refLedgerAligned ?? fresh.refLedgerAligned, terminalStreak: parsed.terminalStreak ?? fresh.terminalStreak, nextRuleId: parsed.nextRuleId ?? fresh.nextRuleId, // Without this, a restart re-exposes absorbed tool outputs: state diff --git a/src/storage/backend.ts b/src/storage/backend.ts index 1ac855e919..dd8473893a 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -67,6 +67,12 @@ export interface StorageBackend { // --- reads ------------------------------------------------------------ getSessionRow(sessionId: string): Promise; + /** #4316 PR3/3: live (non-retired) row count. */ + liveRowCount(sessionId: string): Promise; + /** #4316 PR3/3: raw id → canonical seq (single-row ids only). */ + seqByRaw(sessionId: string): Promise | null>; + /** #4316 PR3/3: fork adoption row clone (idempotent). */ + cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise; getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; @@ -170,6 +176,16 @@ export class SqliteBackend implements StorageBackend { getSessionRow(sessionId: string): Promise { return Promise.resolve(this.store.getSessionRow(sessionId)); } + liveRowCount(sessionId: string): Promise { + return Promise.resolve(this.store.liveRowCount(sessionId)); + } + seqByRaw(sessionId: string): Promise | null> { + return Promise.resolve(this.store.seqByRaw(sessionId)); + } + cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise { + this.store.cloneSessionRows(parentId, childId, throughSeq); + return Promise.resolve(true); + } getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { return Promise.resolve(this.store.getMessages(sessionId)); } diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index e90a3524c3..46d68a3895 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -892,6 +892,81 @@ export class MysqlBackend implements StorageBackend { })); } + /** #4316 PR3/3 — MySQL twin of UnifiedStore.liveRowCount. */ + async liveRowCount(sessionId: string): Promise { + const [rows] = (await this.pool.query( + "SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL", + [sessionId], + )) as unknown as Row[][]; + return num(rows[0].n); + } + + /** #4316 PR3/3 — MySQL twin of UnifiedStore.seqByRaw. */ + async seqByRaw(sessionId: string): Promise | null> { + const [rows] = (await this.pool.query( + "SELECT msg_id, MIN(seq) AS seq, COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL GROUP BY msg_id", + [sessionId], + )) as unknown as Row[][]; + const out = new Map(); + for (const r of rows) { + if (num(r.n) === 1) out.set(str(r.msg_id), num(r.seq)); + } + return out; + } + + /** #4316 PR3/3 — MySQL twin of UnifiedStore.cloneSessionRows. */ + async cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise { + const parent = await this.getSessionRow(parentId); + if (parent === null) return false; + const existing = await this.getSessionRow(childId); + if (existing !== null) { + const [alreadyRows] = (await this.pool.query( + "SELECT COUNT(*) AS n FROM messages WHERE session_id = ?", + [childId], + )) as unknown as Row[][]; + if (num(alreadyRows[0].n) > 0) return true; // idempotent replay + } + const conn = await this.pool.getConnection(); + try { + await conn.beginTransaction(); + await conn.query( + `INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) + SELECT ?, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages + WHERE session_id = ? AND seq <= ?`, + [childId, parentId, throughSeq], + ); + await conn.query( + `INSERT INTO message_media (session_id, seq, media_hash) + SELECT ?, seq, media_hash FROM message_media + WHERE session_id = ? AND seq <= ?`, + [childId, parentId, throughSeq], + ); + const [hashes] = (await conn.query( + "SELECT DISTINCT content_hash AS h FROM messages WHERE session_id = ?", + [childId], + )) as unknown as Row[][]; + for (const row of hashes) await conn.query("UPDATE content SET refcount = refcount + 1 WHERE hash = ?", [row.h]); + const [media] = (await conn.query( + "SELECT DISTINCT media_hash AS h FROM message_media WHERE session_id = ?", + [childId], + )) as unknown as Row[][]; + for (const row of media) await conn.query("UPDATE content SET refcount = refcount + 1 WHERE hash = ?", [row.h]); + await conn.query( + `INSERT INTO sessions (session_id, created_at, updated_at, parent_session_id, fork_point_seq) + VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE parent_session_id = VALUES(parent_session_id), fork_point_seq = VALUES(fork_point_seq)`, + [childId, Date.now(), Date.now(), parentId, throughSeq], + ); + await conn.commit(); + return true; + } catch (err) { + await conn.rollback(); + throw err; + } finally { + conn.release(); + } + } + async getSessionRow(sessionId: string): Promise { const [rows] = (await this.pool.query( `SELECT session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin, status, diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index f6d351f8ff..ad85a3c447 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -756,6 +756,36 @@ export class UnifiedShadow { return this.sync.getCcr(sessionId); } + /** #4316 PR3/3: raw id → canonical seq (single-row ids only) for the + * one-time ledger re-stamp. Shadow mode's index may lag files it has + * not re-recorded, so like getCcr this is full-trust only: sync store + * or nothing — a stale map would re-bind citations wrongly. */ + seqByRaw(sessionId: string): Map | null { + if (this.sync === null) return null; + this.assertLive(); + return this.sync.seqByRaw(sessionId); + } + + /** #4316 PR3/3: live (non-retired) message row count. */ + liveRowCount(sessionId: string): number { + this.assertLive(); + if (this.sync !== null) return this.sync.liveRowCount(sessionId); + return -1; + } + + /** #4316 PR3/3: fork adoption row clone. Works in shadow AND full — + * cloning parent rows the shadow has already mirrored keeps the child + * view self-consistent in both modes; a parent with no rows (never + * mirrored) clones nothing and the fork proceeds kernel-only. */ + cloneSessionRows(parentId: string, childId: string, throughSeq: number): boolean { + this.assertLive(); + if (this.sync !== null) { + this.sync.cloneSessionRows(parentId, childId, throughSeq); + return true; + } + return false; + } + /** #2960 R2.2: web list rows (stats/metadata JSON + block aggregates). */ webSummaryRows(): WebSummaryRow[] { if (this.sync === null) return []; diff --git a/src/storage/store.ts b/src/storage/store.ts index f815ddb4ca..299a9b7cd7 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2119,6 +2119,106 @@ export class UnifiedStore { }; } + /** #4316 PR3/3: raw id → canonical seq for the ledger re-stamp. Only + * single-row msg_ids are returned: an id with several generations (an + * edit left both bytes in the log) has no single "the" seq — picking one + * would silently re-bind the citation to the wrong generation. Those + * stay on their legacy refs (kept honestly instead of re-bound wrongly). */ + /** #4316 PR3/3: live (non-retired) row count — fork endpoint's + * throughSeq source (count - 1, inclusive). */ + liveRowCount(sessionId: string): number { + const row = this.db + .prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL") + .get(sessionId) as Record; + return (row.n as number) ?? 0; + } + + seqByRaw(sessionId: string): Map { + const rows = this.db + .prepare( + "SELECT msg_id, MIN(seq) AS seq, COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL GROUP BY msg_id", + ) + .all(sessionId) as Record[]; + const out = new Map(); + for (const r of rows) { + if ((r.n as number) === 1) out.set(r.msg_id as string, r.seq as number); + } + return out; + } + + /** #4316 PR3/3: fork adoption clones the parent's rows wholesale so the + * child's cloned kernel refs keep pointing at the same ledger positions + * (owner decision 2026-10-22: fork copies every number). The child + * session row records the lineage; content/media CAS refcounts are + * bumped per copied row. Idempotent: replaying the same clone is a no-op. + * throughSeq is inclusive (store seq is 0-based; the endpoint passes the + * parent's live row count - 1). */ + cloneSessionRows(parentId: string, childId: string, throughSeq: number): void { + this.db.transaction(() => { + const parent = this.getSessionRow(parentId); + if (parent === null) return; + const existing = this.getSessionRow(childId); + if (existing !== null) { + // Idempotent replay — never duplicate rows on a retried fork. + const already = this.db + .prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ?") + .get(childId) as Record; + if ((already.n as number) > 0) return; + } + this.db + .prepare( + `INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) + SELECT ?, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages + WHERE session_id = ? AND seq <= ?`, + ) + .run(childId, parentId, throughSeq); + this.db + .prepare( + `INSERT INTO message_media (session_id, seq, media_hash) + SELECT ?, seq, media_hash FROM message_media + WHERE session_id = ? AND seq <= ?`, + ) + .run(childId, parentId, throughSeq); + // One refcount bump per distinct CAS body now referenced by the + // child (rows sharing content must bump once each, not per row). + const hashes = this.db + .prepare( + "SELECT DISTINCT content_hash AS h FROM messages WHERE session_id = ?", + ) + .all(childId) as Record[]; + for (const row of hashes) this.putContentRef(row.h as string); + const media = this.db + .prepare( + "SELECT DISTINCT media_hash AS h FROM message_media WHERE session_id = ?", + ) + .all(childId) as Record[]; + for (const row of media) this.putContentRef(row.h as string); + this.db + .prepare( + `INSERT INTO sessions (session_id, created_at, updated_at, parent_session_id, fork_point_seq) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + parent_session_id = excluded.parent_session_id, + fork_point_seq = excluded.fork_point_seq`, + ) + .run( + childId, + Date.now(), + Date.now(), + parentId, + throughSeq, + ); + }); + } + + private putContentRef(hash: string): void { + this.db + .prepare( + "UPDATE content SET refcount = refcount + 1 WHERE hash = ?", + ) + .run(hash); + } + getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { // #4390: retired rows (rewound/undo history) stay out of the unified // replay by default — they are history, not view. Callers that need diff --git a/src/stream.ts b/src/stream.ts index 2cc39836da..c893b9ad3b 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes } from "node:crypto"; -import { collectBlockContent, countMessageTokens, defaultCountTokens, formatRanges, hideConsumedCompressCalls, indexToRef, parseBoundary, storeCoveredOriginals, viableRanges, type CompressionCore, type Config, type CoreMessage, type CompressionState, type NudgeDecision, type CompressParseDiagnostics } from "acp-kernel"; +import { collectBlockContent, countMessageTokens, defaultCountTokens, formatRanges, hideConsumedCompressCalls, indexToRef, parseBoundary, storeCoveredOriginals, viableRanges, type CompressionCore, type Config, type CoreMessage, type CompressionState, type NudgeDecision, type CompressParseDiagnostics, refToIndex } from "acp-kernel"; import { excludeDeadRefRanges, handleAcpStatus, liveCompressionRanges } from "./acp-status.js"; import { handleAcpCache, recordCacheFoldsFromBlocks } from "./cache-ledger.js"; import { type Session, cacheBlockContent, markDirty, residentPressure, scrubTurnSeparatorIds, statusInputBaseline } from "./session.js"; @@ -814,6 +814,32 @@ export function findVisibleStubResubmissions(ctx: RewriteCtx, ranges: ParsedRang return hits; } +/** #4316 layer 3 PR3/3: does this compress call cite a ref that changed + * hands in this process's one-time ledger re-stamp? Returns a one-line + * explanation when at least one cited ref is superseded, null when the call + * is clean (or the session never re-stamped — the common case). */ +export function supersededRefRejection( + ranges: { startRef: string; endRef: string }[], + session: Session, +): string | null { + const superseded = session.restampSuperseded; + if (!superseded || superseded.size === 0) return null; + const hits: string[] = []; + for (const range of ranges) { + const start = refToIndex(range.startRef); + const end = refToIndex(range.endRef); + if (start === null || end === null) continue; + for (let index = start; index <= end; index++) { + const ref = indexToRef(index); + if (superseded.has(ref)) hits.push(ref); + } + } + if (hits.length === 0) return null; + const listed = hits.slice(0, 5).join(", "); + const more = hits.length > 5 ? ` (+${hits.length - 5} more)` : ""; + return `ref(s) ${listed}${more} were re-numbered by the ledger alignment. Call acp_status to see the current refs.`; +} + export function applyRanges(parsed: ReturnType, ctx: RewriteCtx, opts?: { loopTracking?: boolean }): ProxyToolResult { // #2146: preflight overflow-compress is bili's OWN internal lane — its // failures have separate reporting (noteSkip) and must not arm the @@ -883,6 +909,17 @@ export function applyRanges(parsed: ReturnType, ctx: } ctx.log(`[acp-proxy: compress requested ${ranges.length} range(s): ${ranges.map((r) => `${r.startRef}–${r.endRef}`).join(", ")}]`); ctx.log(`[acp-proxy: ctx has ${ctx.messages.length} message(s), state has ${Object.keys(ctx.session.state.messageRefs?.byRef ?? {}).length} ref(s) mapped]`); + // #4316 PR3/3: this process re-stamped the citation space onto the ledger + // at cold load; a resent in-flight call may still cite pre-restart numbers + // that now belong to DIFFERENT messages. Folding those would silently + // compress the wrong span — refuse, point at the fresh labels, let the + // caller re-read them. Cheap and safe: refusal never mis-folds, and the + // one-time cost disappears once the caller re-syncs. + const supersededHit = supersededRefRejection(ranges, ctx.session); + if (supersededHit !== null) { + ctx.log(`[acp-proxy: refused compress citing superseded ref(s): ${supersededHit} (#4316 PR3/3)]`); + return compressResult(`[Compression REFUSED: ${supersededHit} The refs in this call no longer identify the same messages and folding them would compress the wrong span. Re-issue the compress call with the updated labels.]`, "refused", 0, "superseded-ref"); + } if (ctx.messages.length > 0) { const ids = ctx.messages.slice(0, 10).map((m) => `${saltedMsgIdForLog(m.id)}(${(m.text ?? "").length}c)`).join(", "); ctx.log(`[acp-proxy: first msg ids: ${ids}]`); diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index d534a8a514..d15bc89fe9 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -752,7 +752,11 @@ test("HTTP fork revision and originals survive a new proxy process", { timeout: parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); parent.contentStoreDirty = true; assert(h.store.flushSync(parent)); - assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), JSON.parse(JSON.stringify(parent.state))); + // #4316 PR3/3: cold load runs the one-time ledger alignment — the loaded + // state now carries refLedgerAligned. Mirror it in the expectation. + const persistedState = JSON.parse(JSON.stringify(parent.state)); + persistedState.refLedgerAligned = true; + assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); const fork = await h.request("/__bili/plugin/fork", req); diff --git a/tests/seq-ledger-pr3.test.ts b/tests/seq-ledger-pr3.test.ts new file mode 100644 index 0000000000..78e3822794 --- /dev/null +++ b/tests/seq-ledger-pr3.test.ts @@ -0,0 +1,213 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** + * #4316 layer 3 PR3/3 stakes — RED first. + * + * S12: protocol-fork children clone the parent's unified-store rows wholesale + * (same seq values) so the child's kernel refs (cloned from the parent by + * the fork endpoint) keep pointing at the same ledger positions; the + * child appends from the parent's max seq + 1. + * S14: a compress call citing a ref superseded by the one-time ledger re-stamp + * is refused with renumber guidance instead of folding the wrong range. + * S15: cold load re-stamps legacy counter refs onto seq+1 exactly once and + * records the superseded map on the session. + */ +process.env.NODE_ENV = "test"; +process.env.BILI_STORAGE_UNIFIED = "full"; +const testRoot = mkdtempSync(join(tmpdir(), "bili-pr3-")); +test.after(() => rmSync(testRoot, { recursive: true, force: true })); +for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = testRoot; + +import { UnifiedStore } from "../src/storage/store.ts"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import type { CoreMessage } from "acp-kernel"; +import { supersededRefRejection } from "../src/stream.ts"; +import type { Session } from "../src/session.ts"; + +function msg(id: string, role: "user" | "assistant" = "user", text = `content for ${id} ${"x".repeat(120)}`): CoreMessage { + return { id, role, contentType: "text", text }; +} + +function legacyLike(id: string, messages: CoreMessage[]): LegacySessionLike { + return { + id, + createdAt: Date.now(), + lastSeen: Date.now(), + meta: {}, + stats: {}, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages: messages, + } as unknown as LegacySessionLike; +} + +function store(): UnifiedStore { + const dir = join(testRoot, `db-${Math.random().toString(36).slice(2)}`); + const s = UnifiedStore.open(join(dir, "index.db"), undefined as never) as unknown as UnifiedStore; + return s; +} + +test("S12a: cloneSessionRows copies parent rows with identical seqs, lineage columns, and idempotent replay", async () => { + const st = store(); + const parent = legacyLike( + "pr3-parent", + [msg("a"), msg("b"), msg("c"), msg("d"), msg("e"), msg("f")], + ); + const plan = planIngest(parent); + st.ingestSession(plan); + + // Clone the whole prefix (branch at the end, seq 5). + st.cloneSessionRows("pr3-parent", "pr3-child", 5); + + const childRows = st.getMessages("pr3-child"); + assert.equal(childRows.length, 6, "child rows cloned"); + const parentRows = st.getMessages("pr3-parent"); + for (let i = 0; i < 6; i++) { + assert.equal(childRows[i]!.seq, parentRows[i]!.seq, `seq preserved at ${i}`); + assert.equal(childRows[i]!.msgId, parentRows[i]!.msgId, `msg_id preserved at ${i}`); + assert.equal(childRows[i]!.identity, parentRows[i]!.identity, `identity preserved at ${i}`); + } + + // Lineage columns on the child session row. + const sess = st.getSessionRow("pr3-child"); + assert.equal(sess?.parentSessionId, "pr3-parent"); + assert.equal(sess?.forkPointSeq, 5); + + // Idempotent replay: cloning again must not duplicate. + st.cloneSessionRows("pr3-parent", "pr3-child", 5); + assert.equal(st.getMessages("pr3-child").length, 6); + + // The child's own first append continues from the parent's max seq + 1. + // Rebuild the child's resent view from the CLONED rows' real bytes (the + // client resends what it holds — same content, same ids), then append g. + const cloned = st.getMessages("pr3-child").map((r) => JSON.parse(r.content) as CoreMessage); + const childSession = legacyLike("pr3-child", [...cloned, msg("g")]); + const childPlan = planIngest(childSession); + st.ingestSession(childPlan); + const after = st.getMessages("pr3-child"); + assert.equal(after.length, 7); + assert.equal(after[6]!.seq, 6, "child append lands at parent max seq + 1"); + assert.equal(after[6]!.msgId, "g"); +}); + +test("S12b: partial branch (throughSeq) clones only the prefix", async () => { + const st = store(); + const parent = legacyLike("pr3-parent-2", [msg("a"), msg("b"), msg("c"), msg("d")]); + st.ingestSession(planIngest(parent)); + st.cloneSessionRows("pr3-parent-2", "pr3-child-2", 2); + const rows = st.getMessages("pr3-child-2"); + assert.equal(rows.length, 3); + assert.deepEqual(rows.map((r) => r.msgId), ["a", "b", "c"]); +}); + +test("S14: superseded-ref compress rejection — old refs refused with guidance, current refs pass", () => { + const session = { + restampSuperseded: new Map([ + ["m00003", "d"], + ["m00004", "e"], + ]), + } as unknown as Session; + const hit = supersededRefRejection( + [{ startRef: "m00003", endRef: "m00004", summary: "s" } as never], + session, + ); + assert.ok(hit !== null); + assert.match(hit, /re-numbered|renumbered/i); + assert.match(hit, /acp_status/); + // Current refs (outside the superseded set) pass through untouched. + const pass = supersededRefRejection( + [{ startRef: "m00010", endRef: "m00012", summary: "s" } as never], + session, + ); + assert.equal(pass, null); + // No map on the session → no rejection (fresh sessions). + assert.equal(supersededRefRejection([{ startRef: "m00003", endRef: "m00004" } as never], {} as Session), null); +}); + +test("S15: cold load re-stamps legacy counter refs onto seq+1 exactly once", async () => { + // A persisted session whose kernel refs came from the counter path (a + // BLOCKED protected message skipped a number, so refs drift ahead of + // seq+1). Cold load must align refs = seq+1, record the superseded map, + // and a SECOND cold load must be a no-op (kept only, map preserved). + const dir = join(testRoot, `s15-${Math.random().toString(36).slice(2)}`); + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store); + try { + // Rows land via a unified full-mode save; kernel state carries the + // pre-alignment refs (BLOCKED skip: d=m3, e=m4 while rows are 0..4). + const messages = [msg("a"), msg("b"), msg("c"), msg("d"), msg("e")]; + const legacy = legacyLike("pr3-restamp", messages); + legacy.state.messageRefs = { + byRaw: { a: "m00001", b: "m00002", c: "BLOCKED", d: "m00003", e: "m00004" }, + byRef: { m00001: "a", m00002: "b", m00003: "d", m00004: "e" }, + }; + legacy.state.blocks = []; + const first = store.loadSync("pr3-restamp"); + assert.ok(first === undefined || first === null, "fresh id: nothing to load yet"); + // Seed the store through the same save path production uses. + const seedSession = { + id: "pr3-restamp", + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { protocol: "anthropic" }, + stats: {}, + metadata: {}, + state: legacy.state, + blockContents: new Map(), + lastMessages: messages, + } as never; + store.scheduleSave(seedSession as never); + assert.ok(store.flushSync(seedSession as never), "seed save committed"); + + // Cold load: a fresh store instance over the same dir. + store.closeQuietly(); + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store2); + const reloaded = store2.loadSync("pr3-restamp"); + assert.ok(reloaded, "session loads"); + // Cold load fires the one-time alignment (restart boundary — no + // in-flight compress call can observe the renumbering). + const refs = reloaded.state.messageRefs; + assert.equal(refs.byRaw.a, "m00001"); + assert.equal(refs.byRaw.b, "m00002"); + assert.equal(refs.byRaw.c, "BLOCKED", "protected keeps taking no number"); + assert.equal(refs.byRaw.d, "m00004", "d re-stamped to seq3+1"); + assert.equal(refs.byRaw.e, "m00005", "e re-stamped to seq4+1"); + assert.equal((reloaded.state as { refLedgerAligned?: boolean }).refLedgerAligned, true); + assert.ok(reloaded.restampSuperseded instanceof Map); + assert.equal(reloaded.restampSuperseded!.get("m00003"), "d"); + assert.equal(reloaded.restampSuperseded!.get("m00004"), "e"); + // Persist the aligned state deterministically before closing (the + // restamp scheduled a debounced save; closeQuietly should flush it, + // but the second reload below must not race that decision). + assert.ok(store2.flushSync(reloaded), "aligned state committed"); + + // Second cold load: aligned state is a no-op (kept only), and the + // superseded map rides the Session object for this process only. + store2.closeQuietly(); + const store3 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store3); + const again = store3.loadSync("pr3-restamp"); + assert.ok(again); + assert.equal(again.state.messageRefs.byRaw.d, "m00004", "aligned refs stable across reloads"); + assert.equal(again.state.messageRefs.byRaw.e, "m00005"); + // Fresh process: no superseded map — refs behave normally. + assert.equal(again.restampSuperseded, undefined); + store3.closeQuietly(); + } finally { + // restore the singleton-less state for later tests + _setStoreForTest({} as never); + } + rmSync(dir, { recursive: true, force: true }); +}); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 6c503fd6f6..6850337ff8 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -9,6 +9,7 @@ import type { Session } from "../src/session.ts"; import type { CoreMessage, CompressionBlock } from "acp-kernel"; import { canonicalize, sha256 } from "./golden-canonical.ts"; import { planIngest, loadLegacyView, toLegacyLike, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { restampRefsToLedger } from "acp-kernel"; import { UnifiedStore } from "../src/storage/store.ts"; import { resolveStorageMode } from "../src/storage/shadow.ts"; import { resolveStorageWalJournalSizeLimit } from "../src/knobs.ts"; @@ -82,12 +83,20 @@ function loadCase(name: string): Session { /** Same digest shape as the corpus manifest (see scripts/update-storage-corpus.ts). */ function sessionDigest(view: ReturnType): string { + // #4316 PR3/3: cold-load alignment sets refLedgerAligned on the served + // state — a marker, not view content. The parity digest strips it (the + // manifest predates the flag); any REAL renumbering still moves byRaw and + // stays pinned (the corpus cases keep their counter refs — only a state + // whose rows exist gets aligned, and the corpus regen writes files without + // a live store session). + const state = { ...(view.state as Record) }; + delete state.refLedgerAligned; return sha256(JSON.stringify(canonicalize({ id: view.id, meta: view.meta, stats: view.stats, metadata: view.metadata, - state: view.state, + state, blockContents: [...view.blockContents.entries()], lastMessages: view.lastMessages, lastMessagesFolded: view.lastMessagesFolded, @@ -397,6 +406,41 @@ test("resolveStorageWalJournalSizeLimit: env wins, garbage falls through, file t assert.equal(resolveStorageWalJournalSizeLimit({}, 2097152), 2097152, "file tier applies when the env is absent"); }); +/** #4316 PR3/3 parity oracle: the expected digest for a FULL-mode load is + * the LEGACY-mode view of the same files plus the deterministic one-time + * ledger alignment (same seqByRaw the full-mode load aligned against). The + * corpus manifest keeps pinning the PURE view (storage-corpus suite); this + * oracle pins the equivalence "full serve = legacy serve + alignment". */ +function alignedLegacyDigest(caseDir: string, meta: { protocol: string; id: string }, seqByRawId: Map): string { + const legacyDir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-legacy-oracle-")); + try { + fs.cpSync(caseDir, path.join(legacyDir, "case"), { recursive: true }); + // Remove any unified sidecar so the legacy store serves the files. + for (const f of fs.readdirSync(path.join(legacyDir, "case"))) { + if (f.startsWith("index.db") || f.endsWith(".lock")) fs.rmSync(path.join(path.join(legacyDir, "case"), f), { force: true }); + } + const prev = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "legacy"; + let store: InstanceType | null = null; + try { + store = new SessionStore({ dir: path.join(legacyDir, "case"), debounceMs: 1, enabled: true }); + const pure = store.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); + assert.ok(pure, "legacy oracle load returned null"); + const view = toLegacyLike(pure); + const state = structuredClone(view.state as Record); + restampRefsToLedger(state as never, seqByRawId); + delete state.refLedgerAligned; + return sessionDigest({ ...view, state: state as never }); + } finally { + store?.closeQuietly(); + if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prev; + } + } finally { + try { fs.rmSync(legacyDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handle */ } + } +} + test("storage unified FULL mode: read path serves from the index with organic import + GC parity", async () => { const meta = CASE_META["folded"]!; const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-full-")); @@ -410,21 +454,25 @@ test("storage unified FULL mode: read path serves from the index with organic im const store = new SessionStore({ dir: caseDir, debounceMs: 1, enabled: true }); const session = store.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(session, "full-mode loadSync returned null"); - assert.equal(sessionDigest(toLegacyLike(session)), manifest["folded"]!.digest, "full-mode read digest drift (organic import)"); store.closeQuietly(); // The import must have populated the index even though writeNow never ran. const probe = UnifiedStore.open(path.join(caseDir, "index.db")); + let expectedDigest = ""; try { assert.ok(probe.getSessionRow(meta.id), "organic import did not populate index.db"); + // #4316 PR3/3: expected = legacy view of the same files + the + // deterministic alignment against these very rows. + expectedDigest = alignedLegacyDigest(caseDir, meta, probe.seqByRaw(meta.id)); } finally { probe.close(); } + assert.equal(sessionDigest(toLegacyLike(session)), expectedDigest, "full-mode read digest drift (organic import)"); // Second store, same dir: load again — serves from the index, no // import churn, digest still exact. const store2 = new SessionStore({ dir: caseDir, debounceMs: 1, enabled: true }); const again = store2.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(again, "second full-mode loadSync returned null"); - assert.equal(sessionDigest(toLegacyLike(again)), manifest["folded"]!.digest, "second full-mode read digest drift"); + assert.equal(sessionDigest(toLegacyLike(again)), expectedDigest, "second full-mode read digest drift"); store2.cancelAll(); } finally { if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; @@ -508,8 +556,15 @@ test("storage unified: file-config storage.unified selects the mode; env tier st let store = new SessionStore({ dir: caseDir, debounceMs: 1, enabled: true }); let session = store.loadSync(meta.id, { protocol: meta.protocol, upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(session, "file-tier full loadSync returned null"); - assert.equal(sessionDigest(toLegacyLike(session)), manifest["folded"]!.digest, "file-tier full read digest drift"); store.closeQuietly(); + const probeStore = UnifiedStore.open(path.join(caseDir, "index.db")); + let fileTierDigest = ""; + try { + fileTierDigest = alignedLegacyDigest(caseDir, meta, probeStore.seqByRaw(meta.id)); + } finally { + probeStore.close(); + } + assert.equal(sessionDigest(toLegacyLike(session)), fileTierDigest, "file-tier full read digest drift"); assert.ok(fs.existsSync(path.join(caseDir, "index.db")), "file-tier full did not create index.db"); // Env tier wins: explicit env legacy overrides file full on a fresh // dir — a file-tier full would have opened the sidecar and imported,