diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c6375674f..481521a6b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,21 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] +## [0.1.207] — 2026-10-24 + +### Storage — 行身份与史官化(#4390,storage review 四个 p2 + 三个顺路 p3) + +- **行身份去污染(D4-1)**:统一存储 messages 行新增 `identity` 列——跨保存 JOIN 键改为**位置无关**序列化(剥离 `id`/`wireIndex` 后哈希)。此前身份掺入 wire 位置元数据,客户端头部截断一条消息就会让后续全部行换血(旧行被清扫、seq 重铸、#4335 的区间账记随之抖动)。`content_hash` 保持字节忠实 CAS 键不变。存量库 v8 迁移:回填 identity 并合并 wireIndex 分裂世代(保最低 seq,重复行退还 CAS 引用)。MySQL 同款(information_schema 探测幂等 ALTER + 回填)。 +- **权威集清扫改墓碑(D4-2)**:proxy 会话每轮全量保存不再物理 DELETE 缺席行,改为 `retired_at` 墓碑——rewind/undo 的历史在统一存储里**耐久保留**;同一身份回来时原地复活(保原 seq,含 completeLog-less 保存路径 remap 复活)。读面(getMessages/统一重放/视图)默认过滤 retired;CAS 引用不释放。deleteSession/archive/GC 继续拥有物理生命周期。 +- **渲染接缝图片真指纹(D4-3)**:snapshot-render 的 identitySig 图片段从 `长度` 改为 `sha256(base64) 前 24 hex`——同尺寸截图不再可能连击假接缝造成静默截断。 +- **fork 谱系入正史(D3-2)**:sessions 行的 `parent_session_id` 现在从信封 `metadata.derivedFromSessionId` 落库(此前恒 null,谱系只活在信封里,信封丢失=家族失忆)。 +- **顺路**:gc full vacuum 走 `vacuum('full')`(带 TRUNCATE checkpoint,文件真正缩小,D5-1);loadLegacyView 尾指针缺行从静默缩水改为计数 + 三处加载路径 warn(D4-5);fold-reconcile 的 Pass 0 位置副本失步(存储 ids 与顺序骨架不再 1:1)从静默降级改为响亮 warn(此前无任何日志痕迹)。 +- **审查修复(review)**:identity JOIN 迁移后,全量 ingest 对「identity 相同、字节不同」的行原本靠 INSERT OR IGNORE 静默保留旧世代——头部截断后存储世代带着旧 wireIndex 与活视图永久失配(采样校验漂移→水位反复清零→#4323 信封冻结的数据丢失向量),外部 CAS 损坏也永远无法自愈。现在全量路径在**同一 seq** 上采纳更新的字节(seq 稳定性不变,delta 追加路径仍只增不改);`tailGaps` 诊断键从 parity 摘要面剔除(存在性差异不再误报漂移)。 +- 测试:`tests/storage-review-p2s.test.ts` 7 桩(截断保号/墓碑+复活/接缝指纹/谱系/迁移去重/gc checkpoint/缺行响亮);storage 全家族 + golden/snapshot/fold-reconcile/incremental/web 回归绿。 + ### Fixed +- **master CI 全红修复:#4390 historian 的媒体引用释放 + #3250 WAL cap 断言预算(#4506)。** bisect 定位 f161c518..02f544da 窗口 first bad = #4390(row-identity-historian),两个破口:① 权威集清扫改墓碑后,retired 行的 message_media 映射与媒体 CAS 引用永远不释放(默认要等 archive 365 天)——图片重度会话里即 #3762/#3901 的字节泄漏,`media-cas` 家族在 windows 分片稳定红。现在 retire 时释放媒体映射+引用(文本字节照旧史官保留;图片体积大、历史价值低),rewind-redo 复活时由重摄取路径幂等重建映射与引用(sqlite+mysql 双后端同款)。② `#3250: configured cap` 断言的 CAP_SLACK=4KiB(一页余量)预算不住 #4390 新增的每行 WAL 体积(identity 列 64hex + idx_messages_identity 索引项,~192 行超 12 页/47,376B)——截断不变量本身完好,坏的是页级余量的算术;custom-cap 用例改用 64KiB 的 inter-reset 累积预算(注释引用 #4390 行体积),close-converges 等其余断言维持一页余量不变。 - **增量车道:拉取回执修复 ack 传输盲区 + 漂移回退取富侧(#4352/#4323)**。2026-10-22 事故双根因修复:① pi 的 after_provider_response 只在部分 provider 传输下暴露响应头,x-bili-applied-seq 丢失时 noteIncrementalAck 永不执行,后续每轮都整包 re-seed(sinceSeq=0 重发 21K 条历史)。现在 GET /__bili/plugin/turn?conversation= 返回热 canonical 的 {appliedSeq, boundary}(boundary=末条 wire 消息哈希,冷重建诚实返回 null),pi 扩展在规划下一轮前拉取回执,仅在 boundary 证明 canonical 吸收了我方末条消息时采纳——绝不按 seq 前进盲目采纳(外来的全量重发也会推进 canonical);② #4323 多实例写同一目录时,漂移回退无条件吃旧信封,把索引里更新的折叠世代(26,971 refs/16 active)换成旧世代(11,849/12),覆盖与 canonical 失配→0 可压区间→preflight 8.29M 假性死局 502。现在漂移两侧按 mapped ref 数取富侧(平局信封赢,平局语义=原子整文件)。新增 tests/incremental-ack-pull.test.ts 3 例(响应头 ack 回归钉/拉取采纳/冷 boundary null 拒采纳)+ storage-full-flip 4b(陈旧信封让位新索引世代)。 ### Changes diff --git a/package-lock.json b/package-lock.json index eaafa7f83b..50e080dc29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "billion-context", - "version": "0.1.206", + "version": "0.1.207", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "billion-context", - "version": "0.1.206", + "version": "0.1.207", "license": "MIT", "dependencies": { "mysql2": "^3.15.3", diff --git a/package.json b/package.json index 1e6ea3d534..0c3a03784f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "billion-context", - "version": "0.1.206", + "version": "0.1.207", "description": "A context-compression plugin \u2014 small context windows (a 100K context is enough), 5x fewer tokens, month-long single sessions (billions of tokens), and compression quality \u2014 for all agents: pi, OpenCode, Codex, Claude Code, and more. billion-context is all you need.", "type": "module", "main": "dist/index.js", diff --git a/release-notes/package.json b/release-notes/package.json index b7daf583f0..d8b72b55e1 100644 --- a/release-notes/package.json +++ b/release-notes/package.json @@ -9,8 +9,14 @@ }, "billionContextReleaseNotes": { "schema": 1, - "updated": "2026-10-21", + "updated": "2026-10-24", "releases": [ + { + "version": "0.1.207", + "date": "2026-10-24", + "tier": "recommended", + "summary": "存储可靠性修复(0.1.203–0.1.204 用户请直接升到本版):折叠上下文不再因模型/窗口切换悄悄膨胀直至会话超窗;修复图片会话媒体引用泄漏与存储校验误报(#4390/#4506/#4444)。EN: storage reliability fixes — folded context no longer silently regrows after model/window switches; media-ref leak and false digest drift fixed. v0.1.203–204 users upgrade immediately." + }, { "version": "0.1.203", "date": "2026-10-21", diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index 5fbc417342..9c7fd663b1 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -1202,22 +1202,99 @@ const dshForkAdopter = createForkAdopter((line) => { persistClientEvent(`bili-native-dsh: ${line}`); }); -/** A seeded fork child of the current initiator, or undefined. The gate is - * header.isSeeded === true (set only by dsh's fork seed path): subagent - * sessions carry parentSession WITHOUT isSeeded — they run fresh task - * contexts, never replay the parent history, and must NOT inherit. */ -function forkChildOf(ctx: PluginContext): { sid: string; parent: string } | undefined { - let session: AgentLike["session"]; +// #4215/C10a: structured diagnostics for every skipped fork-adoption entry +// gate. The #4215 incident (seeded child with the correct parentSession, ZERO +// adoption trace in the whole log) showed that silent early returns make the +// fired predicate undiagnosable after the fact: this path had eight ways out +// and none of them wrote anything, so "not a fork" vs "a fork we could not +// see" was unanswerable from the log. The verdict names every shape; the +// caller warns once per session for seeded candidates blocked at a gate and +// notes the ambient-blind shapes ONCE per process (they fire on every +// agentless dispatch — title lanes, disposed scopes — so per-request lines +// would be pure noise). Plain non-fork traffic stays completely silent. +export type ForkChildSkipReason = + | "initiator-threw" + | "initiator-missing" + | "session-missing" + | "sid-missing" + | "header-missing" + | "not-seeded" + | "parent-missing" + | "parent-is-self"; + +export type ForkChildVerdict = + | { ok: true; sid: string; parent: string } + | { ok: false; reason: ForkChildSkipReason; sid?: string; detail?: string }; + +/** A seeded fork child of the current initiator, or the named reason it is + * not (or cannot be determined). The fork gate is header.isSeeded === true + * (set only by dsh's fork seed path): subagent sessions carry parentSession + * WITHOUT isSeeded — they run fresh task contexts, never replay the parent + * history, and must NOT inherit ("not-seeded" is intentional and stays + * silent — it is the documented majority of all traffic). */ +export function forkChildVerdict(ctx: PluginContext): ForkChildVerdict { + let init: AgentLike | undefined; try { - session = ctx.agents?.currentInitiator?.()?.session; - } catch { - return undefined; + init = ctx.agents?.currentInitiator?.(); + } catch (err) { + return { ok: false, reason: "initiator-threw", detail: err instanceof Error ? err.message : String(err) }; } - const sid = typeof session?.id === "string" && session.id.length > 0 ? session.id : undefined; - if (sid === undefined || session?.header === undefined) return undefined; + if (init === undefined || init === null) return { ok: false, reason: "initiator-missing" }; + const session = init.session; + if (session === undefined || session === null) return { ok: false, reason: "session-missing" }; + const sid = typeof session.id === "string" && session.id.length > 0 ? session.id : undefined; + if (sid === undefined) return { ok: false, reason: "sid-missing" }; + if (session.header === undefined) return { ok: false, reason: "header-missing", sid }; const parent = session.header.parentSession; - if (session.header.isSeeded !== true || typeof parent !== "string" || parent.length === 0 || parent === sid) return undefined; - return { sid, parent }; + if (session.header.isSeeded !== true) return { ok: false, reason: "not-seeded", sid }; + if (typeof parent !== "string" || parent.length === 0) return { ok: false, reason: "parent-missing", sid }; + if (parent === sid) return { ok: false, reason: "parent-is-self", sid }; + return { ok: true, sid, parent }; +} + +const forkGateNoted = new Set(); +let noticedForkBlindSpot = false; + +function noteForkGateOnce(key: string, level: "info" | "warn", line: string): void { + if (forkGateNoted.has(key)) return; + forkGateNoted.add(key); + logHostEvent(level, "[dsh-client]", line); +} + +function noteForkVerdict(verdict: Extract): void { + switch (verdict.reason) { + case "not-seeded": + return; // intentional non-inheritance — see forkChildVerdict + case "parent-missing": + noteForkGateOnce(`lineage:${verdict.sid}`, "warn", + `bili-native-dsh: fork adoption for ${verdict.sid} skipped — the session header says isSeeded but carries no usable parentSession; the child starts without inherited compression state (#4215)`); + return; + case "parent-is-self": + noteForkGateOnce(`lineage:${verdict.sid}`, "warn", + `bili-native-dsh: fork adoption for ${verdict.sid} skipped — the session header carries its own id as parentSession; the child starts without inherited compression state (#4215)`); + return; + default: { + // Blind shapes: the ambient read could not determine whether this + // dispatch was a seeded fork child at all. Once per process — these + // fire on EVERY agentless dispatch, so per-request lines would be + // noise. The scoped-attribution state is part of the message: with + // #4082 active the sid rides the request object while fork + // detection still reads only the ambient initiator — that split is + // exactly what #4215 needs named in the log (C10b). + if (noticedForkBlindSpot) return; + noticedForkBlindSpot = true; + const scoped = sidScope.getStore(); + const scopedActive = typeof scoped === "string" && scoped.length > 0; + const shape = + verdict.reason === "initiator-threw" ? `threw (${verdict.detail ?? "unknown"})` + : verdict.reason === "initiator-missing" ? "returned no agent" + : verdict.reason === "session-missing" ? "the agent has no session" + : verdict.reason === "sid-missing" ? "the session has no usable id" + : "the session has no epoch header"; + logHostEvent("info", "[dsh-client]", + `bili-native-dsh: fork discovery blind on this dispatch — ambient currentInitiator ${shape}${scopedActive ? "; scoped llm/stream attribution WAS active but fork detection does not use it yet (#4082)" : ""}; a seeded fork child dispatched this way leaves NO adoption trace (#4215)`); + } + } } function bodyJsonOf(init: RequestInit | undefined): unknown { @@ -1241,14 +1318,33 @@ function bodyJsonOf(init: RequestInit | undefined): unknown { async function maybeForkAdoptBeforeSend(ctx: PluginContext, init: RequestInit | undefined): Promise { - if (!register.toolsReady) return; - const child = forkChildOf(ctx); - if (child === undefined) return; + const child = forkChildVerdict(ctx); + if (!child.ok) { + noteForkVerdict(child); + return; + } + if (!register.toolsReady) { + noteForkGateOnce(`gate:tools-ready:${child.sid}`, "warn", + `bili-native-dsh: fork adoption for ${child.sid} skipped at the tools-ready gate — this request arrived before bili tool registration completed; later requests can still adopt (#4215)`); + return; + } const body = bodyJsonOf(init); - if (body === undefined) return; - if (sideShapedBody(body)) return; + if (body === undefined) { + noteForkGateOnce(`gate:body-parse:${child.sid}`, "warn", + `bili-native-dsh: fork adoption for ${child.sid} skipped at the body-parse gate — the request body was not parseable JSON, so the replay cannot be projected (#4215)`); + return; + } + if (sideShapedBody(body)) { + noteForkGateOnce(`gate:side-shaped:${child.sid}`, "info", + `bili-native-dsh: fork adoption for ${child.sid} deferred for a side-shaped request (no tools, budget <= 200) — the adoption window stays open for the main request (#4215)`); + return; + } const base = register.base; - if (base === undefined) return; + if (base === undefined) { + noteForkGateOnce(`gate:proxy-base:${child.sid}`, "warn", + `bili-native-dsh: fork adoption for ${child.sid} skipped at the proxy-base gate — the proxy origin is not known yet, so the fork protocol cannot be reached (#4215)`); + return; + } await dshForkAdopter.maybeAdopt({ base, parent: child.parent, child: child.sid, body }); } @@ -2417,8 +2513,9 @@ export function apply(ctx: PluginContext): void { } /** Test hook: reset the module-level registration lifecycle so suites can - * drive apply() repeatedly with a fresh mock ctx. */ -export function _resetRegisterForTest(base: string | undefined): void { + * drive apply() repeatedly with a fresh mock ctx. (#4215: optional + * toolsReady so gate scenarios can arm the registration state directly.) */ +export function _resetRegisterForTest(base: string | undefined, toolsReady = false): void { // #3616: invalidate every in-flight recovery chain from earlier scenarios — // their late landings must not write this scenario's register/state. originGen += 1; @@ -2433,7 +2530,7 @@ export function _resetRegisterForTest(base: string | undefined): void { state.respawn = undefined; state.onGiveUp = undefined; register.base = base; - register.toolsReady = false; + register.toolsReady = toolsReady; register.dead = false; register.retryAt = 0; register.pending = undefined; @@ -2454,6 +2551,18 @@ export function maybeRetryForTest(ctx: PluginContext): void { maybeRetry(ctx); } +/** Test hook (#4215): drive the fork-adoption entry gate directly without + * arming apply() (the fetch patch is not installed under NODE_TEST_CONTEXT). */ +export function _maybeForkAdoptBeforeSendForTest(ctx: PluginContext, init: RequestInit | undefined): Promise { + return maybeForkAdoptBeforeSend(ctx, init); +} + +/** Test hook (#4215): reset the fork-gate diagnostics throttle state. */ +export function _resetForkGateDiagnosticsForTest(): void { + forkGateNoted.clear(); + noticedForkBlindSpot = false; +} + /** Test hook (#2604): reset the host-title channel (dedup map + resolved * services) so scenarios start un-armed. */ export function _resetTitlesForTest(): void { diff --git a/src/prefix-affinity.ts b/src/prefix-affinity.ts index e7987445de..83bcb55709 100644 --- a/src/prefix-affinity.ts +++ b/src/prefix-affinity.ts @@ -520,6 +520,85 @@ function containsRun(haystack: string[], needle: string[]): boolean { return false; } +/** #4264: one judged candidate inside a ResumeMissReport. */ +interface ResumeMissCandidate { + sessionId: string; + /** Stored chain depth (neutral turns for hv2, raw messages for hv1). */ + chainDepth: number; + /** Incoming depth in the candidate's hash domain. */ + incomingDepth: number; + /** 2 = wire-neutral domain, 1 = pre-upgrade raw-wire domain. */ + hashDomain: 1 | 2; + /** Why this candidate was not accepted as a resume parent — the same + * gates findResumeParent applies, reported: "equal-depth" is a + * byte-exact replay at exactly the stored depth under another id (a + * duplicate conversation, rejected by design #1486); "prefix-break" is + * a deeper incoming history whose head-anchored progressive hash at + * the stored depth differs (the shared prefix broke somewhere); + * "truncated" is an incoming history SHORTER than the stored chain + * (cannot be a resume — a resume adds at least one message). */ + kind: "equal-depth" | "prefix-break" | "truncated"; + /** First cell index whose per-item hash differs from the stored chain, + * when verifiable from the stored trailing window. null: the whole + * inspectable overlap matched (the break sits before windowStart) or + * no overlap exists at all. */ + firstDivergence: number | null; + /** First stored position inspectable via the trailing itemHashes window + * (0 for chains ≤ MAX_STORED_ITEMS deep). */ + windowStart: number; + /** true when the inspectable overlap is non-empty and every overlapped + * cell matches byte-for-byte. */ + tailWindowIntact: boolean; + /** 8-hex fragments of the diverging cells (log correlation only). */ + incomingCell?: string; + storedCell?: string; +} + +/** #4264: diagnostics-only report for a resume-inheritance miss. */ +interface ResumeMissReport { + /** Tracked chains that passed the pre-screen (depth ≥ + * MIN_RESUME_PREFIX, id ≠ self, in-domain floor) and were actually + * judged. 0 with no declaredParentState = nothing eligible to begin + * with (parent chain absent — LRU eviction or never noted). */ + eligible: number; + /** Scoped match only: state of the declared parent's own chain when it + * could not even be judged. */ + declaredParentState?: "untracked" | "too-shallow"; + /** Best near-miss candidate (closest by kind rank, then deepest chain, + * then most recently seen), when any eligible candidate existed. */ + best?: ResumeMissCandidate; +} + +/** #4264: render a ResumeMissReport as the bounded log-line body. Pure + * formatting — the caller picks the level (info when report.best exists, + * debug otherwise). */ +export function describeResumeMiss(report: ResumeMissReport): string { + const b = report.best; + if (!b) { + const scope = report.declaredParentState !== undefined ? `; declared parent chain ${report.declaredParentState}` : ""; + return `no eligible tracked chain (floor ${MIN_RESUME_PREFIX}${scope})`; + } + switch (b.kind) { + case "equal-depth": + return `no byte-exact resume among ${report.eligible} eligible chain(s); closest ${b.sessionId} is a byte-exact EQUAL-DEPTH replay (${b.chainDepth} msgs == incoming ${b.incomingDepth}) — duplicate conversation, not a resume by design`; + case "truncated": + return `no byte-exact resume among ${report.eligible} eligible chain(s); closest ${b.sessionId} (chain ${b.chainDepth}) is LONGER than the incoming history (${b.incomingDepth} msgs)${b.tailWindowIntact ? `, visible cells ${b.windowStart}..${b.incomingDepth - 1} intact — truncated replay` : ""}`; + case "prefix-break": { + let detail: string; + if (b.firstDivergence !== null) { + detail = b.windowStart > 0 + ? `first divergence found at cell ${b.firstDivergence} (stored ${b.storedCell} vs incoming ${b.incomingCell}; cells 0..${b.windowStart - 1} uninspectable — chain deeper than the ${MAX_STORED_ITEMS}-cell window)` + : `first divergence at cell ${b.firstDivergence} (stored ${b.storedCell} vs incoming ${b.incomingCell})`; + } else if (b.tailWindowIntact) { + detail = `tail window (cells ${b.windowStart}..${Math.min(b.chainDepth, b.incomingDepth) - 1}) byte-intact — break sits BEFORE cell ${b.windowStart}`; + } else { + detail = "head-anchored prefix broken (no inspectable overlap)"; + } + return `no byte-exact resume among ${report.eligible} eligible chain(s); closest ${b.sessionId} (chain ${b.chainDepth}, incoming ${b.incomingDepth}): ${detail}`; + } + } +} + export class PrefixAffinityResolver { private trackedChains = new Map(); @@ -990,6 +1069,91 @@ export class PrefixAffinityResolver { return { sessionId: parentId, sharedDepth: entry.depth }; } + /** #4264: diagnostics-only explanation of a resume-inheritance MISS + * (the gate passed but findResumeParent/findResumeParentWithin returned + * null). It re-runs the SAME gates in reporting mode and names why the + * closest near-miss was rejected; it never influences matching. The + * three causes need completely different fixes, so they are reported + * distinctly: equal-depth replay (a duplicate conversation by design), + * parent-chain absence (LRU eviction / never noted), head break (host + * rebuild drift desynchronizes the byte-exact prefix). Cost-bounded: + * the candidate scan does only O(1) progressive-hash checks per entry; + * the per-item divergence scan runs ONCE on the single best near-miss, + * over its stored trailing window (≤ MAX_STORED_ITEMS compares). */ + diagnoseResumeMiss(messages: unknown[], selfSessionId: string, declaredParent?: string): ResumeMissReport { + const msgs = normalizeAffinityMessages(messages); + if (!hasUserMessage(msgs)) return { eligible: 0 }; + // #2791: judge each candidate in the hash domain it was noted in — + // same dual-domain discipline as findResumeParent. #2824: hashes-only + // consumers pay nothing for sketches; itemHashes come from the same + // fused pass at no extra cost. + const turns = neutralTurns(messages); + const fp2 = neutralFingerprints(turns, 0); + const hashesV2 = fp2.hashes; + const hashesV1 = chainHashes(msgs); + if (hashesV2.length === 0 && hashesV1.length === 0) return { eligible: 0 }; + type Candidate = { id: string; entry: ChainEntry; kind: ResumeMissCandidate["kind"]; rank: number }; + const cands: Candidate[] = []; + let declaredParentState: "untracked" | "too-shallow" | undefined; + const consider = (id: string, entry: ChainEntry): void => { + if (id === selfSessionId) return; + if (entry.depth < MIN_RESUME_PREFIX) return; + const hv2 = entry.hv === 2; + const hashes = hv2 ? hashesV2 : hashesV1; + const len = hv2 ? turns.length : msgs.length; + if (hashes.length < MIN_RESUME_PREFIX) return; + let kind: ResumeMissCandidate["kind"]; + if (len < entry.depth) kind = "truncated"; + else if (len === entry.depth && hashes[entry.depth - 1] === entry.tailHash) kind = "equal-depth"; + else kind = "prefix-break"; + cands.push({ id, entry, kind, rank: kind === "equal-depth" ? 0 : kind === "prefix-break" ? 1 : 2 }); + }; + if (declaredParent !== undefined && declaredParent !== selfSessionId) { + const entry = this.trackedChains.get(declaredParent); + if (entry === undefined) declaredParentState = "untracked"; + else if (entry.depth < MIN_RESUME_PREFIX) declaredParentState = "too-shallow"; + else consider(declaredParent, entry); + } else if (declaredParent === undefined) { + for (const [id, entry] of this.trackedChains) consider(id, entry); + } + if (cands.length === 0) return { eligible: 0, ...(declaredParentState !== undefined ? { declaredParentState } : {}) }; + cands.sort((a, b) => a.rank - b.rank || b.entry.depth - a.entry.depth || b.entry.lastSeen - a.entry.lastSeen); + const top = cands[0]!; + const hv2 = top.entry.hv === 2; + const depth = top.entry.depth; + const len = hv2 ? turns.length : msgs.length; + // Lazy v1 per-item pass: only paid when the winner is a legacy raw-wire entry. + const itemIncoming = hv2 ? fp2.itemHashes : perItemHashes(msgs); + const windowStart = Math.max(0, depth - top.entry.itemHashes.length); + const end = Math.min(depth, len); + let firstDivergence: number | null = null; + let tailWindowIntact = false; + if (windowStart < end) { + tailWindowIntact = true; + for (let i = windowStart; i < end; i++) { + if (itemIncoming[i] !== top.entry.itemHashes[i - windowStart]) { + firstDivergence = i; + tailWindowIntact = false; + break; + } + } + } + const best: ResumeMissCandidate = { + sessionId: top.id, + chainDepth: depth, + incomingDepth: len, + hashDomain: hv2 ? 2 : 1, + kind: top.kind, + firstDivergence, + windowStart, + tailWindowIntact, + ...(firstDivergence !== null + ? { incomingCell: itemIncoming[firstDivergence]?.slice(0, 8), storedCell: top.entry.itemHashes[firstDivergence - windowStart]?.slice(0, 8) } + : {}), + }; + return { eligible: cands.length, ...(declaredParentState !== undefined ? { declaredParentState } : {}), best }; + } + /** #2241: does `messages` continue the tracked chain of `sessionId`? * True when the session's stored chain is a byte-exact head-anchored * prefix of the incoming list — the progressive hash at the stored diff --git a/src/server/handle.ts b/src/server/handle.ts index a1d5f4bcb6..bd9874edb6 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -34,7 +34,7 @@ import { emitPreflightError } from "../stream-error.js"; import { agentProviderRecipes } from "../agent-providers.js"; import { affinityToken, conversationHeaderSource, claudeSubagentAgentId, claudeSubagentSplit, clientConversationHeader, codexTurnIdentity, dshPersonaFingerprintApplies, instructionsFingerprintApplies, openaiSystemTextForPersona, preferPromptCacheKeyIdentity } from "../session-id.js"; import { personaConversationOrder } from "../session-resolve.js"; -import { prefixAffinity, type AnonymousAffinity } from "../prefix-affinity.js"; +import { prefixAffinity, describeResumeMiss, type AnonymousAffinity } from "../prefix-affinity.js"; import { buildAnonymousProvenance, type AnonymousProvenance } from "../anon-provenance.js"; import { maybeAdoptForkBlocks, maybeAdoptResume, maybeAdoptDerivedFork } from "../fork-adoption.js"; import { executeThreadsReforge } from "./threads-lane.js"; @@ -1579,10 +1579,22 @@ async function handleRequest( } } if (!resume) { + const affinityList = affinityMessageList(); const matched = derivedParent !== undefined - ? prefixAffinity.findResumeParentWithin(derivedParent, affinityMessageList(), sessionId) - : prefixAffinity.findResumeParent(affinityMessageList(), sessionId); - if (matched) resume = { sessionId: matched.sessionId, sharedDepth: matched.sharedDepth, via: `${matched.sharedDepth} msg(s) byte-exact prefix` }; + ? prefixAffinity.findResumeParentWithin(derivedParent, affinityList, sessionId) + : prefixAffinity.findResumeParent(affinityList, sessionId); + if (matched) { + resume = { sessionId: matched.sessionId, sharedDepth: matched.sharedDepth, via: `${matched.sharedDepth} msg(s) byte-exact prefix` }; + } else { + // #4264: a miss must not be silent — equal-depth replay, + // parent-chain absence and head break need completely + // different fixes, so name the closest near-miss. + // Diagnostics only: matching above is untouched. + const miss = derivedParent !== undefined + ? prefixAffinity.diagnoseResumeMiss(affinityList, sessionId, derivedParent) + : prefixAffinity.diagnoseResumeMiss(affinityList, sessionId); + log(miss.best !== undefined ? "info" : "debug", `[${sessionId}] [resume-inheritance] miss: ${describeResumeMiss(miss)} (#4264)`); + } } if (resume) { const resumeParent = loadParent(resume.sessionId); @@ -1610,6 +1622,18 @@ async function handleRequest( log("info", `[${sessionId}] [resume-inheritance] matched tracked chain ${resume.sessionId} but the parent session is not loadable — starting fresh (#1486)`); } } + } else if (clientProvided && session.stats.requests === 0) { + // #4264: a SKIPPED gate must not be silent either — without this + // line, zero [resume-inheritance] output stays ambiguous between + // "gate skipped" and "pre-#4265 build / silent miss". First + // request only: stats.requests increments later in the prepare + // lanes, so this fires at most once per session. anonAffinity is + // structurally null here (only the anonymous lane sets it), + // leaving exactly three skip reasons. + const skipReason = session.metadata.derivedFromSessionId !== undefined ? "derivedFromSessionId" + : session.metadata.publicForkReceipt !== undefined ? "publicForkReceipt" + : "resumeInheritance=false"; + log("debug", `[${sessionId}] [resume-inheritance] skipped: ${skipReason} (#4264)`); } // [#1333] explicitly derived conversations (pi RLM child, omp fork, // opencode subagent: the plugin reported its parent at register) diff --git a/src/server/snapshot-render.ts b/src/server/snapshot-render.ts index ed8f4c85b4..de1cd2baae 100644 --- a/src/server/snapshot-render.ts +++ b/src/server/snapshot-render.ts @@ -60,6 +60,8 @@ const MAX_CHAIN_WALK = 8192; * MIN_RESYNC_RUN in fold-reconcile (there 3, because reconcile matches on * position alone; here the second consecutive match must ALSO line up with * the view, which a stray duplicate cannot provide). */ +import { sha256Hex } from "../storage/store.js"; + const MIN_SEAM_RUN = 2; /** #3658 canonical identity signature minus the hash: the tuple the kernel @@ -69,10 +71,19 @@ const MIN_SEAM_RUN = 2; * same bytes ⇒ same length; different images practically always differ). */ function identitySig(m: BiliMessage): string { const text = typeof m.text === "string" ? m.text : ""; - const img = m.imagePart === true ? `|img:${(m.imageBase64 ?? "").length}` : ""; + // #4390: hash the image BYTES, never the length — same-size screenshots + // used to collide and chain a false seam (silent truncation). The walk + // only touches the tail region, so per-request hashing is bounded; a + // 24-hex prefix is far beyond collision reach for seam evidence. + const img = m.imagePart === true ? `|img:${sha256Hex(m.imageBase64 ?? "").slice(0, 24)}` : ""; return `${m.role}|${m.contentType}|${m.toolName ?? ""}|${text}${img}`; } +/** Test seam (#4390): the seam fingerprint under test — see identitySig. */ +export function identitySigForTest(m: BiliMessage): string { + return identitySig(m); +} + interface ViewRow { seq: number; msg: BiliMessage; diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index eda69b2677..4d8b449e7e 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -21,10 +21,12 @@ import { createHash } from "node:crypto"; * provenance (#2671), but its PRESENCE is new (corpus pins predate it), * so masking the value like __VER__ cannot keep old pins green; the key * is removed from digests altogether. - * - * NOT normalized (deliberately contract): every other byte — ref numbers, - * block ids, summaries, stats, metadata. A change to any of those in the - * round-trip is a migration regression and must fail the golden. + * - tailGaps (#4390) → dropped entirely — read-time diagnostic added by + * loadLegacyView (count of tail pointers whose row is missing); the live + * legacy leg of any parity comparison structurally lacks the key, so + * digesting its presence would report drift on every sampled save. Same + * class as storageMode: the PRESENCE, not the value, is the problem. A + * genuinely missing row still diverges the message arrays themselves. */ export function canonicalize(value: unknown): unknown { return canonicalizeWithContext(value, undefined); @@ -42,6 +44,11 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno // unlike biliVersion its PRESENCE is new, so masking the value // is not enough; drop the key from digests entirely. if (k === "storageMode") continue; + // tailGaps (#4390) is read-time gap-count metadata from + // loadLegacyView — the live legacy leg structurally lacks the + // key, so digesting its presence would flag drift on every + // sampled save; drop it like storageMode. + if (k === "tailGaps") continue; out[k] = canonicalizeWithContext(v, k); } return out; diff --git a/src/storage/engine-bootstrap.ts b/src/storage/engine-bootstrap.ts index e056b7a0d3..3f7611ad64 100644 --- a/src/storage/engine-bootstrap.ts +++ b/src/storage/engine-bootstrap.ts @@ -353,14 +353,14 @@ async function runBootstrap(opts: EnsureSqliteEngineOptions): Promise;base64, with the payload swapped for its CAS hash. */ +const MEDIA_URL_MARKER_RE = new RegExp( + `data:([a-zA-Z0-9.+\\/-]+);base64,([A-Za-z0-9+/=]{${MIN_MEDIA_URL_SPLIT_BYTES},})`, + "g", +); + +export function splitMediaContent(content: string): { content: string; hash: string; media: { hash: string; bytes: string }[] } { + let media: { hash: string; bytes: string }[] = []; + let split = content; + if (content.includes("\"imageBase64\"")) { + let parsed: { imageBase64?: unknown }; + try { + parsed = JSON.parse(content) as { imageBase64?: unknown }; + } catch { + parsed = undefined as unknown as { imageBase64?: unknown }; + } + if (parsed) { + const bytes = parsed.imageBase64; + if (typeof bytes === "string" && bytes.length > 0) { + const mediaHash = sha256Hex(bytes); + parsed.imageBase64 = { $media: mediaHash }; + split = JSON.stringify(parsed); + media.push({ hash: mediaHash, bytes }); + } + } + } + // #4069: raw wire sidecars (rawOpenaiContent / rawOpenaiContentParts / + // rawResponsesItem / rawGoogleParts, …) carry the ORIGINAL part verbatim — + // for image parts that means the same base64 payload a second time, as a + // data URL. Swap every sufficiently large payload for its CAS hash, mime + // prefix and JSON structure untouched, so hydration is byte-identical. + if (split.includes(";base64,")) { + const byHash = new Map(media.map((m) => [m.hash, m.bytes])); + split = split.replace(MEDIA_URL_MARKER_RE, (_all: string, mime: string, payload: string) => { + const mediaHash = sha256Hex(payload); + if (!byHash.has(mediaHash)) { + byHash.set(mediaHash, payload); + media = [...media, { hash: mediaHash, bytes: payload }]; + } + return `data:${mime};base64,{$media:${mediaHash}}`; + }); + } + return { content: split, hash: sha256Hex(split), media }; +} + +/** Strip the position/identity metadata from a serialized CoreMessage. + * JSON.parse preserves key insertion order, so the stripped serialization + * is deterministic for the same object shape. */ +function identityInputOf(raw: string): string { + let parsed: { id?: unknown; wireIndex?: unknown }; + try { + parsed = JSON.parse(raw) as { id?: unknown; wireIndex?: unknown }; + } catch { + return raw; + } + if (parsed.id === undefined && parsed.wireIndex === undefined) return raw; + delete parsed.id; + delete parsed.wireIndex; + return JSON.stringify(parsed); +} + +/** Identity from a STORED content column (already media-split; may still + * carry id/wireIndex from pre-#4390 bytes). Strip and media-split + * commute, so this equals the identity the live intern pass computes for + * the same message — the migration backfill and the direct-caller + * fallbacks in store.ts rely on that equivalence. */ +export function identityOfStoredContent(content: string): string { + return sha256Hex(identityInputOf(content)); +} diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index d9470406b5..46eb1b07fd 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -29,6 +29,7 @@ import type { CoreMessage, CompressionBlock } from "../../kernel/src/types.js"; import type { BlockView } from "../session.js"; import { refToIndex } from "../../kernel/src/refs.js"; import { sha256Hex, type BlockRow, type CcrEntryRow, type IngestSessionInput, type MessageRow, type RefRow, type SessionRow } from "./store.js"; +import { splitMediaContent, identityOfStoredContent } from "./identity.js"; /** * Field order of the hydration merge output — DERIVED from mergeState itself @@ -87,61 +88,12 @@ export const ingestSerializations = { count: 0 }; * reader (getMessages hydrates before returning) sees the verbatim original. * No media (or empty) → passthrough with the plain content hash. */ -/** - * Media payloads below this size ride inline even inside raw sidecars — - * splitting them costs a CAS row + a join for a few hundred bytes (#4069). - */ -const MIN_MEDIA_URL_SPLIT_BYTES = 2048; -/** data:;base64, with the payload swapped for its CAS hash. */ -const MEDIA_URL_MARKER_RE = new RegExp( - `data:([a-zA-Z0-9.+\\/-]+);base64,([A-Za-z0-9+/=]{${MIN_MEDIA_URL_SPLIT_BYTES},})`, - "g", -); - -function splitMediaContent(content: string): { content: string; hash: string; media: { hash: string; bytes: string }[] } { - let media: { hash: string; bytes: string }[] = []; - let split = content; - if (content.includes("\"imageBase64\"")) { - let parsed: { imageBase64?: unknown }; - try { - parsed = JSON.parse(content) as { imageBase64?: unknown }; - } catch { - parsed = undefined as unknown as { imageBase64?: unknown }; - } - if (parsed) { - const bytes = parsed.imageBase64; - if (typeof bytes === "string" && bytes.length > 0) { - const mediaHash = sha256Hex(bytes); - parsed.imageBase64 = { $media: mediaHash }; - split = JSON.stringify(parsed); - media.push({ hash: mediaHash, bytes }); - } - } - } - // #4069: raw wire sidecars (rawOpenaiContent / rawOpenaiContentParts / - // rawResponsesItem / rawGoogleParts, …) carry the ORIGINAL part verbatim — - // for image parts that means the same base64 payload a second time, as a - // data URL. Swap every sufficiently large payload for its CAS hash, mime - // prefix and JSON structure untouched, so hydration is byte-identical. - if (split.includes(";base64,")) { - const byHash = new Map(media.map((m) => [m.hash, m.bytes])); - split = split.replace(MEDIA_URL_MARKER_RE, (_all: string, mime: string, payload: string) => { - const mediaHash = sha256Hex(payload); - if (!byHash.has(mediaHash)) { - byHash.set(mediaHash, payload); - media = [...media, { hash: mediaHash, bytes: payload }]; - } - return `data:${mime};base64,{$media:${mediaHash}}`; - }); - } - return { content: split, hash: sha256Hex(split), media }; -} - interface LogEntry { seq: number; msgId: string; content: string; hash: string; + identity: string; role: string; kind: string; ref: string | null; @@ -200,12 +152,16 @@ export function planIngest(session: LegacySessionLike): IngestPlan { const parsed = JSON.parse(raw) as { role: string; contentType?: string }; const split = splitMediaContent(raw); const entry = { msgId: m.id, content: split.content, hash: split.hash, role: parsed.role, kind: parsed.contentType ?? "message", media: split.media }; - const key = `${m.id}\u0000${entry.hash}`; + // #4390: the cross-save JOIN key is the POSITION-INDEPENDENT identity + // (stored-bytes serialization minus id/wireIndex). content_hash stays + // the byte-faithful CAS key; only the join moves to identity. + const identity = identityOfStoredContent(entry.content); + const key = `${m.id}\u0000${identity}`; const existing = seqByIdentity.get(key); if (existing !== undefined) return existing; const seq = log.length; seqByIdentity.set(key, seq); - log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null, ...(entry.media.length > 0 ? { media: entry.media } : {}) }); + log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, identity, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null, ...(entry.media.length > 0 ? { media: entry.media } : {}) }); return seq; }; const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null; @@ -308,6 +264,7 @@ export function planIngestDelta( msgId: m.id, content: split.content, hash: split.hash, + identity: identityOfStoredContent(split.content), role: parsed.role, kind: parsed.contentType ?? "message", ref: byRawRef[m.id] ?? null, @@ -353,6 +310,7 @@ function finishPlan( kind: e.kind, content: e.content, contentHash: e.hash, + identity: e.identity, ...(e.media && e.media.length > 0 ? { media: e.media } : {}), })); @@ -416,7 +374,12 @@ function finishPlan( const sessionRow: SessionRow = { sessionId: session.id, - parentSessionId: null, + // #4390: fork lineage into the unified row — the envelope's + // metadata.derivedFromSessionId (written by handle.ts at fork time) + // used to be the ONLY carrier; envelope loss orphaned the family. + parentSessionId: typeof session.metadata.derivedFromSessionId === "string" && session.metadata.derivedFromSessionId.length > 0 + ? session.metadata.derivedFromSessionId + : null, forkPointSeq: null, protocol: typeof session.meta.protocol === "string" ? session.meta.protocol : null, label: typeof session.meta.label === "string" ? session.meta.label : null, @@ -500,6 +463,8 @@ export function loadLegacyView( lastMessagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; contentStore?: LegacySessionLike["contentStore"]; + /** #4390: tail/pluginSnapshot pointers whose backing row is missing. */ + tailGaps?: number; } { const s = rows.session; const kstate = JSON.parse(s.kstateJson) as Record; @@ -569,19 +534,27 @@ export function loadLegacyView( } const messagesBySeq = new Map(rows.messages.map((m) => [m.seq, JSON.parse(m.content) as CoreMessage])); - const lastMessages = tail.tailSeqs - ? tail.tailSeqs.map((seq) => messagesBySeq.get(seq)).filter((m): m is CoreMessage => m !== undefined) - : undefined; - const pluginSnapshot = tail.pluginSnapshotSeqs - ? tail.pluginSnapshotSeqs - .map((seq) => messagesBySeq.get(seq)) - .filter((m): m is CoreMessage => m !== undefined) - : undefined; + // #4390: a tail pointer whose row is gone used to be silently filtered + // out — the view shrank with no trace. Count the gaps so callers can + // warn (persist logs them); the surviving members still load. + let tailGaps = 0; + const resolveSeqs = (seqs: number[]): CoreMessage[] => { + const out: CoreMessage[] = []; + for (const seq of seqs) { + const m = messagesBySeq.get(seq); + if (m === undefined) tailGaps++; + else out.push(m); + } + return out; + }; + const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; + const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; const out: ReturnType = { id: s.sessionId, createdAt: s.createdAt, lastSeen: s.updatedAt, + tailGaps, meta: JSON.parse(s.metaJson), stats: JSON.parse(s.statsJson), metadata: JSON.parse(s.metadataJson), diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 6750f46b7e..59c9aa5d25 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -32,6 +32,7 @@ import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync } fr import { join } from "node:path"; import { log as loggerLog } from "../logger.js"; import { sha256Hex } from "./store.js"; +import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; import type { StorageBackend } from "./backend.js"; import type { @@ -105,10 +106,13 @@ CREATE TABLE IF NOT EXISTS messages ( role VARCHAR(64) NOT NULL, kind VARCHAR(64) NOT NULL DEFAULT 'message', content_hash VARCHAR(64) NOT NULL, + identity VARCHAR(64) NOT NULL DEFAULT '', + retired_at BIGINT NULL, PRIMARY KEY (session_id, seq), UNIQUE KEY uq_messages_identity (session_id, msg_id, content_hash), KEY idx_messages_session (session_id, seq), - KEY idx_messages_msg_id (session_id, msg_id) + KEY idx_messages_msg_id (session_id, msg_id), + KEY idx_messages_identity (session_id, msg_id, identity) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS blocks ( @@ -252,6 +256,37 @@ export class MysqlBackend implements StorageBackend { "ON DUPLICATE KEY UPDATE meta_value = meta_value", [String(MYSQL_SCHEMA_VERSION)], ); + // #4390 v8: position-independent identity + historian tombstones. + // CREATE TABLE IF NOT EXISTS does not upgrade an existing v7 + // table — probe information_schema and ALTER idempotently + // (MySQL has no ADD COLUMN IF NOT EXISTS). + const [cols] = (await conn.query( + "SELECT COLUMN_NAME AS col FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'messages'", + )) as unknown as Row[][]; + const have = new Set((cols as unknown as { col?: unknown }[]).map((r) => String(r.col ?? ""))); + if (!have.has("identity")) { + await conn.query("ALTER TABLE messages ADD COLUMN identity VARCHAR(64) NOT NULL DEFAULT ''"); + await conn.query("ALTER TABLE messages ADD INDEX idx_messages_identity (session_id, msg_id, identity)"); + } + if (!have.has("retired_at")) { + await conn.query("ALTER TABLE messages ADD COLUMN retired_at BIGINT NULL"); + } + if (!have.has("identity") || !have.has("retired_at")) { + // Backfill identities from the CAS bytes (old rows keep the + // polluted formula's hashes until rewritten; heal them now). + const [rows] = (await conn.query( + "SELECT m.session_id AS sid, m.seq AS seq, m.content_hash AS content_hash, c.bytes AS body " + + "FROM messages m JOIN content c ON c.hash = m.content_hash WHERE m.identity = ''", + )) as unknown as Row[][]; + for (const r of rows as unknown as Row[]) { + const body = typeof r.body === "string" ? r.body : Buffer.from(r.body as Uint8Array).toString("utf8"); + await conn.query("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?", [ + identityOfStoredContent(body), + str(r.sid), + num(r.seq), + ]); + } + } } finally { conn.release(); } @@ -328,9 +363,11 @@ export class MysqlBackend implements StorageBackend { } for (const m of input.messages) { if (!m.contentHash) return false; + // #4390: identity join; retired rows count as existing + // (rewind-redo falls back to the full path). const [hit] = (await c.query( - "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND content_hash = ? LIMIT 1", - [s.sessionId, m.msgId, m.contentHash], + "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ? LIMIT 1", + [s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content)], )) as unknown as Row[][]; if (hit.length > 0) return false; } @@ -353,22 +390,36 @@ export class MysqlBackend implements StorageBackend { if (opts?.completeLog === true && !(await this.hasLaneExtOn(c, s.sessionId))) { const incomingIds = new Set(); for (const m of messages) { - incomingIds.add(`${m.msgId}\u0000${m.contentHash ?? sha256Hex(m.content)}`); + incomingIds.add(`${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`); } const staleRows = (await c.query( - "SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?", + "SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?", [s.sessionId], )) as unknown as Row[][]; for (const row of staleRows[0]) { - if (incomingIds.has(`${row.msg_id}\u0000${row.content_hash}`)) continue; - await this.releaseContent(c, str(row.content_hash)); - const media = (await c.query( + if (incomingIds.has(`${row.msg_id}\u0000${row.identity}`)) { + // #4390 rewind-redo: reactivate the tombstone in place. + if (row.retired_at !== null && row.retired_at !== undefined) { + await c.query("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?", [ + s.sessionId, + num(row.seq), + ]); + } + continue; + } + // #4390 historian sweep: retire, don't delete — + // TEXT CAS refs stay owned by the retired row; MEDIA + // refs release (#4506 — same rationale as sqlite twin). + await c.query( + "UPDATE messages SET retired_at = ? WHERE session_id = ? AND seq = ? AND retired_at IS NULL", + [Date.now(), s.sessionId, num(row.seq)], + ); + const retiredMedia = (await c.query( "SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?", [s.sessionId, num(row.seq)], )) as unknown as Row[][]; - for (const mm of media[0]) await this.releaseContent(c, str(mm.media_hash)); - await c.query("DELETE FROM messages WHERE session_id = ? AND seq = ?", [s.sessionId, num(row.seq)]); await c.query("DELETE FROM message_media WHERE session_id = ? AND seq = ?", [s.sessionId, num(row.seq)]); + for (const rm of retiredMedia[0]) await this.releaseContent(c, String(rm.media_hash)); } } } @@ -428,11 +479,23 @@ export class MysqlBackend implements StorageBackend { for (const m of messages) { const hash = await this.putContent(c, m.content, m.contentHash); const [res] = (await c.query( - "INSERT IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - [s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash], + "INSERT IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + [s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash, m.identity ?? identityOfStoredContent(m.content)], )) as unknown as [{ affectedRows: number }, unknown]; if (res.affectedRows === 0) { await this.releaseContent(c, hash); + // #4506: tombstone reactivation — restore media mappings + // + refcounts the retire released (idempotent upsert). + if (m.media && m.media.length > 0) { + for (const md of m.media) { + await this.putContent(c, md.bytes, md.hash); + const [mm] = (await c.query( + "INSERT IGNORE INTO message_media (session_id, seq, media_hash) VALUES (?, ?, ?)", + [s.sessionId, m.seq, md.hash], + )) as unknown as [{ affectedRows: number }, unknown]; + if (mm.affectedRows === 0) await this.releaseContent(c, md.hash); + } + } } else if (m.media && m.media.length > 0) { for (const md of m.media) { await this.putContent(c, md.bytes, md.hash); @@ -550,27 +613,52 @@ export class MysqlBackend implements StorageBackend { ): Promise<{ messages: MessageRow[]; watermark: number | null }> { const s = input.session; const [rows] = (await c.query( - "SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?", + "SELECT seq, msg_id, identity, content_hash, retired_at FROM messages WHERE session_id = ?", [s.sessionId], )) as unknown as Row[][]; const seqByIdentity = new Map(); + const hashBySeq = new Map(); + const retiredSeqs: number[] = []; let nextSeq = 0; for (const row of rows) { - seqByIdentity.set(`${row.msg_id}\u0000${row.content_hash}`, num(row.seq)); + // #4390: identity join — position-independent; retired rows keep + // claiming their seq until reactivation clears the tombstone. + if (row.retired_at !== null && row.retired_at !== undefined) retiredSeqs.push(num(row.seq)); + seqByIdentity.set(`${row.msg_id}\u0000${row.identity}`, num(row.seq)); + hashBySeq.set(num(row.seq), str(row.content_hash)); if (num(row.seq) + 1 > nextSeq) nextSeq = num(row.seq) + 1; } const planToAbs = new Map(); + const repairs: { seq: number; from: string; to: string; content: string }[] = []; const messages = input.messages.map((m) => { - const hash = m.contentHash ?? sha256Hex(m.content); - const key = `${m.msgId}\u0000${hash}`; + const key = `${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`; let abs = seqByIdentity.get(key); if (abs === undefined) { abs = nextSeq++; seqByIdentity.set(key, abs); + } else { + // #4390 review: same semantics as the sqlite lane — an + // identity match with changed bytes must adopt the fresher + // bytes at the same seq (stale-wireIndex parity drift and + // CAS-corruption healing), never keep the stored generation. + const fresh = m.contentHash ?? sha256Hex(m.content); + const stored = hashBySeq.get(abs); + if (stored !== undefined && stored !== fresh) repairs.push({ seq: abs, from: stored, to: fresh, content: m.content }); } planToAbs.set(m.seq, abs); return { ...m, seq: abs }; }); + for (const r of repairs) { + await this.putContent(c, r.content, r.to); + await c.query("UPDATE messages SET content_hash = ? WHERE session_id = ? AND seq = ?", [r.to, s.sessionId, r.seq]); + await this.releaseContent(c, r.from); + } + // #4390 rewind-redo: reactivate in place (no duplicate row). + for (const seq of retiredSeqs) { + if (messages.some((m) => m.seq === seq)) { + await c.query("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?", [s.sessionId, seq]); + } + } const remapSeqs = (seqs: number[] | null | undefined): number[] | null | undefined => seqs === null || seqs === undefined ? seqs : seqs.map((p) => planToAbs.get(p) ?? p); const pluginSnapshotSeqs = remapSeqs(input.pluginSnapshotSeqs); @@ -806,12 +894,15 @@ export class MysqlBackend implements StorageBackend { }; } - async getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { + async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + // #4390: retired rows stay out of the replay by default. const [rows] = (await this.pool.query( - "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq", + opts?.includeRetired === true + ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" + : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", [sessionId], )) as unknown as Row[][]; - const out: (MessageRow & { contentHash: string })[] = []; + const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = []; for (const r of rows) { const contentHash = str(r.content_hash); out.push({ @@ -822,6 +913,7 @@ export class MysqlBackend implements StorageBackend { role: str(r.role), kind: str(r.kind), contentHash, + retiredAt: nullableNum(r.retired_at), content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? ""), }); } @@ -1066,8 +1158,8 @@ export class MysqlBackend implements StorageBackend { for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); for (const m of dump.messages) { await c.query( - "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash], + "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, m.identity ?? ""], ); } for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) { diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index 6a1253486c..f2c0aee365 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -174,7 +174,10 @@ function degradeReason(): string { // sidecar open ALSO fails with the generic no-engine error — surface the real // reason first, or the sidecar-open-error branch below would shadow it (#3894). if (state.bootstrap?.status === "unsupported-node") { - return `no self-downloaded SQLite engine supports this Node.js (${state.bootstrap.detail ?? "unknown"})`; + // #4444: the specific coverage detail outranks the generic no-engine reason + // (#3894), but must still carry the Node.js upgrade lever the sidecar-open + // branch below carries — an unsupported Node is exactly where it's actionable. + return `no self-downloaded SQLite engine supports this Node.js (${state.bootstrap.detail ?? "unknown"})${nodeSqliteUpgradeHint()}`; } if (state.sidecarOpenError) { const e = shortenError(state.sidecarOpenError); diff --git a/src/storage/schema.ts b/src/storage/schema.ts index a354971ca2..6f253e7c36 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -63,7 +63,7 @@ // index. Both directions compatible: an older build opening a v5 // file no-ops at the version gate (extra column ignored); a newer // build upgrades a v4 file with one idempotent ALTER TABLE. -const STORAGE_SCHEMA_VERSION = 7; +const STORAGE_SCHEMA_VERSION = 8; /** #3250: cap for `journal_size_limit` (see STORAGE_PRAGMAS). Passive and * auto checkpoints move frames into the main db and reset the write offset @@ -97,6 +97,8 @@ export function storagePragmas(walJournalSizeLimitBytes: number): string[] { export const STORAGE_PRAGMAS = storagePragmas(WAL_JOURNAL_SIZE_LIMIT); +import { identityOfStoredContent } from "./identity.js"; + const STORAGE_DDL_V1 = `CREATE TABLE IF NOT EXISTS sessions ( session_id TEXT PRIMARY KEY, parent_session_id TEXT, @@ -139,6 +141,8 @@ CREATE TABLE IF NOT EXISTS messages ( role TEXT NOT NULL, kind TEXT NOT NULL DEFAULT 'message', content_hash TEXT NOT NULL REFERENCES content(hash), + identity TEXT NOT NULL DEFAULT '', + retired_at INTEGER, PRIMARY KEY (session_id, seq), UNIQUE (session_id, msg_id, content_hash) ); @@ -271,7 +275,7 @@ CREATE INDEX IF NOT EXISTS idx_checkpoints_session ON checkpoints(session_id, cr /** Applied inside one transaction whenever user_version < STORAGE_SCHEMA_VERSION. */ export function migrateSchema(db: { exec(sql: string): void; - prepare(sql: string): { get(...a: unknown[]): unknown; all(...a: unknown[]): unknown[] }; + prepare(sql: string): { get(...a: unknown[]): unknown; all(...a: unknown[]): unknown[]; run(...a: unknown[]): unknown }; transaction(fn: () => T): T; }): { from: number; to: number } { const row = db.prepare("PRAGMA user_version").get() as { user_version: number } | undefined; @@ -336,6 +340,84 @@ export function migrateSchema(db: { db.exec("ALTER TABLE sessions ADD COLUMN orphans_json TEXT"); } } + if (from < 8) { + // v8 position-independent identity + historian tombstones + // (#4390). identity: the cross-save JOIN key (stored-bytes + // serialization minus id/wireIndex) — content_hash stays the + // byte-faithful CAS key. retired_at: the authoritative-set sweep + // tombstones absent rows instead of deleting them (rewind/undo + // history survives; getMessages filters retired rows). + // Probe-first: dev-window builds whose DDL already carried the + // columns must not fail on duplicate ALTERs (v2 discipline). + const msgCols = (db.prepare("PRAGMA table_info(messages)").all() as unknown as { name?: unknown }[]) + .map((c) => (typeof c?.name === "string" ? c.name : "")); + if (!msgCols.includes("identity")) { + db.exec("ALTER TABLE messages ADD COLUMN identity TEXT NOT NULL DEFAULT ''"); + } + if (!msgCols.includes("retired_at")) { + db.exec("ALTER TABLE messages ADD COLUMN retired_at INTEGER"); + } + db.exec("CREATE INDEX IF NOT EXISTS idx_messages_identity ON messages (session_id, msg_id, identity)"); + // Backfill from stored bytes, healing the wireIndex-split + // generations the polluted formula minted: same (msg_id, + // identity) after the strip → keep the FIRST arrival (lowest + // seq), drop the later duplicates and hand their CAS refs back. + // Message bytes live in the content CAS (messages carry only the + // hash) — join in, decompress nothing (bytes are stored raw). + const rows = db + .prepare("SELECT m.session_id AS session_id, m.seq AS seq, m.msg_id AS msg_id, m.content_hash AS content_hash, c.bytes AS content FROM messages m JOIN content c ON c.hash = m.content_hash") + .all() as { session_id: string; seq: number; msg_id: string; content: string; content_hash: string }[]; + const keep = new Map(); + const drop: { session_id: string; seq: number; content_hash: string }[] = []; + const rawByIdentityKey = new Map(); + for (const r of rows) { + // node:sqlite returns BLOB as Uint8Array — normalize to utf8 + // before hashing (JSON.parse would coerce it to a comma list + // and silently hash the wrong input). + const raw = typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"); + const identity = identityOfStoredContent(raw); + rawByIdentityKey.set(`${r.session_id}\u0000${r.msg_id}\u0000${identity}`, raw); + const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`; + const seen = keep.get(key); + if (seen === undefined) { + keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); + } else if (r.seq < seen.seq) { + // an earlier arrival discovered late — swap the keeper + drop.push({ session_id: seen.session_id, seq: seen.seq, content_hash: seen.content_hash }); + keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); + } else { + drop.push({ session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); + } + } + const identityBySeq = new Map(); + for (const r of rows) { + const raw = rawByIdentityKey.get(`${r.session_id}\u0000${r.msg_id}\u0000${identityOfStoredContent(typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"))}`) ?? (typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8")); + const identity = identityOfStoredContent(raw); + const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`; + if (keep.get(key)?.seq === r.seq) identityBySeq.set(`${r.session_id}\u0000${r.seq}`, identity); + } + const upd = db.prepare("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?"); + for (const [k, identity] of identityBySeq) { + const [sessionId, seqStr] = k.split("\u0000"); + upd.run(identity, sessionId, Number(seqStr)); + } + // Duplicates: delete their media mappings + message rows and + // decrement CAS refcounts (delete referencing rows FIRST — the + // #4242 discipline; refcount may reach 0, the next gc run + // collects those content rows). + const decRef = db.prepare("UPDATE content SET refcount = refcount - 1 WHERE hash = ?"); + const delMedia = db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?"); + const mediaOf = db.prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?"); + const delMsg = db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?"); + for (const d of drop) { + for (const mm of mediaOf.all(d.session_id, d.seq) as { media_hash: string }[]) { + decRef.run(mm.media_hash); + } + delMedia.run(d.session_id, d.seq); + delMsg.run(d.session_id, d.seq); + decRef.run(d.content_hash); + } + } db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`); }); return { from, to: STORAGE_SCHEMA_VERSION }; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 8712291834..f6d351f8ff 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -230,9 +230,15 @@ export class UnifiedShadow { // construction. No auto-delete: the index may hold pinned checkpoint // blobs (#2671 Phase 2) — destructive recovery is operator policy. const errStr = String(err); + // Deleting the index cannot fix a no-engine failure (this Node.js has neither builtin + // node:sqlite nor a loadable better-sqlite3) — that advice only fits a genuine open + // failure; the Node.js upgrade lever is surfaced via /__bili/status + the bootstrap warn. + const tail = /no SQLite engine available/i.test(errStr) + ? "" + : `${sqliteOpenFailureHint(errStr)}; delete the file to reset the index`; loggerLog( "warn", - `[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${sqliteOpenFailureHint(errStr)}; delete the file to reset the index`, + `[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${tail}`, ); recordSidecarOpenFailure(errStr); return null; @@ -369,6 +375,7 @@ export class UnifiedShadow { refs: this.sync.getRefs(legacy.id), }; const replayed = loadLegacyView(rows); + warnTailGaps(legacy.id, replayed); const expectView = { ...legacy, blockContents: orderedBlockContents(completeBlockContents(legacy.blockContents, legacy.state.blocks)) }; const gotView = { ...replayed, blockContents: orderedBlockContents(replayed.blockContents) }; const expect = shadowDigest(expectView); @@ -514,7 +521,9 @@ export class UnifiedShadow { // The view's `state` object carries every MERGE_STATE_FIELDS key by // construction; the declared Record just needs a // widening cast to reach LegacySessionLike. - return loadLegacyView(rows) as LegacySessionLike; + const unified = loadLegacyView(rows) as LegacySessionLike; + warnTailGaps(id, unified as unknown as { tailGaps?: number }); + return unified; } catch (err) { loggerLog("warn", `[storage] unified load failed for ${id} (${String(err)}) — falling back to the legacy file`); return null; @@ -912,13 +921,15 @@ export class UnifiedShadow { const backend = this.backend!; const sessionRow = await backend.getSessionRow(id); if (!sessionRow) return null; - return loadLegacyView({ + const loaded = loadLegacyView({ session: sessionRow, messages: await backend.getMessages(id), blocks: await backend.getBlocks(id), deadRefs: await backend.getDeadRefs(id), refs: await backend.getRefs(id), }) as LegacySessionLike; + warnTailGaps(id, loaded as unknown as { tailGaps?: number }); + return loaded; } catch (err) { loggerLog("warn", `[storage] unified load failed for ${id} (${String(err)}) — falling back to the legacy file`); return null; @@ -1091,6 +1102,15 @@ export class UnifiedShadow { } } +/** #4390: tail/pluginSnapshot pointers whose backing row is gone used to be + * silently dropped — the view shrank with no trace (legacy tail 缺行静默 + * 缩水). Make it loud: every load path reports the gap count. */ +function warnTailGaps(id: string, view: { tailGaps?: number }): void { + if ((view.tailGaps ?? 0) > 0) { + loggerLog("warn", `[storage] ${view.tailGaps} tail pointer(s) for ${id} reference missing rows — view is shorter than the recorded tail (#4390)`); + } +} + function shadowDigest(view: unknown): string { return sha256(JSON.stringify(canonicalize(view))); } @@ -1127,6 +1147,10 @@ export function describeViewDivergence(expectView: unknown, gotView: unknown): D const ser = (v: unknown): string | null => (v === undefined ? null : JSON.stringify(canonicalize(v))); const keys: string[] = []; for (const k of [...new Set([...Object.keys(e), ...Object.keys(g)])].sort()) { + // tailGaps is read-time diagnostic metadata, not parity surface (#4390) — + // the live legacy leg structurally lacks the key, so it must never be + // named as a diverged dimension (a real gap diverges lastMessages itself). + if (k === "tailGaps") continue; if (ser(e[k]) !== ser(g[k])) keys.push(k); } const diagnostics: DriftDiagnostics = { keys }; diff --git a/src/storage/store.ts b/src/storage/store.ts index e6a0c7a838..96e4cb0ea2 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -22,6 +22,7 @@ import { dirname, join } from "node:path"; import { log as loggerLog } from "../logger.js"; import { openSqliteDatabase, type SqliteDatabase, type SqliteEngineName, type SqliteEngineSource } from "./driver.js"; import { migrateSchema, storagePragmas, WAL_JOURNAL_SIZE_LIMIT } from "./schema.js"; +import { identityOfStoredContent } from "./identity.js"; import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } from "./runtime-status.js"; import { listInstances, type RegistryEntry } from "../instance.js"; import type { MessageContentStore } from "acp-kernel"; @@ -100,6 +101,12 @@ export interface MessageRow { * content CAS, refcounted via message_media rows). Ingest-only — * readers always receive hydrated content. */ media?: { hash: string; bytes: string }[]; + /** #4390: position-independent identity (stored-bytes serialization + * minus id/wireIndex) — the cross-save JOIN key. content_hash stays + * the byte-faithful CAS key; identity is what remap/sweep/guard join + * on, so a client head-truncation (wireIndex shift) no longer re-mints + * the surviving generation. Optional for direct callers. */ + identity?: string; } export interface BlockRow { @@ -622,11 +629,14 @@ export class UnifiedStore { // at nothing and the read view would drop it. One indexed // probe per NEW message; any hit → full ingest instead. const existingIdentity = this.db.prepare( - "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND content_hash = ?", + "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ?", ); for (const m of input.messages) { if (!m.contentHash) return false; - if (existingIdentity.get(s.sessionId, m.msgId, m.contentHash)) return false; + // #4390: join on identity (position-independent). Retired + // rows count as existing — a rewind-redo falls back to the + // full path, which reactivates the tombstone in place. + if (existingIdentity.get(s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content))) return false; } // Placeholder→absolute renumber: the plan emitted // watermark+1.. under the aligned assumption; anchoring at @@ -680,26 +690,48 @@ export class UnifiedStore { if (opts?.completeLog === true && !this.hasLaneExt(s.sessionId)) { const incomingIds = new Set(); for (const m of messages) { - incomingIds.add(`${m.msgId}\u0000${m.contentHash ?? sha256Hex(m.content)}`); + incomingIds.add(`${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`); } const staleRows = this.db - .prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?") - .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; + .prepare("SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; identity: string; retired_at: number | null }[]; for (const row of staleRows) { - if (incomingIds.has(`${row.msg_id}\u0000${row.content_hash}`)) continue; - // #4242 delete-on-zero discipline: remove the - // referencing rows FIRST, then release the hashes — - // a release that hits zero now DELETEs the content - // row, which an FK-referencing leftover row would - // reject. (Media mappings ride the message sweep, - // #3762 review.) - const staleMedia = this.db + if (incomingIds.has(`${row.msg_id}\u0000${row.identity}`)) { + // #4390 rewind-redo: the identity came back — + // reactivate the tombstone at its original seq. + if (row.retired_at !== null) { + this.db + .prepare("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?") + .run(s.sessionId, row.seq); + } + continue; + } + // #4390 historian sweep: the row is absent from the + // authoritative set — RETIRE it, don't delete. The + // unified store is the durable record of everything + // the client ever sent; a rewind/undo that drops a + // message from the view must tombstone it, not + // destroy the bytes (epoch-gate replays #3644 and + // rewind-redo both need the row to still exist — + // reactivation above restores it at the same seq). + // Retired rows leave the unified replay (getMessages + // filters them) so drift healing and the DIGEST + // reconcile keep working exactly as the DELETE did. + // TEXT CAS refs are NOT released: retired rows still + // own their bytes. MEDIA refs ARE released (#4506): + // image bytes are bulky and low-value as history — + // keeping them until archive (365d default) is the + // unbounded leak #3762/#3901 called out. The + // re-ingest path below re-establishes mappings and + // refcounts if the row ever reactivates. + this.db + .prepare("UPDATE messages SET retired_at = ? WHERE session_id = ? AND seq = ? AND retired_at IS NULL") + .run(Date.now(), s.sessionId, row.seq); + const retiredMedia = this.db .prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?") .all(s.sessionId, row.seq) as { media_hash: string }[]; - this.db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); this.db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); - this.releaseContent(row.content_hash); - for (const mm of staleMedia) this.releaseContent(mm.media_hash); + for (const rm of retiredMedia) this.releaseContent(rm.media_hash); } } } @@ -775,13 +807,27 @@ export class UnifiedStore { const hash = this.putContent(m.content, m.contentHash); const res = this.db .prepare( - "INSERT OR IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT OR IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ) - .run(s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash); + .run(s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash, m.identity ?? identityOfStoredContent(m.content)); if (res.changes === 0) { // Idempotent re-receipt: the CAS row is still referenced // by the existing message row — undo our refcount bump. this.releaseContent(hash); + // #4506: an existing row may be a tombstone REACTIVATING + // (its message_media mappings were released at retire). + // The same idempotent upsert below restores mapping + + // refcount; for ordinary re-ingests (mapping still there) + // the release-on-no-change undoes the bump — net zero. + if (m.media && m.media.length > 0) { + for (const md of m.media) { + this.putContent(md.bytes, md.hash); + const mmr = this.db + .prepare("INSERT OR IGNORE INTO message_media (session_id, seq, media_hash) VALUES (?, ?, ?)") + .run(s.sessionId, m.seq, md.hash); + if (mmr.changes === 0) this.releaseContent(md.hash); + } + } } else if (m.media && m.media.length > 0) { // #3762: put the media bytes + mapping rows; refcount per // message row that actually landed (INSERT OR IGNORE on @@ -984,12 +1030,21 @@ export class UnifiedStore { // stable absolute seqs: same identity (msgId + content hash) // keeps its old seq; new identities continue after max(seq). const existing = this.db - .prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?") - .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; + .prepare("SELECT seq, msg_id, identity, content_hash, retired_at FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; identity: string; content_hash: string; retired_at: number | null }[]; const seqByIdentity = new Map(); + const hashBySeq = new Map(); + const retiredSeqs: number[] = []; let nextSeq = 0; for (const row of existing) { - seqByIdentity.set(`${row.msg_id}\u0000${row.content_hash}`, row.seq); + // #4390: identity join — same logical message keeps its seq even + // when its bytes were re-minted with shifted wire positions. + // Retired rows still claim their seq (INSERT OR IGNORE would + // silently drop a re-mint over their PK) and reactivation below + // clears the tombstone when the identity comes back. + if (row.retired_at !== null) retiredSeqs.push(row.seq); + seqByIdentity.set(`${row.msg_id}\u0000${row.identity}`, row.seq); + hashBySeq.set(row.seq, row.content_hash); if (row.seq + 1 > nextSeq) nextSeq = row.seq + 1; } // #3133 watermark gate (full path): the watermark names the abs seq @@ -999,19 +1054,44 @@ export class UnifiedStore { // when the plan carries no tail (envelope-only imports): the next // save runs the full ingest. const planToAbs = new Map(); + const repairs: { seq: number; from: string; to: string; content: string }[] = []; const messages = input.messages.map((m) => { // #3133: plan rows carry the intern-pass hash; only direct // callers without a planner pay the sha256 here. - const hash = m.contentHash ?? sha256Hex(m.content); - const key = `${m.msgId}\u0000${hash}`; + const key = `${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`; let abs = seqByIdentity.get(key); if (abs === undefined) { abs = nextSeq++; seqByIdentity.set(key, abs); + } else { + // #4390 review: identity match + changed bytes (a wireIndex + // re-mint after head truncation, sidecar drift, or external + // CAS corruption) — INSERT OR IGNORE would silently keep the + // stored generation forever: the parity oracle then drifts on + // every sampled save (stale wireIndex vs live view) and the + // heal path can never converge. Adopt the fresher bytes at + // the SAME seq instead: seq stability (D4-1) is preserved, + // the replay matches the authoritative view again, and a + // corrupted row heals on the next full ingest. + const fresh = m.contentHash ?? sha256Hex(m.content); + const stored = hashBySeq.get(abs); + if (stored !== undefined && stored !== fresh) repairs.push({ seq: abs, from: stored, to: fresh, content: m.content }); } planToAbs.set(m.seq, abs); return { ...m, seq: abs }; }); + for (const r of repairs) { + this.putContent(r.content, r.to); + this.db.prepare("UPDATE messages SET content_hash = ? WHERE session_id = ? AND seq = ?").run(r.to, s.sessionId, r.seq); + this.releaseContent(r.from); + } + // #4390: a rewound message that comes back reactivates in place — + // the tombstone clears, the original seq is reused, and the view + // heals without a duplicate row even on completeLog-less saves. + for (const seq of retiredSeqs) { + const back = messages.some((m) => m.seq === seq); + if (back) this.db.prepare("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?").run(s.sessionId, seq); + } const remapSeqs = (seqs: number[] | null | undefined): number[] | null | undefined => seqs === null || seqs === undefined ? seqs : seqs.map((p) => planToAbs.get(p) ?? p); const pluginSnapshotSeqs = remapSeqs(input.pluginSnapshotSeqs); @@ -1547,9 +1627,11 @@ export class UnifiedStore { } if (opts.vacuum === "full") { loggerLog("info", "[#3901] gc: running full VACUUM (this can take a while on large databases)"); - this.db.exec("VACUUM"); + // #4390: delegate to vacuum() — a bare VACUUM leaves the compacted + // image in the WAL; only the TRUNCATE checkpoint shrinks the files. + this.vacuum("full"); } else if (opts.vacuum === "incremental") { - this.db.exec("PRAGMA incremental_vacuum"); + this.vacuum("incremental"); } return { archived, contentRowsDeleted }; } @@ -2003,9 +2085,16 @@ export class UnifiedStore { }; } - getMessages(sessionId: string): (MessageRow & { contentHash: string })[] { + getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { + // #4390: retired rows (rewound/undo history) stay out of the unified + // replay by default — they are history, not view. Callers that need + // the full historian record (export/diagnostics) opt in. const rows = this.db - .prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq") + .prepare( + opts?.includeRetired === true + ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" + : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", + ) .all(sessionId) as Record[]; return rows.map((r) => ({ seq: r.seq as number, @@ -2015,6 +2104,7 @@ export class UnifiedStore { role: r.role as string, kind: r.kind as string, contentHash: r.content_hash as string, + retiredAt: (r.retired_at as number | null) ?? null, content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), })); } diff --git a/tests/acp-status-storage-degraded.test.ts b/tests/acp-status-storage-degraded.test.ts index 27bcb1a41a..ebcc60f4cd 100644 --- a/tests/acp-status-storage-degraded.test.ts +++ b/tests/acp-status-storage-degraded.test.ts @@ -142,3 +142,40 @@ test("#3894: an unsupported Node surfaces the artifact-table reason, not generic rmrf(base); } }); + +test("#4444: an unsupported Node surfaces the Node.js upgrade lever on the degrade reason", () => { + // Node 21 has NO pinned band (neither builtin node:sqlite nor a prebuild), so + // bootstrap is refused before any download -> unsupported-node. The degrade + // reason must carry BOTH the specific coverage detail (the #3894 ordering) AND + // the "upgrade Node.js" lever the sidecar-open branch carries — pre-#4444 the + // unsupported-node branch outranked it and dropped the lever entirely. The + // diagnosed runtime is forced to 21.6.1 so the lever is non-empty regardless + // of the runner's own (builtin-bearing) Node version. + const base = mkdtempSync(path.join(tmpdir(), "acp-status-4444-")); + const snap = isolate(base); + const origNodeDesc = Object.getOwnPropertyDescriptor(process.versions, "node"); + let store: SessionStore | null = null; + try { + setSqliteEnginesUnavailableForTests(true); + _resetStorageRuntimeStatusForTest(); + recordEngineBootstrap({ status: "unsupported-node", detail: "no pinned SQLite engine runs on Node.js 21.6.1 (ABI 120)" }); + store = new SessionStore({ dir: process.env.BILI_SESSIONS_DIR, enabled: true }); + _setStoreForTest(store); + + Object.defineProperty(process.versions, "node", { value: "21.6.1", configurable: true, writable: true }); + const text = handleAcpStatus({}, makeCtx()).text ?? ""; + assert.ok(text.includes("STORAGE —"), "the storage line is present"); + assert.ok(text.includes("⚠ degraded —"), `the degrade warning line is present (got: ${text.slice(-400)})`); + assert.ok(text.includes("no self-downloaded SQLite engine supports this Node.js"), "the specific coverage detail surfaces"); + assert.ok(/Node\.js 21\.6\.1/.test(text), "the reason names the runtime the table could not serve"); + assert.ok(/Node\.js >= 22\.5/.test(text), `#4444: the unsupported-node reason must carry the Node.js upgrade lever (got: ${text.slice(-400)})`); + } finally { + if (origNodeDesc) Object.defineProperty(process.versions, "node", origNodeDesc); + store?.closeQuietly(); + _setStoreForTest(null as unknown as SessionStore); + setSqliteEnginesUnavailableForTests(false); + _resetStorageRuntimeStatusForTest(); + restore(snap); + rmrf(base); + } +}); diff --git a/tests/issue3250-wal-checkpoint.test.ts b/tests/issue3250-wal-checkpoint.test.ts index e010cb8a79..7559fc04cb 100644 --- a/tests/issue3250-wal-checkpoint.test.ts +++ b/tests/issue3250-wal-checkpoint.test.ts @@ -122,6 +122,16 @@ const WAL_CAP = 8 * MiB; const CHECKPOINT_EVERY_N = 20; /** Truncation lands page-aligned; allow one page of measurement slack. */ const CAP_SLACK = 4 * KiB; +/** + * #4506: the custom-cap case observes the plateau BETWEEN resets — after the + * autocheckpoint truncates to the configured cap, subsequent commits keep + * appending frames until the next reset. #4390's row identity (64-hex column + * + idx_messages_identity entry per row) adds ~+250B of WAL frames per + * message row, so ~192 rows of post-reset writes now overshoot by ~12 pages + * (observed 1095952 vs 1048576). The truncate-at-checkpoint INVARIANT is + * intact; budget the inter-reset accumulation instead of pinning one page. + */ +const CUSTOM_CAP_SLACK = 64 * KiB; test("#3250: close() converges the WAL so a clean exit leaves nothing to replay", () => { const dir = tmpdir("bili-3250-close-"); @@ -289,7 +299,7 @@ test("#3250: a configured cap flows open() → journal_size_limit (custom limit // the observable proof the configured value reached the pragma.) for (let i = 2; i <= 10; i++) ingest(store, `small${i}`, 128 * KiB, 8); assert.ok( - walSize(dir) <= customCap + CAP_SLACK, + walSize(dir) <= customCap + CUSTOM_CAP_SLACK, `WAL must be truncated to the configured cap at the autocheckpoint (left ${walSize(dir)} bytes, cap ${customCap})`, ); assert.equal(store.getMessages("big").length, 120, "all burst rows must land"); diff --git a/tests/issue4215-fork-gate-diagnostics.test.ts b/tests/issue4215-fork-gate-diagnostics.test.ts new file mode 100644 index 0000000000..d1be978187 --- /dev/null +++ b/tests/issue4215-fork-gate-diagnostics.test.ts @@ -0,0 +1,181 @@ +// #4215/C10a: every skipped fork-adoption entry gate must name itself. The +// incident (seeded child with the correct parentSession, ZERO adoption trace +// in the whole log) made "not a fork" vs "a fork we could not see" +// unanswerable from the log. This suite pins both directions of every gate +// (#3617 discipline): the skip line fires when it should, exactly once, and +// plain non-fork traffic stays completely silent. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import { once } from "node:events"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { forkChildVerdict, _maybeForkAdoptBeforeSendForTest, _resetForkGateDiagnosticsForTest, _resetRegisterForTest, type ForkChildVerdict, type PluginContext } from "../src/agent/dsh-native.ts"; +import { resetForkCapabilityCacheForTest } from "../src/agent/fork-adopt.ts"; + +process.env.NODE_ENV = "test"; +const testRoot = mkdtempSync(join(tmpdir(), "bili-issue4215-")); +test.after(() => rmSync(testRoot, { recursive: true, force: true })); +for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = testRoot; +const logFile = join(testRoot, "billion-context", "bili.log"); + +type SessionShape = { id?: unknown; header?: { parentSession?: unknown; isSeeded?: unknown } }; +type AgentShape = { session?: SessionShape }; + +function ctxWithInitiator(fn: () => AgentShape | undefined): PluginContext { + return { + tools: { register: () => undefined }, + commands: { register: () => undefined }, + agents: { currentInitiator: fn }, + }; +} + +const ctxNoAgents = (): PluginContext => ({ tools: { register: () => undefined }, commands: { register: () => undefined }, agents: {} }); +const ctxNoInitiator = (): PluginContext => ctxWithInitiator(() => undefined); +const ctxThrowing = (): PluginContext => ctxWithInitiator(() => { throw new Error("disposed scope"); }); +const ctxPlain = (): PluginContext => ctxWithInitiator(() => ({ session: { id: "plain-sid", header: {} } })); +const ctxSubagent = (): PluginContext => ctxWithInitiator(() => ({ session: { id: "sub-sid", header: { parentSession: "p1", isSeeded: false } } })); +const ctxSeeded = (): PluginContext => ctxWithInitiator(() => ({ session: { id: "child-sid", header: { isSeeded: true, parentSession: "parent-sid" } } })); + +const validInit: RequestInit = { body: JSON.stringify({ model: "m", messages: [{ role: "user", content: "hi" }] }) }; + +function freshScenario(base: string | undefined, toolsReady: boolean): void { + _resetRegisterForTest(base, toolsReady); + _resetForkGateDiagnosticsForTest(); + fs.rmSync(logFile, { force: true }); +} + +function logLines(): string[] { + try { + return fs.readFileSync(logFile, "utf8").split("\n").filter((l) => l.length > 0); + } catch { + return []; + } +} + +const GATE_LINE = /fork adoption for .* (skipped at the \S+ gate|deferred for a side-shaped request)/; + +test("forkChildVerdict names every shape (#4215)", () => { + const cases: Array<[string, () => PluginContext, ForkChildVerdict]> = [ + ["threw", ctxThrowing, { ok: false, reason: "initiator-threw", detail: "disposed scope" }], + ["missing-initiator", ctxNoInitiator, { ok: false, reason: "initiator-missing" }], + ["no-initiator-accessor", ctxNoAgents, { ok: false, reason: "initiator-missing" }], + ["agent-no-session", () => ctxWithInitiator(() => ({})), { ok: false, reason: "session-missing" }], + ["session-no-id", () => ctxWithInitiator(() => ({ session: {} })), { ok: false, reason: "sid-missing" }], + ["session-bad-id", () => ctxWithInitiator(() => ({ session: { id: 42 } })), { ok: false, reason: "sid-missing" }], + ["session-empty-id", () => ctxWithInitiator(() => ({ session: { id: "" } })), { ok: false, reason: "sid-missing" }], + ["no-header", () => ctxWithInitiator(() => ({ session: { id: "s1" } })), { ok: false, reason: "header-missing", sid: "s1" }], + ["plain-session", ctxPlain, { ok: false, reason: "not-seeded", sid: "plain-sid" }], + ["subagent-carries-parent-without-isSeeded", ctxSubagent, { ok: false, reason: "not-seeded", sid: "sub-sid" }], + ["seeded-no-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true } } })), { ok: false, reason: "parent-missing", sid: "s1" }], + ["seeded-empty-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true, parentSession: "" } } })), { ok: false, reason: "parent-missing", sid: "s1" }], + ["seeded-non-string-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true, parentSession: 7 } } })), { ok: false, reason: "parent-missing", sid: "s1" }], + ["seeded-self-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true, parentSession: "s1" } } })), { ok: false, reason: "parent-is-self", sid: "s1" }], + ["ok", ctxSeeded, { ok: true, sid: "child-sid", parent: "parent-sid" }], + ]; + for (const [name, make, expected] of cases) { + assert.deepEqual(forkChildVerdict(make()), expected, name); + } +}); + +test("seeded child blocked at tools-ready: one warn naming the gate; repeats stay one (#4215)", async () => { + freshScenario("http://proxy.invalid", false); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + let lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*bili-native-dsh: fork adoption for child-sid skipped at the tools-ready gate.*#4215/); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, "throttle holds across requests"); +}); + +test("seeded child with unparseable body: one warn naming the body-parse gate (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), { body: "not-json" }); + let lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*skipped at the body-parse gate/); + // a missing body hits the same gate and does not re-log per shape + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), undefined); + lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1); +}); + +test("side-shaped request: info-level deferral, window stays open (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), { body: JSON.stringify({ model: "m", messages: [], max_tokens: 100 }) }); + const lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[info\].*deferred for a side-shaped request/); + // the main request after the sidecar reaches the adopter untouched + resetForkCapabilityCacheForTest(); + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ capabilities: {} })); }); + srv.listen(0, "127.0.0.1"); + await once(srv, "listening"); + const addr = srv.address(); + assert(addr && typeof addr === "object"); + try { + _resetRegisterForTest(`http://127.0.0.1:${addr.port}`, true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + } finally { + srv.close(); + srv.closeAllConnections?.(); + await once(srv, "close").catch(() => undefined); + } + const after = logLines(); + assert.equal(after.filter((l) => /does not advertise the fork capability/.test(l)).length, 1, "main request reached the adopter protocol"); + assert.equal(after.filter((l) => /skipped at the .* gate/.test(l)).length, 0, "no gate fired on the main request"); +}); + +test("seeded child before the proxy origin is known: one warn naming the proxy-base gate (#4215)", async () => { + freshScenario(undefined, true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + const lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*skipped at the proxy-base gate/); +}); + +test("malformed seeded lineage: one warn per anomaly naming what is missing (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: { id: "a1", header: { isSeeded: true } } })), validInit); + let lines = logLines(); + assert.equal(lines.filter((l) => /carries no usable parentSession/.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*fork adoption for a1 skipped/); + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: { id: "b1", header: { isSeeded: true, parentSession: "b1" } } })), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /carries its own id as parentSession/.test(l)).length, 1); + // repeat of the same anomaly stays one line + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: { id: "a1", header: { isSeeded: true } } })), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /carries no usable parentSession/.test(l)).length, 1); +}); + +test("ambient-blind shapes: one info notice per process, naming the split when #4082 is active (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxThrowing(), validInit); + let lines = logLines(); + assert.equal(lines.filter((l) => /fork discovery blind/.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[info\].*ambient currentInitiator threw \(disposed scope\)/); + // a DIFFERENT blind shape on a later dispatch adds nothing + await _maybeForkAdoptBeforeSendForTest(ctxNoInitiator(), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /fork discovery blind/.test(l)).length, 1); + // resetting the throttle re-arms the notice (truncate so only post-reset lines count) + _resetForkGateDiagnosticsForTest(); + fs.rmSync(logFile, { force: true }); + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: {} })), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /fork discovery blind/.test(l)).length, 1); + assert.match(lines[0]!, /the session has no usable id/); +}); + +test("plain non-fork traffic stays completely silent (#4215 negative direction)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxPlain(), validInit); + await _maybeForkAdoptBeforeSendForTest(ctxSubagent(), validInit); + await _maybeForkAdoptBeforeSendForTest(ctxPlain(), validInit); + const lines = logLines(); + assert.equal(lines.length, 0, JSON.stringify(lines)); +}); diff --git a/tests/media-cas.test.ts b/tests/media-cas.test.ts index dea8486d45..4c5d7f8611 100644 --- a/tests/media-cas.test.ts +++ b/tests/media-cas.test.ts @@ -127,17 +127,29 @@ test("media CAS: stale-row sweep releases media refs and mappings (full re-inges assert.equal(store.mediaRefCount(IMG_B64), 1, "one media reference before re-ingest"); // The session's raw tail no longer contains the image message: the - // full path sweeps the stale row — its media mapping and CAS refcount - // must go with it, or the bytes leak forever (#3762 review, #3901). + // full path sweeps the stale row into a #4390 historian tombstone — + // its media mapping and CAS refcount must still go with it, or the + // bulky image bytes leak until archive (#3762 review, #3901, #4506). + // The TEXT row itself stays retired (retired_at set), never deleted. store.ingestSession(planIngest(mkSession("sess-sweep", [mTxt])), undefined, { completeLog: true }); assert.equal(store.mediaRefCount(IMG_B64), 0, "swept message released its media ref"); const mappings = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) .db.prepare("SELECT COUNT(*) AS n FROM message_media WHERE session_id = ?").all("sess-sweep"); assert.equal(mappings[0].n, 0, "swept media mappings deleted"); - const rows = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) - .db.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ?").all("sess-sweep"); - assert.equal(rows[0].n, 1, "only the surviving message row remains"); + const rows = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number; retired: number }[] } } }) + .db.prepare("SELECT COUNT(*) AS n, SUM(CASE WHEN retired_at IS NULL THEN 1 ELSE 0 END) AS retired FROM messages WHERE session_id = ?").all("sess-sweep"); + assert.equal(rows[0].n, 2, "historian keeps both rows (one live, one tombstoned)"); + assert.equal(rows[0].retired, 1, "only the surviving message is live"); + + // #4506 rewind-redo: the SAME image message comes back (undo of the + // undo). The tombstone reactivates in place AND its media mapping + + // refcount are re-established by the re-ingest — no manual repair. + store.ingestSession(planIngest(mkSession("sess-sweep", [mImg, mTxt])), undefined, { completeLog: true }); + assert.equal(store.mediaRefCount(IMG_B64), 1, "reactivated message re-owns its media ref"); + const live = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) + .db.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL").all("sess-sweep"); + assert.equal(live[0].n, 2, "both rows live again after rewind-redo"); store.close(); } finally { try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } diff --git a/tests/resume-inheritance-gate-skip.test.ts b/tests/resume-inheritance-gate-skip.test.ts new file mode 100644 index 0000000000..2a87e950df --- /dev/null +++ b/tests/resume-inheritance-gate-skip.test.ts @@ -0,0 +1,152 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { setLogCapture, muteStderrLeg } from "../src/logger.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import type { ProxyOptions } from "../src/config.ts"; + +/** + * #4264: a SKIPPED resume-inheritance gate must not be silent either. The + * miss diagnostics (#4265) only fire when the gate RUNS; when a conjunct + * short-circuits (resumeInheritance=false, derivedFromSessionId already + * linked, publicForkReceipt present) the session produced zero + * [resume-inheritance] output — indistinguishable in the log from a + * pre-#4265 build. Both directions are pinned: skip → exactly one bounded + * debug line naming the reason (first request only), no skip → the existing + * miss line instead (never "skipped"). + */ + +function listen(server: http.Server): Promise { + if (server.listening) return Promise.resolve(); + return once(server, "listening").then(() => undefined); +} + +function close(server: http.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +function sseLine(obj: unknown): string { + return `data: ${JSON.stringify(obj)}\n\n`; +} + +function startMockUpstream(): http.Server { + const server = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(sseLine({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "ok" } }] })); + res.write(sseLine({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 2 } })); + res.write("data: [DONE]\n\n"); + res.end(); + }); + }); + server.listen(0, "127.0.0.1"); + return server; +} + +function proxyOpts(resumeInheritance: boolean | undefined): ProxyOptions { + return { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: {} as ProxyOptions["routes"], + modelContextLimit: 1_000_000, + kernelConfig: defaultConfig(1_000_000, { + preserveRecentMessages: 2, + preserveRecentTokens: 400, + compress: { minCompressRange: 200, minSummaryLength: 20, maxSummaryLength: 5000 }, + }), + compress: { injectTool: true, injectNudge: false }, + promptCache: { routing: "auto" }, + forkAdoption: true, + resumeInheritance, + sessionHeader: "x-claude-code-session-id", + log: true, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; +} + +async function chat(url: string, sessionId: string, text: string): Promise { + const res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-claude-code-session-id": sessionId }, + body: JSON.stringify({ model: "gpt-test", stream: true, messages: [{ role: "user", content: text }] }), + }); + if (!res.ok) { + const errBody = await res.text(); + assert.fail(`HTTP ${res.status}: ${errBody}`); + } + let raw = ""; + for await (const chunk of res.body!) raw += Buffer.from(chunk).toString("utf8"); + assert.ok(raw.includes("[DONE]"), "the mock upstream must stream a complete reply"); +} + +async function startHarness(resumeInheritance: boolean | undefined): Promise<{ proxy: http.Server; relay: http.Server; url: string }> { + process.env.XDG_STATE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "res-gate-")); + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + _resetPluginStateForTest(); + const relay = startMockUpstream(); + await listen(relay); + const relayPort = (relay.address() as { port: number }).port; + const proxy = await startServer(proxyOpts(resumeInheritance)); + await listen(proxy); + const proxyPort = (proxy.address() as { port: number }).port; + return { proxy, relay, url: `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${relayPort}/v1/chat/completions` }; +} + +test("a skipped resume-inheritance gate logs exactly one bounded debug line naming the reason (#4264)", async () => { + muteStderrLeg(); + const h = await startHarness(false); + const captured: Array<{ level: string; msg: string }> = []; + try { + setLogCapture((level, msg) => captured.push({ level, msg })); + await chat(h.url, "gate-skip-disabled", "first turn"); + await chat(h.url, "gate-skip-disabled", "second turn"); + } finally { + setLogCapture(null); + await close(h.proxy); + await close(h.relay); + } + const lines = captured.filter(({ msg }) => msg.includes("[resume-inheritance]")); + assert.equal(lines.length, 1, "the skip line fires once per session, not per request"); + assert.equal(lines[0]!.level, "debug", "skip observability stays debug-level — zero steady-state noise"); + assert.match(lines[0]!.msg, /\[gate-skip-disabled\] \[resume-inheritance\] skipped: resumeInheritance=false \(#4264\)/); +}); + +test("an unskipped gate keeps the existing miss line and never says skipped (#4264 negative direction)", async () => { + muteStderrLeg(); + const h = await startHarness(undefined); + const captured: Array<{ level: string; msg: string }> = []; + try { + setLogCapture((level, msg) => captured.push({ level, msg })); + await chat(h.url, "gate-open-miss", "first turn"); + } finally { + setLogCapture(null); + await close(h.proxy); + await close(h.relay); + } + const lines = captured.filter(({ msg }) => msg.includes("[resume-inheritance]")); + assert.equal(lines.length, 1, "the gate ran — exactly the existing miss diagnostic, nothing else"); + assert.equal(lines[0]!.level, "debug", "no eligible chain: debug granularity as before"); + assert.match(lines[0]!.msg, /\[gate-open-miss\] \[resume-inheritance\] miss: no eligible tracked chain \(floor 8\) \(#4264\)/); + assert.ok(!lines[0]!.msg.includes("skipped"), "the miss line must stay distinguishable from the skip line"); +}); diff --git a/tests/resume-inheritance-miss-diagnostics.test.ts b/tests/resume-inheritance-miss-diagnostics.test.ts new file mode 100644 index 0000000000..455235ebc8 --- /dev/null +++ b/tests/resume-inheritance-miss-diagnostics.test.ts @@ -0,0 +1,171 @@ +import assert from "node:assert"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { PrefixAffinityResolver, describeResumeMiss } from "../src/prefix-affinity.ts"; + +// #4264: a resume-inheritance miss must not be silent. These tests pin the +// diagnostics-only near-miss report: each of the three causes (equal-depth +// replay / parent-chain absence / head break) is asserted distinctly, plus +// the hit-path regression (matching behavior and results stay unchanged). + +function user(text: string): Record { + return { role: "user", content: text }; +} + +test("resume-inheritance-miss: equal-depth replay under another id reports kind equal-depth, matching still null (#4264)", () => { + const r = new PrefixAffinityResolver(); + const transcript = Array.from({ length: 10 }, (_, i) => user(`duplicate conversation turn ${i} with substance`)); + const fp = r.chainFingerprint(transcript)!; + r.note("sess-one", fp.depth, fp.tailHash, fp.itemHashes); + + const before = r.findResumeParent(transcript, "sess-two"); + const report = r.diagnoseResumeMiss(transcript, "sess-two"); + const after = r.findResumeParent(transcript, "sess-two"); + assert.equal(before, null, "byte-exact equal-depth replay is not a resume (existing behavior)"); + assert.equal(after, null, "diagnostics must not mutate matching state"); + assert.equal(report.eligible, 1); + assert.ok(report.best, "an eligible candidate must yield a near-miss report"); + assert.equal(report.best!.kind, "equal-depth"); + assert.equal(report.best!.sessionId, "sess-one"); + assert.equal(report.best!.chainDepth, 10); + assert.equal(report.best!.incomingDepth, 10); + assert.equal(report.best!.firstDivergence, null, "every inspectable cell matches"); + assert.equal(report.best!.tailWindowIntact, true); + assert.match(describeResumeMiss(report), /EQUAL-DEPTH replay \(10 msgs == incoming 10\)/); +}); + +test("resume-inheritance-miss: head break names the first diverging cell with hash digests (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => user(`stable parent turn ${i} with substance`)); + const fp = r.chainFingerprint(parent)!; + r.note("sess-parent", fp.depth, fp.tailHash, fp.itemHashes); + + const edited = [...parent]; + edited[2] = user("edited third turn — different bytes"); + const incoming = [...edited, user("and the conversation goes on")]; + assert.equal(r.findResumeParent(incoming, "child"), null, "a mid-history edit breaks the head-anchored match"); + + const report = r.diagnoseResumeMiss(incoming, "child"); + assert.equal(report.eligible, 1); + assert.equal(report.best!.kind, "prefix-break"); + assert.equal(report.best!.sessionId, "sess-parent"); + assert.equal(report.best!.chainDepth, 10); + assert.equal(report.best!.incomingDepth, 11); + assert.equal(report.best!.windowStart, 0); + assert.equal(report.best!.firstDivergence, 2, "divergence at exactly the edited cell"); + assert.equal(report.best!.tailWindowIntact, false); + assert.match(report.best!.incomingCell!, /^[0-9a-f]{8}$/); + assert.match(report.best!.storedCell!, /^[0-9a-f]{8}$/); + assert.notEqual(report.best!.incomingCell, report.best!.storedCell); + assert.match(describeResumeMiss(report), /first divergence at cell 2 \(stored [0-9a-f]{8} vs incoming [0-9a-f]{8}\)/); +}); + +test("resume-inheritance-miss: deep chain broken before the stored window reports the pre-window bound (#4264)", () => { + const r = new PrefixAffinityResolver(); + const long = Array.from({ length: 130 }, (_, i) => user(`long chain message number ${i} with substance`)); + const fp = r.chainFingerprint(long)!; + assert.equal(fp.itemHashes.length, 128, "per-item hashes stay capped"); + r.note("sess-long", fp.depth, fp.tailHash, fp.itemHashes); + + const drifted = [...long]; + drifted[0] = user("re-encoded first message — host rebuild drift"); + const incoming = [...drifted, user("thirteen-one")]; + assert.equal(r.findResumeParent(incoming, "child"), null); + + const report = r.diagnoseResumeMiss(incoming, "child"); + assert.equal(report.best!.kind, "prefix-break"); + assert.equal(report.best!.windowStart, 2, "cells 0..1 uninspectable for a 130-deep chain"); + assert.equal(report.best!.firstDivergence, null, "the whole inspectable overlap matched"); + assert.equal(report.best!.tailWindowIntact, true); + assert.match(describeResumeMiss(report), /tail window \(cells 2\.\.129\) byte-intact — break sits BEFORE cell 2/); +}); + +test("resume-inheritance-miss: truncated replay is reported distinctly from a head break (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => user(`full parent turn ${i} with substance`)); + const fp = r.chainFingerprint(parent)!; + r.note("sess-parent", fp.depth, fp.tailHash, fp.itemHashes); + + const trimmed = parent.slice(0, 9); + assert.equal(r.findResumeParent(trimmed, "child"), null, "trimmed history is not a resume"); + const report = r.diagnoseResumeMiss(trimmed, "child"); + assert.equal(report.best!.kind, "truncated"); + assert.equal(report.best!.chainDepth, 10); + assert.equal(report.best!.incomingDepth, 9); + assert.equal(report.best!.tailWindowIntact, true, "the visible cells form a clean prefix"); + assert.match(describeResumeMiss(report), /LONGER than the incoming history \(9 msgs\)/); +}); + +test("resume-inheritance-miss: absent parent chains report zero eligible at debug granularity (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => user(`orphaned branch turn ${i} with substance`)); + const report = r.diagnoseResumeMiss(parent, "child"); + assert.deepEqual(report, { eligible: 0 }, "nothing tracked — no near miss exists"); + assert.match(describeResumeMiss(report), /^no eligible tracked chain \(floor 8\)$/); +}); + +test("resume-inheritance-miss: scoped match reports the declared parent's own state (#4264)", () => { + const r = new PrefixAffinityResolver(); + const other = Array.from({ length: 10 }, (_, i) => user(`unrelated tracked turn ${i} with substance`)); + const ofp = r.chainFingerprint(other)!; + r.note("sess-other", ofp.depth, ofp.tailHash, ofp.itemHashes); + const short = Array.from({ length: 7 }, (_, i) => user(`short declared turn ${i} with substance`)); + const sfp = r.chainFingerprint(short)!; + r.note("sess-short", sfp.depth, sfp.tailHash, sfp.itemHashes); + + const candidate = Array.from({ length: 11 }, (_, i) => user(`declared child turn ${i} with substance`)); + assert.equal(r.findResumeParentWithin("sess-ghost", candidate, "child"), null); + assert.deepEqual(r.diagnoseResumeMiss(candidate, "child", "sess-ghost"), { eligible: 0, declaredParentState: "untracked" }); + assert.match(describeResumeMiss(r.diagnoseResumeMiss(candidate, "child", "sess-ghost")), /declared parent chain untracked/); + + assert.equal(r.findResumeParentWithin("sess-short", candidate, "child"), null, "7-deep declared parent below the floor cannot match"); + const shallow = r.diagnoseResumeMiss(candidate, "child", "sess-short"); + assert.deepEqual(shallow, { eligible: 0, declaredParentState: "too-shallow" }); +}); + +test("resume-inheritance-miss: scoped match judges only the declared parent's chain (#4264)", () => { + const r = new PrefixAffinityResolver(); + const base = Array.from({ length: 10 }, (_, i) => user(`common base turn ${i} with substance`)); + const extA = [...base, user("extension A")]; + const extB = [...base, user("extension B")]; + const fpA = r.chainFingerprint(extA)!; + r.note("sess-a", fpA.depth, fpA.tailHash, fpA.itemHashes); + const fpB = r.chainFingerprint(extB)!; + r.note("sess-b", fpB.depth, fpB.tailHash, fpB.itemHashes); + + const candidate = [...extA, user("and further along")]; + // Hit-path regression: the matching scoped parent still resolves exactly as before. + assert.deepEqual(r.findResumeParentWithin("sess-a", candidate, "child"), { sessionId: "sess-a", sharedDepth: 11 }); + // Miss-path: the OTHER chain is out of scope even though it is tracked. + assert.equal(r.findResumeParentWithin("sess-b", candidate, "child"), null); + const report = r.diagnoseResumeMiss(candidate, "child", "sess-b"); + assert.equal(report.eligible, 1, "only the declared parent's chain is judged"); + assert.equal(report.best!.sessionId, "sess-b"); + assert.equal(report.best!.kind, "prefix-break"); + assert.equal(report.best!.firstDivergence, 10, "the two extensions differ at cell 10"); + assert.equal(report.best!.windowStart, 0); + assert.match(describeResumeMiss(report), /closest sess-b \(chain 11, incoming 12\)/); +}); + +test("resume-inheritance-miss: the caller's own id stays excluded from the near-miss report (#4264)", () => { + const r = new PrefixAffinityResolver(); + const transcript = Array.from({ length: 10 }, (_, i) => user(`self replay turn ${i} with substance`)); + const fp = r.chainFingerprint(transcript)!; + r.note("self-id", fp.depth, fp.tailHash, fp.itemHashes); + assert.equal(r.findResumeParent(transcript, "self-id"), null); + assert.deepEqual(r.diagnoseResumeMiss(transcript, "self-id"), { eligible: 0 }, "self-exclusion mirrors the matcher"); +}); + +test("resume-inheritance-miss: hit paths are byte-for-byte unchanged by the diagnostics surface (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => (i % 2 === 0 ? user(`parent turn ${i} with real substance`) : { role: "assistant", content: `parent reply ${i}` })); + const fp = r.chainFingerprint(parent)!; + r.note("sess-parent", fp.depth, fp.tailHash, fp.itemHashes); + const resumed = [...parent, user("resumed conversation next question")]; + assert.deepEqual(r.findResumeParent(resumed, "sess-child"), { sessionId: "sess-parent", sharedDepth: 10 }); + // A HIT produces no miss report at all — the server only calls diagnose + // on a null match, so nothing here may change the hit result. + assert.deepEqual(r.findResumeParent(resumed, "sess-child"), { sessionId: "sess-parent", sharedDepth: 10 }); +}); diff --git a/tests/storage-ext.test.ts b/tests/storage-ext.test.ts index 3ee171c76a..b837cf0fd9 100644 --- a/tests/storage-ext.test.ts +++ b/tests/storage-ext.test.ts @@ -155,11 +155,11 @@ test("guards reject hostile keys and oversize values", async () => { } }); -test("v3 databases migrate to v7 in place, idempotently", async () => { +test("v3 databases migrate to v8 in place, idempotently", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ext-migrate-")); try { const dbPath = path.join(tmp, "index.db"); - // Build a current (v7) database, then roll the extension tables and + // Build a current (v8) database, then roll the extension tables and // version marker back to the v3 shape — exactly what an upgraded // production lane's file looks like to the new code. const fresh = UnifiedStore.open(dbPath); @@ -174,17 +174,17 @@ test("v3 databases migrate to v7 in place, idempotently", async () => { const migrated = UnifiedStore.open(dbPath); try { const ver = (migrated as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 7, "open() runs migrateSchema: v3 → v7"); + assert.equal(ver.user_version, 8, "open() runs migrateSchema: v3 → v8"); migrated.setKv("probe", "alive", true); assert.deepEqual(migrated.getKv("probe", "alive"), true); assert.deepEqual(migrated.listSessionIds(), [], "existing v3 content untouched"); } finally { migrated.close(); } - // Reopening a v7 database is a no-op (idempotent migration). + // Reopening a v8 database is a no-op (idempotent migration). const again = UnifiedStore.open(dbPath); const ver2 = (again as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver2.user_version, 7); + assert.equal(ver2.user_version, 8); again.close(); } finally { fs.rmSync(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 1d2f523bb1..2fb846e810 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -532,7 +532,7 @@ test("watermark: sampled verification still catches drift through the delta path } }); -test("watermark: v4 database migrates to v7 additively; newer user_version no-ops", async () => { +test("watermark: v4 database migrates to v8 additively; newer user_version no-ops", async () => { const dir = tmpdir("bili-wm-migrate-"); try { const dbPath = path.join(dir, "index.db"); @@ -561,7 +561,7 @@ test("watermark: v4 database migrates to v7 additively; newer user_version no-op const store = UnifiedStore.open(dbPath); try { const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 7, "open() migrates v4 → v7"); + assert.equal(ver.user_version, 8, "open() migrates v4 → v8"); const row = store.getSessionRow("v4-sess"); assert.ok(row, "legacy session row survives the migration"); assert.equal(row!.lastIngestedSeq, null, "legacy rows read back with a NULL watermark (≡ full ingest)"); diff --git a/tests/storage-review-p2s.test.ts b/tests/storage-review-p2s.test.ts new file mode 100644 index 0000000000..9bb782227c --- /dev/null +++ b/tests/storage-review-p2s.test.ts @@ -0,0 +1,241 @@ +/** + * #4390: storage foundation defects found by the five-dimension review. + * Stakes FIRST (RED on master), then the fixes: + * S1 identity must not include position metadata — a client head-truncation + * (wireIndex shift) must NOT re-mint rows/seqs for surviving messages. + * S2 the authoritative-set sweep must RETIRE (tombstone) absent rows + * instead of DELETE-ing them; a rewind-redo reactivates the SAME seq. + * S3 snapshot-render seam fingerprints must hash image bytes, not length: + * same-length different images must NOT match; identical images MUST. + * S4 fork lineage (metadata.derivedFromSessionId) must land in the + * sessions row instead of being dropped (dead columns). + * S5 schema migration backfills the identity column and heals + * wireIndex-split duplicate generations, preserving survivor seqs. + * S6 gc(vacuum:'full') must delegate to vacuum() (wal_checkpoint TRUNCATE). + * S7 loadLegacyView must REPORT tail pointer gaps instead of silently + * shrinking the tail view. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { createInitialState } from "acp-kernel"; +import { planIngest, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { identitySigForTest } from "../src/server/snapshot-render.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "legacy"; +process.env.BILI_CONFIG_FILE = join(mkdtempSync(join(tmpdir(), "p2s-")), "missing.json"); + +/** CoreMessage-shaped fixture; wireIndex simulates the positional metadata the + * wire codecs stamp on every parse (the D4-1 pollutant). */ +function wmsg(id: string, text: string, wireIndex: number) { + return { id, role: "user", contentType: "text", text, wireIndex }; +} + +function histLike(id: string, requests: number, msgs: { id: string; text: string; wireIndex: number }[], parent?: string): LegacySessionLike { + const state = { ...createInitialState(), requests }; + state.nextBlockId = 1; + return { + id, + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { label: id, protocol: "anthropic", upstreamOrigin: "https://api.anthropic.com" }, + stats: { requests }, + metadata: parent ? { pluginAgent: "proxy", derivedFromSessionId: parent } : { pluginAgent: "proxy" }, + state: state as never, + blockContents: new Map() as never, + lastMessages: msgs.map((m) => wmsg(m.id, m.text, m.wireIndex)) as never, + lastMessagesFolded: false, + }; +} + +type Row = { seq: number; msg_id: string; content_hash: string; identity: string; retired_at: number | null }; + +function rowsOf(store: UnifiedStore, sessionId: string): Row[] { + const db = (store as unknown as { db: { prepare: (q: string) => { all: (...a: unknown[]) => unknown[] } } }).db; + return db + .prepare("SELECT seq, msg_id, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq") + .all(sessionId) as Row[]; +} + +function liveOf(store: UnifiedStore, sessionId: string): string[] { + // getMessages is the unified replay surface — after the fix it filters retired rows. + return store + .getMessages(sessionId) + .map((r) => (JSON.parse(r.content as unknown as string) as { id?: string }).id ?? "?"); +} + +test("S1+S2: head truncation keeps identities/seqs stable; sweep retires (not deletes); rewind-redo reactivates the same seq", () => { + const dir = mkdtempSync(join(tmpdir(), "s1-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + const msgs = [ + { id: "id-a", text: "alpha", wireIndex: 0 }, + { id: "id-b", text: "bravo", wireIndex: 1 }, + { id: "id-c", text: "charlie", wireIndex: 2 }, + { id: "id-d", text: "delta", wireIndex: 3 }, + { id: "id-e", text: "echo", wireIndex: 4 }, + ]; + store.ingestSession(planIngest(histLike("s12", 10, msgs)), undefined, { completeLog: true }); + const seq1 = new Map(rowsOf(store, "s12").map((r) => [r.msg_id, r.seq])); + assert.equal(seq1.size, 5); + + // Client truncated the first message → the re-parse shifts every + // surviving wireIndex down by one. Same ids, same content, new wire + // positions. Old behavior: whole row generation re-minted. + const truncated = msgs.slice(1).map((m, i) => ({ ...m, wireIndex: i })); + store.ingestSession(planIngest(histLike("s12", 11, truncated)), undefined, { completeLog: true }); + + const rows2 = rowsOf(store, "s12"); + // b..e survive at their ORIGINAL seqs (identity ignores wireIndex). + for (const m of truncated) { + const row = rows2.find((r) => r.msg_id === m.id); + assert.ok(row, `row for ${m.id} exists`); + assert.equal(row.seq, seq1.get(m.id), `${m.id} keeps its save-1 seq (identity stability, #4390)`); + } + // a is RETIRED (tombstoned), not deleted — the historian keeps the bytes. + const a2 = rows2.find((r) => r.msg_id === "id-a"); + assert.ok(a2, "retired row for id-a still exists (tombstone, #4390)"); + assert.notEqual(a2.retired_at, null, "id-a is retired"); + // And no duplicate generation was minted for the survivors. + assert.equal(rows2.length, 5, "no re-minted rows (5 originals: 4 live + 1 retired)"); + assert.deepEqual(liveOf(store, "s12"), ["id-b", "id-c", "id-d", "id-e"]); + + // Rewind-redo: the truncated head comes back. Same identity → same seq. + store.ingestSession(planIngest(histLike("s12", 12, msgs)), undefined, { completeLog: true }); + const rows3 = rowsOf(store, "s12"); + assert.equal(rows3.length, 5, "still 5 rows after redo"); + const a3 = rows3.find((r) => r.msg_id === "id-a")!; + assert.equal(a3.retired_at, null, "id-a reactivated"); + assert.equal(a3.seq, seq1.get("id-a"), "id-a reactivated at its ORIGINAL seq"); + assert.deepEqual(liveOf(store, "s12"), ["id-a", "id-b", "id-c", "id-d", "id-e"]); + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("S3: seam fingerprint hashes image bytes, not their length (#4390)", () => { + const base = (byte: number, n: number) => Buffer.alloc(n, byte).toString("base64"); + const sameLen = base(1, 4096); + const sameLen2 = base(2, 4096); + const mk = (b64: string) => ({ role: "user", contentType: "image", text: "[image]", imagePart: true, imageBase64: b64 }) as never; + // Different content, identical length → signatures MUST differ. + assert.notEqual(identitySigForTest(mk(sameLen)), identitySigForTest(mk(sameLen2)), "same-length different images must not collide"); + // Identical content → identical signature (seam chaining works). + assert.equal(identitySigForTest(mk(sameLen)), identitySigForTest(mk(sameLen))); + // And a different-length crop of the SAME image stays a different sig from its parent. + assert.notEqual(identitySigForTest(mk(base(1, 2048))), identitySigForTest(mk(base(1, 4096)))); +}); + +test("S4: fork lineage lands in the sessions row (parent_session_id)", () => { + const plan = planIngest(histLike("child-1", 5, [{ id: "id-a", text: "hi", wireIndex: 0 }], "parent-9")); + const row = plan.session as unknown as { parentSessionId: string | null }; + assert.equal(row.parentSessionId, "parent-9", "metadata.derivedFromSessionId must reach the unified row (#4390)"); + const noParent = planIngest(histLike("plain-1", 5, [{ id: "id-a", text: "hi", wireIndex: 0 }])); + assert.equal((noParent.session as unknown as { parentSessionId: string | null }).parentSessionId, null); +}); + +test("S5: v8 migration backfills identity and heals wireIndex-split generations", () => { + const dir = mkdtempSync(join(tmpdir(), "s5-")); + try { + // 1. Write rows with the CURRENT code (post-fix schema v8). + const store = UnifiedStore.open(join(dir, "index.db")); + store.ingestSession(planIngest(histLike("mig", 3, [ + { id: "id-a", text: "alpha", wireIndex: 0 }, + { id: "id-b", text: "bravo", wireIndex: 1 }, + ])), undefined, { completeLog: true }); + // Simulate a PRE-v8 database: the split-generation bug already + // happened historically — id-b re-minted at a later seq with a + // shifted wireIndex inside its content bytes, identity column empty. + const db = (store as unknown as { db: { prepare: (q: string) => { run: (...a: unknown[]) => unknown }; exec: (q: string) => void } }).db; + const dupContent = JSON.stringify(wmsg("id-b", "bravo", 9)); + // The dup generation carries its own CAS bytes under its own hash + // (the polluted formula minted a different content_hash too). + const dupBuf = Buffer.from(dupContent, "utf8"); + db.prepare("INSERT INTO content (hash, byte_len, refcount, algo, bytes) VALUES ('legacy-dup-hash', ?, 1, 'sha256', ?)").run(dupBuf.byteLength, dupBuf); + db.prepare( + "INSERT INTO messages (session_id, seq, msg_id, ref, role, kind, content_hash, identity) VALUES (?, 2, 'id-b', NULL, 'user', 'text', 'legacy-dup-hash', '')", + ).run("mig"); + // Reset to pre-v8 shape: version 7 + empty identity on every row. + db.exec("UPDATE messages SET identity = ''"); + db.exec("PRAGMA user_version = 7"); + store.close(); + + // 2. Reopen — the v7→v8 migration must backfill identity from content + // bytes (id/wireIndex stripped) and dedupe split generations, + // keeping the ORIGINAL arrival (lowest seq). + const reopened = UnifiedStore.open(join(dir, "index.db")); + const rows = rowsOf(reopened, "mig"); + assert.equal(rows.length, 2, "split generation healed (dup removed)"); + const b = rows.find((r) => r.msg_id === "id-b")!; + assert.equal(b.seq, 1, "survivor keeps the original arrival seq"); + assert.ok(b.identity.length >= 16, "identity backfilled"); + // Backfilled identity must match what the live intern computes for + // the same message regardless of wireIndex position. + const store2 = UnifiedStore.open(join(dir, "second.db")); + try { + store2.ingestSession(planIngest(histLike("mig2", 3, [{ id: "id-b", text: "bravo", wireIndex: 42 }]))); + const live = rowsOf(store2, "mig2").find((r) => r.msg_id === "id-b")!; + assert.equal(live.identity, b.identity, "backfilled identity equals live-computed identity (wireIndex-independent)"); + } finally { + store2.close(); + } + reopened.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("S6: gc full vacuum delegates to vacuum() (wal_checkpoint TRUNCATE included)", () => { + const dir = mkdtempSync(join(tmpdir(), "s6-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + store.ingestSession(planIngest(histLike("gcv", 2, [{ id: "id-a", text: "x", wireIndex: 0 }]))); + const calls: string[] = []; + (store as unknown as { vacuum: (m: string) => void }).vacuum = (m: string) => { calls.push(m); }; + store.gc({ archiveAfterDays: 0, archiveDir: join(dir, "arch"), vacuum: "full" }); + assert.deepEqual(calls, ["full"], "gc must route through vacuum() so the TRUNCATE checkpoint runs (#4390)"); + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("S7: loadLegacyView reports tail pointer gaps instead of silently shrinking", () => { + const core = { id: "id-a", role: "user", contentType: "text", text: "alpha" }; + const out = loadLegacyView({ + session: { + sessionId: "gap", + createdAt: 1, + updatedAt: 2, + nextBlockId: 1, + nextRunId: 1, + metaJson: "{}", + statsJson: "{}", + metadataJson: "{}", + kstateJson: JSON.stringify(createInitialState()), + tailJson: JSON.stringify({ pluginSnapshotSeqs: null, tailSeqs: [0, 7, 8], tailFolded: false }), + ccrJson: null, + orphansJson: null, + }, + messages: [ + { seq: 0, msgId: "id-a", ref: null, content: JSON.stringify(core) }, + { seq: 8, msgId: "id-b", ref: null, content: JSON.stringify({ ...core, id: "id-b", text: "b" }) }, + ], + blocks: [], + deadRefs: [], + refs: [], + }); + // seq 7 is pointed at but missing → must be REPORTED (and the tail keeps + // its surviving members rather than pretending nothing happened). + assert.equal(out.lastMessages!.length, 2); + assert.equal((out as unknown as { tailGaps?: number }).tailGaps, 1, "missing tail rows must be counted, not swallowed (#4390)"); +}); + +// keep the hash import honest for fixtures that grow media later +void createHash; diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 3f7592aa49..6c503fd6f6 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -739,7 +739,7 @@ test("schema v2: dev-window v1 database upgrades in place (identity columns adde assert.equal(row!.namespace, "proxy", "namespace backfills with its constant default"); assert.equal(row!.origin, null, "origin backfills NULL"); const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 7, "schema version stamped to 7 (v2 columns + v3 identity index + v4 extension namespaces + v5 watermark column + v6 orphans journal + v7 media map)"); + assert.equal(ver.user_version, 8, "schema version stamped to 8 (v2 columns + v3 identity index + v4 extension namespaces + v5 watermark column + v6 orphans journal + v7 media map + v8 identity/tombstones)"); // v3 backfill: every pre-existing sessions row claims its // (namespace, conversation_id) identity binding. const binding = (store as unknown as { lookupConversation(ns: string, id: string): string | null }).lookupConversation("proxy", "dev-window-1");