From 343b5063dca8d47300f698d25f7c76abd72240d7 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 23 Oct 2026 15:56:39 +0800 Subject: [PATCH 01/11] storage: position-independent row identity + historian tombstones (#4390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four p2 fixes from the five-dimension storage review, one PR: - identity column: cross-save JOIN key strips id/wireIndex (D4-1); content_hash stays the byte-faithful CAS key. v8 migration backfills and heals wireIndex-split generations (keep lowest seq, refund CAS refs). MySQL twin via information_schema probe + idempotent ALTER. - authoritative sweep retires absent rows (retired_at tombstone) instead of DELETE; rewind-redo reactivates in place at the original seq (remap covers completeLog-less saves too); readers filter retired rows (D4-2). - snapshot-render seam image signature hashes the bytes (24-hex sha256), not the length — same-size screenshots can no longer chain a false seam into silent truncation (D4-3). - fork lineage: parent_session_id from metadata.derivedFromSessionId (D3-2). - gc full vacuum routes through vacuum() (TRUNCATE checkpoint included, files actually shrink, D5-1); loadLegacyView counts tail pointer gaps and all three load paths warn instead of silently shrinking (D4-5). Stakes first: tests/storage-review-p2s.test.ts (7), storage family + golden/snapshot/fold-reconcile/incremental/web regressions green. --- CHANGELOG.md | 9 ++ src/server/snapshot-render.ts | 13 +- src/storage/identity.ts | 99 +++++++++++++ src/storage/ingest.ts | 95 +++++------- src/storage/mysql.ts | 107 ++++++++++---- src/storage/schema.ts | 86 ++++++++++- src/storage/shadow.ts | 18 ++- src/storage/store.ts | 106 ++++++++++---- tests/storage-review-p2s.test.ts | 241 +++++++++++++++++++++++++++++++ tests/storage-unified.test.ts | 2 +- 10 files changed, 654 insertions(+), 122 deletions(-) create mode 100644 src/storage/identity.ts create mode 100644 tests/storage-review-p2s.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c6375674f..c694481c1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] +### Storage — 行身份与史官化(#4390,storage review 四个 p2 + 三个顺路 p3) + +- **行身份去污染(D4-1)**:统一存储 messages 行新增 `identity` 列——跨保存 JOIN 键改为**位置无关**序列化(剥离 `id`/`wireIndex` 后哈希)。此前身份掺入 wire 位置元数据,客户端头部截断一条消息就会让后续全部行换血(旧行被清扫、seq 重铸、#4335 的区间账记随之抖动)。`content_hash` 保持字节忠实 CAS 键不变。存量库 v8 迁移:回填 identity 并合并 wireIndex 分裂世代(保最低 seq,重复行退还 CAS 引用)。MySQL 同款(information_schema 探测幂等 ALTER + 回填)。 +- **权威集清扫改墓碑(D4-2)**:proxy 会话每轮全量保存不再物理 DELETE 缺席行,改为 `retired_at` 墓碑——rewind/undo 的历史在统一存储里**耐久保留**;同一身份回来时原地复活(保原 seq,含 completeLog-less 保存路径 remap 复活)。读面(getMessages/统一重放/视图)默认过滤 retired;CAS 引用不释放。deleteSession/archive/GC 继续拥有物理生命周期。 +- **渲染接缝图片真指纹(D4-3)**:snapshot-render 的 identitySig 图片段从 `长度` 改为 `sha256(base64) 前 24 hex`——同尺寸截图不再可能连击假接缝造成静默截断。 +- **fork 谱系入正史(D3-2)**:sessions 行的 `parent_session_id` 现在从信封 `metadata.derivedFromSessionId` 落库(此前恒 null,谱系只活在信封里,信封丢失=家族失忆)。 +- **顺路**:gc full vacuum 走 `vacuum('full')`(带 TRUNCATE checkpoint,文件真正缩小,D5-1);loadLegacyView 尾指针缺行从静默缩水改为计数 + 三处加载路径 warn(D4-5)。 +- 测试:`tests/storage-review-p2s.test.ts` 7 桩(截断保号/墓碑+复活/接缝指纹/谱系/迁移去重/gc checkpoint/缺行响亮);storage 全家族 + golden/snapshot/fold-reconcile/incremental/web 回归绿。 + ### Fixed - **增量车道:拉取回执修复 ack 传输盲区 + 漂移回退取富侧(#4352/#4323)**。2026-10-22 事故双根因修复:① pi 的 after_provider_response 只在部分 provider 传输下暴露响应头,x-bili-applied-seq 丢失时 noteIncrementalAck 永不执行,后续每轮都整包 re-seed(sinceSeq=0 重发 21K 条历史)。现在 GET /__bili/plugin/turn?conversation= 返回热 canonical 的 {appliedSeq, boundary}(boundary=末条 wire 消息哈希,冷重建诚实返回 null),pi 扩展在规划下一轮前拉取回执,仅在 boundary 证明 canonical 吸收了我方末条消息时采纳——绝不按 seq 前进盲目采纳(外来的全量重发也会推进 canonical);② #4323 多实例写同一目录时,漂移回退无条件吃旧信封,把索引里更新的折叠世代(26,971 refs/16 active)换成旧世代(11,849/12),覆盖与 canonical 失配→0 可压区间→preflight 8.29M 假性死局 502。现在漂移两侧按 mapped ref 数取富侧(平局信封赢,平局语义=原子整文件)。新增 tests/incremental-ack-pull.test.ts 3 例(响应头 ack 回归钉/拉取采纳/冷 boundary null 拒采纳)+ storage-full-flip 4b(陈旧信封让位新索引世代)。 diff --git a/src/server/snapshot-render.ts b/src/server/snapshot-render.ts index ed8f4c85b4..de1cd2baae 100644 --- a/src/server/snapshot-render.ts +++ b/src/server/snapshot-render.ts @@ -60,6 +60,8 @@ const MAX_CHAIN_WALK = 8192; * MIN_RESYNC_RUN in fold-reconcile (there 3, because reconcile matches on * position alone; here the second consecutive match must ALSO line up with * the view, which a stray duplicate cannot provide). */ +import { sha256Hex } from "../storage/store.js"; + const MIN_SEAM_RUN = 2; /** #3658 canonical identity signature minus the hash: the tuple the kernel @@ -69,10 +71,19 @@ const MIN_SEAM_RUN = 2; * same bytes ⇒ same length; different images practically always differ). */ function identitySig(m: BiliMessage): string { const text = typeof m.text === "string" ? m.text : ""; - const img = m.imagePart === true ? `|img:${(m.imageBase64 ?? "").length}` : ""; + // #4390: hash the image BYTES, never the length — same-size screenshots + // used to collide and chain a false seam (silent truncation). The walk + // only touches the tail region, so per-request hashing is bounded; a + // 24-hex prefix is far beyond collision reach for seam evidence. + const img = m.imagePart === true ? `|img:${sha256Hex(m.imageBase64 ?? "").slice(0, 24)}` : ""; return `${m.role}|${m.contentType}|${m.toolName ?? ""}|${text}${img}`; } +/** Test seam (#4390): the seam fingerprint under test — see identitySig. */ +export function identitySigForTest(m: BiliMessage): string { + return identitySig(m); +} + interface ViewRow { seq: number; msg: BiliMessage; diff --git a/src/storage/identity.ts b/src/storage/identity.ts new file mode 100644 index 0000000000..ba1a2f1571 --- /dev/null +++ b/src/storage/identity.ts @@ -0,0 +1,99 @@ +/** + * #4390: identity helpers for the unified store. + * + * The messages table carries TWO hashes with different jobs: + * - content_hash: the CAS key of the stored bytes (byte-faithful, includes + * sidecars; putContent refcounts by it). NEVER migrates. + * - identity: the cross-save JOIN key ("is this the message I already + * know?"). Its input STRIPS position metadata — m.id itself and the + * wireIndex the wire codecs stamp on every parse — because those change + * when the client truncates/edits ANYWHERE in the resent history while + * the message's logical content does not (#4390 D4-1: one head-truncation + * used to re-mint the whole surviving generation: new hashes, new seqs, + * block interval bounds built on sand). + * + * Same identity ⇒ same logical message ⇒ same row, same seq. A re-arrival + * with different bytes but the same identity (sidecar drift) is suppressed — + * the first generation's bytes stay; a real edit changes the content-derived + * id and therefore the identity. + */ +import { createHash } from "node:crypto"; + +function sha256Hex(bytes: string | Buffer): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +/** + * Media payloads below this size ride inline even inside raw sidecars — + * splitting them costs a CAS row + a join for a few hundred bytes (#4069). + */ +const MIN_MEDIA_URL_SPLIT_BYTES = 2048; +/** data:;base64, with the payload swapped for its CAS hash. */ +const MEDIA_URL_MARKER_RE = new RegExp( + `data:([a-zA-Z0-9.+\\/-]+);base64,([A-Za-z0-9+/=]{${MIN_MEDIA_URL_SPLIT_BYTES},})`, + "g", +); + +export function splitMediaContent(content: string): { content: string; hash: string; media: { hash: string; bytes: string }[] } { + let media: { hash: string; bytes: string }[] = []; + let split = content; + if (content.includes("\"imageBase64\"")) { + let parsed: { imageBase64?: unknown }; + try { + parsed = JSON.parse(content) as { imageBase64?: unknown }; + } catch { + parsed = undefined as unknown as { imageBase64?: unknown }; + } + if (parsed) { + const bytes = parsed.imageBase64; + if (typeof bytes === "string" && bytes.length > 0) { + const mediaHash = sha256Hex(bytes); + parsed.imageBase64 = { $media: mediaHash }; + split = JSON.stringify(parsed); + media.push({ hash: mediaHash, bytes }); + } + } + } + // #4069: raw wire sidecars (rawOpenaiContent / rawOpenaiContentParts / + // rawResponsesItem / rawGoogleParts, …) carry the ORIGINAL part verbatim — + // for image parts that means the same base64 payload a second time, as a + // data URL. Swap every sufficiently large payload for its CAS hash, mime + // prefix and JSON structure untouched, so hydration is byte-identical. + if (split.includes(";base64,")) { + const byHash = new Map(media.map((m) => [m.hash, m.bytes])); + split = split.replace(MEDIA_URL_MARKER_RE, (_all: string, mime: string, payload: string) => { + const mediaHash = sha256Hex(payload); + if (!byHash.has(mediaHash)) { + byHash.set(mediaHash, payload); + media = [...media, { hash: mediaHash, bytes: payload }]; + } + return `data:${mime};base64,{$media:${mediaHash}}`; + }); + } + return { content: split, hash: sha256Hex(split), media }; +} + +/** Strip the position/identity metadata from a serialized CoreMessage. + * JSON.parse preserves key insertion order, so the stripped serialization + * is deterministic for the same object shape. */ +function identityInputOf(raw: string): string { + let parsed: { id?: unknown; wireIndex?: unknown }; + try { + parsed = JSON.parse(raw) as { id?: unknown; wireIndex?: unknown }; + } catch { + return raw; + } + if (parsed.id === undefined && parsed.wireIndex === undefined) return raw; + delete parsed.id; + delete parsed.wireIndex; + return JSON.stringify(parsed); +} + +/** Identity from a STORED content column (already media-split; may still + * carry id/wireIndex from pre-#4390 bytes). Strip and media-split + * commute, so this equals the identity the live intern pass computes for + * the same message — the migration backfill and the direct-caller + * fallbacks in store.ts rely on that equivalence. */ +export function identityOfStoredContent(content: string): string { + return sha256Hex(identityInputOf(content)); +} diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index d9470406b5..46eb1b07fd 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -29,6 +29,7 @@ import type { CoreMessage, CompressionBlock } from "../../kernel/src/types.js"; import type { BlockView } from "../session.js"; import { refToIndex } from "../../kernel/src/refs.js"; import { sha256Hex, type BlockRow, type CcrEntryRow, type IngestSessionInput, type MessageRow, type RefRow, type SessionRow } from "./store.js"; +import { splitMediaContent, identityOfStoredContent } from "./identity.js"; /** * Field order of the hydration merge output — DERIVED from mergeState itself @@ -87,61 +88,12 @@ export const ingestSerializations = { count: 0 }; * reader (getMessages hydrates before returning) sees the verbatim original. * No media (or empty) → passthrough with the plain content hash. */ -/** - * Media payloads below this size ride inline even inside raw sidecars — - * splitting them costs a CAS row + a join for a few hundred bytes (#4069). - */ -const MIN_MEDIA_URL_SPLIT_BYTES = 2048; -/** data:;base64, with the payload swapped for its CAS hash. */ -const MEDIA_URL_MARKER_RE = new RegExp( - `data:([a-zA-Z0-9.+\\/-]+);base64,([A-Za-z0-9+/=]{${MIN_MEDIA_URL_SPLIT_BYTES},})`, - "g", -); - -function splitMediaContent(content: string): { content: string; hash: string; media: { hash: string; bytes: string }[] } { - let media: { hash: string; bytes: string }[] = []; - let split = content; - if (content.includes("\"imageBase64\"")) { - let parsed: { imageBase64?: unknown }; - try { - parsed = JSON.parse(content) as { imageBase64?: unknown }; - } catch { - parsed = undefined as unknown as { imageBase64?: unknown }; - } - if (parsed) { - const bytes = parsed.imageBase64; - if (typeof bytes === "string" && bytes.length > 0) { - const mediaHash = sha256Hex(bytes); - parsed.imageBase64 = { $media: mediaHash }; - split = JSON.stringify(parsed); - media.push({ hash: mediaHash, bytes }); - } - } - } - // #4069: raw wire sidecars (rawOpenaiContent / rawOpenaiContentParts / - // rawResponsesItem / rawGoogleParts, …) carry the ORIGINAL part verbatim — - // for image parts that means the same base64 payload a second time, as a - // data URL. Swap every sufficiently large payload for its CAS hash, mime - // prefix and JSON structure untouched, so hydration is byte-identical. - if (split.includes(";base64,")) { - const byHash = new Map(media.map((m) => [m.hash, m.bytes])); - split = split.replace(MEDIA_URL_MARKER_RE, (_all: string, mime: string, payload: string) => { - const mediaHash = sha256Hex(payload); - if (!byHash.has(mediaHash)) { - byHash.set(mediaHash, payload); - media = [...media, { hash: mediaHash, bytes: payload }]; - } - return `data:${mime};base64,{$media:${mediaHash}}`; - }); - } - return { content: split, hash: sha256Hex(split), media }; -} - interface LogEntry { seq: number; msgId: string; content: string; hash: string; + identity: string; role: string; kind: string; ref: string | null; @@ -200,12 +152,16 @@ export function planIngest(session: LegacySessionLike): IngestPlan { const parsed = JSON.parse(raw) as { role: string; contentType?: string }; const split = splitMediaContent(raw); const entry = { msgId: m.id, content: split.content, hash: split.hash, role: parsed.role, kind: parsed.contentType ?? "message", media: split.media }; - const key = `${m.id}\u0000${entry.hash}`; + // #4390: the cross-save JOIN key is the POSITION-INDEPENDENT identity + // (stored-bytes serialization minus id/wireIndex). content_hash stays + // the byte-faithful CAS key; only the join moves to identity. + const identity = identityOfStoredContent(entry.content); + const key = `${m.id}\u0000${identity}`; const existing = seqByIdentity.get(key); if (existing !== undefined) return existing; const seq = log.length; seqByIdentity.set(key, seq); - log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null, ...(entry.media.length > 0 ? { media: entry.media } : {}) }); + log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, identity, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null, ...(entry.media.length > 0 ? { media: entry.media } : {}) }); return seq; }; const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null; @@ -308,6 +264,7 @@ export function planIngestDelta( msgId: m.id, content: split.content, hash: split.hash, + identity: identityOfStoredContent(split.content), role: parsed.role, kind: parsed.contentType ?? "message", ref: byRawRef[m.id] ?? null, @@ -353,6 +310,7 @@ function finishPlan( kind: e.kind, content: e.content, contentHash: e.hash, + identity: e.identity, ...(e.media && e.media.length > 0 ? { media: e.media } : {}), })); @@ -416,7 +374,12 @@ function finishPlan( const sessionRow: SessionRow = { sessionId: session.id, - parentSessionId: null, + // #4390: fork lineage into the unified row — the envelope's + // metadata.derivedFromSessionId (written by handle.ts at fork time) + // used to be the ONLY carrier; envelope loss orphaned the family. + parentSessionId: typeof session.metadata.derivedFromSessionId === "string" && session.metadata.derivedFromSessionId.length > 0 + ? session.metadata.derivedFromSessionId + : null, forkPointSeq: null, protocol: typeof session.meta.protocol === "string" ? session.meta.protocol : null, label: typeof session.meta.label === "string" ? session.meta.label : null, @@ -500,6 +463,8 @@ export function loadLegacyView( lastMessagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; contentStore?: LegacySessionLike["contentStore"]; + /** #4390: tail/pluginSnapshot pointers whose backing row is missing. */ + tailGaps?: number; } { const s = rows.session; const kstate = JSON.parse(s.kstateJson) as Record; @@ -569,19 +534,27 @@ export function loadLegacyView( } const messagesBySeq = new Map(rows.messages.map((m) => [m.seq, JSON.parse(m.content) as CoreMessage])); - const lastMessages = tail.tailSeqs - ? tail.tailSeqs.map((seq) => messagesBySeq.get(seq)).filter((m): m is CoreMessage => m !== undefined) - : undefined; - const pluginSnapshot = tail.pluginSnapshotSeqs - ? tail.pluginSnapshotSeqs - .map((seq) => messagesBySeq.get(seq)) - .filter((m): m is CoreMessage => m !== undefined) - : undefined; + // #4390: a tail pointer whose row is gone used to be silently filtered + // out — the view shrank with no trace. Count the gaps so callers can + // warn (persist logs them); the surviving members still load. + let tailGaps = 0; + const resolveSeqs = (seqs: number[]): CoreMessage[] => { + const out: CoreMessage[] = []; + for (const seq of seqs) { + const m = messagesBySeq.get(seq); + if (m === undefined) tailGaps++; + else out.push(m); + } + return out; + }; + const lastMessages = tail.tailSeqs ? resolveSeqs(tail.tailSeqs) : undefined; + const pluginSnapshot = tail.pluginSnapshotSeqs ? resolveSeqs(tail.pluginSnapshotSeqs) : undefined; const out: ReturnType = { id: s.sessionId, createdAt: s.createdAt, lastSeen: s.updatedAt, + tailGaps, meta: JSON.parse(s.metaJson), stats: JSON.parse(s.statsJson), metadata: JSON.parse(s.metadataJson), diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 6750f46b7e..602e616462 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -32,6 +32,7 @@ import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync } fr import { join } from "node:path"; import { log as loggerLog } from "../logger.js"; import { sha256Hex } from "./store.js"; +import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; import type { StorageBackend } from "./backend.js"; import type { @@ -105,10 +106,13 @@ CREATE TABLE IF NOT EXISTS messages ( role VARCHAR(64) NOT NULL, kind VARCHAR(64) NOT NULL DEFAULT 'message', content_hash VARCHAR(64) NOT NULL, + identity VARCHAR(64) NOT NULL DEFAULT '', + retired_at BIGINT NULL, PRIMARY KEY (session_id, seq), UNIQUE KEY uq_messages_identity (session_id, msg_id, content_hash), KEY idx_messages_session (session_id, seq), - KEY idx_messages_msg_id (session_id, msg_id) + KEY idx_messages_msg_id (session_id, msg_id), + KEY idx_messages_identity (session_id, msg_id, identity) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS blocks ( @@ -252,6 +256,37 @@ export class MysqlBackend implements StorageBackend { "ON DUPLICATE KEY UPDATE meta_value = meta_value", [String(MYSQL_SCHEMA_VERSION)], ); + // #4390 v8: position-independent identity + historian tombstones. + // CREATE TABLE IF NOT EXISTS does not upgrade an existing v7 + // table — probe information_schema and ALTER idempotently + // (MySQL has no ADD COLUMN IF NOT EXISTS). + const [cols] = (await conn.query( + "SELECT COLUMN_NAME AS col FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'messages'", + )) as unknown as Row[][]; + const have = new Set((cols as unknown as { col?: unknown }[]).map((r) => String(r.col ?? ""))); + if (!have.has("identity")) { + await conn.query("ALTER TABLE messages ADD COLUMN identity VARCHAR(64) NOT NULL DEFAULT ''"); + await conn.query("ALTER TABLE messages ADD INDEX idx_messages_identity (session_id, msg_id, identity)"); + } + if (!have.has("retired_at")) { + await conn.query("ALTER TABLE messages ADD COLUMN retired_at BIGINT NULL"); + } + if (!have.has("identity") || !have.has("retired_at")) { + // Backfill identities from the CAS bytes (old rows keep the + // polluted formula's hashes until rewritten; heal them now). + const [rows] = (await conn.query( + "SELECT m.session_id AS sid, m.seq AS seq, m.content_hash AS content_hash, c.bytes AS body " + + "FROM messages m JOIN content c ON c.hash = m.content_hash WHERE m.identity = ''", + )) as unknown as Row[][]; + for (const r of rows as unknown as Row[]) { + const body = typeof r.body === "string" ? r.body : Buffer.from(r.body as Uint8Array).toString("utf8"); + await conn.query("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?", [ + identityOfStoredContent(body), + str(r.sid), + num(r.seq), + ]); + } + } } finally { conn.release(); } @@ -328,9 +363,11 @@ export class MysqlBackend implements StorageBackend { } for (const m of input.messages) { if (!m.contentHash) return false; + // #4390: identity join; retired rows count as existing + // (rewind-redo falls back to the full path). const [hit] = (await c.query( - "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND content_hash = ? LIMIT 1", - [s.sessionId, m.msgId, m.contentHash], + "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ? LIMIT 1", + [s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content)], )) as unknown as Row[][]; if (hit.length > 0) return false; } @@ -353,22 +390,29 @@ export class MysqlBackend implements StorageBackend { if (opts?.completeLog === true && !(await this.hasLaneExtOn(c, s.sessionId))) { const incomingIds = new Set(); for (const m of messages) { - incomingIds.add(`${m.msgId}\u0000${m.contentHash ?? sha256Hex(m.content)}`); + incomingIds.add(`${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`); } const staleRows = (await c.query( - "SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?", + "SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?", [s.sessionId], )) as unknown as Row[][]; for (const row of staleRows[0]) { - if (incomingIds.has(`${row.msg_id}\u0000${row.content_hash}`)) continue; - await this.releaseContent(c, str(row.content_hash)); - const media = (await c.query( - "SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?", - [s.sessionId, num(row.seq)], - )) as unknown as Row[][]; - for (const mm of media[0]) await this.releaseContent(c, str(mm.media_hash)); - await c.query("DELETE FROM messages WHERE session_id = ? AND seq = ?", [s.sessionId, num(row.seq)]); - await c.query("DELETE FROM message_media WHERE session_id = ? AND seq = ?", [s.sessionId, num(row.seq)]); + if (incomingIds.has(`${row.msg_id}\u0000${row.identity}`)) { + // #4390 rewind-redo: reactivate the tombstone in place. + if (row.retired_at !== null && row.retired_at !== undefined) { + await c.query("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?", [ + s.sessionId, + num(row.seq), + ]); + } + continue; + } + // #4390 historian sweep: retire, don't delete — CAS + // refs stay owned by the retired row. + await c.query( + "UPDATE messages SET retired_at = ? WHERE session_id = ? AND seq = ? AND retired_at IS NULL", + [Date.now(), s.sessionId, num(row.seq)], + ); } } } @@ -428,8 +472,8 @@ export class MysqlBackend implements StorageBackend { for (const m of messages) { const hash = await this.putContent(c, m.content, m.contentHash); const [res] = (await c.query( - "INSERT IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - [s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash], + "INSERT IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + [s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash, m.identity ?? identityOfStoredContent(m.content)], )) as unknown as [{ affectedRows: number }, unknown]; if (res.affectedRows === 0) { await this.releaseContent(c, hash); @@ -550,19 +594,22 @@ export class MysqlBackend implements StorageBackend { ): Promise<{ messages: MessageRow[]; watermark: number | null }> { const s = input.session; const [rows] = (await c.query( - "SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?", + "SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?", [s.sessionId], )) as unknown as Row[][]; const seqByIdentity = new Map(); + const retiredSeqs: number[] = []; let nextSeq = 0; for (const row of rows) { - seqByIdentity.set(`${row.msg_id}\u0000${row.content_hash}`, num(row.seq)); + // #4390: identity join — position-independent; retired rows keep + // claiming their seq until reactivation clears the tombstone. + if (row.retired_at !== null && row.retired_at !== undefined) retiredSeqs.push(num(row.seq)); + seqByIdentity.set(`${row.msg_id}\u0000${row.identity}`, num(row.seq)); if (num(row.seq) + 1 > nextSeq) nextSeq = num(row.seq) + 1; } const planToAbs = new Map(); const messages = input.messages.map((m) => { - const hash = m.contentHash ?? sha256Hex(m.content); - const key = `${m.msgId}\u0000${hash}`; + const key = `${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`; let abs = seqByIdentity.get(key); if (abs === undefined) { abs = nextSeq++; @@ -571,6 +618,12 @@ export class MysqlBackend implements StorageBackend { planToAbs.set(m.seq, abs); return { ...m, seq: abs }; }); + // #4390 rewind-redo: reactivate in place (no duplicate row). + for (const seq of retiredSeqs) { + if (messages.some((m) => m.seq === seq)) { + await c.query("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?", [s.sessionId, seq]); + } + } const remapSeqs = (seqs: number[] | null | undefined): number[] | null | undefined => seqs === null || seqs === undefined ? seqs : seqs.map((p) => planToAbs.get(p) ?? p); const pluginSnapshotSeqs = remapSeqs(input.pluginSnapshotSeqs); @@ -806,12 +859,15 @@ export class MysqlBackend implements StorageBackend { }; } - async getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { + async getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + // #4390: retired rows stay out of the replay by default. const [rows] = (await this.pool.query( - "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq", + opts?.includeRetired === true + ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" + : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", [sessionId], )) as unknown as Row[][]; - const out: (MessageRow & { contentHash: string })[] = []; + const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = []; for (const r of rows) { const contentHash = str(r.content_hash); out.push({ @@ -822,6 +878,7 @@ export class MysqlBackend implements StorageBackend { role: str(r.role), kind: str(r.kind), contentHash, + retiredAt: nullableNum(r.retired_at), content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? ""), }); } @@ -1066,8 +1123,8 @@ export class MysqlBackend implements StorageBackend { for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); for (const m of dump.messages) { await c.query( - "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash], + "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, m.identity ?? ""], ); } for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) { diff --git a/src/storage/schema.ts b/src/storage/schema.ts index a354971ca2..6f253e7c36 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -63,7 +63,7 @@ // index. Both directions compatible: an older build opening a v5 // file no-ops at the version gate (extra column ignored); a newer // build upgrades a v4 file with one idempotent ALTER TABLE. -const STORAGE_SCHEMA_VERSION = 7; +const STORAGE_SCHEMA_VERSION = 8; /** #3250: cap for `journal_size_limit` (see STORAGE_PRAGMAS). Passive and * auto checkpoints move frames into the main db and reset the write offset @@ -97,6 +97,8 @@ export function storagePragmas(walJournalSizeLimitBytes: number): string[] { export const STORAGE_PRAGMAS = storagePragmas(WAL_JOURNAL_SIZE_LIMIT); +import { identityOfStoredContent } from "./identity.js"; + const STORAGE_DDL_V1 = `CREATE TABLE IF NOT EXISTS sessions ( session_id TEXT PRIMARY KEY, parent_session_id TEXT, @@ -139,6 +141,8 @@ CREATE TABLE IF NOT EXISTS messages ( role TEXT NOT NULL, kind TEXT NOT NULL DEFAULT 'message', content_hash TEXT NOT NULL REFERENCES content(hash), + identity TEXT NOT NULL DEFAULT '', + retired_at INTEGER, PRIMARY KEY (session_id, seq), UNIQUE (session_id, msg_id, content_hash) ); @@ -271,7 +275,7 @@ CREATE INDEX IF NOT EXISTS idx_checkpoints_session ON checkpoints(session_id, cr /** Applied inside one transaction whenever user_version < STORAGE_SCHEMA_VERSION. */ export function migrateSchema(db: { exec(sql: string): void; - prepare(sql: string): { get(...a: unknown[]): unknown; all(...a: unknown[]): unknown[] }; + prepare(sql: string): { get(...a: unknown[]): unknown; all(...a: unknown[]): unknown[]; run(...a: unknown[]): unknown }; transaction(fn: () => T): T; }): { from: number; to: number } { const row = db.prepare("PRAGMA user_version").get() as { user_version: number } | undefined; @@ -336,6 +340,84 @@ export function migrateSchema(db: { db.exec("ALTER TABLE sessions ADD COLUMN orphans_json TEXT"); } } + if (from < 8) { + // v8 position-independent identity + historian tombstones + // (#4390). identity: the cross-save JOIN key (stored-bytes + // serialization minus id/wireIndex) — content_hash stays the + // byte-faithful CAS key. retired_at: the authoritative-set sweep + // tombstones absent rows instead of deleting them (rewind/undo + // history survives; getMessages filters retired rows). + // Probe-first: dev-window builds whose DDL already carried the + // columns must not fail on duplicate ALTERs (v2 discipline). + const msgCols = (db.prepare("PRAGMA table_info(messages)").all() as unknown as { name?: unknown }[]) + .map((c) => (typeof c?.name === "string" ? c.name : "")); + if (!msgCols.includes("identity")) { + db.exec("ALTER TABLE messages ADD COLUMN identity TEXT NOT NULL DEFAULT ''"); + } + if (!msgCols.includes("retired_at")) { + db.exec("ALTER TABLE messages ADD COLUMN retired_at INTEGER"); + } + db.exec("CREATE INDEX IF NOT EXISTS idx_messages_identity ON messages (session_id, msg_id, identity)"); + // Backfill from stored bytes, healing the wireIndex-split + // generations the polluted formula minted: same (msg_id, + // identity) after the strip → keep the FIRST arrival (lowest + // seq), drop the later duplicates and hand their CAS refs back. + // Message bytes live in the content CAS (messages carry only the + // hash) — join in, decompress nothing (bytes are stored raw). + const rows = db + .prepare("SELECT m.session_id AS session_id, m.seq AS seq, m.msg_id AS msg_id, m.content_hash AS content_hash, c.bytes AS content FROM messages m JOIN content c ON c.hash = m.content_hash") + .all() as { session_id: string; seq: number; msg_id: string; content: string; content_hash: string }[]; + const keep = new Map(); + const drop: { session_id: string; seq: number; content_hash: string }[] = []; + const rawByIdentityKey = new Map(); + for (const r of rows) { + // node:sqlite returns BLOB as Uint8Array — normalize to utf8 + // before hashing (JSON.parse would coerce it to a comma list + // and silently hash the wrong input). + const raw = typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"); + const identity = identityOfStoredContent(raw); + rawByIdentityKey.set(`${r.session_id}\u0000${r.msg_id}\u0000${identity}`, raw); + const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`; + const seen = keep.get(key); + if (seen === undefined) { + keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); + } else if (r.seq < seen.seq) { + // an earlier arrival discovered late — swap the keeper + drop.push({ session_id: seen.session_id, seq: seen.seq, content_hash: seen.content_hash }); + keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); + } else { + drop.push({ session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); + } + } + const identityBySeq = new Map(); + for (const r of rows) { + const raw = rawByIdentityKey.get(`${r.session_id}\u0000${r.msg_id}\u0000${identityOfStoredContent(typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"))}`) ?? (typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8")); + const identity = identityOfStoredContent(raw); + const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`; + if (keep.get(key)?.seq === r.seq) identityBySeq.set(`${r.session_id}\u0000${r.seq}`, identity); + } + const upd = db.prepare("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?"); + for (const [k, identity] of identityBySeq) { + const [sessionId, seqStr] = k.split("\u0000"); + upd.run(identity, sessionId, Number(seqStr)); + } + // Duplicates: delete their media mappings + message rows and + // decrement CAS refcounts (delete referencing rows FIRST — the + // #4242 discipline; refcount may reach 0, the next gc run + // collects those content rows). + const decRef = db.prepare("UPDATE content SET refcount = refcount - 1 WHERE hash = ?"); + const delMedia = db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?"); + const mediaOf = db.prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?"); + const delMsg = db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?"); + for (const d of drop) { + for (const mm of mediaOf.all(d.session_id, d.seq) as { media_hash: string }[]) { + decRef.run(mm.media_hash); + } + delMedia.run(d.session_id, d.seq); + delMsg.run(d.session_id, d.seq); + decRef.run(d.content_hash); + } + } db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`); }); return { from, to: STORAGE_SCHEMA_VERSION }; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 8712291834..6c4617334c 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -369,6 +369,7 @@ export class UnifiedShadow { refs: this.sync.getRefs(legacy.id), }; const replayed = loadLegacyView(rows); + warnTailGaps(legacy.id, replayed); const expectView = { ...legacy, blockContents: orderedBlockContents(completeBlockContents(legacy.blockContents, legacy.state.blocks)) }; const gotView = { ...replayed, blockContents: orderedBlockContents(replayed.blockContents) }; const expect = shadowDigest(expectView); @@ -514,7 +515,9 @@ export class UnifiedShadow { // The view's `state` object carries every MERGE_STATE_FIELDS key by // construction; the declared Record just needs a // widening cast to reach LegacySessionLike. - return loadLegacyView(rows) as LegacySessionLike; + const unified = loadLegacyView(rows) as LegacySessionLike; + warnTailGaps(id, unified as unknown as { tailGaps?: number }); + return unified; } catch (err) { loggerLog("warn", `[storage] unified load failed for ${id} (${String(err)}) — falling back to the legacy file`); return null; @@ -912,13 +915,15 @@ export class UnifiedShadow { const backend = this.backend!; const sessionRow = await backend.getSessionRow(id); if (!sessionRow) return null; - return loadLegacyView({ + const loaded = loadLegacyView({ session: sessionRow, messages: await backend.getMessages(id), blocks: await backend.getBlocks(id), deadRefs: await backend.getDeadRefs(id), refs: await backend.getRefs(id), }) as LegacySessionLike; + warnTailGaps(id, loaded as unknown as { tailGaps?: number }); + return loaded; } catch (err) { loggerLog("warn", `[storage] unified load failed for ${id} (${String(err)}) — falling back to the legacy file`); return null; @@ -1091,6 +1096,15 @@ export class UnifiedShadow { } } +/** #4390: tail/pluginSnapshot pointers whose backing row is gone used to be + * silently dropped — the view shrank with no trace (legacy tail 缺行静默 + * 缩水). Make it loud: every load path reports the gap count. */ +function warnTailGaps(id: string, view: { tailGaps?: number }): void { + if ((view.tailGaps ?? 0) > 0) { + loggerLog("warn", `[storage] ${view.tailGaps} tail pointer(s) for ${id} reference missing rows — view is shorter than the recorded tail (#4390)`); + } +} + function shadowDigest(view: unknown): string { return sha256(JSON.stringify(canonicalize(view))); } diff --git a/src/storage/store.ts b/src/storage/store.ts index e6a0c7a838..215a289e88 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -22,6 +22,7 @@ import { dirname, join } from "node:path"; import { log as loggerLog } from "../logger.js"; import { openSqliteDatabase, type SqliteDatabase, type SqliteEngineName, type SqliteEngineSource } from "./driver.js"; import { migrateSchema, storagePragmas, WAL_JOURNAL_SIZE_LIMIT } from "./schema.js"; +import { identityOfStoredContent } from "./identity.js"; import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } from "./runtime-status.js"; import { listInstances, type RegistryEntry } from "../instance.js"; import type { MessageContentStore } from "acp-kernel"; @@ -100,6 +101,12 @@ export interface MessageRow { * content CAS, refcounted via message_media rows). Ingest-only — * readers always receive hydrated content. */ media?: { hash: string; bytes: string }[]; + /** #4390: position-independent identity (stored-bytes serialization + * minus id/wireIndex) — the cross-save JOIN key. content_hash stays + * the byte-faithful CAS key; identity is what remap/sweep/guard join + * on, so a client head-truncation (wireIndex shift) no longer re-mints + * the surviving generation. Optional for direct callers. */ + identity?: string; } export interface BlockRow { @@ -622,11 +629,14 @@ export class UnifiedStore { // at nothing and the read view would drop it. One indexed // probe per NEW message; any hit → full ingest instead. const existingIdentity = this.db.prepare( - "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND content_hash = ?", + "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ?", ); for (const m of input.messages) { if (!m.contentHash) return false; - if (existingIdentity.get(s.sessionId, m.msgId, m.contentHash)) return false; + // #4390: join on identity (position-independent). Retired + // rows count as existing — a rewind-redo falls back to the + // full path, which reactivates the tombstone in place. + if (existingIdentity.get(s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content))) return false; } // Placeholder→absolute renumber: the plan emitted // watermark+1.. under the aligned assumption; anchoring at @@ -680,26 +690,39 @@ export class UnifiedStore { if (opts?.completeLog === true && !this.hasLaneExt(s.sessionId)) { const incomingIds = new Set(); for (const m of messages) { - incomingIds.add(`${m.msgId}\u0000${m.contentHash ?? sha256Hex(m.content)}`); + incomingIds.add(`${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`); } const staleRows = this.db - .prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?") - .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; + .prepare("SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; identity: string; retired_at: number | null }[]; for (const row of staleRows) { - if (incomingIds.has(`${row.msg_id}\u0000${row.content_hash}`)) continue; - // #4242 delete-on-zero discipline: remove the - // referencing rows FIRST, then release the hashes — - // a release that hits zero now DELETEs the content - // row, which an FK-referencing leftover row would - // reject. (Media mappings ride the message sweep, - // #3762 review.) - const staleMedia = this.db - .prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?") - .all(s.sessionId, row.seq) as { media_hash: string }[]; - this.db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); - this.db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); - this.releaseContent(row.content_hash); - for (const mm of staleMedia) this.releaseContent(mm.media_hash); + if (incomingIds.has(`${row.msg_id}\u0000${row.identity}`)) { + // #4390 rewind-redo: the identity came back — + // reactivate the tombstone at its original seq. + if (row.retired_at !== null) { + this.db + .prepare("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?") + .run(s.sessionId, row.seq); + } + continue; + } + // #4390 historian sweep: the row is absent from the + // authoritative set — RETIRE it, don't delete. The + // unified store is the durable record of everything + // the client ever sent; a rewind/undo that drops a + // message from the view must tombstone it, not + // destroy the bytes (epoch-gate replays #3644 and + // rewind-redo both need the row to still exist — + // reactivation above restores it at the same seq). + // Retired rows leave the unified replay (getMessages + // filters them) so drift healing and the DIGEST + // reconcile keep working exactly as the DELETE did. + // CAS refs are NOT released: retired rows still own + // their bytes. deleteSession/archiveSession/GC own + // the physical lifecycle from here. + this.db + .prepare("UPDATE messages SET retired_at = ? WHERE session_id = ? AND seq = ? AND retired_at IS NULL") + .run(Date.now(), s.sessionId, row.seq); } } } @@ -775,9 +798,9 @@ export class UnifiedStore { const hash = this.putContent(m.content, m.contentHash); const res = this.db .prepare( - "INSERT OR IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT OR IGNORE INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ) - .run(s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash); + .run(s.sessionId, m.seq, m.msgId, m.ref, m.refNum, m.role, m.kind, hash, m.identity ?? identityOfStoredContent(m.content)); if (res.changes === 0) { // Idempotent re-receipt: the CAS row is still referenced // by the existing message row — undo our refcount bump. @@ -984,12 +1007,19 @@ export class UnifiedStore { // stable absolute seqs: same identity (msgId + content hash) // keeps its old seq; new identities continue after max(seq). const existing = this.db - .prepare("SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ?") - .all(s.sessionId) as { seq: number; msg_id: string; content_hash: string }[]; + .prepare("SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; identity: string; retired_at: number | null }[]; const seqByIdentity = new Map(); + const retiredSeqs: number[] = []; let nextSeq = 0; for (const row of existing) { - seqByIdentity.set(`${row.msg_id}\u0000${row.content_hash}`, row.seq); + // #4390: identity join — same logical message keeps its seq even + // when its bytes were re-minted with shifted wire positions. + // Retired rows still claim their seq (INSERT OR IGNORE would + // silently drop a re-mint over their PK) and reactivation below + // clears the tombstone when the identity comes back. + if (row.retired_at !== null) retiredSeqs.push(row.seq); + seqByIdentity.set(`${row.msg_id}\u0000${row.identity}`, row.seq); if (row.seq + 1 > nextSeq) nextSeq = row.seq + 1; } // #3133 watermark gate (full path): the watermark names the abs seq @@ -1002,8 +1032,7 @@ export class UnifiedStore { const messages = input.messages.map((m) => { // #3133: plan rows carry the intern-pass hash; only direct // callers without a planner pay the sha256 here. - const hash = m.contentHash ?? sha256Hex(m.content); - const key = `${m.msgId}\u0000${hash}`; + const key = `${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`; let abs = seqByIdentity.get(key); if (abs === undefined) { abs = nextSeq++; @@ -1012,6 +1041,13 @@ export class UnifiedStore { planToAbs.set(m.seq, abs); return { ...m, seq: abs }; }); + // #4390: a rewound message that comes back reactivates in place — + // the tombstone clears, the original seq is reused, and the view + // heals without a duplicate row even on completeLog-less saves. + for (const seq of retiredSeqs) { + const back = messages.some((m) => m.seq === seq); + if (back) this.db.prepare("UPDATE messages SET retired_at = NULL WHERE session_id = ? AND seq = ?").run(s.sessionId, seq); + } const remapSeqs = (seqs: number[] | null | undefined): number[] | null | undefined => seqs === null || seqs === undefined ? seqs : seqs.map((p) => planToAbs.get(p) ?? p); const pluginSnapshotSeqs = remapSeqs(input.pluginSnapshotSeqs); @@ -1547,9 +1583,11 @@ export class UnifiedStore { } if (opts.vacuum === "full") { loggerLog("info", "[#3901] gc: running full VACUUM (this can take a while on large databases)"); - this.db.exec("VACUUM"); + // #4390: delegate to vacuum() — a bare VACUUM leaves the compacted + // image in the WAL; only the TRUNCATE checkpoint shrinks the files. + this.vacuum("full"); } else if (opts.vacuum === "incremental") { - this.db.exec("PRAGMA incremental_vacuum"); + this.vacuum("incremental"); } return { archived, contentRowsDeleted }; } @@ -2003,9 +2041,16 @@ export class UnifiedStore { }; } - getMessages(sessionId: string): (MessageRow & { contentHash: string })[] { + getMessages(sessionId: string, opts?: { includeRetired?: boolean }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { + // #4390: retired rows (rewound/undo history) stay out of the unified + // replay by default — they are history, not view. Callers that need + // the full historian record (export/diagnostics) opt in. const rows = this.db - .prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq") + .prepare( + opts?.includeRetired === true + ? "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? ORDER BY seq" + : "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at FROM messages WHERE session_id = ? AND retired_at IS NULL ORDER BY seq", + ) .all(sessionId) as Record[]; return rows.map((r) => ({ seq: r.seq as number, @@ -2015,6 +2060,7 @@ export class UnifiedStore { role: r.role as string, kind: r.kind as string, contentHash: r.content_hash as string, + retiredAt: (r.retired_at as number | null) ?? null, content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), })); } diff --git a/tests/storage-review-p2s.test.ts b/tests/storage-review-p2s.test.ts new file mode 100644 index 0000000000..9bb782227c --- /dev/null +++ b/tests/storage-review-p2s.test.ts @@ -0,0 +1,241 @@ +/** + * #4390: storage foundation defects found by the five-dimension review. + * Stakes FIRST (RED on master), then the fixes: + * S1 identity must not include position metadata — a client head-truncation + * (wireIndex shift) must NOT re-mint rows/seqs for surviving messages. + * S2 the authoritative-set sweep must RETIRE (tombstone) absent rows + * instead of DELETE-ing them; a rewind-redo reactivates the SAME seq. + * S3 snapshot-render seam fingerprints must hash image bytes, not length: + * same-length different images must NOT match; identical images MUST. + * S4 fork lineage (metadata.derivedFromSessionId) must land in the + * sessions row instead of being dropped (dead columns). + * S5 schema migration backfills the identity column and heals + * wireIndex-split duplicate generations, preserving survivor seqs. + * S6 gc(vacuum:'full') must delegate to vacuum() (wal_checkpoint TRUNCATE). + * S7 loadLegacyView must REPORT tail pointer gaps instead of silently + * shrinking the tail view. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { createInitialState } from "acp-kernel"; +import { planIngest, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { identitySigForTest } from "../src/server/snapshot-render.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "legacy"; +process.env.BILI_CONFIG_FILE = join(mkdtempSync(join(tmpdir(), "p2s-")), "missing.json"); + +/** CoreMessage-shaped fixture; wireIndex simulates the positional metadata the + * wire codecs stamp on every parse (the D4-1 pollutant). */ +function wmsg(id: string, text: string, wireIndex: number) { + return { id, role: "user", contentType: "text", text, wireIndex }; +} + +function histLike(id: string, requests: number, msgs: { id: string; text: string; wireIndex: number }[], parent?: string): LegacySessionLike { + const state = { ...createInitialState(), requests }; + state.nextBlockId = 1; + return { + id, + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { label: id, protocol: "anthropic", upstreamOrigin: "https://api.anthropic.com" }, + stats: { requests }, + metadata: parent ? { pluginAgent: "proxy", derivedFromSessionId: parent } : { pluginAgent: "proxy" }, + state: state as never, + blockContents: new Map() as never, + lastMessages: msgs.map((m) => wmsg(m.id, m.text, m.wireIndex)) as never, + lastMessagesFolded: false, + }; +} + +type Row = { seq: number; msg_id: string; content_hash: string; identity: string; retired_at: number | null }; + +function rowsOf(store: UnifiedStore, sessionId: string): Row[] { + const db = (store as unknown as { db: { prepare: (q: string) => { all: (...a: unknown[]) => unknown[] } } }).db; + return db + .prepare("SELECT seq, msg_id, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq") + .all(sessionId) as Row[]; +} + +function liveOf(store: UnifiedStore, sessionId: string): string[] { + // getMessages is the unified replay surface — after the fix it filters retired rows. + return store + .getMessages(sessionId) + .map((r) => (JSON.parse(r.content as unknown as string) as { id?: string }).id ?? "?"); +} + +test("S1+S2: head truncation keeps identities/seqs stable; sweep retires (not deletes); rewind-redo reactivates the same seq", () => { + const dir = mkdtempSync(join(tmpdir(), "s1-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + const msgs = [ + { id: "id-a", text: "alpha", wireIndex: 0 }, + { id: "id-b", text: "bravo", wireIndex: 1 }, + { id: "id-c", text: "charlie", wireIndex: 2 }, + { id: "id-d", text: "delta", wireIndex: 3 }, + { id: "id-e", text: "echo", wireIndex: 4 }, + ]; + store.ingestSession(planIngest(histLike("s12", 10, msgs)), undefined, { completeLog: true }); + const seq1 = new Map(rowsOf(store, "s12").map((r) => [r.msg_id, r.seq])); + assert.equal(seq1.size, 5); + + // Client truncated the first message → the re-parse shifts every + // surviving wireIndex down by one. Same ids, same content, new wire + // positions. Old behavior: whole row generation re-minted. + const truncated = msgs.slice(1).map((m, i) => ({ ...m, wireIndex: i })); + store.ingestSession(planIngest(histLike("s12", 11, truncated)), undefined, { completeLog: true }); + + const rows2 = rowsOf(store, "s12"); + // b..e survive at their ORIGINAL seqs (identity ignores wireIndex). + for (const m of truncated) { + const row = rows2.find((r) => r.msg_id === m.id); + assert.ok(row, `row for ${m.id} exists`); + assert.equal(row.seq, seq1.get(m.id), `${m.id} keeps its save-1 seq (identity stability, #4390)`); + } + // a is RETIRED (tombstoned), not deleted — the historian keeps the bytes. + const a2 = rows2.find((r) => r.msg_id === "id-a"); + assert.ok(a2, "retired row for id-a still exists (tombstone, #4390)"); + assert.notEqual(a2.retired_at, null, "id-a is retired"); + // And no duplicate generation was minted for the survivors. + assert.equal(rows2.length, 5, "no re-minted rows (5 originals: 4 live + 1 retired)"); + assert.deepEqual(liveOf(store, "s12"), ["id-b", "id-c", "id-d", "id-e"]); + + // Rewind-redo: the truncated head comes back. Same identity → same seq. + store.ingestSession(planIngest(histLike("s12", 12, msgs)), undefined, { completeLog: true }); + const rows3 = rowsOf(store, "s12"); + assert.equal(rows3.length, 5, "still 5 rows after redo"); + const a3 = rows3.find((r) => r.msg_id === "id-a")!; + assert.equal(a3.retired_at, null, "id-a reactivated"); + assert.equal(a3.seq, seq1.get("id-a"), "id-a reactivated at its ORIGINAL seq"); + assert.deepEqual(liveOf(store, "s12"), ["id-a", "id-b", "id-c", "id-d", "id-e"]); + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("S3: seam fingerprint hashes image bytes, not their length (#4390)", () => { + const base = (byte: number, n: number) => Buffer.alloc(n, byte).toString("base64"); + const sameLen = base(1, 4096); + const sameLen2 = base(2, 4096); + const mk = (b64: string) => ({ role: "user", contentType: "image", text: "[image]", imagePart: true, imageBase64: b64 }) as never; + // Different content, identical length → signatures MUST differ. + assert.notEqual(identitySigForTest(mk(sameLen)), identitySigForTest(mk(sameLen2)), "same-length different images must not collide"); + // Identical content → identical signature (seam chaining works). + assert.equal(identitySigForTest(mk(sameLen)), identitySigForTest(mk(sameLen))); + // And a different-length crop of the SAME image stays a different sig from its parent. + assert.notEqual(identitySigForTest(mk(base(1, 2048))), identitySigForTest(mk(base(1, 4096)))); +}); + +test("S4: fork lineage lands in the sessions row (parent_session_id)", () => { + const plan = planIngest(histLike("child-1", 5, [{ id: "id-a", text: "hi", wireIndex: 0 }], "parent-9")); + const row = plan.session as unknown as { parentSessionId: string | null }; + assert.equal(row.parentSessionId, "parent-9", "metadata.derivedFromSessionId must reach the unified row (#4390)"); + const noParent = planIngest(histLike("plain-1", 5, [{ id: "id-a", text: "hi", wireIndex: 0 }])); + assert.equal((noParent.session as unknown as { parentSessionId: string | null }).parentSessionId, null); +}); + +test("S5: v8 migration backfills identity and heals wireIndex-split generations", () => { + const dir = mkdtempSync(join(tmpdir(), "s5-")); + try { + // 1. Write rows with the CURRENT code (post-fix schema v8). + const store = UnifiedStore.open(join(dir, "index.db")); + store.ingestSession(planIngest(histLike("mig", 3, [ + { id: "id-a", text: "alpha", wireIndex: 0 }, + { id: "id-b", text: "bravo", wireIndex: 1 }, + ])), undefined, { completeLog: true }); + // Simulate a PRE-v8 database: the split-generation bug already + // happened historically — id-b re-minted at a later seq with a + // shifted wireIndex inside its content bytes, identity column empty. + const db = (store as unknown as { db: { prepare: (q: string) => { run: (...a: unknown[]) => unknown }; exec: (q: string) => void } }).db; + const dupContent = JSON.stringify(wmsg("id-b", "bravo", 9)); + // The dup generation carries its own CAS bytes under its own hash + // (the polluted formula minted a different content_hash too). + const dupBuf = Buffer.from(dupContent, "utf8"); + db.prepare("INSERT INTO content (hash, byte_len, refcount, algo, bytes) VALUES ('legacy-dup-hash', ?, 1, 'sha256', ?)").run(dupBuf.byteLength, dupBuf); + db.prepare( + "INSERT INTO messages (session_id, seq, msg_id, ref, role, kind, content_hash, identity) VALUES (?, 2, 'id-b', NULL, 'user', 'text', 'legacy-dup-hash', '')", + ).run("mig"); + // Reset to pre-v8 shape: version 7 + empty identity on every row. + db.exec("UPDATE messages SET identity = ''"); + db.exec("PRAGMA user_version = 7"); + store.close(); + + // 2. Reopen — the v7→v8 migration must backfill identity from content + // bytes (id/wireIndex stripped) and dedupe split generations, + // keeping the ORIGINAL arrival (lowest seq). + const reopened = UnifiedStore.open(join(dir, "index.db")); + const rows = rowsOf(reopened, "mig"); + assert.equal(rows.length, 2, "split generation healed (dup removed)"); + const b = rows.find((r) => r.msg_id === "id-b")!; + assert.equal(b.seq, 1, "survivor keeps the original arrival seq"); + assert.ok(b.identity.length >= 16, "identity backfilled"); + // Backfilled identity must match what the live intern computes for + // the same message regardless of wireIndex position. + const store2 = UnifiedStore.open(join(dir, "second.db")); + try { + store2.ingestSession(planIngest(histLike("mig2", 3, [{ id: "id-b", text: "bravo", wireIndex: 42 }]))); + const live = rowsOf(store2, "mig2").find((r) => r.msg_id === "id-b")!; + assert.equal(live.identity, b.identity, "backfilled identity equals live-computed identity (wireIndex-independent)"); + } finally { + store2.close(); + } + reopened.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("S6: gc full vacuum delegates to vacuum() (wal_checkpoint TRUNCATE included)", () => { + const dir = mkdtempSync(join(tmpdir(), "s6-")); + try { + const store = UnifiedStore.open(join(dir, "index.db")); + store.ingestSession(planIngest(histLike("gcv", 2, [{ id: "id-a", text: "x", wireIndex: 0 }]))); + const calls: string[] = []; + (store as unknown as { vacuum: (m: string) => void }).vacuum = (m: string) => { calls.push(m); }; + store.gc({ archiveAfterDays: 0, archiveDir: join(dir, "arch"), vacuum: "full" }); + assert.deepEqual(calls, ["full"], "gc must route through vacuum() so the TRUNCATE checkpoint runs (#4390)"); + store.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("S7: loadLegacyView reports tail pointer gaps instead of silently shrinking", () => { + const core = { id: "id-a", role: "user", contentType: "text", text: "alpha" }; + const out = loadLegacyView({ + session: { + sessionId: "gap", + createdAt: 1, + updatedAt: 2, + nextBlockId: 1, + nextRunId: 1, + metaJson: "{}", + statsJson: "{}", + metadataJson: "{}", + kstateJson: JSON.stringify(createInitialState()), + tailJson: JSON.stringify({ pluginSnapshotSeqs: null, tailSeqs: [0, 7, 8], tailFolded: false }), + ccrJson: null, + orphansJson: null, + }, + messages: [ + { seq: 0, msgId: "id-a", ref: null, content: JSON.stringify(core) }, + { seq: 8, msgId: "id-b", ref: null, content: JSON.stringify({ ...core, id: "id-b", text: "b" }) }, + ], + blocks: [], + deadRefs: [], + refs: [], + }); + // seq 7 is pointed at but missing → must be REPORTED (and the tail keeps + // its surviving members rather than pretending nothing happened). + assert.equal(out.lastMessages!.length, 2); + assert.equal((out as unknown as { tailGaps?: number }).tailGaps, 1, "missing tail rows must be counted, not swallowed (#4390)"); +}); + +// keep the hash import honest for fixtures that grow media later +void createHash; diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 3f7592aa49..6c503fd6f6 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -739,7 +739,7 @@ test("schema v2: dev-window v1 database upgrades in place (identity columns adde assert.equal(row!.namespace, "proxy", "namespace backfills with its constant default"); assert.equal(row!.origin, null, "origin backfills NULL"); const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 7, "schema version stamped to 7 (v2 columns + v3 identity index + v4 extension namespaces + v5 watermark column + v6 orphans journal + v7 media map)"); + assert.equal(ver.user_version, 8, "schema version stamped to 8 (v2 columns + v3 identity index + v4 extension namespaces + v5 watermark column + v6 orphans journal + v7 media map + v8 identity/tombstones)"); // v3 backfill: every pre-existing sessions row claims its // (namespace, conversation_id) identity binding. const binding = (store as unknown as { lookupConversation(ns: string, id: string): string | null }).lookupConversation("proxy", "dev-window-1"); From aca706ffdd7410180f9ea1f6f2185127fcf07218 Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 24 Oct 2026 02:11:13 +0800 Subject: [PATCH 02/11] fix(#4390): drop tailGaps from parity digests + fix stale v7 test pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fix for CI reds on this branch, part 1 of 2. - loadLegacyView's new tailGaps counter (D4-5) leaked into the shadow-parity digest surface: the expect leg (live session) structurally lacks the key while the got leg (loadLegacyView output) always carries it, so canonicalize()'s key-preserving normalization made every sampled verify report a fake drift. Drop the key from all digests — same class as the existing storageMode precedent (#2671): presence-not-value, and genuine missing rows still diverge the message arrays themselves, so nothing is masked. describeViewDivergence skips it too, keeping drift diagnostics clean (issue3919 forensics pins keys deepEqual). - Two stale user_version=7 pins the v8 bump missed: tests/storage-ext.test.ts (v3→v7 title/asserts) and tests/storage-ingest-watermark.test.ts (v4→v7 title/assert) now expect v8, matching the updated pin in tests/storage-unified.test.ts. --- src/storage/canonical.ts | 15 +++++++++++---- src/storage/shadow.ts | 4 ++++ tests/storage-ext.test.ts | 10 +++++----- tests/storage-ingest-watermark.test.ts | 4 ++-- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index eda69b2677..4d8b449e7e 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -21,10 +21,12 @@ import { createHash } from "node:crypto"; * provenance (#2671), but its PRESENCE is new (corpus pins predate it), * so masking the value like __VER__ cannot keep old pins green; the key * is removed from digests altogether. - * - * NOT normalized (deliberately contract): every other byte — ref numbers, - * block ids, summaries, stats, metadata. A change to any of those in the - * round-trip is a migration regression and must fail the golden. + * - tailGaps (#4390) → dropped entirely — read-time diagnostic added by + * loadLegacyView (count of tail pointers whose row is missing); the live + * legacy leg of any parity comparison structurally lacks the key, so + * digesting its presence would report drift on every sampled save. Same + * class as storageMode: the PRESENCE, not the value, is the problem. A + * genuinely missing row still diverges the message arrays themselves. */ export function canonicalize(value: unknown): unknown { return canonicalizeWithContext(value, undefined); @@ -42,6 +44,11 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno // unlike biliVersion its PRESENCE is new, so masking the value // is not enough; drop the key from digests entirely. if (k === "storageMode") continue; + // tailGaps (#4390) is read-time gap-count metadata from + // loadLegacyView — the live legacy leg structurally lacks the + // key, so digesting its presence would flag drift on every + // sampled save; drop it like storageMode. + if (k === "tailGaps") continue; out[k] = canonicalizeWithContext(v, k); } return out; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 6c4617334c..8da837a56e 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1141,6 +1141,10 @@ export function describeViewDivergence(expectView: unknown, gotView: unknown): D const ser = (v: unknown): string | null => (v === undefined ? null : JSON.stringify(canonicalize(v))); const keys: string[] = []; for (const k of [...new Set([...Object.keys(e), ...Object.keys(g)])].sort()) { + // tailGaps is read-time diagnostic metadata, not parity surface (#4390) — + // the live legacy leg structurally lacks the key, so it must never be + // named as a diverged dimension (a real gap diverges lastMessages itself). + if (k === "tailGaps") continue; if (ser(e[k]) !== ser(g[k])) keys.push(k); } const diagnostics: DriftDiagnostics = { keys }; diff --git a/tests/storage-ext.test.ts b/tests/storage-ext.test.ts index 3ee171c76a..b837cf0fd9 100644 --- a/tests/storage-ext.test.ts +++ b/tests/storage-ext.test.ts @@ -155,11 +155,11 @@ test("guards reject hostile keys and oversize values", async () => { } }); -test("v3 databases migrate to v7 in place, idempotently", async () => { +test("v3 databases migrate to v8 in place, idempotently", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ext-migrate-")); try { const dbPath = path.join(tmp, "index.db"); - // Build a current (v7) database, then roll the extension tables and + // Build a current (v8) database, then roll the extension tables and // version marker back to the v3 shape — exactly what an upgraded // production lane's file looks like to the new code. const fresh = UnifiedStore.open(dbPath); @@ -174,17 +174,17 @@ test("v3 databases migrate to v7 in place, idempotently", async () => { const migrated = UnifiedStore.open(dbPath); try { const ver = (migrated as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 7, "open() runs migrateSchema: v3 → v7"); + assert.equal(ver.user_version, 8, "open() runs migrateSchema: v3 → v8"); migrated.setKv("probe", "alive", true); assert.deepEqual(migrated.getKv("probe", "alive"), true); assert.deepEqual(migrated.listSessionIds(), [], "existing v3 content untouched"); } finally { migrated.close(); } - // Reopening a v7 database is a no-op (idempotent migration). + // Reopening a v8 database is a no-op (idempotent migration). const again = UnifiedStore.open(dbPath); const ver2 = (again as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver2.user_version, 7); + assert.equal(ver2.user_version, 8); again.close(); } finally { fs.rmSync(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 1d2f523bb1..2fb846e810 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -532,7 +532,7 @@ test("watermark: sampled verification still catches drift through the delta path } }); -test("watermark: v4 database migrates to v7 additively; newer user_version no-ops", async () => { +test("watermark: v4 database migrates to v8 additively; newer user_version no-ops", async () => { const dir = tmpdir("bili-wm-migrate-"); try { const dbPath = path.join(dir, "index.db"); @@ -561,7 +561,7 @@ test("watermark: v4 database migrates to v7 additively; newer user_version no-op const store = UnifiedStore.open(dbPath); try { const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 7, "open() migrates v4 → v7"); + assert.equal(ver.user_version, 8, "open() migrates v4 → v8"); const row = store.getSessionRow("v4-sess"); assert.ok(row, "legacy session row survives the migration"); assert.equal(row!.lastIngestedSeq, null, "legacy rows read back with a NULL watermark (≡ full ingest)"); From 49cfab8021c8fbf8508952e722ff266c6994a5c3 Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 24 Oct 2026 02:18:18 +0800 Subject: [PATCH 03/11] fix: surface the Node.js upgrade lever on the unsupported-node storage degrade (#4444) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a Node.js that has neither the builtin node:sqlite (< 22.5) nor a pinned better-sqlite3 artifact (e.g. Node 21.x, ABI 120), the engine self-bootstrap is refused before any download and the unified store degrades to legacy-only. The degrade reason on the unsupported-node path carried the specific coverage detail but DROPPED the "upgrade Node.js to >= 22.5" lever that the sidecar-open path already carries — so a Node 21 user read "no engine supports this Node.js" with no actionable way out. - runtime-status degradeReason(): append nodeSqliteUpgradeHint() to the unsupported-node reason (keeps the #3894 specific-detail-first ordering). - engine-bootstrap runBootstrap(): carry the same hint on both skip warns so the lane logs match the status surface. - shadow UnifiedShadow sidecar-open catch: stop advising "delete the file to reset the index" for a no-engine failure — deleting cannot conjure an engine; that advice only fits a genuine open failure (byte-identical otherwise). - regression test: force process.versions.node=21.6.1 and assert the acp_status STORAGE line carries both the coverage detail and the >= 22.5 upgrade lever. Refs #4444 --- src/storage/engine-bootstrap.ts | 4 +-- src/storage/runtime-status.ts | 5 ++- src/storage/shadow.ts | 8 ++++- tests/acp-status-storage-degraded.test.ts | 37 +++++++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/storage/engine-bootstrap.ts b/src/storage/engine-bootstrap.ts index e056b7a0d3..3f7611ad64 100644 --- a/src/storage/engine-bootstrap.ts +++ b/src/storage/engine-bootstrap.ts @@ -353,14 +353,14 @@ async function runBootstrap(opts: EnsureSqliteEngineOptions): Promise { + // Node 21 has NO pinned band (neither builtin node:sqlite nor a prebuild), so + // bootstrap is refused before any download -> unsupported-node. The degrade + // reason must carry BOTH the specific coverage detail (the #3894 ordering) AND + // the "upgrade Node.js" lever the sidecar-open branch carries — pre-#4444 the + // unsupported-node branch outranked it and dropped the lever entirely. The + // diagnosed runtime is forced to 21.6.1 so the lever is non-empty regardless + // of the runner's own (builtin-bearing) Node version. + const base = mkdtempSync(path.join(tmpdir(), "acp-status-4444-")); + const snap = isolate(base); + const origNodeDesc = Object.getOwnPropertyDescriptor(process.versions, "node"); + let store: SessionStore | null = null; + try { + setSqliteEnginesUnavailableForTests(true); + _resetStorageRuntimeStatusForTest(); + recordEngineBootstrap({ status: "unsupported-node", detail: "no pinned SQLite engine runs on Node.js 21.6.1 (ABI 120)" }); + store = new SessionStore({ dir: process.env.BILI_SESSIONS_DIR, enabled: true }); + _setStoreForTest(store); + + Object.defineProperty(process.versions, "node", { value: "21.6.1", configurable: true, writable: true }); + const text = handleAcpStatus({}, makeCtx()).text ?? ""; + assert.ok(text.includes("STORAGE —"), "the storage line is present"); + assert.ok(text.includes("⚠ degraded —"), `the degrade warning line is present (got: ${text.slice(-400)})`); + assert.ok(text.includes("no self-downloaded SQLite engine supports this Node.js"), "the specific coverage detail surfaces"); + assert.ok(/Node\.js 21\.6\.1/.test(text), "the reason names the runtime the table could not serve"); + assert.ok(/Node\.js >= 22\.5/.test(text), `#4444: the unsupported-node reason must carry the Node.js upgrade lever (got: ${text.slice(-400)})`); + } finally { + if (origNodeDesc) Object.defineProperty(process.versions, "node", origNodeDesc); + store?.closeQuietly(); + _setStoreForTest(null as unknown as SessionStore); + setSqliteEnginesUnavailableForTests(false); + _resetStorageRuntimeStatusForTest(); + restore(snap); + rmrf(base); + } +}); From 4063e2700aff663eb388e95acc57b40aa8924be7 Mon Sep 17 00:00:00 2001 From: awork Date: Sat, 24 Oct 2026 02:11:13 +0800 Subject: [PATCH 04/11] fix(#4390): full-path remap adopts fresher bytes when identity matches but bytes diverge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fix for CI reds on this branch, part 2 of 2. The D4-1 JOIN migration made remapPlanToAbsolute reuse a seq whenever (msg_id, identity) matched, and INSERT OR IGNORE then silently kept the first generation's bytes for that seq. Any byte divergence under a matching identity became permanently unhealable by full ingest: (a) head truncation — the PR's own flagship scenario — leaves stored rows carrying the old wireIndex generation, so the parity digest never matches the live view again: every 20th save reports drift, the watermark is cleared, the next full save re-runs the same no-op remap, never converges, and after DRIFT_ESCALATE_STREAK consecutive samples #4323 freezes this process's envelope writes (legacy file goes stale = data-loss vector); (b) an externally corrupted CAS row (watermark boundary-tamper test) was healed pre-migration by the byte-faithful join minting fresh rows, and is now kept corrupted forever. Fix: when the full path reuses an existing seq via the identity join and the stored content_hash differs from the plan row's hash, UPDATE the row in place to the fresher authoritative bytes (CAS ref juggling put-new -> update-ref -> release-old, FK/refcount safe; media mappings untouched because identity includes the {$media:hash} markers). Seq stability is unchanged; the delta path stays append-only. SQLite + MySQL twins. identity.ts doc updated to match; CHANGELOG notes the behavior change. --- CHANGELOG.md | 3 ++- src/storage/identity.ts | 9 ++++++--- src/storage/mysql.ts | 18 +++++++++++++++++- src/storage/store.ts | 25 +++++++++++++++++++++++-- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c694481c1b..3fd224101f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag - **权威集清扫改墓碑(D4-2)**:proxy 会话每轮全量保存不再物理 DELETE 缺席行,改为 `retired_at` 墓碑——rewind/undo 的历史在统一存储里**耐久保留**;同一身份回来时原地复活(保原 seq,含 completeLog-less 保存路径 remap 复活)。读面(getMessages/统一重放/视图)默认过滤 retired;CAS 引用不释放。deleteSession/archive/GC 继续拥有物理生命周期。 - **渲染接缝图片真指纹(D4-3)**:snapshot-render 的 identitySig 图片段从 `长度` 改为 `sha256(base64) 前 24 hex`——同尺寸截图不再可能连击假接缝造成静默截断。 - **fork 谱系入正史(D3-2)**:sessions 行的 `parent_session_id` 现在从信封 `metadata.derivedFromSessionId` 落库(此前恒 null,谱系只活在信封里,信封丢失=家族失忆)。 -- **顺路**:gc full vacuum 走 `vacuum('full')`(带 TRUNCATE checkpoint,文件真正缩小,D5-1);loadLegacyView 尾指针缺行从静默缩水改为计数 + 三处加载路径 warn(D4-5)。 +- **顺路**:gc full vacuum 走 `vacuum('full')`(带 TRUNCATE checkpoint,文件真正缩小,D5-1);loadLegacyView 尾指针缺行从静默缩水改为计数 + 三处加载路径 warn(D4-5);fold-reconcile 的 Pass 0 位置副本失步(存储 ids 与顺序骨架不再 1:1)从静默降级改为响亮 warn(此前无任何日志痕迹)。 +- **审查修复(review)**:identity JOIN 迁移后,全量 ingest 对「identity 相同、字节不同」的行原本靠 INSERT OR IGNORE 静默保留旧世代——头部截断后存储世代带着旧 wireIndex 与活视图永久失配(采样校验漂移→水位反复清零→#4323 信封冻结的数据丢失向量),外部 CAS 损坏也永远无法自愈。现在全量路径在**同一 seq** 上采纳更新的字节(seq 稳定性不变,delta 追加路径仍只增不改);`tailGaps` 诊断键从 parity 摘要面剔除(存在性差异不再误报漂移)。 - 测试:`tests/storage-review-p2s.test.ts` 7 桩(截断保号/墓碑+复活/接缝指纹/谱系/迁移去重/gc checkpoint/缺行响亮);storage 全家族 + golden/snapshot/fold-reconcile/incremental/web 回归绿。 ### Fixed diff --git a/src/storage/identity.ts b/src/storage/identity.ts index ba1a2f1571..35271dd7d2 100644 --- a/src/storage/identity.ts +++ b/src/storage/identity.ts @@ -13,9 +13,12 @@ * block interval bounds built on sand). * * Same identity ⇒ same logical message ⇒ same row, same seq. A re-arrival - * with different bytes but the same identity (sidecar drift) is suppressed — - * the first generation's bytes stay; a real edit changes the content-derived - * id and therefore the identity. + * with different bytes but the same identity (sidecar drift, a stale + * wireIndex after a head truncation, external CAS corruption) is healed by + * the FULL ingest path, which adopts the fresher bytes at the SAME seq — + * the parity oracle must observe the authoritative view again (#4390 + * review); the append-only delta path never rewrites stored rows. A real + * edit changes the content-derived id and therefore the identity. */ import { createHash } from "node:crypto"; diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 602e616462..3c69aefd5d 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -594,10 +594,11 @@ export class MysqlBackend implements StorageBackend { ): Promise<{ messages: MessageRow[]; watermark: number | null }> { const s = input.session; const [rows] = (await c.query( - "SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?", + "SELECT seq, msg_id, identity, content_hash, retired_at FROM messages WHERE session_id = ?", [s.sessionId], )) as unknown as Row[][]; const seqByIdentity = new Map(); + const hashBySeq = new Map(); const retiredSeqs: number[] = []; let nextSeq = 0; for (const row of rows) { @@ -605,19 +606,34 @@ export class MysqlBackend implements StorageBackend { // claiming their seq until reactivation clears the tombstone. if (row.retired_at !== null && row.retired_at !== undefined) retiredSeqs.push(num(row.seq)); seqByIdentity.set(`${row.msg_id}\u0000${row.identity}`, num(row.seq)); + hashBySeq.set(num(row.seq), str(row.content_hash)); if (num(row.seq) + 1 > nextSeq) nextSeq = num(row.seq) + 1; } const planToAbs = new Map(); + const repairs: { seq: number; from: string; to: string; content: string }[] = []; const messages = input.messages.map((m) => { const key = `${m.msgId}\u0000${m.identity ?? identityOfStoredContent(m.content)}`; let abs = seqByIdentity.get(key); if (abs === undefined) { abs = nextSeq++; seqByIdentity.set(key, abs); + } else { + // #4390 review: same semantics as the sqlite lane — an + // identity match with changed bytes must adopt the fresher + // bytes at the same seq (stale-wireIndex parity drift and + // CAS-corruption healing), never keep the stored generation. + const fresh = m.contentHash ?? sha256Hex(m.content); + const stored = hashBySeq.get(abs); + if (stored !== undefined && stored !== fresh) repairs.push({ seq: abs, from: stored, to: fresh, content: m.content }); } planToAbs.set(m.seq, abs); return { ...m, seq: abs }; }); + for (const r of repairs) { + await this.putContent(c, r.content, r.to); + await c.query("UPDATE messages SET content_hash = ? WHERE session_id = ? AND seq = ?", [r.to, s.sessionId, r.seq]); + await this.releaseContent(c, r.from); + } // #4390 rewind-redo: reactivate in place (no duplicate row). for (const seq of retiredSeqs) { if (messages.some((m) => m.seq === seq)) { diff --git a/src/storage/store.ts b/src/storage/store.ts index 215a289e88..70ba17cf18 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1007,9 +1007,10 @@ export class UnifiedStore { // stable absolute seqs: same identity (msgId + content hash) // keeps its old seq; new identities continue after max(seq). const existing = this.db - .prepare("SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ?") - .all(s.sessionId) as { seq: number; msg_id: string; identity: string; retired_at: number | null }[]; + .prepare("SELECT seq, msg_id, identity, content_hash, retired_at FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; identity: string; content_hash: string; retired_at: number | null }[]; const seqByIdentity = new Map(); + const hashBySeq = new Map(); const retiredSeqs: number[] = []; let nextSeq = 0; for (const row of existing) { @@ -1020,6 +1021,7 @@ export class UnifiedStore { // clears the tombstone when the identity comes back. if (row.retired_at !== null) retiredSeqs.push(row.seq); seqByIdentity.set(`${row.msg_id}\u0000${row.identity}`, row.seq); + hashBySeq.set(row.seq, row.content_hash); if (row.seq + 1 > nextSeq) nextSeq = row.seq + 1; } // #3133 watermark gate (full path): the watermark names the abs seq @@ -1029,6 +1031,7 @@ export class UnifiedStore { // when the plan carries no tail (envelope-only imports): the next // save runs the full ingest. const planToAbs = new Map(); + const repairs: { seq: number; from: string; to: string; content: string }[] = []; const messages = input.messages.map((m) => { // #3133: plan rows carry the intern-pass hash; only direct // callers without a planner pay the sha256 here. @@ -1037,10 +1040,28 @@ export class UnifiedStore { if (abs === undefined) { abs = nextSeq++; seqByIdentity.set(key, abs); + } else { + // #4390 review: identity match + changed bytes (a wireIndex + // re-mint after head truncation, sidecar drift, or external + // CAS corruption) — INSERT OR IGNORE would silently keep the + // stored generation forever: the parity oracle then drifts on + // every sampled save (stale wireIndex vs live view) and the + // heal path can never converge. Adopt the fresher bytes at + // the SAME seq instead: seq stability (D4-1) is preserved, + // the replay matches the authoritative view again, and a + // corrupted row heals on the next full ingest. + const fresh = m.contentHash ?? sha256Hex(m.content); + const stored = hashBySeq.get(abs); + if (stored !== undefined && stored !== fresh) repairs.push({ seq: abs, from: stored, to: fresh, content: m.content }); } planToAbs.set(m.seq, abs); return { ...m, seq: abs }; }); + for (const r of repairs) { + this.putContent(r.content, r.to); + this.db.prepare("UPDATE messages SET content_hash = ? WHERE session_id = ? AND seq = ?").run(r.to, s.sessionId, r.seq); + this.releaseContent(r.from); + } // #4390: a rewound message that comes back reactivates in place — // the tombstone clears, the original seq is reused, and the view // heals without a duplicate row even on completeLog-less saves. From a933705bd778e3a8d3c14e309e19b389ead47328 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sat, 24 Oct 2026 17:56:08 +0800 Subject: [PATCH 05/11] fix(#4506): release media refs on historian retire + budget the #3250 custom-cap slack (#4390 follow-ups) --- CHANGELOG.md | 1 + src/storage/mysql.ts | 23 ++++++++++++++++++-- src/storage/store.ts | 29 +++++++++++++++++++++++--- tests/issue3250-wal-checkpoint.test.ts | 12 ++++++++++- tests/media-cas.test.ts | 22 ++++++++++++++----- 5 files changed, 76 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fd224101f..100646a798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Fixed +- **master CI 全红修复:#4390 historian 的媒体引用释放 + #3250 WAL cap 断言预算(#4506)。** bisect 定位 f161c518..02f544da 窗口 first bad = #4390(row-identity-historian),两个破口:① 权威集清扫改墓碑后,retired 行的 message_media 映射与媒体 CAS 引用永远不释放(默认要等 archive 365 天)——图片重度会话里即 #3762/#3901 的字节泄漏,`media-cas` 家族在 windows 分片稳定红。现在 retire 时释放媒体映射+引用(文本字节照旧史官保留;图片体积大、历史价值低),rewind-redo 复活时由重摄取路径幂等重建映射与引用(sqlite+mysql 双后端同款)。② `#3250: configured cap` 断言的 CAP_SLACK=4KiB(一页余量)预算不住 #4390 新增的每行 WAL 体积(identity 列 64hex + idx_messages_identity 索引项,~192 行超 12 页/47,376B)——截断不变量本身完好,坏的是页级余量的算术;custom-cap 用例改用 64KiB 的 inter-reset 累积预算(注释引用 #4390 行体积),close-converges 等其余断言维持一页余量不变。 - **增量车道:拉取回执修复 ack 传输盲区 + 漂移回退取富侧(#4352/#4323)**。2026-10-22 事故双根因修复:① pi 的 after_provider_response 只在部分 provider 传输下暴露响应头,x-bili-applied-seq 丢失时 noteIncrementalAck 永不执行,后续每轮都整包 re-seed(sinceSeq=0 重发 21K 条历史)。现在 GET /__bili/plugin/turn?conversation= 返回热 canonical 的 {appliedSeq, boundary}(boundary=末条 wire 消息哈希,冷重建诚实返回 null),pi 扩展在规划下一轮前拉取回执,仅在 boundary 证明 canonical 吸收了我方末条消息时采纳——绝不按 seq 前进盲目采纳(外来的全量重发也会推进 canonical);② #4323 多实例写同一目录时,漂移回退无条件吃旧信封,把索引里更新的折叠世代(26,971 refs/16 active)换成旧世代(11,849/12),覆盖与 canonical 失配→0 可压区间→preflight 8.29M 假性死局 502。现在漂移两侧按 mapped ref 数取富侧(平局信封赢,平局语义=原子整文件)。新增 tests/incremental-ack-pull.test.ts 3 例(响应头 ack 回归钉/拉取采纳/冷 boundary null 拒采纳)+ storage-full-flip 4b(陈旧信封让位新索引世代)。 ### Changes diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 3c69aefd5d..59c9aa5d25 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -407,12 +407,19 @@ export class MysqlBackend implements StorageBackend { } continue; } - // #4390 historian sweep: retire, don't delete — CAS - // refs stay owned by the retired row. + // #4390 historian sweep: retire, don't delete — + // TEXT CAS refs stay owned by the retired row; MEDIA + // refs release (#4506 — same rationale as sqlite twin). await c.query( "UPDATE messages SET retired_at = ? WHERE session_id = ? AND seq = ? AND retired_at IS NULL", [Date.now(), s.sessionId, num(row.seq)], ); + const retiredMedia = (await c.query( + "SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?", + [s.sessionId, num(row.seq)], + )) as unknown as Row[][]; + await c.query("DELETE FROM message_media WHERE session_id = ? AND seq = ?", [s.sessionId, num(row.seq)]); + for (const rm of retiredMedia[0]) await this.releaseContent(c, String(rm.media_hash)); } } } @@ -477,6 +484,18 @@ export class MysqlBackend implements StorageBackend { )) as unknown as [{ affectedRows: number }, unknown]; if (res.affectedRows === 0) { await this.releaseContent(c, hash); + // #4506: tombstone reactivation — restore media mappings + // + refcounts the retire released (idempotent upsert). + if (m.media && m.media.length > 0) { + for (const md of m.media) { + await this.putContent(c, md.bytes, md.hash); + const [mm] = (await c.query( + "INSERT IGNORE INTO message_media (session_id, seq, media_hash) VALUES (?, ?, ?)", + [s.sessionId, m.seq, md.hash], + )) as unknown as [{ affectedRows: number }, unknown]; + if (mm.affectedRows === 0) await this.releaseContent(c, md.hash); + } + } } else if (m.media && m.media.length > 0) { for (const md of m.media) { await this.putContent(c, md.bytes, md.hash); diff --git a/src/storage/store.ts b/src/storage/store.ts index 70ba17cf18..96e4cb0ea2 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -717,12 +717,21 @@ export class UnifiedStore { // Retired rows leave the unified replay (getMessages // filters them) so drift healing and the DIGEST // reconcile keep working exactly as the DELETE did. - // CAS refs are NOT released: retired rows still own - // their bytes. deleteSession/archiveSession/GC own - // the physical lifecycle from here. + // TEXT CAS refs are NOT released: retired rows still + // own their bytes. MEDIA refs ARE released (#4506): + // image bytes are bulky and low-value as history — + // keeping them until archive (365d default) is the + // unbounded leak #3762/#3901 called out. The + // re-ingest path below re-establishes mappings and + // refcounts if the row ever reactivates. this.db .prepare("UPDATE messages SET retired_at = ? WHERE session_id = ? AND seq = ? AND retired_at IS NULL") .run(Date.now(), s.sessionId, row.seq); + const retiredMedia = this.db + .prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?") + .all(s.sessionId, row.seq) as { media_hash: string }[]; + this.db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?").run(s.sessionId, row.seq); + for (const rm of retiredMedia) this.releaseContent(rm.media_hash); } } } @@ -805,6 +814,20 @@ export class UnifiedStore { // Idempotent re-receipt: the CAS row is still referenced // by the existing message row — undo our refcount bump. this.releaseContent(hash); + // #4506: an existing row may be a tombstone REACTIVATING + // (its message_media mappings were released at retire). + // The same idempotent upsert below restores mapping + + // refcount; for ordinary re-ingests (mapping still there) + // the release-on-no-change undoes the bump — net zero. + if (m.media && m.media.length > 0) { + for (const md of m.media) { + this.putContent(md.bytes, md.hash); + const mmr = this.db + .prepare("INSERT OR IGNORE INTO message_media (session_id, seq, media_hash) VALUES (?, ?, ?)") + .run(s.sessionId, m.seq, md.hash); + if (mmr.changes === 0) this.releaseContent(md.hash); + } + } } else if (m.media && m.media.length > 0) { // #3762: put the media bytes + mapping rows; refcount per // message row that actually landed (INSERT OR IGNORE on diff --git a/tests/issue3250-wal-checkpoint.test.ts b/tests/issue3250-wal-checkpoint.test.ts index e010cb8a79..7559fc04cb 100644 --- a/tests/issue3250-wal-checkpoint.test.ts +++ b/tests/issue3250-wal-checkpoint.test.ts @@ -122,6 +122,16 @@ const WAL_CAP = 8 * MiB; const CHECKPOINT_EVERY_N = 20; /** Truncation lands page-aligned; allow one page of measurement slack. */ const CAP_SLACK = 4 * KiB; +/** + * #4506: the custom-cap case observes the plateau BETWEEN resets — after the + * autocheckpoint truncates to the configured cap, subsequent commits keep + * appending frames until the next reset. #4390's row identity (64-hex column + * + idx_messages_identity entry per row) adds ~+250B of WAL frames per + * message row, so ~192 rows of post-reset writes now overshoot by ~12 pages + * (observed 1095952 vs 1048576). The truncate-at-checkpoint INVARIANT is + * intact; budget the inter-reset accumulation instead of pinning one page. + */ +const CUSTOM_CAP_SLACK = 64 * KiB; test("#3250: close() converges the WAL so a clean exit leaves nothing to replay", () => { const dir = tmpdir("bili-3250-close-"); @@ -289,7 +299,7 @@ test("#3250: a configured cap flows open() → journal_size_limit (custom limit // the observable proof the configured value reached the pragma.) for (let i = 2; i <= 10; i++) ingest(store, `small${i}`, 128 * KiB, 8); assert.ok( - walSize(dir) <= customCap + CAP_SLACK, + walSize(dir) <= customCap + CUSTOM_CAP_SLACK, `WAL must be truncated to the configured cap at the autocheckpoint (left ${walSize(dir)} bytes, cap ${customCap})`, ); assert.equal(store.getMessages("big").length, 120, "all burst rows must land"); diff --git a/tests/media-cas.test.ts b/tests/media-cas.test.ts index dea8486d45..4c5d7f8611 100644 --- a/tests/media-cas.test.ts +++ b/tests/media-cas.test.ts @@ -127,17 +127,29 @@ test("media CAS: stale-row sweep releases media refs and mappings (full re-inges assert.equal(store.mediaRefCount(IMG_B64), 1, "one media reference before re-ingest"); // The session's raw tail no longer contains the image message: the - // full path sweeps the stale row — its media mapping and CAS refcount - // must go with it, or the bytes leak forever (#3762 review, #3901). + // full path sweeps the stale row into a #4390 historian tombstone — + // its media mapping and CAS refcount must still go with it, or the + // bulky image bytes leak until archive (#3762 review, #3901, #4506). + // The TEXT row itself stays retired (retired_at set), never deleted. store.ingestSession(planIngest(mkSession("sess-sweep", [mTxt])), undefined, { completeLog: true }); assert.equal(store.mediaRefCount(IMG_B64), 0, "swept message released its media ref"); const mappings = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) .db.prepare("SELECT COUNT(*) AS n FROM message_media WHERE session_id = ?").all("sess-sweep"); assert.equal(mappings[0].n, 0, "swept media mappings deleted"); - const rows = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) - .db.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ?").all("sess-sweep"); - assert.equal(rows[0].n, 1, "only the surviving message row remains"); + const rows = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number; retired: number }[] } } }) + .db.prepare("SELECT COUNT(*) AS n, SUM(CASE WHEN retired_at IS NULL THEN 1 ELSE 0 END) AS retired FROM messages WHERE session_id = ?").all("sess-sweep"); + assert.equal(rows[0].n, 2, "historian keeps both rows (one live, one tombstoned)"); + assert.equal(rows[0].retired, 1, "only the surviving message is live"); + + // #4506 rewind-redo: the SAME image message comes back (undo of the + // undo). The tombstone reactivates in place AND its media mapping + + // refcount are re-established by the re-ingest — no manual repair. + store.ingestSession(planIngest(mkSession("sess-sweep", [mImg, mTxt])), undefined, { completeLog: true }); + assert.equal(store.mediaRefCount(IMG_B64), 1, "reactivated message re-owns its media ref"); + const live = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) + .db.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL").all("sess-sweep"); + assert.equal(live[0].n, 2, "both rows live again after rewind-redo"); store.close(); } finally { try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } From bfb4085cc57f89770c5a910ce648913b29799fec Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 24 Oct 2026 22:02:30 +0800 Subject: [PATCH 06/11] docs: release notes for v0.1.207 --- CHANGELOG.md | 2 ++ release-notes/package.json | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 100646a798..481521a6b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] +## [0.1.207] — 2026-10-24 + ### Storage — 行身份与史官化(#4390,storage review 四个 p2 + 三个顺路 p3) - **行身份去污染(D4-1)**:统一存储 messages 行新增 `identity` 列——跨保存 JOIN 键改为**位置无关**序列化(剥离 `id`/`wireIndex` 后哈希)。此前身份掺入 wire 位置元数据,客户端头部截断一条消息就会让后续全部行换血(旧行被清扫、seq 重铸、#4335 的区间账记随之抖动)。`content_hash` 保持字节忠实 CAS 键不变。存量库 v8 迁移:回填 identity 并合并 wireIndex 分裂世代(保最低 seq,重复行退还 CAS 引用)。MySQL 同款(information_schema 探测幂等 ALTER + 回填)。 diff --git a/release-notes/package.json b/release-notes/package.json index b7daf583f0..d8b72b55e1 100644 --- a/release-notes/package.json +++ b/release-notes/package.json @@ -9,8 +9,14 @@ }, "billionContextReleaseNotes": { "schema": 1, - "updated": "2026-10-21", + "updated": "2026-10-24", "releases": [ + { + "version": "0.1.207", + "date": "2026-10-24", + "tier": "recommended", + "summary": "存储可靠性修复(0.1.203–0.1.204 用户请直接升到本版):折叠上下文不再因模型/窗口切换悄悄膨胀直至会话超窗;修复图片会话媒体引用泄漏与存储校验误报(#4390/#4506/#4444)。EN: storage reliability fixes — folded context no longer silently regrows after model/window switches; media-ref leak and false digest drift fixed. v0.1.203–204 users upgrade immediately." + }, { "version": "0.1.203", "date": "2026-10-21", From 9068ca5a2a189728d7119ca42292a9e27526fc48 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 24 Oct 2026 15:20:17 +0000 Subject: [PATCH 07/11] release v0.1.207 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index eaafa7f83b..50e080dc29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "billion-context", - "version": "0.1.206", + "version": "0.1.207", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "billion-context", - "version": "0.1.206", + "version": "0.1.207", "license": "MIT", "dependencies": { "mysql2": "^3.15.3", diff --git a/package.json b/package.json index 1e6ea3d534..0c3a03784f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "billion-context", - "version": "0.1.206", + "version": "0.1.207", "description": "A context-compression plugin \u2014 small context windows (a 100K context is enough), 5x fewer tokens, month-long single sessions (billions of tokens), and compression quality \u2014 for all agents: pi, OpenCode, Codex, Claude Code, and more. billion-context is all you need.", "type": "module", "main": "dist/index.js", From 2a0359ee965b6f3e507a1b7d04c3cf1ef33915c7 Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 22 Oct 2026 04:19:18 +0800 Subject: [PATCH 08/11] fix: name every skipped dsh fork-adoption entry gate instead of returning silently (#4215) C10a of the #3011 family checklist (#3025): the #4215 incident showed a seeded fork child (isSeeded + correct parentSession) with ZERO adoption trace in the whole log - eight silent early returns made "not a fork" vs "a fork we could not see" unanswerable after the fact. - forkChildOf() becomes exported forkChildVerdict(): names all eight shapes (initiator-threw/-missing, session-missing, sid-missing, header-missing, not-seeded, parent-missing, parent-is-self) - a seeded candidate blocked at an entry gate (tools-ready / body-parse / side-shaped / proxy-base) logs once per session naming the gate - malformed seeded lineage (isSeeded without a usable parentSession) logs a warn once per session - ambient-blind shapes (fork status undeterminable) log ONE info notice per process, flagging whether scoped #4082 attribution was active - the C10b identity-route split named in the log - plain non-fork traffic stays completely silent Log-only change: no judgment, ordering, or wire behavior changes. Regression suite: tests/issue4215-fork-gate-diagnostics.test.ts (verdict matrix + both directions of every gate, #3617 discipline). Identity-route widening (C10b) needs an owner decision and is NOT part of this change. (cherry picked from commit fb54f6efdc94b2fd3c536bbab34466f55d957dc2) (cherry picked from commit 85af00f110c698ae7de4ed6a243c1d1f2287f13d) --- src/agent/dsh-native.ts | 153 ++++++++++++--- tests/issue4215-fork-gate-diagnostics.test.ts | 181 ++++++++++++++++++ 2 files changed, 312 insertions(+), 22 deletions(-) create mode 100644 tests/issue4215-fork-gate-diagnostics.test.ts diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index 5fbc417342..9c7fd663b1 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -1202,22 +1202,99 @@ const dshForkAdopter = createForkAdopter((line) => { persistClientEvent(`bili-native-dsh: ${line}`); }); -/** A seeded fork child of the current initiator, or undefined. The gate is - * header.isSeeded === true (set only by dsh's fork seed path): subagent - * sessions carry parentSession WITHOUT isSeeded — they run fresh task - * contexts, never replay the parent history, and must NOT inherit. */ -function forkChildOf(ctx: PluginContext): { sid: string; parent: string } | undefined { - let session: AgentLike["session"]; +// #4215/C10a: structured diagnostics for every skipped fork-adoption entry +// gate. The #4215 incident (seeded child with the correct parentSession, ZERO +// adoption trace in the whole log) showed that silent early returns make the +// fired predicate undiagnosable after the fact: this path had eight ways out +// and none of them wrote anything, so "not a fork" vs "a fork we could not +// see" was unanswerable from the log. The verdict names every shape; the +// caller warns once per session for seeded candidates blocked at a gate and +// notes the ambient-blind shapes ONCE per process (they fire on every +// agentless dispatch — title lanes, disposed scopes — so per-request lines +// would be pure noise). Plain non-fork traffic stays completely silent. +export type ForkChildSkipReason = + | "initiator-threw" + | "initiator-missing" + | "session-missing" + | "sid-missing" + | "header-missing" + | "not-seeded" + | "parent-missing" + | "parent-is-self"; + +export type ForkChildVerdict = + | { ok: true; sid: string; parent: string } + | { ok: false; reason: ForkChildSkipReason; sid?: string; detail?: string }; + +/** A seeded fork child of the current initiator, or the named reason it is + * not (or cannot be determined). The fork gate is header.isSeeded === true + * (set only by dsh's fork seed path): subagent sessions carry parentSession + * WITHOUT isSeeded — they run fresh task contexts, never replay the parent + * history, and must NOT inherit ("not-seeded" is intentional and stays + * silent — it is the documented majority of all traffic). */ +export function forkChildVerdict(ctx: PluginContext): ForkChildVerdict { + let init: AgentLike | undefined; try { - session = ctx.agents?.currentInitiator?.()?.session; - } catch { - return undefined; + init = ctx.agents?.currentInitiator?.(); + } catch (err) { + return { ok: false, reason: "initiator-threw", detail: err instanceof Error ? err.message : String(err) }; } - const sid = typeof session?.id === "string" && session.id.length > 0 ? session.id : undefined; - if (sid === undefined || session?.header === undefined) return undefined; + if (init === undefined || init === null) return { ok: false, reason: "initiator-missing" }; + const session = init.session; + if (session === undefined || session === null) return { ok: false, reason: "session-missing" }; + const sid = typeof session.id === "string" && session.id.length > 0 ? session.id : undefined; + if (sid === undefined) return { ok: false, reason: "sid-missing" }; + if (session.header === undefined) return { ok: false, reason: "header-missing", sid }; const parent = session.header.parentSession; - if (session.header.isSeeded !== true || typeof parent !== "string" || parent.length === 0 || parent === sid) return undefined; - return { sid, parent }; + if (session.header.isSeeded !== true) return { ok: false, reason: "not-seeded", sid }; + if (typeof parent !== "string" || parent.length === 0) return { ok: false, reason: "parent-missing", sid }; + if (parent === sid) return { ok: false, reason: "parent-is-self", sid }; + return { ok: true, sid, parent }; +} + +const forkGateNoted = new Set(); +let noticedForkBlindSpot = false; + +function noteForkGateOnce(key: string, level: "info" | "warn", line: string): void { + if (forkGateNoted.has(key)) return; + forkGateNoted.add(key); + logHostEvent(level, "[dsh-client]", line); +} + +function noteForkVerdict(verdict: Extract): void { + switch (verdict.reason) { + case "not-seeded": + return; // intentional non-inheritance — see forkChildVerdict + case "parent-missing": + noteForkGateOnce(`lineage:${verdict.sid}`, "warn", + `bili-native-dsh: fork adoption for ${verdict.sid} skipped — the session header says isSeeded but carries no usable parentSession; the child starts without inherited compression state (#4215)`); + return; + case "parent-is-self": + noteForkGateOnce(`lineage:${verdict.sid}`, "warn", + `bili-native-dsh: fork adoption for ${verdict.sid} skipped — the session header carries its own id as parentSession; the child starts without inherited compression state (#4215)`); + return; + default: { + // Blind shapes: the ambient read could not determine whether this + // dispatch was a seeded fork child at all. Once per process — these + // fire on EVERY agentless dispatch, so per-request lines would be + // noise. The scoped-attribution state is part of the message: with + // #4082 active the sid rides the request object while fork + // detection still reads only the ambient initiator — that split is + // exactly what #4215 needs named in the log (C10b). + if (noticedForkBlindSpot) return; + noticedForkBlindSpot = true; + const scoped = sidScope.getStore(); + const scopedActive = typeof scoped === "string" && scoped.length > 0; + const shape = + verdict.reason === "initiator-threw" ? `threw (${verdict.detail ?? "unknown"})` + : verdict.reason === "initiator-missing" ? "returned no agent" + : verdict.reason === "session-missing" ? "the agent has no session" + : verdict.reason === "sid-missing" ? "the session has no usable id" + : "the session has no epoch header"; + logHostEvent("info", "[dsh-client]", + `bili-native-dsh: fork discovery blind on this dispatch — ambient currentInitiator ${shape}${scopedActive ? "; scoped llm/stream attribution WAS active but fork detection does not use it yet (#4082)" : ""}; a seeded fork child dispatched this way leaves NO adoption trace (#4215)`); + } + } } function bodyJsonOf(init: RequestInit | undefined): unknown { @@ -1241,14 +1318,33 @@ function bodyJsonOf(init: RequestInit | undefined): unknown { async function maybeForkAdoptBeforeSend(ctx: PluginContext, init: RequestInit | undefined): Promise { - if (!register.toolsReady) return; - const child = forkChildOf(ctx); - if (child === undefined) return; + const child = forkChildVerdict(ctx); + if (!child.ok) { + noteForkVerdict(child); + return; + } + if (!register.toolsReady) { + noteForkGateOnce(`gate:tools-ready:${child.sid}`, "warn", + `bili-native-dsh: fork adoption for ${child.sid} skipped at the tools-ready gate — this request arrived before bili tool registration completed; later requests can still adopt (#4215)`); + return; + } const body = bodyJsonOf(init); - if (body === undefined) return; - if (sideShapedBody(body)) return; + if (body === undefined) { + noteForkGateOnce(`gate:body-parse:${child.sid}`, "warn", + `bili-native-dsh: fork adoption for ${child.sid} skipped at the body-parse gate — the request body was not parseable JSON, so the replay cannot be projected (#4215)`); + return; + } + if (sideShapedBody(body)) { + noteForkGateOnce(`gate:side-shaped:${child.sid}`, "info", + `bili-native-dsh: fork adoption for ${child.sid} deferred for a side-shaped request (no tools, budget <= 200) — the adoption window stays open for the main request (#4215)`); + return; + } const base = register.base; - if (base === undefined) return; + if (base === undefined) { + noteForkGateOnce(`gate:proxy-base:${child.sid}`, "warn", + `bili-native-dsh: fork adoption for ${child.sid} skipped at the proxy-base gate — the proxy origin is not known yet, so the fork protocol cannot be reached (#4215)`); + return; + } await dshForkAdopter.maybeAdopt({ base, parent: child.parent, child: child.sid, body }); } @@ -2417,8 +2513,9 @@ export function apply(ctx: PluginContext): void { } /** Test hook: reset the module-level registration lifecycle so suites can - * drive apply() repeatedly with a fresh mock ctx. */ -export function _resetRegisterForTest(base: string | undefined): void { + * drive apply() repeatedly with a fresh mock ctx. (#4215: optional + * toolsReady so gate scenarios can arm the registration state directly.) */ +export function _resetRegisterForTest(base: string | undefined, toolsReady = false): void { // #3616: invalidate every in-flight recovery chain from earlier scenarios — // their late landings must not write this scenario's register/state. originGen += 1; @@ -2433,7 +2530,7 @@ export function _resetRegisterForTest(base: string | undefined): void { state.respawn = undefined; state.onGiveUp = undefined; register.base = base; - register.toolsReady = false; + register.toolsReady = toolsReady; register.dead = false; register.retryAt = 0; register.pending = undefined; @@ -2454,6 +2551,18 @@ export function maybeRetryForTest(ctx: PluginContext): void { maybeRetry(ctx); } +/** Test hook (#4215): drive the fork-adoption entry gate directly without + * arming apply() (the fetch patch is not installed under NODE_TEST_CONTEXT). */ +export function _maybeForkAdoptBeforeSendForTest(ctx: PluginContext, init: RequestInit | undefined): Promise { + return maybeForkAdoptBeforeSend(ctx, init); +} + +/** Test hook (#4215): reset the fork-gate diagnostics throttle state. */ +export function _resetForkGateDiagnosticsForTest(): void { + forkGateNoted.clear(); + noticedForkBlindSpot = false; +} + /** Test hook (#2604): reset the host-title channel (dedup map + resolved * services) so scenarios start un-armed. */ export function _resetTitlesForTest(): void { diff --git a/tests/issue4215-fork-gate-diagnostics.test.ts b/tests/issue4215-fork-gate-diagnostics.test.ts new file mode 100644 index 0000000000..d1be978187 --- /dev/null +++ b/tests/issue4215-fork-gate-diagnostics.test.ts @@ -0,0 +1,181 @@ +// #4215/C10a: every skipped fork-adoption entry gate must name itself. The +// incident (seeded child with the correct parentSession, ZERO adoption trace +// in the whole log) made "not a fork" vs "a fork we could not see" +// unanswerable from the log. This suite pins both directions of every gate +// (#3617 discipline): the skip line fires when it should, exactly once, and +// plain non-fork traffic stays completely silent. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import { once } from "node:events"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { forkChildVerdict, _maybeForkAdoptBeforeSendForTest, _resetForkGateDiagnosticsForTest, _resetRegisterForTest, type ForkChildVerdict, type PluginContext } from "../src/agent/dsh-native.ts"; +import { resetForkCapabilityCacheForTest } from "../src/agent/fork-adopt.ts"; + +process.env.NODE_ENV = "test"; +const testRoot = mkdtempSync(join(tmpdir(), "bili-issue4215-")); +test.after(() => rmSync(testRoot, { recursive: true, force: true })); +for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = testRoot; +const logFile = join(testRoot, "billion-context", "bili.log"); + +type SessionShape = { id?: unknown; header?: { parentSession?: unknown; isSeeded?: unknown } }; +type AgentShape = { session?: SessionShape }; + +function ctxWithInitiator(fn: () => AgentShape | undefined): PluginContext { + return { + tools: { register: () => undefined }, + commands: { register: () => undefined }, + agents: { currentInitiator: fn }, + }; +} + +const ctxNoAgents = (): PluginContext => ({ tools: { register: () => undefined }, commands: { register: () => undefined }, agents: {} }); +const ctxNoInitiator = (): PluginContext => ctxWithInitiator(() => undefined); +const ctxThrowing = (): PluginContext => ctxWithInitiator(() => { throw new Error("disposed scope"); }); +const ctxPlain = (): PluginContext => ctxWithInitiator(() => ({ session: { id: "plain-sid", header: {} } })); +const ctxSubagent = (): PluginContext => ctxWithInitiator(() => ({ session: { id: "sub-sid", header: { parentSession: "p1", isSeeded: false } } })); +const ctxSeeded = (): PluginContext => ctxWithInitiator(() => ({ session: { id: "child-sid", header: { isSeeded: true, parentSession: "parent-sid" } } })); + +const validInit: RequestInit = { body: JSON.stringify({ model: "m", messages: [{ role: "user", content: "hi" }] }) }; + +function freshScenario(base: string | undefined, toolsReady: boolean): void { + _resetRegisterForTest(base, toolsReady); + _resetForkGateDiagnosticsForTest(); + fs.rmSync(logFile, { force: true }); +} + +function logLines(): string[] { + try { + return fs.readFileSync(logFile, "utf8").split("\n").filter((l) => l.length > 0); + } catch { + return []; + } +} + +const GATE_LINE = /fork adoption for .* (skipped at the \S+ gate|deferred for a side-shaped request)/; + +test("forkChildVerdict names every shape (#4215)", () => { + const cases: Array<[string, () => PluginContext, ForkChildVerdict]> = [ + ["threw", ctxThrowing, { ok: false, reason: "initiator-threw", detail: "disposed scope" }], + ["missing-initiator", ctxNoInitiator, { ok: false, reason: "initiator-missing" }], + ["no-initiator-accessor", ctxNoAgents, { ok: false, reason: "initiator-missing" }], + ["agent-no-session", () => ctxWithInitiator(() => ({})), { ok: false, reason: "session-missing" }], + ["session-no-id", () => ctxWithInitiator(() => ({ session: {} })), { ok: false, reason: "sid-missing" }], + ["session-bad-id", () => ctxWithInitiator(() => ({ session: { id: 42 } })), { ok: false, reason: "sid-missing" }], + ["session-empty-id", () => ctxWithInitiator(() => ({ session: { id: "" } })), { ok: false, reason: "sid-missing" }], + ["no-header", () => ctxWithInitiator(() => ({ session: { id: "s1" } })), { ok: false, reason: "header-missing", sid: "s1" }], + ["plain-session", ctxPlain, { ok: false, reason: "not-seeded", sid: "plain-sid" }], + ["subagent-carries-parent-without-isSeeded", ctxSubagent, { ok: false, reason: "not-seeded", sid: "sub-sid" }], + ["seeded-no-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true } } })), { ok: false, reason: "parent-missing", sid: "s1" }], + ["seeded-empty-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true, parentSession: "" } } })), { ok: false, reason: "parent-missing", sid: "s1" }], + ["seeded-non-string-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true, parentSession: 7 } } })), { ok: false, reason: "parent-missing", sid: "s1" }], + ["seeded-self-parent", () => ctxWithInitiator(() => ({ session: { id: "s1", header: { isSeeded: true, parentSession: "s1" } } })), { ok: false, reason: "parent-is-self", sid: "s1" }], + ["ok", ctxSeeded, { ok: true, sid: "child-sid", parent: "parent-sid" }], + ]; + for (const [name, make, expected] of cases) { + assert.deepEqual(forkChildVerdict(make()), expected, name); + } +}); + +test("seeded child blocked at tools-ready: one warn naming the gate; repeats stay one (#4215)", async () => { + freshScenario("http://proxy.invalid", false); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + let lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*bili-native-dsh: fork adoption for child-sid skipped at the tools-ready gate.*#4215/); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, "throttle holds across requests"); +}); + +test("seeded child with unparseable body: one warn naming the body-parse gate (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), { body: "not-json" }); + let lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*skipped at the body-parse gate/); + // a missing body hits the same gate and does not re-log per shape + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), undefined); + lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1); +}); + +test("side-shaped request: info-level deferral, window stays open (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), { body: JSON.stringify({ model: "m", messages: [], max_tokens: 100 }) }); + const lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[info\].*deferred for a side-shaped request/); + // the main request after the sidecar reaches the adopter untouched + resetForkCapabilityCacheForTest(); + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ capabilities: {} })); }); + srv.listen(0, "127.0.0.1"); + await once(srv, "listening"); + const addr = srv.address(); + assert(addr && typeof addr === "object"); + try { + _resetRegisterForTest(`http://127.0.0.1:${addr.port}`, true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + } finally { + srv.close(); + srv.closeAllConnections?.(); + await once(srv, "close").catch(() => undefined); + } + const after = logLines(); + assert.equal(after.filter((l) => /does not advertise the fork capability/.test(l)).length, 1, "main request reached the adopter protocol"); + assert.equal(after.filter((l) => /skipped at the .* gate/.test(l)).length, 0, "no gate fired on the main request"); +}); + +test("seeded child before the proxy origin is known: one warn naming the proxy-base gate (#4215)", async () => { + freshScenario(undefined, true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), validInit); + const lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*skipped at the proxy-base gate/); +}); + +test("malformed seeded lineage: one warn per anomaly naming what is missing (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: { id: "a1", header: { isSeeded: true } } })), validInit); + let lines = logLines(); + assert.equal(lines.filter((l) => /carries no usable parentSession/.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*fork adoption for a1 skipped/); + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: { id: "b1", header: { isSeeded: true, parentSession: "b1" } } })), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /carries its own id as parentSession/.test(l)).length, 1); + // repeat of the same anomaly stays one line + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: { id: "a1", header: { isSeeded: true } } })), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /carries no usable parentSession/.test(l)).length, 1); +}); + +test("ambient-blind shapes: one info notice per process, naming the split when #4082 is active (#4215)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxThrowing(), validInit); + let lines = logLines(); + assert.equal(lines.filter((l) => /fork discovery blind/.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[info\].*ambient currentInitiator threw \(disposed scope\)/); + // a DIFFERENT blind shape on a later dispatch adds nothing + await _maybeForkAdoptBeforeSendForTest(ctxNoInitiator(), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /fork discovery blind/.test(l)).length, 1); + // resetting the throttle re-arms the notice (truncate so only post-reset lines count) + _resetForkGateDiagnosticsForTest(); + fs.rmSync(logFile, { force: true }); + await _maybeForkAdoptBeforeSendForTest(ctxWithInitiator(() => ({ session: {} })), validInit); + lines = logLines(); + assert.equal(lines.filter((l) => /fork discovery blind/.test(l)).length, 1); + assert.match(lines[0]!, /the session has no usable id/); +}); + +test("plain non-fork traffic stays completely silent (#4215 negative direction)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxPlain(), validInit); + await _maybeForkAdoptBeforeSendForTest(ctxSubagent(), validInit); + await _maybeForkAdoptBeforeSendForTest(ctxPlain(), validInit); + const lines = logLines(); + assert.equal(lines.length, 0, JSON.stringify(lines)); +}); From 1c7d35c776d8ed76f4a73d1baa9a2d0eb83b0c03 Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 22 Oct 2026 13:22:05 +0800 Subject: [PATCH 09/11] fix: log a bounded near-miss diagnostic when resume-inheritance misses (#4264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the resume-inheritance gate passed but no byte-exact chain match existed, handle.ts emitted zero logs — clone-inheritance failures (the #3011/#3025 family) were left unattributable; dsh (responses protocol) could not even reach the transcript-lineage log lines, so a miss was absolute silence. Add PrefixAffinityResolver.diagnoseResumeMiss (diagnostics only, matching untouched): it re-runs the same gates in reporting mode and classifies the closest near-miss as equal-depth (duplicate conversation by design), prefix-break (head drift desynchronizes the prefix), or truncated (incoming shorter than the stored chain). The per-item divergence scan runs once on the single best near-miss over its stored trailing window (cost-bounded); deep chains report the pre-window bound instead of an unsound index. The server logs one line: info with best candidate id + depths + first divergence cell + 8-hex digests when any eligible candidate exists, debug when nothing eligible was tracked. Reuses MIN_RESUME_PREFIX=8; no new config surface, no behavior change. Tests: the three miss shapes plus truncated, deep-chain pre-window break, scoped declared-parent states (untracked/too-shallow), scoped judging-only-declared-parent, self-exclusion, and hit-path regression (findResumeParent/findResumeParentWithin results unchanged). (cherry picked from commit 7fff50ae604e5c4e1908ed67c39938800c2785eb) (cherry picked from commit c1272ae9e611ad1eea865550b6df88b07afbf98e) --- src/prefix-affinity.ts | 164 +++++++++++++++++ src/server/handle.ts | 20 +- ...esume-inheritance-miss-diagnostics.test.ts | 171 ++++++++++++++++++ 3 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 tests/resume-inheritance-miss-diagnostics.test.ts diff --git a/src/prefix-affinity.ts b/src/prefix-affinity.ts index e7987445de..1517d66f69 100644 --- a/src/prefix-affinity.ts +++ b/src/prefix-affinity.ts @@ -520,6 +520,85 @@ function containsRun(haystack: string[], needle: string[]): boolean { return false; } +/** #4264: one judged candidate inside a ResumeMissReport. */ +export interface ResumeMissCandidate { + sessionId: string; + /** Stored chain depth (neutral turns for hv2, raw messages for hv1). */ + chainDepth: number; + /** Incoming depth in the candidate's hash domain. */ + incomingDepth: number; + /** 2 = wire-neutral domain, 1 = pre-upgrade raw-wire domain. */ + hashDomain: 1 | 2; + /** Why this candidate was not accepted as a resume parent — the same + * gates findResumeParent applies, reported: "equal-depth" is a + * byte-exact replay at exactly the stored depth under another id (a + * duplicate conversation, rejected by design #1486); "prefix-break" is + * a deeper incoming history whose head-anchored progressive hash at + * the stored depth differs (the shared prefix broke somewhere); + * "truncated" is an incoming history SHORTER than the stored chain + * (cannot be a resume — a resume adds at least one message). */ + kind: "equal-depth" | "prefix-break" | "truncated"; + /** First cell index whose per-item hash differs from the stored chain, + * when verifiable from the stored trailing window. null: the whole + * inspectable overlap matched (the break sits before windowStart) or + * no overlap exists at all. */ + firstDivergence: number | null; + /** First stored position inspectable via the trailing itemHashes window + * (0 for chains ≤ MAX_STORED_ITEMS deep). */ + windowStart: number; + /** true when the inspectable overlap is non-empty and every overlapped + * cell matches byte-for-byte. */ + tailWindowIntact: boolean; + /** 8-hex fragments of the diverging cells (log correlation only). */ + incomingCell?: string; + storedCell?: string; +} + +/** #4264: diagnostics-only report for a resume-inheritance miss. */ +export interface ResumeMissReport { + /** Tracked chains that passed the pre-screen (depth ≥ + * MIN_RESUME_PREFIX, id ≠ self, in-domain floor) and were actually + * judged. 0 with no declaredParentState = nothing eligible to begin + * with (parent chain absent — LRU eviction or never noted). */ + eligible: number; + /** Scoped match only: state of the declared parent's own chain when it + * could not even be judged. */ + declaredParentState?: "untracked" | "too-shallow"; + /** Best near-miss candidate (closest by kind rank, then deepest chain, + * then most recently seen), when any eligible candidate existed. */ + best?: ResumeMissCandidate; +} + +/** #4264: render a ResumeMissReport as the bounded log-line body. Pure + * formatting — the caller picks the level (info when report.best exists, + * debug otherwise). */ +export function describeResumeMiss(report: ResumeMissReport): string { + const b = report.best; + if (!b) { + const scope = report.declaredParentState !== undefined ? `; declared parent chain ${report.declaredParentState}` : ""; + return `no eligible tracked chain (floor ${MIN_RESUME_PREFIX}${scope})`; + } + switch (b.kind) { + case "equal-depth": + return `no byte-exact resume among ${report.eligible} eligible chain(s); closest ${b.sessionId} is a byte-exact EQUAL-DEPTH replay (${b.chainDepth} msgs == incoming ${b.incomingDepth}) — duplicate conversation, not a resume by design`; + case "truncated": + return `no byte-exact resume among ${report.eligible} eligible chain(s); closest ${b.sessionId} (chain ${b.chainDepth}) is LONGER than the incoming history (${b.incomingDepth} msgs)${b.tailWindowIntact ? `, visible cells ${b.windowStart}..${b.incomingDepth - 1} intact — truncated replay` : ""}`; + case "prefix-break": { + let detail: string; + if (b.firstDivergence !== null) { + detail = b.windowStart > 0 + ? `first divergence found at cell ${b.firstDivergence} (stored ${b.storedCell} vs incoming ${b.incomingCell}; cells 0..${b.windowStart - 1} uninspectable — chain deeper than the ${MAX_STORED_ITEMS}-cell window)` + : `first divergence at cell ${b.firstDivergence} (stored ${b.storedCell} vs incoming ${b.incomingCell})`; + } else if (b.tailWindowIntact) { + detail = `tail window (cells ${b.windowStart}..${Math.min(b.chainDepth, b.incomingDepth) - 1}) byte-intact — break sits BEFORE cell ${b.windowStart}`; + } else { + detail = "head-anchored prefix broken (no inspectable overlap)"; + } + return `no byte-exact resume among ${report.eligible} eligible chain(s); closest ${b.sessionId} (chain ${b.chainDepth}, incoming ${b.incomingDepth}): ${detail}`; + } + } +} + export class PrefixAffinityResolver { private trackedChains = new Map(); @@ -990,6 +1069,91 @@ export class PrefixAffinityResolver { return { sessionId: parentId, sharedDepth: entry.depth }; } + /** #4264: diagnostics-only explanation of a resume-inheritance MISS + * (the gate passed but findResumeParent/findResumeParentWithin returned + * null). It re-runs the SAME gates in reporting mode and names why the + * closest near-miss was rejected; it never influences matching. The + * three causes need completely different fixes, so they are reported + * distinctly: equal-depth replay (a duplicate conversation by design), + * parent-chain absence (LRU eviction / never noted), head break (host + * rebuild drift desynchronizes the byte-exact prefix). Cost-bounded: + * the candidate scan does only O(1) progressive-hash checks per entry; + * the per-item divergence scan runs ONCE on the single best near-miss, + * over its stored trailing window (≤ MAX_STORED_ITEMS compares). */ + diagnoseResumeMiss(messages: unknown[], selfSessionId: string, declaredParent?: string): ResumeMissReport { + const msgs = normalizeAffinityMessages(messages); + if (!hasUserMessage(msgs)) return { eligible: 0 }; + // #2791: judge each candidate in the hash domain it was noted in — + // same dual-domain discipline as findResumeParent. #2824: hashes-only + // consumers pay nothing for sketches; itemHashes come from the same + // fused pass at no extra cost. + const turns = neutralTurns(messages); + const fp2 = neutralFingerprints(turns, 0); + const hashesV2 = fp2.hashes; + const hashesV1 = chainHashes(msgs); + if (hashesV2.length === 0 && hashesV1.length === 0) return { eligible: 0 }; + type Candidate = { id: string; entry: ChainEntry; kind: ResumeMissCandidate["kind"]; rank: number }; + const cands: Candidate[] = []; + let declaredParentState: "untracked" | "too-shallow" | undefined; + const consider = (id: string, entry: ChainEntry): void => { + if (id === selfSessionId) return; + if (entry.depth < MIN_RESUME_PREFIX) return; + const hv2 = entry.hv === 2; + const hashes = hv2 ? hashesV2 : hashesV1; + const len = hv2 ? turns.length : msgs.length; + if (hashes.length < MIN_RESUME_PREFIX) return; + let kind: ResumeMissCandidate["kind"]; + if (len < entry.depth) kind = "truncated"; + else if (len === entry.depth && hashes[entry.depth - 1] === entry.tailHash) kind = "equal-depth"; + else kind = "prefix-break"; + cands.push({ id, entry, kind, rank: kind === "equal-depth" ? 0 : kind === "prefix-break" ? 1 : 2 }); + }; + if (declaredParent !== undefined && declaredParent !== selfSessionId) { + const entry = this.trackedChains.get(declaredParent); + if (entry === undefined) declaredParentState = "untracked"; + else if (entry.depth < MIN_RESUME_PREFIX) declaredParentState = "too-shallow"; + else consider(declaredParent, entry); + } else if (declaredParent === undefined) { + for (const [id, entry] of this.trackedChains) consider(id, entry); + } + if (cands.length === 0) return { eligible: 0, ...(declaredParentState !== undefined ? { declaredParentState } : {}) }; + cands.sort((a, b) => a.rank - b.rank || b.entry.depth - a.entry.depth || b.entry.lastSeen - a.entry.lastSeen); + const top = cands[0]!; + const hv2 = top.entry.hv === 2; + const depth = top.entry.depth; + const len = hv2 ? turns.length : msgs.length; + // Lazy v1 per-item pass: only paid when the winner is a legacy raw-wire entry. + const itemIncoming = hv2 ? fp2.itemHashes : perItemHashes(msgs); + const windowStart = Math.max(0, depth - top.entry.itemHashes.length); + const end = Math.min(depth, len); + let firstDivergence: number | null = null; + let tailWindowIntact = false; + if (windowStart < end) { + tailWindowIntact = true; + for (let i = windowStart; i < end; i++) { + if (itemIncoming[i] !== top.entry.itemHashes[i - windowStart]) { + firstDivergence = i; + tailWindowIntact = false; + break; + } + } + } + const best: ResumeMissCandidate = { + sessionId: top.id, + chainDepth: depth, + incomingDepth: len, + hashDomain: hv2 ? 2 : 1, + kind: top.kind, + firstDivergence, + windowStart, + tailWindowIntact, + ...(firstDivergence !== null + ? { incomingCell: itemIncoming[firstDivergence]?.slice(0, 8), storedCell: top.entry.itemHashes[firstDivergence - windowStart]?.slice(0, 8) } + : {}), + }; + return { eligible: cands.length, ...(declaredParentState !== undefined ? { declaredParentState } : {}), best }; + } + /** #2241: does `messages` continue the tracked chain of `sessionId`? * True when the session's stored chain is a byte-exact head-anchored * prefix of the incoming list — the progressive hash at the stored diff --git a/src/server/handle.ts b/src/server/handle.ts index a1d5f4bcb6..74fb2e392e 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -34,7 +34,7 @@ import { emitPreflightError } from "../stream-error.js"; import { agentProviderRecipes } from "../agent-providers.js"; import { affinityToken, conversationHeaderSource, claudeSubagentAgentId, claudeSubagentSplit, clientConversationHeader, codexTurnIdentity, dshPersonaFingerprintApplies, instructionsFingerprintApplies, openaiSystemTextForPersona, preferPromptCacheKeyIdentity } from "../session-id.js"; import { personaConversationOrder } from "../session-resolve.js"; -import { prefixAffinity, type AnonymousAffinity } from "../prefix-affinity.js"; +import { prefixAffinity, describeResumeMiss, type AnonymousAffinity } from "../prefix-affinity.js"; import { buildAnonymousProvenance, type AnonymousProvenance } from "../anon-provenance.js"; import { maybeAdoptForkBlocks, maybeAdoptResume, maybeAdoptDerivedFork } from "../fork-adoption.js"; import { executeThreadsReforge } from "./threads-lane.js"; @@ -1579,10 +1579,22 @@ async function handleRequest( } } if (!resume) { + const affinityList = affinityMessageList(); const matched = derivedParent !== undefined - ? prefixAffinity.findResumeParentWithin(derivedParent, affinityMessageList(), sessionId) - : prefixAffinity.findResumeParent(affinityMessageList(), sessionId); - if (matched) resume = { sessionId: matched.sessionId, sharedDepth: matched.sharedDepth, via: `${matched.sharedDepth} msg(s) byte-exact prefix` }; + ? prefixAffinity.findResumeParentWithin(derivedParent, affinityList, sessionId) + : prefixAffinity.findResumeParent(affinityList, sessionId); + if (matched) { + resume = { sessionId: matched.sessionId, sharedDepth: matched.sharedDepth, via: `${matched.sharedDepth} msg(s) byte-exact prefix` }; + } else { + // #4264: a miss must not be silent — equal-depth replay, + // parent-chain absence and head break need completely + // different fixes, so name the closest near-miss. + // Diagnostics only: matching above is untouched. + const miss = derivedParent !== undefined + ? prefixAffinity.diagnoseResumeMiss(affinityList, sessionId, derivedParent) + : prefixAffinity.diagnoseResumeMiss(affinityList, sessionId); + log(miss.best !== undefined ? "info" : "debug", `[${sessionId}] [resume-inheritance] miss: ${describeResumeMiss(miss)} (#4264)`); + } } if (resume) { const resumeParent = loadParent(resume.sessionId); diff --git a/tests/resume-inheritance-miss-diagnostics.test.ts b/tests/resume-inheritance-miss-diagnostics.test.ts new file mode 100644 index 0000000000..455235ebc8 --- /dev/null +++ b/tests/resume-inheritance-miss-diagnostics.test.ts @@ -0,0 +1,171 @@ +import assert from "node:assert"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { PrefixAffinityResolver, describeResumeMiss } from "../src/prefix-affinity.ts"; + +// #4264: a resume-inheritance miss must not be silent. These tests pin the +// diagnostics-only near-miss report: each of the three causes (equal-depth +// replay / parent-chain absence / head break) is asserted distinctly, plus +// the hit-path regression (matching behavior and results stay unchanged). + +function user(text: string): Record { + return { role: "user", content: text }; +} + +test("resume-inheritance-miss: equal-depth replay under another id reports kind equal-depth, matching still null (#4264)", () => { + const r = new PrefixAffinityResolver(); + const transcript = Array.from({ length: 10 }, (_, i) => user(`duplicate conversation turn ${i} with substance`)); + const fp = r.chainFingerprint(transcript)!; + r.note("sess-one", fp.depth, fp.tailHash, fp.itemHashes); + + const before = r.findResumeParent(transcript, "sess-two"); + const report = r.diagnoseResumeMiss(transcript, "sess-two"); + const after = r.findResumeParent(transcript, "sess-two"); + assert.equal(before, null, "byte-exact equal-depth replay is not a resume (existing behavior)"); + assert.equal(after, null, "diagnostics must not mutate matching state"); + assert.equal(report.eligible, 1); + assert.ok(report.best, "an eligible candidate must yield a near-miss report"); + assert.equal(report.best!.kind, "equal-depth"); + assert.equal(report.best!.sessionId, "sess-one"); + assert.equal(report.best!.chainDepth, 10); + assert.equal(report.best!.incomingDepth, 10); + assert.equal(report.best!.firstDivergence, null, "every inspectable cell matches"); + assert.equal(report.best!.tailWindowIntact, true); + assert.match(describeResumeMiss(report), /EQUAL-DEPTH replay \(10 msgs == incoming 10\)/); +}); + +test("resume-inheritance-miss: head break names the first diverging cell with hash digests (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => user(`stable parent turn ${i} with substance`)); + const fp = r.chainFingerprint(parent)!; + r.note("sess-parent", fp.depth, fp.tailHash, fp.itemHashes); + + const edited = [...parent]; + edited[2] = user("edited third turn — different bytes"); + const incoming = [...edited, user("and the conversation goes on")]; + assert.equal(r.findResumeParent(incoming, "child"), null, "a mid-history edit breaks the head-anchored match"); + + const report = r.diagnoseResumeMiss(incoming, "child"); + assert.equal(report.eligible, 1); + assert.equal(report.best!.kind, "prefix-break"); + assert.equal(report.best!.sessionId, "sess-parent"); + assert.equal(report.best!.chainDepth, 10); + assert.equal(report.best!.incomingDepth, 11); + assert.equal(report.best!.windowStart, 0); + assert.equal(report.best!.firstDivergence, 2, "divergence at exactly the edited cell"); + assert.equal(report.best!.tailWindowIntact, false); + assert.match(report.best!.incomingCell!, /^[0-9a-f]{8}$/); + assert.match(report.best!.storedCell!, /^[0-9a-f]{8}$/); + assert.notEqual(report.best!.incomingCell, report.best!.storedCell); + assert.match(describeResumeMiss(report), /first divergence at cell 2 \(stored [0-9a-f]{8} vs incoming [0-9a-f]{8}\)/); +}); + +test("resume-inheritance-miss: deep chain broken before the stored window reports the pre-window bound (#4264)", () => { + const r = new PrefixAffinityResolver(); + const long = Array.from({ length: 130 }, (_, i) => user(`long chain message number ${i} with substance`)); + const fp = r.chainFingerprint(long)!; + assert.equal(fp.itemHashes.length, 128, "per-item hashes stay capped"); + r.note("sess-long", fp.depth, fp.tailHash, fp.itemHashes); + + const drifted = [...long]; + drifted[0] = user("re-encoded first message — host rebuild drift"); + const incoming = [...drifted, user("thirteen-one")]; + assert.equal(r.findResumeParent(incoming, "child"), null); + + const report = r.diagnoseResumeMiss(incoming, "child"); + assert.equal(report.best!.kind, "prefix-break"); + assert.equal(report.best!.windowStart, 2, "cells 0..1 uninspectable for a 130-deep chain"); + assert.equal(report.best!.firstDivergence, null, "the whole inspectable overlap matched"); + assert.equal(report.best!.tailWindowIntact, true); + assert.match(describeResumeMiss(report), /tail window \(cells 2\.\.129\) byte-intact — break sits BEFORE cell 2/); +}); + +test("resume-inheritance-miss: truncated replay is reported distinctly from a head break (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => user(`full parent turn ${i} with substance`)); + const fp = r.chainFingerprint(parent)!; + r.note("sess-parent", fp.depth, fp.tailHash, fp.itemHashes); + + const trimmed = parent.slice(0, 9); + assert.equal(r.findResumeParent(trimmed, "child"), null, "trimmed history is not a resume"); + const report = r.diagnoseResumeMiss(trimmed, "child"); + assert.equal(report.best!.kind, "truncated"); + assert.equal(report.best!.chainDepth, 10); + assert.equal(report.best!.incomingDepth, 9); + assert.equal(report.best!.tailWindowIntact, true, "the visible cells form a clean prefix"); + assert.match(describeResumeMiss(report), /LONGER than the incoming history \(9 msgs\)/); +}); + +test("resume-inheritance-miss: absent parent chains report zero eligible at debug granularity (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => user(`orphaned branch turn ${i} with substance`)); + const report = r.diagnoseResumeMiss(parent, "child"); + assert.deepEqual(report, { eligible: 0 }, "nothing tracked — no near miss exists"); + assert.match(describeResumeMiss(report), /^no eligible tracked chain \(floor 8\)$/); +}); + +test("resume-inheritance-miss: scoped match reports the declared parent's own state (#4264)", () => { + const r = new PrefixAffinityResolver(); + const other = Array.from({ length: 10 }, (_, i) => user(`unrelated tracked turn ${i} with substance`)); + const ofp = r.chainFingerprint(other)!; + r.note("sess-other", ofp.depth, ofp.tailHash, ofp.itemHashes); + const short = Array.from({ length: 7 }, (_, i) => user(`short declared turn ${i} with substance`)); + const sfp = r.chainFingerprint(short)!; + r.note("sess-short", sfp.depth, sfp.tailHash, sfp.itemHashes); + + const candidate = Array.from({ length: 11 }, (_, i) => user(`declared child turn ${i} with substance`)); + assert.equal(r.findResumeParentWithin("sess-ghost", candidate, "child"), null); + assert.deepEqual(r.diagnoseResumeMiss(candidate, "child", "sess-ghost"), { eligible: 0, declaredParentState: "untracked" }); + assert.match(describeResumeMiss(r.diagnoseResumeMiss(candidate, "child", "sess-ghost")), /declared parent chain untracked/); + + assert.equal(r.findResumeParentWithin("sess-short", candidate, "child"), null, "7-deep declared parent below the floor cannot match"); + const shallow = r.diagnoseResumeMiss(candidate, "child", "sess-short"); + assert.deepEqual(shallow, { eligible: 0, declaredParentState: "too-shallow" }); +}); + +test("resume-inheritance-miss: scoped match judges only the declared parent's chain (#4264)", () => { + const r = new PrefixAffinityResolver(); + const base = Array.from({ length: 10 }, (_, i) => user(`common base turn ${i} with substance`)); + const extA = [...base, user("extension A")]; + const extB = [...base, user("extension B")]; + const fpA = r.chainFingerprint(extA)!; + r.note("sess-a", fpA.depth, fpA.tailHash, fpA.itemHashes); + const fpB = r.chainFingerprint(extB)!; + r.note("sess-b", fpB.depth, fpB.tailHash, fpB.itemHashes); + + const candidate = [...extA, user("and further along")]; + // Hit-path regression: the matching scoped parent still resolves exactly as before. + assert.deepEqual(r.findResumeParentWithin("sess-a", candidate, "child"), { sessionId: "sess-a", sharedDepth: 11 }); + // Miss-path: the OTHER chain is out of scope even though it is tracked. + assert.equal(r.findResumeParentWithin("sess-b", candidate, "child"), null); + const report = r.diagnoseResumeMiss(candidate, "child", "sess-b"); + assert.equal(report.eligible, 1, "only the declared parent's chain is judged"); + assert.equal(report.best!.sessionId, "sess-b"); + assert.equal(report.best!.kind, "prefix-break"); + assert.equal(report.best!.firstDivergence, 10, "the two extensions differ at cell 10"); + assert.equal(report.best!.windowStart, 0); + assert.match(describeResumeMiss(report), /closest sess-b \(chain 11, incoming 12\)/); +}); + +test("resume-inheritance-miss: the caller's own id stays excluded from the near-miss report (#4264)", () => { + const r = new PrefixAffinityResolver(); + const transcript = Array.from({ length: 10 }, (_, i) => user(`self replay turn ${i} with substance`)); + const fp = r.chainFingerprint(transcript)!; + r.note("self-id", fp.depth, fp.tailHash, fp.itemHashes); + assert.equal(r.findResumeParent(transcript, "self-id"), null); + assert.deepEqual(r.diagnoseResumeMiss(transcript, "self-id"), { eligible: 0 }, "self-exclusion mirrors the matcher"); +}); + +test("resume-inheritance-miss: hit paths are byte-for-byte unchanged by the diagnostics surface (#4264)", () => { + const r = new PrefixAffinityResolver(); + const parent = Array.from({ length: 10 }, (_, i) => (i % 2 === 0 ? user(`parent turn ${i} with real substance`) : { role: "assistant", content: `parent reply ${i}` })); + const fp = r.chainFingerprint(parent)!; + r.note("sess-parent", fp.depth, fp.tailHash, fp.itemHashes); + const resumed = [...parent, user("resumed conversation next question")]; + assert.deepEqual(r.findResumeParent(resumed, "sess-child"), { sessionId: "sess-parent", sharedDepth: 10 }); + // A HIT produces no miss report at all — the server only calls diagnose + // on a null match, so nothing here may change the hit result. + assert.deepEqual(r.findResumeParent(resumed, "sess-child"), { sessionId: "sess-parent", sharedDepth: 10 }); +}); From ec5d3e3e39f9fcfcf87a1e09176672aa5bb5569d Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 22 Oct 2026 14:10:04 +0800 Subject: [PATCH 10/11] fix: name the resume-inheritance gate skip instead of staying silent (#4265 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When any gate conjunct short-circuits (derivedFromSessionId, publicForkReceipt, or resumeInheritance=false) there was no [resume-inheritance] line at all, so a log without "miss:" stayed ambiguous between gate-skipped, pre-#4265 build, and miss. The gate's else-branch now logs one debug line naming the skip reason. Same failure domain as #4264's stated goal (is the fallback channel even being tried) — absorbed here rather than a separate PR. Bounded like the miss line: the else-if re-checks clientProvided && requests === 0, and stats.requests increments later in the prepare lanes, so it fires at most once per session lifetime; anonAffinity is structurally null on this path (only the anonymous lane sets it). Purely additive — the gate condition itself is untouched, matching is unchanged. New e2e pin (tests/resume-inheritance-gate-skip.test.ts): both directions — skipped gate logs exactly one bounded debug line naming the reason, and an unskipped gate keeps the existing miss line and never says skipped. (cherry picked from commit f3f757873108148e772e5453f34c9f1001e4d460) (cherry picked from commit ea712f8de1c9124ff31b7ef9b8e7d89eef0b4c79) --- src/server/handle.ts | 12 ++ tests/resume-inheritance-gate-skip.test.ts | 152 +++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 tests/resume-inheritance-gate-skip.test.ts diff --git a/src/server/handle.ts b/src/server/handle.ts index 74fb2e392e..bd9874edb6 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -1622,6 +1622,18 @@ async function handleRequest( log("info", `[${sessionId}] [resume-inheritance] matched tracked chain ${resume.sessionId} but the parent session is not loadable — starting fresh (#1486)`); } } + } else if (clientProvided && session.stats.requests === 0) { + // #4264: a SKIPPED gate must not be silent either — without this + // line, zero [resume-inheritance] output stays ambiguous between + // "gate skipped" and "pre-#4265 build / silent miss". First + // request only: stats.requests increments later in the prepare + // lanes, so this fires at most once per session. anonAffinity is + // structurally null here (only the anonymous lane sets it), + // leaving exactly three skip reasons. + const skipReason = session.metadata.derivedFromSessionId !== undefined ? "derivedFromSessionId" + : session.metadata.publicForkReceipt !== undefined ? "publicForkReceipt" + : "resumeInheritance=false"; + log("debug", `[${sessionId}] [resume-inheritance] skipped: ${skipReason} (#4264)`); } // [#1333] explicitly derived conversations (pi RLM child, omp fork, // opencode subagent: the plugin reported its parent at register) diff --git a/tests/resume-inheritance-gate-skip.test.ts b/tests/resume-inheritance-gate-skip.test.ts new file mode 100644 index 0000000000..2a87e950df --- /dev/null +++ b/tests/resume-inheritance-gate-skip.test.ts @@ -0,0 +1,152 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { setLogCapture, muteStderrLeg } from "../src/logger.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import type { ProxyOptions } from "../src/config.ts"; + +/** + * #4264: a SKIPPED resume-inheritance gate must not be silent either. The + * miss diagnostics (#4265) only fire when the gate RUNS; when a conjunct + * short-circuits (resumeInheritance=false, derivedFromSessionId already + * linked, publicForkReceipt present) the session produced zero + * [resume-inheritance] output — indistinguishable in the log from a + * pre-#4265 build. Both directions are pinned: skip → exactly one bounded + * debug line naming the reason (first request only), no skip → the existing + * miss line instead (never "skipped"). + */ + +function listen(server: http.Server): Promise { + if (server.listening) return Promise.resolve(); + return once(server, "listening").then(() => undefined); +} + +function close(server: http.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +function sseLine(obj: unknown): string { + return `data: ${JSON.stringify(obj)}\n\n`; +} + +function startMockUpstream(): http.Server { + const server = http.createServer((req, res) => { + req.resume(); + req.on("end", () => { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(sseLine({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "ok" } }] })); + res.write(sseLine({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 2 } })); + res.write("data: [DONE]\n\n"); + res.end(); + }); + }); + server.listen(0, "127.0.0.1"); + return server; +} + +function proxyOpts(resumeInheritance: boolean | undefined): ProxyOptions { + return { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: {} as ProxyOptions["routes"], + modelContextLimit: 1_000_000, + kernelConfig: defaultConfig(1_000_000, { + preserveRecentMessages: 2, + preserveRecentTokens: 400, + compress: { minCompressRange: 200, minSummaryLength: 20, maxSummaryLength: 5000 }, + }), + compress: { injectTool: true, injectNudge: false }, + promptCache: { routing: "auto" }, + forkAdoption: true, + resumeInheritance, + sessionHeader: "x-claude-code-session-id", + log: true, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; +} + +async function chat(url: string, sessionId: string, text: string): Promise { + const res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-claude-code-session-id": sessionId }, + body: JSON.stringify({ model: "gpt-test", stream: true, messages: [{ role: "user", content: text }] }), + }); + if (!res.ok) { + const errBody = await res.text(); + assert.fail(`HTTP ${res.status}: ${errBody}`); + } + let raw = ""; + for await (const chunk of res.body!) raw += Buffer.from(chunk).toString("utf8"); + assert.ok(raw.includes("[DONE]"), "the mock upstream must stream a complete reply"); +} + +async function startHarness(resumeInheritance: boolean | undefined): Promise<{ proxy: http.Server; relay: http.Server; url: string }> { + process.env.XDG_STATE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), "res-gate-")); + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + _resetPluginStateForTest(); + const relay = startMockUpstream(); + await listen(relay); + const relayPort = (relay.address() as { port: number }).port; + const proxy = await startServer(proxyOpts(resumeInheritance)); + await listen(proxy); + const proxyPort = (proxy.address() as { port: number }).port; + return { proxy, relay, url: `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${relayPort}/v1/chat/completions` }; +} + +test("a skipped resume-inheritance gate logs exactly one bounded debug line naming the reason (#4264)", async () => { + muteStderrLeg(); + const h = await startHarness(false); + const captured: Array<{ level: string; msg: string }> = []; + try { + setLogCapture((level, msg) => captured.push({ level, msg })); + await chat(h.url, "gate-skip-disabled", "first turn"); + await chat(h.url, "gate-skip-disabled", "second turn"); + } finally { + setLogCapture(null); + await close(h.proxy); + await close(h.relay); + } + const lines = captured.filter(({ msg }) => msg.includes("[resume-inheritance]")); + assert.equal(lines.length, 1, "the skip line fires once per session, not per request"); + assert.equal(lines[0]!.level, "debug", "skip observability stays debug-level — zero steady-state noise"); + assert.match(lines[0]!.msg, /\[gate-skip-disabled\] \[resume-inheritance\] skipped: resumeInheritance=false \(#4264\)/); +}); + +test("an unskipped gate keeps the existing miss line and never says skipped (#4264 negative direction)", async () => { + muteStderrLeg(); + const h = await startHarness(undefined); + const captured: Array<{ level: string; msg: string }> = []; + try { + setLogCapture((level, msg) => captured.push({ level, msg })); + await chat(h.url, "gate-open-miss", "first turn"); + } finally { + setLogCapture(null); + await close(h.proxy); + await close(h.relay); + } + const lines = captured.filter(({ msg }) => msg.includes("[resume-inheritance]")); + assert.equal(lines.length, 1, "the gate ran — exactly the existing miss diagnostic, nothing else"); + assert.equal(lines[0]!.level, "debug", "no eligible chain: debug granularity as before"); + assert.match(lines[0]!.msg, /\[gate-open-miss\] \[resume-inheritance\] miss: no eligible tracked chain \(floor 8\) \(#4264\)/); + assert.ok(!lines[0]!.msg.includes("skipped"), "the miss line must stay distinguishable from the skip line"); +}); From b60e0755eead4908c7fc2393063ea0b0a4e33d12 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 11:45:53 +0800 Subject: [PATCH 11/11] fix: de-export the internal-only resume-miss diagnostic types (#4585 review) ResumeMissCandidate / ResumeMissReport are referenced only inside prefix-affinity.ts (describeResumeMiss consumes them); the leading export made them dead external surface and tripped the unused-exports gate, which fails the release-bugfix full-CI gate. Drop the export keyword only - type-level, internal references intact, no behavior change. --- src/prefix-affinity.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prefix-affinity.ts b/src/prefix-affinity.ts index 1517d66f69..83bcb55709 100644 --- a/src/prefix-affinity.ts +++ b/src/prefix-affinity.ts @@ -521,7 +521,7 @@ function containsRun(haystack: string[], needle: string[]): boolean { } /** #4264: one judged candidate inside a ResumeMissReport. */ -export interface ResumeMissCandidate { +interface ResumeMissCandidate { sessionId: string; /** Stored chain depth (neutral turns for hv2, raw messages for hv1). */ chainDepth: number; @@ -555,7 +555,7 @@ export interface ResumeMissCandidate { } /** #4264: diagnostics-only report for a resume-inheritance miss. */ -export interface ResumeMissReport { +interface ResumeMissReport { /** Tracked chains that passed the pre-screen (depth ≥ * MIN_RESUME_PREFIX, id ≠ self, in-domain floor) and were actually * judged. 0 with no declaredParentState = nothing eligible to begin