From b3286dbf5813a00e9af9564b3ed066d760ffe44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=99=B4=E7=8C=AB?= Date: Thu, 12 Nov 2026 05:37:09 +0900 Subject: [PATCH] fix(stream): arm the ledger align only after a compress created a block applyRanges set session.pendingLedgerAlign right after committing the kernel's returned state, before the blocksCreated === 0 returns. A refused compress (range too small, cannot anchor, already compressed) and the no-op class armed the one-time #4316 alignment too, so the next save ran it after a call that folded nothing. Set the marker past those returns. A compress that created a block arms it exactly as before (#5608-D). --- CHANGELOG.md | 1 + src/stream.ts | 12 +- tests/compress-refused-ledger-align.test.ts | 128 ++++++++++++++++++++ 3 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 tests/compress-refused-ledger-align.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 710ce2fabe..31ee28d756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag - **系统架构文档体系落地(owner 指令 2026-11-08):`docs/architecture/` 成为实时维护的架构枢纽**:新增 `docs/architecture.md` / `docs/architecture.zh-CN.md` 总枢纽(整体架构图中英双版转正 + 8 模块总表),首个样板模块页 `docs/architecture/storage.{md,zh-CN.md}`(统一存储:职责/原理/工作机制/硬不变量,配模块图中英双版)。文件级地图 `reference/architecture.md` 补齐 125 个缺失条目(新存储层 / marker / fastpath / lanes 等全部新模块)并升级为 CI 强制全量覆盖。新增 CI 车道 `architecture-docs`(`tools/check-architecture-docs.mjs`):地图覆盖全 `src/` 文件、总表↔页面成对、图 svg 优先 + png 备选齐备、SVG 基础良构(未转义 `&` 曾让英文图线上破图)。AGENTS.md「Documentation Layout」新增实时维护规则:改结构 ⇒ 同 PR 更新页 + 图 + 总表。其余 7 个模块页(pipeline / lanes / kernel / identity / fastpaths / webui / update)同一 PR 全部补齐——总表 8 行全 ✅(每页 = 中英双版文档 + 中英双版图)。 ### Fixed +- **A compress call that folded nothing no longer rewrites every `` tag on the next request (follow-up to #5608-C/D, #5797).** `applyRanges` armed the deferred one-time ref/ledger alignment (`session.pendingLedgerAlign`) right after committing the kernel's returned state — before the `blocksCreated === 0` returns — so a refused compress (range too small, cannot anchor, already compressed) or a no-op armed it too. The next save then ran `restampRefsToLedger`, which raised `timestampsHighWaterRefSeq` to the highest ref unconditionally: on a session whose refs already sit on their ledger seats (`kept=N restamped=0 rehomed=0`) no number moved, yet every tag minted so far dropped below the mark and lost its `time=` attribute — the whole cached prefix re-billed after a call whose receipt says nothing was folded. Now the marker is armed only once a block was really created; a real fold arms it exactly as before. - **压缩树图:节点标题不再一闪即逝,点击压缩块能真正看到被压缩的消息内容**:此前树节点只有原生 SVG `` 悬停提示——选中节点或 5s 轮询重画时 DOM 一变提示就消失,实测「显示一下马上消失」;右侧 Block details 里块的消息行虽带消息编号(mNNNNN)但内容全是一句「消息已不在会话中」(宿主原生压缩把消息从上下文剪掉后,字节无处可取)。现在:① 悬停提示改为页面内常驻浮层(.g-tip)——跟随指针、显示完整块号/tier/消息范围/完整主题/tokens/消息数,选中节点或轮询重画都不再打断,离开图形区才隐藏;悬停时节点卡片上的主题行同时原地扩宽(~20 字 → 20 字上限的完整截断,离开复原);② 5s 轮询只回传瘦节点(无主题/摘要/tokens),重画前先与详情富块按 blockId 合并——节点标题、浮层元信息、消息数不再在每次 tick 后退化成光秃秃的块号;③ 服务端新增**冻结快照水合**:压缩块落盘时已把原始消息逐条存进 blockContents(`[role]`/`[role • tool]` 头 + 原文,pi 宿主 user 消息还带 mNNNNN 标记行)——水合按**标记锚定**对齐:pi 宿主打进 user 消息正文的 mNNNNN 标记是身份戳,每个标记用两条独立证据之一锚死——**恒等锚**(标记号=该位当前 ref,从未被重编号)或**冻结 span 算术锚**(标记的冻结编号 − 块冻结 span 起点 = 快照位序;span 与标记同批写入,冷加载 restamp(#5196 parked 段换新号)重编了整块 ref 也照样成立——实测 b27 全块 127 条因此恢复,旧规则因标记号已不在 ref 表里整块拒水合);锚点之间的段仅在条数平衡时按位 zip。标记既非恒等也非算术锚(冻结上下文混入外来消息/span 无法证明)、锚点乱序、无锚非叶块(快照 span 的是 effective 超集)一律拒绝,未证明的位置保留诚实的 missing 标记——宁可不显示,不冒错位张冠李戴的风险(预览 + 点击展开取原文,`/message?ref=` 同享该层)。实测大会话(253 块/9771 个直接消息引用):7789 条(80%)恢复出真实预览与原文,其余保持 missing。回归:tests/web-blocks-tree.test.ts 冻结快照水合 7 用例(按位水合 / 条数不符拒 / 标记移位旧锚 / 宿主剪块中消息的锚定恢复 / restamp 整块恢复 / 算术锚优先于巧合 ref 碰撞 / 无锚标记整块拒(含 click-to-open 双向断言)),源码级断言更新为合并后重画(renderCollapSvg(merged)。 - **交接文档「Compressed block summaries」附录不再被吞进最后一条消息卡片**:客户端 handoff 渲染器只认 `### user/assistant/tool` 三种角色分隔线(#2065 语义,正文内其他 `###` 标题是内容),export 端 #845 追加在对话之后的 `## Compressed block summaries` 整节(长会话可达 80+ 个 `### Block bN` 条目、数百行)因此全部落进最后一个 assistant 卡片,看起来像「压缩消息被追加成一条巨型消息」。现在客户端把 Conversation 之后的 `## ` 级标题识别为独立分节:`### Block bN — topic` 逐条渲染为小块卡片(mono 块号 + 主题截断标题 + tier/token 元行 + Markdown 正文),角色卡片只保留真实对话。回归:tests/web-handoff-roles.test.ts 新增附录分节用例(角色卡片数不膨胀、条目内容保真、#2065 不变量不受影响)。 - **会话历史的「条数」维度与超大未压缩请求:按真实维度拦下,判死信息点名超重字段(#5000 #5794 #5885)**:#5000 的 26,903 条消息会话(55.2MiB)穿过了**所有**字节/token 闸门、在上游死于消息条数上限——因为条数这个维度从来没有任何闸(它无法被任何字节/token 估算表达:每条消息还各带 ~69B 的 m-ref 标记,26,903 条 ≈ 1.8MB 纯标记)。现在新增按上游配置的**条数闸**(`network.maxMessages` / `BILI_MAX_MESSAGES`,默认 0=关闭——不存在全局「通用上限」,条数上限因上游而异):超限直接 413,信息点名 COUNT 维度并给出可执行指引(重启会话或回退到更早的时点;折叠在一轮 preflight 内关不掉 ~20K 条的缺口,所以是纯判死、不做条数折叠),且在 token 触发器**之前**运行——token 达标但条数超标的体(正是 #5000 的形状)也只有这道闸看得见。同时所有超窗判死信息(#5932 的三道计量闸)现在**点名超重的是哪个字段**——按 #554 计量器同源分桶(messages / tools[] / system / other),不再只给一个总数:#5000 的形状会显示 `tools[] ~X`,巨大粘贴/工具结果会显示 `messages ~X`,运维一眼知道该删什么。另新增 **32MiB 解码摄入成本带**(`network.ingestBodyBytes` / `BILI_INGEST_BODY_BYTES`):#5794 的 51.2MiB/29,619 条请求每轮独占事件循环 5.5–7.2s(本机 CPU 54.6% → 747s FATAL 自重启、全员缓存冷),成本带在 JSON 解析扫掠**之前**按解码后字节 413(stage=ingest)——任何可解析的**文本为主**的 32MiB+ 体都已是 ≥~8M token,超出所有真实模型窗口,放行必然烧钱再死。带图片的体在所有车道豁免(base64 像素按设计不进 token 计量——#496/#1800/#857「图片单独计费、绝不在原始字节上判死」;#3909 的 ~48MiB 图片密集 chat 会话与 #3701 的 ~38.5MiB WS 截图体都是受支持工作负载,由槽位预算/收件上限/车道自身的预发送字节预算+丢弃逻辑治理;Responses 车道无论如何整体豁免);并发槽位预算与 5GiB 收件上限继续独立生效。回归:tests/issue5000-count-gate(3 e2e:token 达标+条数超标→413 零上游 / 无 model 路径 / 条数达标照常转发)、tests/issue5794-ingest-cap(3 e2e:超带文本→413 stage ingest / 带内照常 / 超带带图不拦)、tests/side-request-isolation 分桶计量单测;既有 332 用例(含 #3701/#857/#496/#4079/#4185/#2490 全部回归面)全绿。 diff --git a/src/stream.ts b/src/stream.ts index 502acda0af..53deb9eebe 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1349,10 +1349,6 @@ export function applyRanges(parsed: ReturnType<typeof parseCompressInput>, ctx: return compressResult(receipt, "refused", 0, "gate:no-net-shrink"); } ctx.session.state = res.state; - // #5608-D: this fold is the session's next legitimate cache-breaking - // boundary — flag the one-time #4316 citation-space alignment to ride - // it (persist.scheduleSave consumes the marker before serializing). - ctx.session.pendingLedgerAlign = true; // #2627 backstop: no committed state may carry outbound-only turn- // separator ids in coverage (a polluted child would otherwise re- // poison higher folds via effectiveMessageIds inheritance). Runs @@ -1500,6 +1496,14 @@ export function applyRanges(parsed: ReturnType<typeof parseCompressInput>, ctx: } return compressResult(receipt, "refused", 0, gateReason); } + // #5608-D: this fold is the session's next legitimate cache-breaking + // boundary — flag the one-time #4316 citation-space alignment to ride + // it (persist.scheduleSave consumes the marker before serializing). + // Armed only HERE, past the blocksCreated === 0 returns above: a + // refused or no-op compress rewrites no history, so it is not a + // boundary — arming on it let the next save run the alignment (and + // its time-attribute mark raise) on a call that folded nothing. + ctx.session.pendingLedgerAlign = true; clearCompressFailures(ctx.session); clearCompressLoopStreak(ctx.session); diff --git a/tests/compress-refused-ledger-align.test.ts b/tests/compress-refused-ledger-align.test.ts new file mode 100644 index 0000000000..c457327d7b --- /dev/null +++ b/tests/compress-refused-ledger-align.test.ts @@ -0,0 +1,128 @@ +// #5608-D follow-up: the one-time #4316 ref/ledger alignment is deferred to +// "the session's next successful fold" so its byte changes ride a boundary +// that already breaks the provider prefix. applyRanges armed the marker +// (session.pendingLedgerAlign) right after committing the kernel's returned +// state — BEFORE the blocksCreated === 0 returns — so a refused compress +// (range too small, cannot anchor, already compressed …) and the no-op class +// armed it as well. The next save then ran the alignment, whose mark raise +// stripped the time attribute from every <acp> tag rendered so far: the whole +// cached prefix changed after a call whose receipt says nothing was folded. +// +// Pinned here: a compress that creates no block leaves the marker unset and +// the next request's rendered bytes identical; a real fold still arms it. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +process.env.NODE_ENV = "test"; +process.env.BILI_STORAGE_UNIFIED = "full"; +const testRoot = mkdtempSync(join(tmpdir(), "bili-refused-align-")); +test.after(() => rmSync(testRoot, { recursive: true, force: true })); +for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = testRoot; + +import type { CoreMessage } from "acp-kernel"; +import { createCore, createInitialState, defaultConfig } from "acp-kernel"; +import { parseCompressInput } from "../src/compress-tool.ts"; +import { applyRanges, type RewriteCtx } from "../src/stream.ts"; +import type { Session } from "../src/session.ts"; + +type Ctx = Omit<RewriteCtx, "log"> & { log: (m: string) => void; logs: string[] }; +type Marked = Session & { pendingLedgerAlign?: boolean }; + +const cfg = defaultConfig(200000, { preserveRecentMessages: 2, preserveRecentTokens: 0 } as never); +const timeOf = (t: string | undefined): string | undefined => / time="([^"]+)"/.exec(t ?? "")?.[1]; + +function conversation(): CoreMessage[] { + // 9 turns of 3000 chars: one message alone sits under minCompressRange + // (5000), four together clear it. + return Array.from({ length: 9 }, (_, i): CoreMessage => ({ + id: `raw_${i + 1}`, + role: i % 2 === 0 ? "user" : "assistant", + contentType: "text", + text: `turn ${i + 1} ` + "lorem ipsum ".repeat(250), + })); +} + +/** A brand-new session exactly as the proxy leaves it after its first rounds: + * the timestamp pipeline has rendered every tag with a time attribute, and + * the rows are saved so each ref already sits on its ledger seat (seq + 1). */ +async function freshSession(tag: string) { + const { SessionStore, _setStoreForTest } = await import("../src/persist.ts"); + const dir = join(testRoot, `${tag}-${Math.random().toString(36).slice(2)}`); + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + _setStoreForTest(store); + const core = createCore({ timestamps: true }); + const messages = conversation(); + const first = core.processTurn({ messages, state: createInitialState(), config: cfg, tokenCount: 9000 }); + const session = { + id: `refused-align-${tag}`, + meta: { protocol: "anthropic" }, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 100000, compressCreditTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, + state: first.state, + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + lastMessages: messages, + } as unknown as Marked; + store.scheduleSave(session); + assert.ok(store.flushSync(session), "seed save committed"); + const seq = store.unifiedSeqByRaw(session.id); + assert.ok(seq !== null && seq.size === messages.length, "rows are in the unified store (the alignment would have something to read)"); + const logs: string[] = []; + const ctx: Ctx = { core, config: cfg, messages: first.messages, session, log: (m) => { logs.push(m); }, logs }; + const render = (): string[] => core + .processTurn({ messages, state: session.state, config: cfg, tokenCount: 9000 }) + .messages.map((m) => m.text ?? ""); + const close = (): void => { store.closeQuietly(); _setStoreForTest({} as never); rmSync(dir, { recursive: true, force: true }); }; + return { store, session, ctx, before: first.messages.map((m) => m.text ?? ""), render, close }; +} + +test("a refused compress does not arm the ledger alignment — next request's tags are byte-identical", async () => { + const s = await freshSession("refused"); + try { + assert.ok(s.before.every((t) => timeOf(t) !== undefined), "precondition: every tag carries a time attribute"); + const res = applyRanges(parseCompressInput({ content: [{ startId: "m00002", endId: "m00002", summary: "one short turn, nothing worth keeping from it at all" }] }), s.ctx); + assert.equal(res.outcome, "refused", res.text.slice(0, 200)); + assert.equal(res.reason, "gate:too-small", res.text.slice(0, 200)); + assert.equal(s.session.state.blocks.length, 0, "nothing was folded"); + assert.equal(s.session.pendingLedgerAlign, undefined, "no fold ⇒ marker never set (persist.ts consumePendingLedgerAlign contract)"); + // The save that follows the tool round, then the next request's render. + s.store.scheduleSave(s.session); + assert.ok(s.store.flushSync(s.session), "post-refusal save committed"); + assert.notEqual((s.session.state as { refLedgerAligned?: boolean }).refLedgerAligned, true, "alignment did not run on a refusal"); + assert.deepEqual(s.render(), s.before, "rendered tags unchanged after a refused compress"); + } finally { s.close(); } +}); + +test("a no-op compress (range inside the protected zone) does not arm it either", async () => { + const s = await freshSession("noop"); + try { + const res = applyRanges(parseCompressInput({ content: [{ startId: "m00008", endId: "m00009", summary: "the two most recent turns, which the protection window keeps raw" }] }), s.ctx); + assert.equal(res.blocksCreated ?? 0, 0, res.text.slice(0, 200)); + assert.equal(s.session.state.blocks.length, 0, "nothing was folded"); + assert.equal(s.session.pendingLedgerAlign, undefined, "no block created ⇒ marker never set"); + s.store.scheduleSave(s.session); + assert.ok(s.store.flushSync(s.session)); + assert.deepEqual(s.render(), s.before, "rendered tags unchanged after a compress that folded nothing"); + } finally { s.close(); } +}); + +test("a real fold still arms the alignment and the next save consumes it (#5608-D unchanged)", async () => { + const s = await freshSession("applied"); + try { + const res = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00004", summary: "four early turns of filler text; nothing in them needs to be kept verbatim" }] }), s.ctx); + assert.equal(res.outcome, "applied", res.text.slice(0, 200)); + assert.ok(s.session.state.blocks.length > 0, "a block was created"); + assert.equal(s.session.pendingLedgerAlign, true, "fold commit arms the one-time alignment"); + s.store.scheduleSave(s.session); + assert.ok(s.store.flushSync(s.session)); + assert.equal(s.session.pendingLedgerAlign, undefined, "marker consumed by the save"); + assert.equal((s.session.state as { refLedgerAligned?: boolean }).refLedgerAligned, true, "alignment ran on the fold boundary"); + } finally { s.close(); } +});