diff --git a/CHANGELOG.md b/CHANGELOG.md index 60fd819367..03604e3050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ## [Unreleased] +### Added + +- **seq ledger foundations — block interval bounds + ledger-numbered refs (#4316 layer 3, phase 1).** Block coverage now carries its ledger interval: `planIngest` fills `blocks.start_seq/end_seq` with the min/max unified-store seq of the covered messages (`-1/-1` when no rows resolve) — coverage bounds are id-independent, so client-side byte churn can never invalidate them. `CoreMessage.ledgerSeq` sidecar: store-served views (snapshot-render prefix rows) stamp the row's `(session, seq)`; `assignRefs` issues the ref as the direct display of that number (seq+1 in the 1-based display space), advancing the allocation cursor past the stamp so counter-issued refs never collide with future stamps. A store-served message's number is assigned once at ingest and is structurally immune to the #4316 churn family (model-switch re-stamps, image re-encodes re-minting every content-hash id). Stakes first in `tests/seq-ledger.test.ts`: interval bounds (S1), ref-set stability across a full id re-mint under snapshot render (S2), and the kernel seam (S3). Follow-ups: incremental-lane stamping, fork `cloneRows` number-preserving copy, and the one-time legacy re-mint window handling — tracked in #4316. + ### Storage — 行身份与史官化(#4390,storage review 四个 p2 + 三个顺路 p3) - **行身份去污染(D4-1)**:统一存储 messages 行新增 `identity` 列——跨保存 JOIN 键改为**位置无关**序列化(剥离 `id`/`wireIndex` 后哈希)。此前身份掺入 wire 位置元数据,客户端头部截断一条消息就会让后续全部行换血(旧行被清扫、seq 重铸、#4335 的区间账记随之抖动)。`content_hash` 保持字节忠实 CAS 键不变。存量库 v8 迁移:回填 identity 并合并 wireIndex 分裂世代(保最低 seq,重复行退还 CAS 引用)。MySQL 同款(information_schema 探测幂等 ALTER + 回填)。 diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index 63c309d2f2..d3a8dea750 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -126,6 +126,29 @@ export function assignRefs( continue; } + // #4316 layer 3: store-served views stamp ledgerSeq — the unified + // transcript position, assigned once and never re-minted. The ref IS the + // ledger number (direct display), so byte churn cannot renumber + // citations. Collisions (should be impossible — (session, seq) is a PK) + // fall through to the counter path; the cursor advances past the stamp so + // later counter-allocated refs never collide with future stamps either. + const ledgerSeq: number | undefined = message.ledgerSeq; + if ( + typeof ledgerSeq === "number" && + Number.isInteger(ledgerSeq) && + ledgerSeq >= MIN_INDEX && + ledgerSeq <= MAX_INDEX + ) { + const text = indexToRef(ledgerSeq); + if (!map.byRef[text]) { + map.byRaw[message.id] = text; + map.byRef[text] = message.id; + newlyAssigned++; + if (ledgerSeq >= cursor) cursor = ledgerSeq + 1; + continue; + } + } + const ref = allocateFreeRef(map, cursor); cursor = ref.index + 1; map.byRaw[message.id] = ref.text; diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 68e729582e..479c987692 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -30,6 +30,16 @@ export interface CoreMessage { * inactive/unknown block folds normally. Reserved-prefix ids * (`acp_summary_*`) stay kernel-internal and never need this marker. */ summaryOfBlockId?: string; + /** #4316 layer 3 — server-ledger numbering. Set by store-served views + * (snapshot-render prefix rows, incremental canonical rows): the unified + * transcript's (session, seq) for this message, assigned once at ingest + * and never re-minted. When present, assignRefs issues the ref as the + * direct display of this number, so client-side byte churn (model-switch + * re-stamps, image re-encodes) cannot renumber the model's citations. + * Callers must guarantee per-session uniqueness; a collision falls back + * to the counter path for the loser. Not serialized into wire payloads — + * it rides CoreMessage sidecars and the persisted store rows verbatim. */ + ledgerSeq?: number; } export type CompressionTier = 1 | 2 | 3; diff --git a/src/server/snapshot-render.ts b/src/server/snapshot-render.ts index de1cd2baae..e5247726e3 100644 --- a/src/server/snapshot-render.ts +++ b/src/server/snapshot-render.ts @@ -120,7 +120,16 @@ function parseRow(content: string, seq: number): BiliMessage | null { try { const parsed = JSON.parse(content) as { id?: unknown; role?: unknown }; if (typeof parsed.id !== "string" || typeof parsed.role !== "string") return null; - return parsed as BiliMessage; + const msg = parsed as BiliMessage; + // #4316 layer 3: the row's (session, seq) is the message's permanent + // ledger position — stamp it so assignRefs issues the ref as the + // direct display of the seq (never re-minted across byte churn). + // Store seqs are 0-based; the ref display space is 1-based + // (MIN_INDEX=1, "m00001" is the first number) — the stamp carries the + // DISPLAY index. Rows written before this change simply lack the + // field; the kernel counter path keeps serving them unchanged. + msg.ledgerSeq = seq + 1; + return msg; } catch { return null; } diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 46eb1b07fd..34b515ff51 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -314,16 +314,33 @@ function finishPlan( ...(e.media && e.media.length > 0 ? { media: e.media } : {}), })); + // #4316 layer 3: the ledger positions of a block's covered messages — + // min/max seq over the rows its coverage resolves to. Ghost ids (no row + // in this plan's log) are skipped; a block whose coverage resolves to no + // row keeps the -1/-1 "unknown bounds" shape (pre-layer-3 + // rows stored 0/0 placeholders; both are simply "no interval evidence"). These bounds are the durable interval form of coverage: + // independent of message ids, so byte churn can never invalidate them. + const seqById = new Map(); + for (const e of log) if (!seqById.has(e.msgId)) seqById.set(e.msgId, e.seq); + const blocks: BlockRow[] = session.state.blocks.map((b, ord) => { const views = completeBlockContents(session.blockContents, session.state.blocks).get(b.blockId); const full = views?.full ?? { text: b.summary, count: 0 }; const one = views?.one ?? null; const covered = { effectiveMessageIds: b.effectiveMessageIds, directMessageIds: b.directMessageIds }; + let startSeq = -1; + let endSeq = -1; + for (const id of b.effectiveMessageIds) { + const s = seqById.get(id); + if (s === undefined) continue; + if (startSeq === -1 || s < startSeq) startSeq = s; + if (s > endSeq) endSeq = s; + } return { blockId: b.blockId, ord, - startSeq: 0, - endSeq: 0, + startSeq, + endSeq, startRef: b.startRef !== undefined ? refToIndex(b.startRef) : null, endRef: b.endRef !== undefined ? refToIndex(b.endRef) : null, tier: b.tier, diff --git a/tests/seq-ledger.test.ts b/tests/seq-ledger.test.ts new file mode 100644 index 0000000000..504614d833 --- /dev/null +++ b/tests/seq-ledger.test.ts @@ -0,0 +1,303 @@ +// #4316 layer 3 — the server ledger IS the numbering. +// +// Contract staked (打桩): +// +// S1 block coverage carries its ledger interval: planIngest fills +// start_seq/end_seq with the min/max unified-store seq of the covered +// messages (pre-layer-3 rows carried 0/0 placeholders). Id-independent +// coverage bounds — byte churn can never invalidate them. +// +// S2 snapshot-render prefix rows are numbered BY THE LEDGER: the ref +// of every store-served message equals indexToRef(rowSeq + 1) (store +// seqs are 0-based, the ref display space is 1-based). The kernel +// counter path stops being the numbering authority for served views — +// the number is assigned once at ingest and is structurally immune to +// client-side byte churn (#4316: model switch / re-stamp / re-encode +// re-mints every content-hash id today and kills the model's numbers). +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +process.env.NODE_ENV = "test"; +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "shadow"; +process.env.BILI_CONFIG_FILE = path.join(mkdtempSync(path.join(tmpdir(), "bili-seq-ledger-")), "no-config.json"); +process.env.BILI_ROLLOUT_SNAPSHOT_RENDER = "100"; + +import { defaultConfig, createInitialState } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest, getStore } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions, _resetSessionsForTest } from "../src/session.ts"; +import { rmrf } from "./tmp-rm.ts"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; + +const SUMMARY = "SEQ-LEDGER-SUMMARY-SETUP-CONTEXT-FOLDED-BY-COMPRESSION-LONG-ENOUGH-FOR-KERNEL-MIN-LENGTH-CHECK"; + +test("S1: block coverage carries its ledger interval (start_seq/end_seq)", () => { + const session: LegacySessionLike = { + id: "seq-bounds-1", + createdAt: 1, + lastSeen: 2, + meta: { label: "seq-bounds-1", protocol: "openai", upstreamOrigin: "http://upstream" }, + stats: { requests: 1 } as LegacySessionLike["stats"], + metadata: {}, + state: { + ...createInitialState(), + messageRefs: { byRaw: {}, byRef: {} }, + blocks: [ + { + blockId: "b1", + runId: "r1", + tier: 1 as const, + summary: SUMMARY, + directMessageIds: ["m3", "m4", "m5"], + effectiveMessageIds: ["m2", "m3", "m4", "m5", "m6"], + directBlockIds: [], + compressedTokens: 500, + createdAt: 3, + survivedCount: 0, + generation: "young", + active: true, + }, + ], + }, + blockContents: new Map([["b1", { one: null, full: { text: SUMMARY, count: 5 } }]]), + lastMessages: Array.from({ length: 10 }, (_, i) => ({ id: `m${i + 1}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text: `MSG_${i + 1}_BODY_`.repeat(40) })), + lastMessagesFolded: false, + }; + const plan = planIngest(session); + // Coverage m2..m6 must resolve to the ledger positions of those exact + // messages — the plan's own seq assignment is the ground truth. + const seqOf: Record = {}; + for (const row of plan.messages) seqOf[row.msgId] = row.seq; + assert.notEqual(seqOf["m2"], undefined, "m2 has a log row"); + assert.notEqual(seqOf["m6"], undefined, "m6 has a log row"); + assert.ok(seqOf["m2"] < seqOf["m6"], "seqs are ordered"); + const b1 = plan.blocks.find((b) => b.blockId === "b1"); + assert.ok(b1, "block row present"); + assert.equal(b1.startSeq, seqOf["m2"], "start_seq = min covered ledger seq"); + assert.equal(b1.endSeq, seqOf["m6"], "end_seq = max covered ledger seq"); + // Ghost-only coverage (no rows) keeps the unknown-bounds shape. + const ghost: LegacySessionLike = { + ...session, + id: "seq-bounds-ghost", + state: { + ...session.state, + blocks: [{ ...(session.state.blocks[0] ?? ({} as never)), blockId: "bG", effectiveMessageIds: ["ghost-1"], directMessageIds: ["ghost-1"] }], + }, + }; + const planG = planIngest(ghost); + const bG = planG.blocks.find((b) => b.blockId === "bG"); + assert.ok(bG); + assert.equal(bG.startSeq, -1, "no covered rows resolve → unknown bounds"); + assert.equal(bG.endSeq, -1); +}); + +// --------------------------------------------------------------------------- +// S2: snapshot-render prefix refs == ledger positions. Harness mirrors +// tests/snapshot-render.test.ts (fake upstream, real server + shadow store). + +function conversation(): Array<{ role: string; content: string }> { + const out: Array<{ role: string; content: string }> = []; + for (let i = 0; i < 23; i++) { + out.push({ role: i % 2 === 0 ? "user" : "assistant", content: `Message ${i} payload follows. ` + `WORK_${i}_content_`.repeat(88) }); + } + return out; +} + +interface Harness { + bodies: string[]; + sessionOf: (label: string) => { id: string; state: { messageRefs: { byRaw: Record }; blocks: Array<{ active: boolean }> } } | undefined; + url: string; + proxy: http.Server; + upstream: http.Server; + store: SessionStore; +} + +async function withHarness(fn: (h: Harness) => Promise): Promise { + _setStoreForTest(new SessionStore({ enabled: false })); + _resetSessionsForTest(); + setRegistryForTest({}); + const root = mkdtempSync(path.join(tmpdir(), "seq-ledger-e2e-")); + const store = new SessionStore({ enabled: true, dir: path.join(root, "sessions"), debounceMs: 0 }); + _setStoreForTest(store); + let n = 0; + const bodies: string[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + n += 1; + bodies.push(Buffer.concat(chunks).toString("utf8")); + res.writeHead(200, { "content-type": "text/event-stream" }); + if (n === 1) { + res.write(`data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "compress", arguments: JSON.stringify({ content: [{ startId: "m00001", endId: "m00008", topic: "setup", summary: SUMMARY }] }) } }] } }] })}\n\n`); + res.write(`data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] })}\n\n`); + } else if (n === 2) { + res.write(`data: ${JSON.stringify({ id: "c2", object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: "done compressing" }, finish_reason: null }] })}\n\n`); + res.write(`data: ${JSON.stringify({ id: "c2", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`); + } else { + res.write(`data: ${JSON.stringify({ id: "c3", object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: `ok ${n}` }, finish_reason: null }] })}\n\n`); + res.write(`data: ${JSON.stringify({ id: "c3", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`); + } + res.write("data: [DONE]\n\n"); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const opts = { + port: 0, + host: "127.0.0.1", + upstream: `http://127.0.0.1:${upstreamPort}`, + routes: {}, + proxy: "", + proxyMode: "direct", + proxySource: "direct", + modelContextLimit: 15000, + kernelConfig: defaultConfig(15000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const port = (proxy.address() as { port: number }).port; + try { + await fn({ bodies, sessionOf: (label) => listSessions().find((s) => s.meta.label === label) as never, url: `http://127.0.0.1:${port}/bili/http://127.0.0.1:${upstreamPort}/v1/chat/completions`, proxy, upstream, store }); + } finally { + delete process.env.BILI_ROLLOUT_SNAPSHOT_RENDER; + (proxy as http.Server).closeAllConnections?.(); + await new Promise((r) => proxy.close(() => r())); + (upstream as http.Server).closeAllConnections?.(); + await new Promise((r) => upstream.close(() => r())); + await store.closeQuietly(); + _resetSessionsForTest(); + rmrf(root); + } +} + +async function turn(h: Harness, label: string, messages: Array<{ role: string; content: string }>, history: Array<{ role: string; content: string }> | null): Promise<{ status: number; body: string; idx: number }> { + const { promise, resolve, reject } = Promise.withResolvers<{ status: number; body: string }>(); + const data = JSON.stringify({ model: "gpt-test", messages, stream: true }); + const req = http.request(h.url, { method: "POST", headers: { "content-type": "application/json", "content-length": String(Buffer.byteLength(data)), "x-acp-session": label } }, (res) => { + let body = ""; + res.on("data", (c: Buffer) => { body += c.toString("utf8"); }); + res.on("end", () => resolve({ status: res.statusCode ?? 0, body })); + }); + req.once("error", reject); + req.end(data); + const r = await promise; + await new Promise((r2) => setTimeout(r2, 120)); // debounceMs=0 write+ingest lands + return { ...r, idx: h.bodies.length - 1 }; +} + +function refOf(s: string): number { + const m = /m(\d{5})<\/acp>/.exec(s); + return m ? Number(m[1]) : 0; +} + +test("S2: snapshot-render prefix refs equal the ledger positions (ref = indexToRef(seq+1))", async () => { + process.env.BILI_ROLLOUT_SNAPSHOT_RENDER = "100"; + await withHarness(async (h) => { + const history = conversation(); + const r1 = await turn(h, "seq-ledger-1", history, history); + assert.equal(r1.status, 200, r1.body); + const r2 = await turn(h, "seq-ledger-1", [...history, { role: "user", content: "TAIL_TURN_TWO_ASKS_FOR_A_SUMMARY. " + "TAIL_A_".repeat(80) }], history); + assert.equal(r2.status, 200, r2.body); + assert.ok(h.sessionOf("seq-ledger-1")!.state.blocks.some((b: { active: boolean }) => b.active), "block armed"); + // settle: the ingest of turn 2's view must land before the D view build + const t0 = Date.now(); + for (;;) { + try { + const rows = getStore().unifiedTailViewRows("seq-ledger-1"); + if (rows !== null && rows.length > 0) break; + } catch { /* not yet */ } + if (Date.now() - t0 > 5000) throw new Error("settleRows: store tail view never appeared"); + await new Promise((r) => setTimeout(r, 50)); + } + const r3 = await turn(h, "seq-ledger-1", [...history, { role: "user", content: "TAIL_TURN_TWO_ASKS_FOR_A_SUMMARY. " + "TAIL_A_".repeat(80) }, { role: "user", content: "TAIL_TURN_THREE_FOLLOWS_UP. " + "TAIL_B_".repeat(80) }], null); + assert.equal(r3.status, 200, r3.body); + const out3 = h.bodies[r3.idx] ?? ""; + const refsOf = (body: string): string[] => { + const all = JSON.parse(body).messages as Array<{ role?: unknown; content?: unknown }>; + return all + .filter((m) => m.role !== "system" && typeof m.content === "string") + .map((m) => (m.content as string).match(/m\d{5}<\/acp>/)?.[0] ?? "") + .filter((s) => s.length > 0); + }; + const refs3 = refsOf(out3); + assert.ok(refs3.length >= 8, `prefix messages carry acp refs (got ${refs3.length})`); + + // The seed-upgrade scenario (#3658/#3752 class): every message id is + // re-minted on identical content. The D view serves the store rows — + // the ref SET (the model's live citations) must be unchanged. + const { __setMessageIdSaltForTests } = await import("acp-kernel/wire"); + __setMessageIdSaltForTests("upgrade-v2"); + try { + const churnHistory = history.map((m, i) => ({ ...m, content: m.content + (i < 10 ? " " : "") })); + const r4 = await turn(h, "seq-ledger-1", [...churnHistory, { role: "user", content: "TAIL_TURN_TWO_ASKS_FOR_A_SUMMARY. " + "TAIL_A_".repeat(80) }, { role: "user", content: "TAIL_TURN_THREE_FOLLOWS_UP. " + "TAIL_B_".repeat(80) }, { role: "user", content: "TAIL_TURN_FOUR_AFTER_CHURN. " + "TAIL_C_".repeat(80) }], null); + assert.equal(r4.status, 200, r4.body); + const refs4 = refsOf(h.bodies[r4.idx] ?? ""); + const prefix3 = refs3.slice(0, Math.min(refs3.length, refs4.length)); + const prefix4 = refs4.slice(0, prefix3.length); + assert.deepEqual(prefix4, prefix3, "model-visible ref set is unchanged across a full id re-mint (ledger numbering)"); + } finally { + __setMessageIdSaltForTests(undefined as never); + } + }); +}); + +// Kernel seam, pinned directly: a store-served message that stamps its +// ledger position takes the ref that IS that position — even when the +// counter would have issued something else, and even at map-rebuild time. +test("S3: assignRefs honors ledgerSeq (ref = direct display of the ledger position)", async () => { + const { assignRefs } = await import("acp-kernel"); + const base = { + byRaw: { known: "m00003" }, + byRef: { m00003: "known" }, + }; + const res = assignRefs( + [ + { id: "known", role: "user", contentType: "text", text: "already numbered" }, + { id: "fresh-ledger", role: "user", contentType: "text", text: "row seq 76 → display m00077", ledgerSeq: 77 }, + { id: "fresh-counter", role: "user", contentType: "text", text: "no stamp → counter path" }, + ], + { existing: base, nextIndex: 1 }, + ); + assert.equal(res.map.byRaw["known"], "m00003", "known id keeps its ref"); + assert.equal(res.map.byRaw["fresh-ledger"], "m00077", "ledgerSeq 77 → m00077 directly"); + // The stamp advanced the cursor past 77, so the next counter allocation + // can never collide with a future ledger stamp — it takes 78. + assert.equal(res.map.byRaw["fresh-counter"], "m00078", "unstamped message takes the counter path, past the stamp"); + assert.ok(res.nextIndex > 77, "cursor advanced past the ledger stamp so future counter refs never collide"); + // Cold-rebuild shape: empty map, stamps arriving OUT OF ORDER — each row + // must claim its own ledger number directly (order-independent), and the + // cursor must land past the highest stamp so a later counter allocation + // cannot collide. This is the case the stamp exists for: no prior map to + // borrow numbers from, so the ledger IS the numbering authority. + const cold = assignRefs( + [ + { id: "row-a", role: "user", contentType: "text", text: "stored at seq 4", ledgerSeq: 5 }, + { id: "row-b", role: "user", contentType: "text", text: "stored at seq 8", ledgerSeq: 9 }, + { id: "row-c", role: "user", contentType: "text", text: "stored at seq 1", ledgerSeq: 2 }, + ], + { existing: { byRaw: {}, byRef: {} }, nextIndex: 1 }, + ); + assert.equal(cold.map.byRaw["row-a"], "m00005", "out-of-order stamp claims its own number"); + assert.equal(cold.map.byRaw["row-b"], "m00009"); + assert.equal(cold.map.byRaw["row-c"], "m00002", "lower stamp below the cursor still claims its slot"); + assert.ok(cold.nextIndex >= 10, "cursor advanced past the highest stamp"); +});