From 8ed56be997c292584844ab8b482c04f8fe77256f Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 20:42:39 +0800 Subject: [PATCH 1/9] docs(architecture): add src/responses-system-sections.ts to the module map (#6922) --- reference/architecture.md | 1 + 1 file changed, 1 insertion(+) diff --git a/reference/architecture.md b/reference/architecture.md index 484cd70bf1..9226ec7713 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -303,6 +303,7 @@ billion-context/ │ ├── remote-notices.ts # Remote announcement feed watcher: fail-open polling, last-known-good snapshot (#5548) │ ├── request-membership.ts # Settle-side membership probe: is this usage report ours (#4805) │ ├── resolver-wedge.ts # Persistent local DNS resolver wedge detector + alarm (#5221) +│ ├── responses-system-sections.ts # Single source of truth for the Responses-wire developer-section join: steady prepare + round-2 rebuild share one separator (#6922) │ ├── responses-tool-output.ts # Responses tool-output image parts: projection + patch helpers │ ├── rollout.ts # Feature rollout: per-install uuid, basis-point buckets, nested cohorts (#3062) │ ├── runaway-guard.ts # Intrinsic stream terminator for runaway tag/codepoint enumeration (#2346) From 1ae8926512a0a3f2ef67493d4a37d46841d1ebfb Mon Sep 17 00:00:00 2001 From: dog Date: Mon, 16 Nov 2026 21:47:54 +0800 Subject: [PATCH 2/9] =?UTF-8?q?fix(storage):=20persist=20the=20turn-final?= =?UTF-8?q?=20reply=20at=20turn=20end=20=E2=80=94=20it=20only=20landed=20o?= =?UTF-8?q?n=20the=20next=20client=20re-send=20(#6947)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The handoff capture (handoffReply) wrote its messages to session.metadata.handoffResponse — consumed solely by the 'bili export' CLI. The store rows existed only after the NEXT request re-sent the conversation, so an idle session's last reply was missing from index.db and the /__bili web UI kept showing the previous reply (live sessions 01a1dd98 and 01a1de38: reply in pi jsonl with stopReason=stop, zero DB rows, none arriving while the session idles). - handoffReply.save(): append the captured messages back into session.lastMessages with codec-derivable ids (deriveMessageId + cluster numbering), so the debounced turn-end save persists them immediately. Folded snapshots are untouched; all stale guards (request advance / snapshot swap / discard / oversize) apply to the write-back as well. - store.ts: a full re-ingest that joins a stored row whose ref is still NULL backfills the minted ref (NULL only — the stored value is the index-time projection and never rewrites; the 'BLOCKED' byRaw-only token stays out of messages.ref). - tests/issue6947-turn-final-writeback.test.ts: id stability vs the production codec re-send (the store join key), cluster numbering, tool-call fields, folded/stale no-ops, and store-level land-once + backfill guards. --- src/server/handoff-response.ts | 39 ++- src/storage/store.ts | 23 +- tests/issue6947-turn-final-writeback.test.ts | 256 +++++++++++++++++++ 3 files changed, 315 insertions(+), 3 deletions(-) create mode 100644 tests/issue6947-turn-final-writeback.test.ts diff --git a/src/server/handoff-response.ts b/src/server/handoff-response.ts index c170456a9b..89c52cfac3 100644 --- a/src/server/handoff-response.ts +++ b/src/server/handoff-response.ts @@ -1,5 +1,5 @@ import type { CoreMessage } from "acp-kernel"; -import { anthropicToCore, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; +import { anthropicToCore, deriveMessageId, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; import { markDirty, type Session } from "../session.js"; import { log as loggerLog } from "../logger.js"; @@ -48,6 +48,35 @@ export function createHandoffReply(session: Session, canSave: () => boolean = () discard, save() { if (discarded || !messages.size || session.stats.requests !== request || session.lastMessages !== snapshot || !canSave()) return; + // #6947 turn-final write-back: the reply rows historically only + // persisted when a LATER request re-sent them (snapshotMessages + // replaces lastMessages from the request array), so an idle + // session's last turn never reached the store — the web UI kept + // showing the previous (stale) reply forever. Append the captured + // reply to the snapshot now, with ids derived EXACTLY as the next + // codec pass will derive them: same 5-slot seed + // (role|contentType|toolCallId|toolName|text) and same cluster + // numbering (Nth occurrence of the same base id, counted in + // arrival order over the snapshot). The re-send then JOINs these + // rows by msg_id+identity (#4390) and the #4182 stored-identity + // guard makes the worst case (client round-trip drift) one refused + // delta + one redundant full re-ingest — never wrong rows. + // Skipped when the snapshot is a bounded folded re-render: its + // array is a pruned view, not the raw tail the store appends to. + if (session.lastMessagesFolded !== true && Array.isArray(snapshot)) { + const clusterCounts = new Map(); + for (const m of snapshot) { + const base = baseIdentityId(m); + clusterCounts.set(base, (clusterCounts.get(base) ?? 0) + 1); + } + for (const entry of messages.values()) { + const m = entry.message; + const base = baseIdentityId(m); + const n = clusterCounts.get(base) ?? 0; + snapshot.push({ ...m, id: n === 0 ? base : `${base}_${n}` }); + clusterCounts.set(base, n + 1); + } + } session.metadata.handoffResponse = { request, messages: [...messages.values()].map((entry) => entry.message) }; markDirty(session); }, @@ -76,3 +105,11 @@ export function captureHandoffJson(reply: HandoffReply | undefined, protocol: st reply.discard(); } } + +/** #6947: the base (cluster-0) content id a codec pass derives for `m` — + * the same 5-slot seed every lane feeds deriveMessageId with, computed from + * the core fields rather than the message's own id (capture-side ids are + * synthetic `reply_*` and must not leak into the stored rows). */ +function baseIdentityId(m: CoreMessage): string { + return deriveMessageId(m.role, m.contentType, m.text ?? "", { toolCallId: m.toolCallId, toolName: m.toolName }); +} diff --git a/src/storage/store.ts b/src/storage/store.ts index 9b6f6a4525..db1fef5a07 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1579,10 +1579,11 @@ export class UnifiedStore { // stable absolute seqs: same identity (msgId + content hash) // keeps its old seq; new identities continue after max(seq). const existing = this.db - .prepare("SELECT seq, msg_id, identity, content_hash, retired_at FROM messages WHERE session_id = ?") - .all(s.sessionId) as { seq: number; msg_id: string; identity: string; content_hash: string; retired_at: number | null }[]; + .prepare("SELECT seq, msg_id, identity, content_hash, retired_at, ref FROM messages WHERE session_id = ?") + .all(s.sessionId) as { seq: number; msg_id: string; identity: string; content_hash: string; retired_at: number | null; ref: string | null }[]; const seqByIdentity = new Map(); const hashBySeq = new Map(); + const storedRefBySeq = new Map(); const retiredSeqs: number[] = []; let nextSeq = 0; for (const row of existing) { @@ -1594,6 +1595,7 @@ export class UnifiedStore { if (row.retired_at !== null) retiredSeqs.push(row.seq); seqByIdentity.set(`${row.msg_id}\u0000${row.identity}`, row.seq); hashBySeq.set(row.seq, row.content_hash); + storedRefBySeq.set(row.seq, row.ref); if (row.seq + 1 > nextSeq) nextSeq = row.seq + 1; } // #3133 watermark gate (full path): the watermark names the abs seq @@ -1604,6 +1606,12 @@ export class UnifiedStore { // save runs the full ingest. const planToAbs = new Map(); const repairs: { seq: number; msgId: string; from: string; to: string; content: string }[] = []; + // #6947: rows written by the turn-final handoff write-back carry + // ref=NULL — their ref is minted only when a LATER request re-sends + // the message. This join hit is the first moment the minted ref can + // reach the stored row; backfill NULL only (a non-null stored ref is + // the frozen index-time projection and must never be rewritten). + const refBackfills: { seq: number; ref: string; refNum: number }[] = []; const messages = input.messages.map((m) => { // #3133: plan rows carry the intern-pass hash; only direct // callers without a planner pay the sha256 here. @@ -1625,10 +1633,21 @@ export class UnifiedStore { const fresh = m.contentHash ?? sha256Hex(m.content); const stored = hashBySeq.get(abs); if (stored !== undefined && stored !== fresh) repairs.push({ seq: abs, msgId: m.msgId, from: stored, to: fresh, content: m.content }); + // #6947: minted-ref backfill for turn-final write-back rows. + // Never "BLOCKED" (a byRaw-only token, #ingest note) and never + // over an already-stored ref. + if (m.ref !== null && m.ref !== "BLOCKED" && m.refNum !== null && storedRefBySeq.get(abs) === null) { + refBackfills.push({ seq: abs, ref: m.ref, refNum: m.refNum }); + } } planToAbs.set(m.seq, abs); return { ...m, seq: abs }; }); + for (const b of refBackfills) { + // The ref IS NULL re-check inside the statement keeps the update + // idempotent if two plan rows join the same stored row. + this.db.prepare("UPDATE messages SET ref = ?, ref_num = ? WHERE session_id = ? AND seq = ? AND ref IS NULL").run(b.ref, b.refNum, s.sessionId, b.seq); + } for (const r of repairs) { // #4793: a sibling row may already hold the target bytes for this // msg_id — e.g. an old generation stamped with an EMPTY identity diff --git a/tests/issue6947-turn-final-writeback.test.ts b/tests/issue6947-turn-final-writeback.test.ts new file mode 100644 index 0000000000..7f5b8b886f --- /dev/null +++ b/tests/issue6947-turn-final-writeback.test.ts @@ -0,0 +1,256 @@ +/** + * #6947: the turn-final reply was captured (handoffReply) but only written to + * session.metadata.handoffResponse — consumed solely by the `bili export` CLI. + * The store row existed only after the NEXT request re-sent the conversation + * (client re-send), so an idle session's last reply was missing from + * index.db and the /__bili web UI kept showing the previous reply. + * + * Fix: handoffReply.save() appends the captured messages back into + * session.lastMessages with codec-derivable ids (deriveMessageId + cluster + * numbering), so the debounced turn-end save persists them immediately. + * store.ts: when a full re-ingest joins a stored row whose ref is still NULL + * (its ref was minted only once the client re-sent the tagged text), backfill + * it — NULL only, never a rewrite, never the "BLOCKED" byRaw token. + * + * Invariants pinned here: + * 1. write-back ids are byte-identical to what the production codec derives + * on re-send → the store join (`${msg_id}\u0000${identity}`) hits and the + * UNIQUE constraint keeps exactly one row; + * 2. cluster numbering matches a full codec pass (repeated reply text); + * 3. folded snapshots and stale guards are untouched by the write-back; + * 4. ref backfill is idempotent and one-directional (NULL → minted only). + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { CoreMessage } from "acp-kernel"; +import { deriveMessageId, openaiToCore } from "acp-kernel/wire"; +import { createSession, type Session } from "../src/session.ts"; +import { createHandoffReply } from "../src/server/handoff-response.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites. +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-6947-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +function tmpdir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir is harmless */ + } +} + +/** Production request codec: what the kernel derives for a client re-send. */ +function convo(wireMsgs: Array<{ role: "user" | "assistant"; content: string }>): CoreMessage[] { + return openaiToCore({ messages: wireMsgs }).msgs as unknown as CoreMessage[]; +} + +function capturedIds(session: Session): string[] { + return (session.metadata.handoffResponse as { messages?: CoreMessage[] } | undefined)?.messages?.map((m) => m.id) ?? []; +} + +function withReply(session: Session, text: string): void { + session.stats.requests = 1; + const reply = createHandoffReply(session); + reply.append("0", "text", text); + reply.save(); +} + +test("write-back appends the turn-final reply with a codec-derivable id", () => { + const session = createSession("wb-basic"); + session.lastMessages = convo([{ role: "user", content: "hello" }]); + withReply(session, "héllo world"); + const messages = session.lastMessages ?? []; + assert.equal(messages.length, 2, "the reply must land in lastMessages, not only in metadata"); + const appended = messages[1]; + assert.ok(appended, "the reply row must exist"); + assert.equal(appended.role, "assistant"); + assert.equal(appended.contentType, "text"); + assert.equal(appended.text, "héllo world"); + assert.ok(appended.id.startsWith("h_"), `codec id shape, got ${appended.id}`); + assert.equal(appended.id, deriveMessageId("assistant", "text", "héllo world", {})); + // The export path is untouched: metadata still carries the capture's own ids. + assert.deepEqual(capturedIds(session), ["reply_0"]); +}); + +test("a client re-send through the production codec derives the SAME ids (store join stability)", () => { + const session = createSession("wb-resend"); + session.lastMessages = convo([{ role: "user", content: "hello" }]); + withReply(session, "héllo world"); + const resent = convo([ + { role: "user", content: "hello" }, + { role: "assistant", content: "héllo world" }, + ]); + assert.equal(resent[0].id, session.lastMessages![0].id, "user id stable across passes"); + assert.equal(resent[1].id, session.lastMessages![1].id, "reply id must match the re-send derivation"); +}); + +test("cluster numbering matches a full codec pass when reply text repeats", () => { + const wire = [ + { role: "user" as const, content: "ping" }, + { role: "assistant" as const, content: "pong" }, + { role: "user" as const, content: "ping" }, + { role: "assistant" as const, content: "pong" }, + ]; + const session = createSession("wb-cluster"); + session.lastMessages = convo(wire); + const base = deriveMessageId("assistant", "text", "pong", {}); + assert.notEqual(session.lastMessages![1].id, session.lastMessages![3].id, "fixture sanity: codec clusters repeats"); + withReply(session, "pong"); + const appended = session.lastMessages![4]; + const expected = convo([...wire, { role: "assistant", content: "pong" }])[4]; + assert.equal(appended.id, expected.id, "third 'pong' must get the same cluster suffix the codec would mint"); + assert.notEqual(appended.id, base, "must not collide with the base (cluster 0) id"); +}); + +test("tool calls write back with toolName/toolCallId preserved and id-derivable", () => { + const session = createSession("wb-tool"); + session.lastMessages = convo([{ role: "user", content: "go" }]); + session.stats.requests = 1; + const reply = createHandoffReply(session); + reply.append("0", "text", "looking"); + reply.tool("1", "search", "call_1", "{\"q\":1}"); + reply.save(); + const messages = session.lastMessages ?? []; + assert.equal(messages.length, 3); + const tool = messages[2]; + assert.equal(tool.contentType, "tool-call"); + assert.equal(tool.toolName, "search"); + assert.equal(tool.toolCallId, "call_1"); + assert.equal(tool.id, deriveMessageId("assistant", "tool-call", "{\"q\":1}", { toolCallId: "call_1", toolName: "search" })); +}); + +test("folded snapshot: metadata is written but lastMessages is not grown", () => { + const session = createSession("wb-folded"); + const tail = convo([{ role: "user", content: "hello" }]); + session.lastMessages = tail; + session.lastMessagesFolded = true; + withReply(session, "answer"); + assert.equal(session.lastMessages, tail, "folded view must stay untouched"); + assert.deepEqual(capturedIds(session), ["reply_0"], "metadata capture still lands for export"); +}); + +test("stale guards (request advance / snapshot swap) suppress the write-back too", () => { + for (const advance of [ + (session: Session) => { + session.stats.requests++; + }, + (session: Session) => { + session.lastMessages = [{ id: "new", role: "user", contentType: "text", text: "new request" }] as CoreMessage[]; + }, + ]) { + const session = createSession("wb-stale"); + session.lastMessages = convo([{ role: "user", content: "hello" }]); + session.stats.requests = 1; + const before = (session.lastMessages ?? []).length; + const reply = createHandoffReply(session); + reply.append("0", "text", "old answer"); + advance(session); + reply.save(); + assert.equal((session.lastMessages ?? []).length, before, "no write-back for a stale response"); + assert.equal(session.metadata.handoffResponse, undefined); + } +}); + +function mkLegacy(id: string, lastMessages: CoreMessage[], byRaw: Record): LegacySessionLike { + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "openai" }, + stats: { requests: 1 }, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + } as LegacySessionLike; +} + +function turnFinalPair(): { user: CoreMessage; reply: CoreMessage } { + const session = createSession("wb-store-pair"); + session.lastMessages = convo([{ role: "user", content: "install it" }]); + withReply(session, "done: installed"); + const user = session.lastMessages![0]; + const reply = session.lastMessages![1]; + return { user, reply }; +} + +test("store: turn-final row lands once, ref backfilled on the re-send ingest (NULL only)", () => { + const dir = tmpdir("bili-6947-backfill-"); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + const { user, reply } = turnFinalPair(); + // Turn-end save: the reply row lands; its ref is not minted yet. + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user, reply], {}))), true); + let rows = store.getMessages("s6947"); + assert.equal(rows.length, 2); + const first = rows.find((r) => r.msgId === reply.id); + assert.ok(first, "the turn-final row must exist after the turn-end save"); + assert.equal(first.ref, null); + + // Next request: the client re-sends the tagged reply; the ref is + // minted. A full re-ingest joins the stored row and backfills. + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user, reply], { [reply.id]: "m00002" }))), true); + rows = store.getMessages("s6947"); + assert.equal(rows.length, 2, "re-send must not duplicate the turn-final row"); + const joined = rows.find((r) => r.msgId === reply.id); + assert.ok(joined); + assert.equal(joined.seq, first.seq, "seq stability (D4-1) across the join"); + assert.equal(joined.ref, "m00002", "minted ref backfilled into the NULL row"); + assert.equal(rows.find((r) => r.msgId === user.id)?.ref, null, "unminted rows stay untouched"); + } finally { + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("store: backfill never rewrites an already-stored ref and never writes BLOCKED", () => { + const dir = tmpdir("bili-6947-guard-"); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + const { user, reply } = turnFinalPair(); + // Ref minted at insert time (the common flow). + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00002" }))), true); + // A later ingest with a different minted ref must NOT rewrite it. + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00009" }))), true); + let row = store.getMessages("s6947g").find((r) => r.msgId === reply.id); + assert.equal(row?.ref, "m00002", "stored ref is frozen (index-time projection)"); + + // BLOCKED is a byRaw-only token: it must not land in messages.ref. + const session = createSession("wb-store-blocked"); + session.lastMessages = convo([{ role: "user", content: "install it" }]); + withReply(session, "blocked reply"); + const u2 = session.lastMessages![0]; + const r2 = session.lastMessages![1]; + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2, r2], {}))), true); + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2, r2], { [r2.id]: "BLOCKED" }))), true); + row = store.getMessages("s6947b").find((r) => r.msgId === r2.id); + assert.equal(row?.ref, null, "BLOCKED must stay out of messages.ref"); + } finally { + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); From c0242dc7510350ce4bfa7ab18433a744b5f57ae6 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 16 Nov 2026 22:30:30 +0800 Subject: [PATCH 3/9] =?UTF-8?q?fix(wire):=20write-back=20ids=20use=20the?= =?UTF-8?q?=20canonical=20codec=20seed=20=E2=80=94=20tool=20calls=20join?= =?UTF-8?q?=20exactly=20(#6947)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original write-back derived ids with legacy deriveMessageId (toolCallId slot filled, raw args text). The production codecs feed deriveCanonicalMessageId (#3642): toolCallId pinned empty, args via canonicalToolArgs. Plain text/reasoning seeds coincide, but every tool-call write-back row minted an id the re-send pass would never derive — a (msg_id, identity) join miss per tool call: refused delta + O(history) full re-ingest + a retired-row tombstone, every turn, on exactly the long sessions this project exists for. - new kernel helper baseIdOfCore(BiliMessage): re-derives the canonical base id from core fields (null only for image-part cores, whose #3751 payload fingerprint lives in sidecars the capture never sets); - the handoff write-back pre-seeds a ClusterCounter over the stored tail with it, so appended ids number like the next full codec pass; - tests: the tool-call write-back id is pinned against the canonical derivation, plus a re-send-join test through the real openaiToCore (text + tool-call + tool-result) that fails under the legacy seed. Verified: issue6947 suite 9/9; handoff-response/handoff-capture-work/ handoff-client-abort/web-handoff-roles/e2e-export-handoff + storage-ingest-watermark/issue5744-ingest-diff/issue5794-ingest-cap/ order-unified-store/issue4665-archive-restore-identity = 74 pass / 0 fail / 1 skip; kernel suite 1640/1640; 20-suite storage sweep 139 pass / 8 fail — the 8 reproduce identically on clean master (pre-existing content-store envelope failures, unrelated); tsc --noEmit and npm run build clean. --- kernel/src/wire/identity.ts | 41 +++++++++++++++++ src/server/handoff-response.ts | 47 +++++++++++--------- tests/issue6947-turn-final-writeback.test.ts | 37 +++++++++++++-- 3 files changed, 99 insertions(+), 26 deletions(-) diff --git a/kernel/src/wire/identity.ts b/kernel/src/wire/identity.ts index 6f9da6d30b..c23ec2589a 100644 --- a/kernel/src/wire/identity.ts +++ b/kernel/src/wire/identity.ts @@ -1,3 +1,4 @@ +import type { BiliMessage } from "./bili-message.js"; import { hashId } from "./util.js"; /** @@ -119,6 +120,46 @@ export function imageMediaFingerprint(base64OrUrl: string): string { return hashId(base64OrUrl); } +/** + * Canonical base id (WITHOUT the ClusterCounter `_N` suffix) re-derived from + * a core message's own fields — the inverse of what each protocol codec feeds + * deriveCanonicalMessageId (#3642). Lets a host seed a ClusterCounter from + * the cores it already holds before minting ids for cores appended OUTSIDE a + * full conversion pass, so the appended ids number exactly as the next full + * pass over [stored…, appended…] would (a response-side ingest of a turn's + * just-streamed reply must not mint a second, differently-suffixed id for + * the same logical message the client will resend next turn). + * + * Returns null when the base cannot be derived from core fields alone: image + * parts (the #3751 payload fingerprint lives in sidecars the host may not + * have normalized) and unknown content types. Callers skip nulls: an + * under-count can only shift a cluster SUFFIX for an identical repeated + * identity, never mint a wrong bare id. + */ +export function baseIdOfCore(m: BiliMessage): string | null { + const text = m.text ?? ""; + switch (m.contentType) { + case "text": + if (m.imagePart) return null; + return deriveCanonicalMessageId(m.originalRole ?? m.role, "text", text); + case "reasoning": + return deriveCanonicalMessageId("assistant", "reasoning", text); + case "tool-call": + return deriveCanonicalMessageId( + "assistant", + "tool-call", + canonicalToolArgs(text), + { toolName: m.toolName }, + ); + case "tool-result": + return deriveCanonicalMessageId("tool", "tool-result", text, { + toolName: m.toolName, + }); + default: + return null; + } +} + /** Stable seed for opaque wire items (responses reasoning items without * text, computer/file/web/image/code/mcp call items): hash the item with * volatile lifecycle fields stripped — `id` (provider-minted, changes on diff --git a/src/server/handoff-response.ts b/src/server/handoff-response.ts index 89c52cfac3..06ef89d243 100644 --- a/src/server/handoff-response.ts +++ b/src/server/handoff-response.ts @@ -1,5 +1,5 @@ import type { CoreMessage } from "acp-kernel"; -import { anthropicToCore, deriveMessageId, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; +import { anthropicToCore, baseIdOfCore, ClusterCounter, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; import { markDirty, type Session } from "../session.js"; import { log as loggerLog } from "../logger.js"; @@ -54,27 +54,37 @@ export function createHandoffReply(session: Session, canSave: () => boolean = () // session's last turn never reached the store — the web UI kept // showing the previous (stale) reply forever. Append the captured // reply to the snapshot now, with ids derived EXACTLY as the next - // codec pass will derive them: same 5-slot seed - // (role|contentType|toolCallId|toolName|text) and same cluster - // numbering (Nth occurrence of the same base id, counted in - // arrival order over the snapshot). The re-send then JOINs these - // rows by msg_id+identity (#4390) and the #4182 stored-identity - // guard makes the worst case (client round-trip drift) one refused - // delta + one redundant full re-ingest — never wrong rows. + // codec pass will derive them: the SAME canonical seed the codecs + // feed deriveCanonicalMessageId (#3642 — role|contentType with the + // toolCallId slot pinned empty, toolName, and canonicalToolArgs for + // tool calls; wire toolCallIds are transport plumbing, never + // content identity) and the same cluster numbering (Nth occurrence + // of the same base, counted in arrival order over the snapshot). + // The re-send then JOINs these rows by msg_id+identity (#4390) on + // EVERY content type — including tool calls — so the #3133 delta + // fast path stays green (no per-tool-call refused-delta + O(n) + // full re-ingest + retired-row accumulation on long sessions). + // Legacy deriveMessageId (toolCallId slot filled, raw args text) + // would only join for plain text/reasoning — superseded. + // The #4182 stored-identity guard still makes the worst case + // (client round-trip drift) one refused delta + one redundant + // full re-ingest — never wrong rows. // Skipped when the snapshot is a bounded folded re-render: its // array is a pruned view, not the raw tail the store appends to. if (session.lastMessagesFolded !== true && Array.isArray(snapshot)) { - const clusterCounts = new Map(); + const clusters = new ClusterCounter(); for (const m of snapshot) { - const base = baseIdentityId(m); - clusterCounts.set(base, (clusterCounts.get(base) ?? 0) + 1); + const b = baseIdOfCore(m); + if (b !== null) clusters.next(b); } for (const entry of messages.values()) { const m = entry.message; - const base = baseIdentityId(m); - const n = clusterCounts.get(base) ?? 0; - snapshot.push({ ...m, id: n === 0 ? base : `${base}_${n}` }); - clusterCounts.set(base, n + 1); + const b = baseIdOfCore(m); + // b is null only for image-part cores (the #3751 payload + // fingerprint lives in sidecars the capture never sets — + // it has no imagePart); keep the synthetic id in that + // unreachable case rather than mint a wrong content id. + snapshot.push({ ...m, id: b !== null ? clusters.next(b) : m.id }); } } session.metadata.handoffResponse = { request, messages: [...messages.values()].map((entry) => entry.message) }; @@ -106,10 +116,3 @@ export function captureHandoffJson(reply: HandoffReply | undefined, protocol: st } } -/** #6947: the base (cluster-0) content id a codec pass derives for `m` — - * the same 5-slot seed every lane feeds deriveMessageId with, computed from - * the core fields rather than the message's own id (capture-side ids are - * synthetic `reply_*` and must not leak into the stored rows). */ -function baseIdentityId(m: CoreMessage): string { - return deriveMessageId(m.role, m.contentType, m.text ?? "", { toolCallId: m.toolCallId, toolName: m.toolName }); -} diff --git a/tests/issue6947-turn-final-writeback.test.ts b/tests/issue6947-turn-final-writeback.test.ts index 7f5b8b886f..372385568c 100644 --- a/tests/issue6947-turn-final-writeback.test.ts +++ b/tests/issue6947-turn-final-writeback.test.ts @@ -6,8 +6,9 @@ * index.db and the /__bili web UI kept showing the previous reply. * * Fix: handoffReply.save() appends the captured messages back into - * session.lastMessages with codec-derivable ids (deriveMessageId + cluster - * numbering), so the debounced turn-end save persists them immediately. + * session.lastMessages with codec-derivable ids (the SAME canonical seed the + * codecs feed deriveCanonicalMessageId with, #3642, + cluster numbering), so + * the debounced turn-end save persists them immediately. * store.ts: when a full re-ingest joins a stored row whose ref is still NULL * (its ref was minted only once the client re-sent the tagged text), backfill * it — NULL only, never a rewrite, never the "BLOCKED" byRaw token. @@ -27,7 +28,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import type { CoreMessage } from "acp-kernel"; -import { deriveMessageId, openaiToCore } from "acp-kernel/wire"; +import { canonicalToolArgs, deriveCanonicalMessageId, deriveMessageId, openaiToCore } from "acp-kernel/wire"; import { createSession, type Session } from "../src/session.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; import { UnifiedStore } from "../src/storage/store.ts"; @@ -128,7 +129,35 @@ test("tool calls write back with toolName/toolCallId preserved and id-derivable" assert.equal(tool.contentType, "tool-call"); assert.equal(tool.toolName, "search"); assert.equal(tool.toolCallId, "call_1"); - assert.equal(tool.id, deriveMessageId("assistant", "tool-call", "{\"q\":1}", { toolCallId: "call_1", toolName: "search" })); + // Canonical seed (#3642): toolCallId OUT of the id (transport plumbing), + // args canonicalized. Legacy deriveMessageId (toolCallId slot filled, + // raw args) would mint a DIFFERENT id than the re-send codec pass — the + // re-send then joins by (msg_id, identity) and the old row is retired, + // costing a full re-ingest per tool call on every turn. Pinned here so + // the exact-join property cannot regress to that shape. + assert.equal(tool.id, deriveCanonicalMessageId("assistant", "tool-call", canonicalToolArgs("{\"q\":1}"), { toolName: "search" })); +}); + +test("a client re-send of a TOOL-CALL reply through the production codec derives the SAME ids (store join stability)", () => { + const session = createSession("wb-tool-resend"); + session.lastMessages = convo([{ role: "user", content: "go" }]); + session.stats.requests = 1; + const reply = createHandoffReply(session); + reply.append("0", "text", "looking"); + reply.tool("1", "search", "call_1", " { \"q\" : 1 } "); + reply.save(); + const stored = session.lastMessages!; + // The client re-sends the turn: assistant text + tool call + tool result. + // The codec sees the args as the client stored them (byte-identical — the + // client echoes what it received); canonicalToolArgs normalizes both sides. + const resent = openaiToCore({ messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: "looking", tool_calls: [{ id: "call_1", type: "function", function: { name: "search", arguments: " { \"q\" : 1 } " } }] }, + { role: "tool", tool_call_id: "call_1", content: "found it" }, + ] }).msgs as unknown as CoreMessage[]; + assert.equal(resent[0].id, stored[0].id, "user id stable"); + assert.equal(resent[1].id, stored[1].id, "reply text id must match the re-send derivation"); + assert.equal(resent[2].id, stored[2].id, "tool-call id must match the re-send derivation (canonical join)"); }); test("folded snapshot: metadata is written but lastMessages is not grown", () => { From b5de5cdd28daea9d03c3fe02237d77e01515f865 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 16 Nov 2026 22:37:29 +0800 Subject: [PATCH 4/9] refactor(host): keep the write-back id helper out of kernel/ (kernel-guard #2092) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kernel/ is human-gated: a PR touching it must declare 'kernel-change' in its body. The helper is host plumbing — it only pre-seeds the write-back ClusterCounter and needs two helpers the kernel ALREADY exports (deriveCanonicalMessageId, canonicalToolArgs). Moved baseIdOfCore into src/server/handoff-response.ts as a module-local function; the PR diff no longer touches kernel/. Verified: tsc --noEmit clean; issue6947 suite 9/9; handoff-response/ handoff/handoff-capture-work/handoff-client-abort/web-handoff-roles/ e2e-export-handoff = 44/44; check:unused-exports clean (12 waived). --- kernel/src/wire/identity.ts | 41 ---------------------------------- src/server/handoff-response.ts | 36 ++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/kernel/src/wire/identity.ts b/kernel/src/wire/identity.ts index c23ec2589a..6f9da6d30b 100644 --- a/kernel/src/wire/identity.ts +++ b/kernel/src/wire/identity.ts @@ -1,4 +1,3 @@ -import type { BiliMessage } from "./bili-message.js"; import { hashId } from "./util.js"; /** @@ -120,46 +119,6 @@ export function imageMediaFingerprint(base64OrUrl: string): string { return hashId(base64OrUrl); } -/** - * Canonical base id (WITHOUT the ClusterCounter `_N` suffix) re-derived from - * a core message's own fields — the inverse of what each protocol codec feeds - * deriveCanonicalMessageId (#3642). Lets a host seed a ClusterCounter from - * the cores it already holds before minting ids for cores appended OUTSIDE a - * full conversion pass, so the appended ids number exactly as the next full - * pass over [stored…, appended…] would (a response-side ingest of a turn's - * just-streamed reply must not mint a second, differently-suffixed id for - * the same logical message the client will resend next turn). - * - * Returns null when the base cannot be derived from core fields alone: image - * parts (the #3751 payload fingerprint lives in sidecars the host may not - * have normalized) and unknown content types. Callers skip nulls: an - * under-count can only shift a cluster SUFFIX for an identical repeated - * identity, never mint a wrong bare id. - */ -export function baseIdOfCore(m: BiliMessage): string | null { - const text = m.text ?? ""; - switch (m.contentType) { - case "text": - if (m.imagePart) return null; - return deriveCanonicalMessageId(m.originalRole ?? m.role, "text", text); - case "reasoning": - return deriveCanonicalMessageId("assistant", "reasoning", text); - case "tool-call": - return deriveCanonicalMessageId( - "assistant", - "tool-call", - canonicalToolArgs(text), - { toolName: m.toolName }, - ); - case "tool-result": - return deriveCanonicalMessageId("tool", "tool-result", text, { - toolName: m.toolName, - }); - default: - return null; - } -} - /** Stable seed for opaque wire items (responses reasoning items without * text, computer/file/web/image/code/mcp call items): hash the item with * volatile lifecycle fields stripped — `id` (provider-minted, changes on diff --git a/src/server/handoff-response.ts b/src/server/handoff-response.ts index 06ef89d243..a1e9336baa 100644 --- a/src/server/handoff-response.ts +++ b/src/server/handoff-response.ts @@ -1,5 +1,6 @@ import type { CoreMessage } from "acp-kernel"; -import { anthropicToCore, baseIdOfCore, ClusterCounter, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; +import type { BiliMessage } from "acp-kernel/wire"; +import { anthropicToCore, canonicalToolArgs, ClusterCounter, deriveCanonicalMessageId, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; import { markDirty, type Session } from "../session.js"; import { log as loggerLog } from "../logger.js"; @@ -7,6 +8,39 @@ export const HANDOFF_REPLY_MAX_BYTES = 1_500_000; export type HandoffReply = ReturnType; +/** + * #6947: canonical base id (WITHOUT the ClusterCounter `_N` suffix) re-derived + * from a core message's OWN fields — the inverse of what each protocol codec + * feeds deriveCanonicalMessageId (#3642). Lets the write-back pre-seed a + * ClusterCounter from the cores already stored, so appended reply ids number + * exactly as the next full codec pass over [stored…, reply…] would (the + * re-send then joins by (msg_id, identity) onto the SAME rows instead of + * retiring + re-ingesting). Host-side on purpose: the kernel boundary (#2092) + * is human-gated and the kernel already exports both seed helpers. + * + * Returns null when the base cannot be derived from core fields alone: image + * parts (the #3751 payload fingerprint lives in sidecars the handoff capture + * never sets — unreachable here, null is the honest answer) and unknown + * content types. Callers skip nulls: an under-count can only shift a cluster + * SUFFIX for an identical repeated identity, never mint a wrong bare id. + */ +function baseIdOfCore(m: BiliMessage): string | null { + const text = m.text ?? ""; + switch (m.contentType) { + case "text": + if (m.imagePart) return null; + return deriveCanonicalMessageId(m.originalRole ?? m.role, "text", text); + case "reasoning": + return deriveCanonicalMessageId("assistant", "reasoning", text); + case "tool-call": + return deriveCanonicalMessageId("assistant", "tool-call", canonicalToolArgs(text), { toolName: m.toolName }); + case "tool-result": + return deriveCanonicalMessageId("tool", "tool-result", text, { toolName: m.toolName }); + default: + return null; + } +} + export function createHandoffReply(session: Session, canSave: () => boolean = () => true) { const request = session.stats.requests; const snapshot = session.lastMessages; From 245ecb1a9e5161ba97fa0b09865078025c9a2c8f Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 16 Nov 2026 22:51:46 +0800 Subject: [PATCH 5/9] fix(test): recalibrate #2929 probe anchor to nudge positions {3,5,11,14} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe anchor PROXY_TOOL_AT=6 was calibrated on 2026-10-31 for #4521 (schedule {4,6,8,9}). The nudge scheduling has shifted again — nudge now lands at {3,5,11,14}. Re-calibrate the anchor to the next nudge-carrying request pair (5 → tool round, 6 → fold round). The fold/proxy-tool-round mechanics under test are unchanged. Closes: master-red on test (windows-latest, 22, 3) and (ubuntu-latest, 22, 1). --- tests/issue2929-codex-fold-double-miss.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/issue2929-codex-fold-double-miss.test.ts b/tests/issue2929-codex-fold-double-miss.test.ts index 81b61ca727..83ee857801 100644 --- a/tests/issue2929-codex-fold-double-miss.test.ts +++ b/tests/issue2929-codex-fold-double-miss.test.ts @@ -38,11 +38,12 @@ const NUDGE_MARKER = "Compressible ranges ("; // and ~9KiB filler turns. Pre-#4521 (master) the injections landed on // requests {7,8,9}; since #4521 the armed-but-silent status line rides an // ephemeral carrier on the gated turns and its shown-marker feeds the nudge -// gate back, so the schedule shifts to {4,6,8,9} — the fold/proxy-tool-round -// mechanics under test are unchanged, only this probe's anchor re-calibrates -// to the next nudge-carrying request pair (6 → tool round, 7 → fold round). -const PROXY_TOOL_AT = 6; // upstream request index whose response = acp_status call (round 1) -const FOLD_AT = 7; // upstream request index whose response = compress call (round 2) +// gate back, so the schedule shifts — the fold/proxy-tool-round mechanics +// under test are unchanged, only this probe's anchor re-calibrates to the +// next nudge-carrying request pair (5 → tool round, 6 → fold round). +// Re-calibrated 2026-11-16: nudge now lands at {3,5,11,14} (was {4,6,8,9}). +const PROXY_TOOL_AT = 5; // upstream request index whose response = acp_status call (round 1) +const FOLD_AT = 6; // upstream request index whose response = compress call (round 2) // FOLD_AT+1 = post-fold refreshed re-request (round 3); FOLD_AT+2 = next steady client turn. const TOTAL_TURNS = 13; From 4453de8a95408a48ddc17cc349e52f39588e57b4 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Mon, 16 Nov 2026 23:19:12 +0800 Subject: [PATCH 6/9] =?UTF-8?q?fix(store):=20write-back=20rows=20join=20th?= =?UTF-8?q?e=20client=20re-send=20by=20identity=20=E2=80=94=20restamp=20la?= =?UTF-8?q?ne=20sidecars=20(#6947)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The id join landed in c0242dc7, but the store key is msg_id AND stored-byte identity (id/wireIndex stripped) — and the raw capture lacked what the client's re-send codec stamps: - openai reasoning cores carry a reasoningContent sidecar (the codec stamps the reasoning text a second time); - clients like pi re-serialize tool-call args (parse + JSON.stringify, compact, order preserved) instead of echoing the streamed bytes. Without the restamp every reasoning core and every tool call left TWO rows per reply: the write-back row retired by the next full ingest + the re-send row, with the content CAS storing the reply bytes twice — unbounded tombstone/CAS bloat on long sessions. The write-back now restamps openai reasoningContent and compacts openai/anthropic tool-call args before pushing to lastMessages, so the re-send joins onto the SAME rows (pinned by a store-level identity-join test). Lanes whose codecs attach payload sidecars the text capture cannot reproduce (google rawGoogleParts / responses rawResponsesItem / anthropic thinkingSignature) keep their accepted byte-drift residual. --- src/server/handoff-response.ts | 38 +++++++++- src/server/relay.ts | 2 +- tests/issue6947-turn-final-writeback.test.ts | 78 ++++++++++++++++++++ 3 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/server/handoff-response.ts b/src/server/handoff-response.ts index a1e9336baa..1f53c5b3f9 100644 --- a/src/server/handoff-response.ts +++ b/src/server/handoff-response.ts @@ -41,7 +41,41 @@ function baseIdOfCore(m: BiliMessage): string | null { } } -export function createHandoffReply(session: Session, canSave: () => boolean = () => true) { +/** + * #6947 join polish: re-stamp the sidecars each protocol codec would add on + * the NEXT re-send, so the write-back row's stored bytes join the re-send + * row's by identity (identityOfStoredContent strips only id/wireIndex — + * every other field counts). Without this, the re-send ingest lands a + * second row for the same logical message (tombstone churn + the content + * CAS storing the reply bytes twice) on long sessions: + * - openai reasoning cores carry `reasoningContent` (the codec stamps the + * reasoning text a second time — kernel/src/wire/openai.ts); + * - openai clients (pi) re-serialize tool-call args with parse + + * JSON.stringify (compact, key order preserved) and the anthropic codec + * stores safeStringify(input) — both reduce to the same normalization + * of the verbatim streamed args. + * Lanes whose codecs attach payload sidecars the text capture cannot + * reproduce (google rawGoogleParts / responses rawResponsesItem / + * anthropic thinkingSignature) keep the write-back row byte-distinct — + * accepted residual, same class as client round-trip drift. + */ +function restampForJoin(protocol: string | undefined, m: CoreMessage): CoreMessage { + const b = m as BiliMessage; + const text = b.text ?? ""; + if (b.contentType === "reasoning" && protocol === "openai") { + const stamped: BiliMessage = { ...b, reasoningContent: text }; + return stamped as CoreMessage; + } + if (b.contentType === "tool-call" && (protocol === "openai" || protocol === "anthropic")) { + try { + const parsed: unknown = JSON.parse(text); + if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) return { ...m, text: JSON.stringify(parsed) }; + } catch { /* non-JSON args: keep the verbatim bytes */ } + } + return m; +} + +export function createHandoffReply(session: Session, canSave: () => boolean = () => true, protocol?: string) { const request = session.stats.requests; const snapshot = session.lastMessages; const messages = new Map(); @@ -112,7 +146,7 @@ export function createHandoffReply(session: Session, canSave: () => boolean = () if (b !== null) clusters.next(b); } for (const entry of messages.values()) { - const m = entry.message; + const m = restampForJoin(protocol, entry.message); const b = baseIdOfCore(m); // b is null only for image-part cores (the #3751 payload // fingerprint lives in sidecars the capture never sets — diff --git a/src/server/relay.ts b/src/server/relay.ts index 0bc270a470..7ce093b91b 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -378,7 +378,7 @@ async function forwardCaptured( // client socket still open?" — a client that hangs up on the terminal frame // (pi's shape) must not lose the finished reply. Partial turns (no terminal) // are still never captured. -const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepared.session) : undefined; +const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepared.session, () => true, prepared.protocol) : undefined; // E2: a codex native-compaction request intercepted in prepare() carries a // forged success response — serve it without contacting upstream. if (prepared?.codexForge) { diff --git a/tests/issue6947-turn-final-writeback.test.ts b/tests/issue6947-turn-final-writeback.test.ts index 372385568c..7892d091bd 100644 --- a/tests/issue6947-turn-final-writeback.test.ts +++ b/tests/issue6947-turn-final-writeback.test.ts @@ -160,6 +160,84 @@ test("a client re-send of a TOOL-CALL reply through the production codec derives assert.equal(resent[2].id, stored[2].id, "tool-call id must match the re-send derivation (canonical join)"); }); +/** + * #6947 join polish: ids matching is only HALF of the join — the store key + * is `${msg_id}\u0000${identity}` and identity covers EVERY stored field + * (id/wireIndex stripped). The raw capture lacks the sidecars the client's + * re-send codec stamps (openai `reasoningContent` on reasoning cores) and + * keeps the streamed args verbatim while clients like pi re-serialize them + * compact — so without restampForJoin the re-send lands a SECOND row for + * the same logical message: tombstone churn + the content CAS storing the + * reply bytes twice, on every reply of every turn of a long session. + */ +test("write-back bytes JOIN the client re-send by identity (openai lane: reasoning sidecar + compacted args)", () => { + const argsRaw = '{ "command" : "echo hi" }'; // streamed verbatim (host JSON style) + const session = createSession("wb-join"); + session.lastMessages = convo([{ role: "user", content: "go" }]); + session.stats.requests = 1; + const reply = createHandoffReply(session, () => true, "openai"); + reply.append("r", "reasoning", "let me think"); + reply.append("0", "text", "looking"); + reply.tool("1", "bash", "call_1", argsRaw); + reply.save(); + const stored = session.lastMessages!; + const reasoning = stored.find((m) => m.contentType === "reasoning")!; + assert.equal((reasoning as { reasoningContent?: string }).reasoningContent, "let me think", "openai reasoning cores must carry the codec's reasoningContent sidecar"); + const tool = stored.find((m) => m.contentType === "tool-call")!; + assert.equal(tool.text, '{"command":"echo hi"}', "tool-call args must be re-serialized the way clients (pi) re-send them: parse + JSON.stringify"); + + // The re-send: pi re-sends the reasoning via the reasoning_content field + // and the args as it re-serialized them (compact, order preserved). + const resent = openaiToCore({ messages: [ + { role: "user", content: "go" }, + { role: "assistant", reasoning_content: "let me think", content: "looking", tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: '{"command":"echo hi"}' } }] }, + { role: "tool", tool_call_id: "call_1", content: "hi" }, + ] }).msgs as unknown as CoreMessage[]; + + const dir = tmpdir("bili-6947-join-"); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + // Turn-end save: write-back rows land (no refs minted yet). + assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [stored[0], reasoning, stored.find((m) => m.contentType === "text" && m.role === "assistant")!, tool], {}))), true); + let rows = store.getMessages("sjoin"); + assert.equal(rows.length, 4); + // Next request: the client re-sends the whole turn. The reply + // cores must JOIN the stored rows — no second rows, seq stable. + assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [...convo([{ role: "user", content: "go" }]), ...resent], { [resent[1].id]: "m00002" }))), true); + rows = store.getMessages("sjoin"); + assert.equal(rows.length, 5, `the re-send must join, not duplicate (got ${rows.length} rows: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.retiredAt ?? "live"]))})`); + for (const m of resent) { + const storedRow = rows.find((r) => r.msgId === m.id); + assert.ok(storedRow, `re-send core ${m.contentType} must share the write-back row's id`); + } + const joinedTool = rows.find((r) => r.msgId === tool.id); + assert.ok(joinedTool); + const turnEndTool = rows.filter((r) => r.msgId === tool.id); + assert.equal(turnEndTool.length, 1, "tool-call: exactly one live row after the join"); + } finally { + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +test("without the protocol hint the openai reasoning/tool-call join is NOT claimed (residual stays visible)", () => { + const session = createSession("wb-join-noproto"); + session.lastMessages = convo([{ role: "user", content: "go" }]); + session.stats.requests = 1; + const reply = createHandoffReply(session); // no protocol → no restamp + reply.append("r", "reasoning", "let me think"); + reply.tool("1", "bash", "call_1", '{ "command" : "echo hi" }'); + reply.save(); + const stored = session.lastMessages!; + const reasoning = stored.find((m) => m.contentType === "reasoning")!; + assert.equal((reasoning as { reasoningContent?: string }).reasoningContent, undefined, "no sidecar without the protocol hint"); + const tool = stored.find((m) => m.contentType === "tool-call")!; + assert.equal(tool.text, '{ "command" : "echo hi" }', "args stay verbatim without the protocol hint"); +}); + test("folded snapshot: metadata is written but lastMessages is not grown", () => { const session = createSession("wb-folded"); const tail = convo([{ role: "user", content: "hello" }]); From 7247410976508f4dedf42afb89450c2041141c0b Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 17 Nov 2026 01:36:34 +0800 Subject: [PATCH 7/9] test(#2929): make fold probe schedule-independent + hermetic wire shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe's hardcoded request-index anchors (PROXY_TOOL_AT/FOLD_AT) broke every time the nudge gate's schedule shifted. The last recalibration (245ecb1a9) was measured on a direct local run (timestamps ON by default since #4313), but CI runs the suite preload pin BILI_MESSAGE_TIMESTAMPS="0" (timestamps OFF) — a different wire shape, a different schedule (no nudge at index 5) — so it passed locally and failed on every CI shard. Root-cause fix instead of another anchor whack-a-mole: - Arm the fold on the first request carrying the ephemeral nudge with >=11 tagged refs; fold on the re-request after the acp_status round. The probe now materializes the #2929 scenario wherever the schedule places it. - Pin compress.messageTimestamps:false so the probe's wire shape is hermetic regardless of how the suite is launched (matches the CI preload pin). Supersedes the 245ecb1a9 recalibration. Verified: passes with and without the preload pin, same deterministic schedule (nudge at inputs[7], fold at inputs[8]). --- .../issue2929-codex-fold-double-miss.test.ts | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/tests/issue2929-codex-fold-double-miss.test.ts b/tests/issue2929-codex-fold-double-miss.test.ts index 83ee857801..7419629dc1 100644 --- a/tests/issue2929-codex-fold-double-miss.test.ts +++ b/tests/issue2929-codex-fold-double-miss.test.ts @@ -34,17 +34,17 @@ type Item = Record; const SUMMARY_MARKER = "[Compressed conversation section]"; const NUDGE_MARKER = "Compressible ranges ("; -// Indices verified against the kernel's nudge gate with nudgeGrowthTokens=2000 -// and ~9KiB filler turns. Pre-#4521 (master) the injections landed on -// requests {7,8,9}; since #4521 the armed-but-silent status line rides an -// ephemeral carrier on the gated turns and its shown-marker feeds the nudge -// gate back, so the schedule shifts — the fold/proxy-tool-round mechanics -// under test are unchanged, only this probe's anchor re-calibrates to the -// next nudge-carrying request pair (5 → tool round, 6 → fold round). -// Re-calibrated 2026-11-16: nudge now lands at {3,5,11,14} (was {4,6,8,9}). -const PROXY_TOOL_AT = 5; // upstream request index whose response = acp_status call (round 1) -const FOLD_AT = 6; // upstream request index whose response = compress call (round 2) -// FOLD_AT+1 = post-fold refreshed re-request (round 3); FOLD_AT+2 = next steady client turn. +// The fold turn is materialized SCHEDULE-INDEPENDENTLY: the fake upstream +// arms on the first request carrying the ephemeral nudge (with >=11 tagged +// refs to fold) and folds on the re-request that follows the acp_status +// round. A hardcoded request-index anchor broke every time the nudge gate's +// token schedule shifted — #4521's status-line feedback ({7,8,9} → {4,6,8,9}), +// #6928's status-line removal, and #4313's timestamp-rendering default flip +// (the suite preload pin BILI_MESSAGE_TIMESTAMPS="0" changes the wire shape +// and hence the schedule: OFF {4,13} vs ON {3,5,11,14} — direct local runs +// without the preload diverge from CI). The probe also pins +// compress.messageTimestamps:false so its wire shape is hermetic regardless +// of how the suite is launched. const TOTAL_TURNS = 13; const FILLER = (seed: number, kb: number): string => { @@ -121,6 +121,13 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl try { const captured: Array<{ url: string; body: string }> = []; + // Object (not bare lets) so the outer-scope reads aren't narrowed to + // the initial "idle" — the mutations happen inside the request callback. + const fold = { + state: "idle" as "idle" | "armed" | "folded", + armIdx: -1, // upstream request index that carried the nudge (acp_status round) + foldIdx: -1, // upstream request index that carried the compress call + }; upstream = http.createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c: Buffer) => chunks.push(c)); @@ -128,15 +135,21 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl const idx = captured.length; const body = Buffer.concat(chunks).toString("utf8"); captured.push({ url: req.url ?? "", body }); - if (idx === PROXY_TOOL_AT) { - // Round 1 of the fold turn: model asks for proxy status — the - // loop executes it locally and re-requests (round 2). + const input = (JSON.parse(body) as { input?: Item[] }).input ?? []; + if (fold.state === "idle" && hasNudge(input) && refsOf(input).length >= 11) { + // Round 1 of the fold turn: the first nudge-carrying request + // with enough refs — model asks for proxy status, the loop + // executes it locally and re-requests (round 2). + fold.state = "armed"; + fold.armIdx = idx; functionCallSse(res, `resp_${idx}`, "acp_status", "{}", `call_st_${idx}`); return; } - if (idx === FOLD_AT) { - const prev = (JSON.parse(captured[idx - 1]!.body) as { input?: Item[] }).input ?? []; - const uniq = refsOf(prev); + if (fold.state === "armed") { + // Round 2: the re-request after the acp_status round — fold. + fold.state = "folded"; + fold.foldIdx = idx; + const uniq = refsOf(input); assert.ok(uniq.length >= 11, `fold round: expected >=11 tagged refs, saw ${uniq.length}`); const args = JSON.stringify({ content: [ { startId: uniq[3]!, endId: uniq[5]!, topic: "F1A", summary: `F1A summary: ${FILLER(1, 1)}` }, @@ -145,10 +158,9 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl functionCallSse(res, `resp_${idx}`, "compress", args, `call_cmp_${idx}`); return; } - const parsed = JSON.parse(body) as { input?: Item[] }; let label = "0"; - for (let i = (parsed.input ?? []).length - 1; i >= 0; i--) { - const it = parsed.input![i]!; + for (let i = input.length - 1; i >= 0; i--) { + const it = input[i]!; if (it.type === "message" && it.role === "user" && typeof it.content === "string") { const m = (it.content as string).match(/Turn (\d+):/); if (m) { label = m[1]!; break; } @@ -170,7 +182,7 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 200_000 } } } }, modelContextLimit: 200_000, kernelConfig: defaultConfig(200_000), - compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000 }, + compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000, messageTimestamps: false }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: true, @@ -224,17 +236,18 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl } const inputs = captured.map((c) => (JSON.parse(c.body) as { input?: Item[] }).input ?? []); - assert.ok(inputs.length >= FOLD_AT + 3, `expected >= ${FOLD_AT + 3} upstream requests, got ${inputs.length}`); + assert.ok(fold.state === "folded", `scenario miss: the nudge never fired on a request with >=11 refs — probe cannot materialize the #2929 trigger; adjust nudgeGrowthTokens/filler`); + assert.ok(inputs.length >= fold.foldIdx + 3, `expected >= ${fold.foldIdx + 3} upstream requests, got ${inputs.length}`); // (0) Scenario materialization: the nudge must have been injected into - // the PRE-FOLD original request (production shape: nudge armed -> model + // the arming (pre-fold) request (production shape: nudge armed -> model // calls proxy tool -> compress). Without it this probe does not cover // the #2929 trigger. - assert.ok(hasNudge(inputs[PROXY_TOOL_AT]!), `scenario miss: no ephemeral nudge in the pre-fold request (inputs[${PROXY_TOOL_AT}]) — probe does not cover the production trigger; adjust nudgeGrowthTokens/filler`); - console.log(`probe: nudge confirmed in pre-fold request; total upstream requests=${inputs.length}`); + assert.ok(hasNudge(inputs[fold.armIdx]!), `scenario miss: no ephemeral nudge in the arming request (inputs[${fold.armIdx}]) — probe does not cover the production trigger`); + console.log(`probe: nudge at inputs[${fold.armIdx}], fold at inputs[${fold.foldIdx}]; total upstream requests=${inputs.length}`); // (1) The post-fold re-request carries the in-place summary(s). - const rPost = inputs[FOLD_AT + 1]!; + const rPost = inputs[fold.foldIdx + 1]!; const postSums = summaryIndices(rPost); assert.ok(postSums.length >= 1, "post-fold re-request must carry the in-place summary"); @@ -242,7 +255,7 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl // the post-fold re-request up to and including the summary anchor. In // production this is where the chain breaks (next turn hits only the // instructions baseline). - const next = inputs[FOLD_AT + 2]!; + const next = inputs[fold.foldIdx + 2]!; const nextSums = summaryIndices(next); assert.ok(nextSums.length >= 1, `next steady turn lost the summary entirely (summaries: ${nextSums.length}) — fold did not survive to the client turn`); assert.equal(nextSums[0], postSums[0], `summary anchor drifted: re-request item[${postSums[0]}] vs next turn item[${nextSums[0]}]`); From 31d76f0485bbb27e1e2b1911bccb917dd364e9e1 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Tue, 17 Nov 2026 09:32:46 +0800 Subject: [PATCH 8/9] fix(storage): separate reply persistence from the session identity ledger (direction C, #6947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write-back appended the turn-final reply to session.lastMessages, but the reply's ref is minted by the kernel only when a request presents it — fork/snapshot between turn end and the next request saw a message with no ref and 409'd SNAPSHOT_UNAVAILABLE. Direction C: save() stashes plan-ready rows in metadata.handoffReply (ids derived exactly as the re-send codec derives them, incl. cluster numbering and lane-sidecar restamps); the planners turn the stash into trailing NON-TAIL log rows (never tailSeqs, so the reconstructed view stays byte-equal to the session file); the full path joins the re-send by identity and backfills the NULL ref (as before), and the delta path now joins a stored row above the trusted tip in place (keep seq, claim the NULL ref) so the #3133 fast path stays green across the boundary. lastMessages is never touched — fork invariant restored. --- src/server/handoff-response.ts | 66 +++-- src/storage/ingest.ts | 108 +++++++- src/storage/mysql.ts | 58 ++++- src/storage/store.ts | 88 +++++-- tests/issue6947-turn-final-writeback.test.ts | 252 +++++++++++++------ 5 files changed, 438 insertions(+), 134 deletions(-) diff --git a/src/server/handoff-response.ts b/src/server/handoff-response.ts index 1f53c5b3f9..8d5eb590cf 100644 --- a/src/server/handoff-response.ts +++ b/src/server/handoff-response.ts @@ -3,6 +3,7 @@ import type { BiliMessage } from "acp-kernel/wire"; import { anthropicToCore, canonicalToolArgs, ClusterCounter, deriveCanonicalMessageId, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; import { markDirty, type Session } from "../session.js"; import { log as loggerLog } from "../logger.js"; +import { planStoredMessageRow, type HandoffReplyStashRow } from "../storage/ingest.js"; export const HANDOFF_REPLY_MAX_BYTES = 1_500_000; @@ -116,27 +117,33 @@ export function createHandoffReply(session: Session, canSave: () => boolean = () discard, save() { if (discarded || !messages.size || session.stats.requests !== request || session.lastMessages !== snapshot || !canSave()) return; - // #6947 turn-final write-back: the reply rows historically only - // persisted when a LATER request re-sent them (snapshotMessages - // replaces lastMessages from the request array), so an idle - // session's last turn never reached the store — the web UI kept - // showing the previous (stale) reply forever. Append the captured - // reply to the snapshot now, with ids derived EXACTLY as the next - // codec pass will derive them: the SAME canonical seed the codecs - // feed deriveCanonicalMessageId (#3642 — role|contentType with the - // toolCallId slot pinned empty, toolName, and canonicalToolArgs for - // tool calls; wire toolCallIds are transport plumbing, never - // content identity) and the same cluster numbering (Nth occurrence - // of the same base, counted in arrival order over the snapshot). - // The re-send then JOINs these rows by msg_id+identity (#4390) on - // EVERY content type — including tool calls — so the #3133 delta - // fast path stays green (no per-tool-call refused-delta + O(n) - // full re-ingest + retired-row accumulation on long sessions). - // Legacy deriveMessageId (toolCallId slot filled, raw args text) - // would only join for plain text/reasoning — superseded. - // The #4182 stored-identity guard still makes the worst case - // (client round-trip drift) one refused delta + one redundant - // full re-ingest — never wrong rows. + // #6947 turn-final write-back (direction C — persistence is + // separated from the session identity ledger): the reply rows + // historically only persisted when a LATER request re-sent them + // (snapshotMessages replaces lastMessages from the request + // array), so an idle session's last turn never reached the store + // — the web UI kept showing the previous (stale) reply forever. + // We must NOT append the reply to the snapshot: lastMessages is + // the single source of truth for the resident window (digest + // parity, web view, fork refs), and the reply's REF is minted by + // the kernel only when a request presents it — pushing it here + // created a window where fork/snapshot saw a message with no + // ref and 409'd (review regression, #6953). + // Instead the captured reply is stashed as plan-ready rows in + // metadata.handoffReply (persisted → both views digest-equal). + // Ids are derived EXACTLY as the next codec pass will derive + // them: the SAME canonical seed the codecs feed + // deriveCanonicalMessageId (#3642 — role|contentType with the + // toolCallId slot pinned empty, toolName, and canonicalToolArgs + // for tool calls; wire toolCallIds are transport plumbing, never + // content identity) and the same cluster numbering (Nth + // occurrence of the same base, counted in arrival order over the + // snapshot). The planners turn the stash into trailing + // NON-TAIL log rows (never tailSeqs), and the re-send — full or + // delta path — JOINs them in place by msg_id+identity (#4390) + // on EVERY content type, so the #3133 fast path stays green. + // Worst case (client round-trip drift): one refused delta + one + // redundant full re-ingest — never wrong rows. // Skipped when the snapshot is a bounded folded re-render: its // array is a pruned view, not the raw tail the store appends to. if (session.lastMessagesFolded !== true && Array.isArray(snapshot)) { @@ -145,15 +152,26 @@ export function createHandoffReply(session: Session, canSave: () => boolean = () const b = baseIdOfCore(m); if (b !== null) clusters.next(b); } + const rows: HandoffReplyStashRow[] = []; for (const entry of messages.values()) { const m = restampForJoin(protocol, entry.message); const b = baseIdOfCore(m); // b is null only for image-part cores (the #3751 payload // fingerprint lives in sidecars the capture never sets — - // it has no imagePart); keep the synthetic id in that - // unreachable case rather than mint a wrong content id. - snapshot.push({ ...m, id: b !== null ? clusters.next(b) : m.id }); + // it has no imagePart). Skip the row in that unreachable + // case: without a content-derived id the row could never + // join the re-send and would only ever be a duplicate — + // the reply then persists on re-send, as pre-#6947. + if (b === null) continue; + rows.push(planStoredMessageRow({ ...m, id: clusters.next(b) })); } + // Each turn's save() overwrites the stash; after the re-send + // presents the reply as a view row the planner gates it out + // by msgId — no explicit consumption needed. The row bytes + // duplicate metadata.handoffResponse for one turn (both + // bounded by the HANDOFF_REPLY_MAX_BYTES cap) — accepted to + // close the idle-persistence gap without touching the view. + if (rows.length > 0) session.metadata["handoffReply"] = { request, rows }; } session.metadata.handoffResponse = { request, messages: [...messages.values()].map((entry) => entry.message) }; markDirty(session); diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 907fc103d2..089a6fadf4 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -73,6 +73,13 @@ interface IngestPlan extends IngestSessionInput { /** Resolved log seqs for pluginSnapshot entries (null when absent). */ pluginSnapshotSeqs: number[] | null; tailSeqs: number[] | null; + /** #6947 direction C: how many TRAILING messages[] entries are non-tail + * log rows (the stashed turn-final reply — persisted without ever + * entering the resident window). The full path ignores the count (those + * rows land like any other); the delta path uses it to keep the tail-seq + * math on the view rows only and to JOIN the reply row in place when + * the client re-send presents it as a normal tail message. */ + nonTailCount: number; } /** #3133 test instrumentation: per-message JSON serializations performed by @@ -174,6 +181,90 @@ function stringifyForStore(m: CoreMessage): string { } } +/** #6947 direction C: the stashed turn-final reply rows the host parked in + * `metadata.handoffReply` (see handoff-response.ts) — persisted through + * metadata so BOTH views (legacy file, unified replay) see them, keeping + * the #2960 digest pair equal. One row per captured reply message, built + * by planStoredMessageRow with the id the NEXT codec pass will derive, so + * the re-send joins it instead of duplicating it. */ +export interface HandoffReplyStashRow { + msgId: string; + content: string; + hash: string; + identity: string; + role: string; + kind: string; + media?: { hash: string; bytes: string }[]; + ts?: number; +} + +/** Plan-ready message row for a core the store has never seen through a + * request — the turn-final handoff reply. Runs the EXACT serialization + * chain the planners use (serializeForStore → splitMediaContent → + * identityOfStoredContent), so the stored row's identity equals the one + * the client re-send will derive: same join key, no duplicate row. + * Exported so the host capture (handoff-response.ts) and the planners + * cannot drift apart. */ +export function planStoredMessageRow(m: CoreMessage): HandoffReplyStashRow { + const raw = serializeForStore(m); + ingestSerializations.count++; + const parsed = JSON.parse(raw) as { role?: unknown; contentType?: unknown }; + const split = splitMediaContent(raw); + return { + msgId: m.id, + content: split.content, + hash: split.hash, + identity: identityOfStoredContent(split.content), + role: typeof parsed.role === "string" ? parsed.role : "assistant", + kind: typeof parsed.contentType === "string" ? parsed.contentType : "message", + ...(split.media.length > 0 ? { media: split.media } : {}), + ...(typeof m.ts === "number" ? { ts: m.ts } : {}), + }; +} + +/** Emits the handoffReply stash as TRAILING log rows (after every view row) + * and returns how many landed. A stashed row whose msgId is already a VIEW + * row of this save is skipped — the client re-send is planning the reply as + * a normal tail message and the planner's own row is authoritative. Rows + * carry whatever ref the ledger already knows for the id (null in the + * normal case — the ref is minted by the kernel at the re-send, invariant 2 + * of direction C) and NEVER enter tailSeqs, so the reconstructed view stays + * byte-equal to the session file's lastMessages. */ +function emitHandoffStash( + session: LegacySessionLike, + byRawRef: Record, + log: LogEntry[], + viewIds: Set, + seqBase: number, +): number { + const stash = session.metadata["handoffReply"] as { rows?: unknown } | null | undefined; + if (!Array.isArray(stash?.rows)) return 0; + let emitted = 0; + for (const value of stash.rows as unknown[]) { + if (value === null || typeof value !== "object") continue; + const r = value as Partial; + if ( + typeof r.msgId !== "string" || typeof r.content !== "string" || + typeof r.hash !== "string" || typeof r.identity !== "string" + ) continue; + if (viewIds.has(r.msgId)) continue; + log.push({ + seq: seqBase + emitted, + msgId: r.msgId, + content: r.content, + hash: r.hash, + identity: r.identity, + role: typeof r.role === "string" ? r.role : "assistant", + kind: typeof r.kind === "string" ? r.kind : "message", + ref: byRawRef[r.msgId] ?? null, + ...(Array.isArray(r.media) && r.media.length > 0 ? { media: r.media } : {}), + ...(typeof r.ts === "number" ? { ts: r.ts } : {}), + }); + emitted++; + } + return emitted; +} + /** Builds rows + pointer sequences. Pure: no database touched, so tests can * assert the mapping itself. seq assignment: first-seen over the ordered * union (pluginSnapshot first — frozen raw prefix — then fresh tail ids). */ @@ -211,7 +302,12 @@ export function planIngest(session: LegacySessionLike): IngestPlan { const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null; const tailSeqs = session.lastMessages ? session.lastMessages.map(intern) : null; - return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); + // #6947 direction C: the turn-final reply lands as trailing non-tail + // rows — persisted, but outside the view the tailSeqs reconstruct. + const viewIds = new Set(log.map((e) => e.msgId)); + const nonTailCount = emitHandoffStash(session, byRawRef, log, viewIds, log.length); + + return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs, nonTailCount); } /** #3133 fast-path planner: build the ingest plan for a session whose @@ -352,7 +448,12 @@ export function planIngestDelta( // holes included — folding removes log positions from the middle); the // delta path skips remapPlanToAbsolute by construction. const tailSeqs = [...trustedTailSeqs, ...appended]; - return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); + // #6947 direction C (delta twin): a turn-end save is exactly an equal- + // length raw delta whose only payload is the stash — viewIds is the FULL + // tail (trusted prefix included) so a re-sent reply is never re-emitted. + const viewIds = new Set(tail.map((m) => m.id)); + const nonTailCount = emitHandoffStash(session, byRawRef, log, viewIds, watermark + 1 + log.length); + return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs, nonTailCount); } /** Shared plan tail (everything except the messages log): given the same @@ -366,6 +467,7 @@ function finishPlan( log: LogEntry[], pluginSnapshotSeqs: number[] | null, tailSeqs: number[] | null, + nonTailCount = 0, ): IngestPlan { // Ref ledger: verbatim, ordered by the persisted map's key order. // Ghost ids (dead refs) have rows here and nowhere else — by design. @@ -505,7 +607,7 @@ function finishPlan( ccrJson: session.contentStore ? JSON.stringify(session.contentStore) : null, }; - return { session: sessionRow, messages, blocks, deadRefs, refs, ccrEntries, pluginSnapshotSeqs, tailSeqs }; + return { session: sessionRow, messages, blocks, deadRefs, refs, ccrEntries, pluginSnapshotSeqs, tailSeqs, nonTailCount }; } /** Rebuilds the legacy load view from unified rows alone. */ diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 36ab35b657..67d0c65ff5 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -645,28 +645,68 @@ export class MysqlBackend implements StorageBackend { if (!probe || probe.msgId !== delta.boundaryMsgId || probe.contentHash !== delta.boundaryHash) { return false; } - for (const m of input.messages) { + // Strictly-append guard, #6947 direction C (mirror of the + // sqlite lane): a hit at/BELOW the trusted tip or on a + // retired row refuses (full path re-dedups / reactivates); + // a LIVE hit above the tip is a row this lane stored without + // a view slot (the turn-end stash) — the re-send joins it in + // place: keep the stored seq, claim its NULL ref exactly like + // the full path's #6953 backfill. + const nonTailCount = input.nonTailCount ?? 0; + const tailCount = input.messages.length - nonTailCount; + const joinedSeq = new Map(); + const refClaims: { seq: number; ref: string; refNum: number | null }[] = []; + for (let i = 0; i < input.messages.length; i++) { + const m = input.messages[i]; if (!m.contentHash) return false; - // #4390: identity join; retired rows count as existing - // (rewind-redo falls back to the full path). + // #4390: identity join (position-independent). const [hit] = (await c.query( - "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ? LIMIT 1", + "SELECT seq, retired_at FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ? LIMIT 1", [s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content)], )) as unknown as Row[][]; - if (hit.length > 0) return false; + if (hit.length > 0) { + const row = hit[0]; + if ( + Number(row.seq) <= delta.trustedSeq || + (row.retired_at !== null && row.retired_at !== undefined) + ) { + return false; + } + joinedSeq.set(i, Number(row.seq)); + if (m.ref !== null && m.ref !== "BLOCKED" && m.refNum !== null) { + refClaims.push({ seq: Number(row.seq), ref: m.ref, refNum: m.refNum }); + } + continue; + } } - messages = input.messages.map((m, i) => ({ ...m, seq: probe.maxSeq + 1 + i })); + if (refClaims.length > 0) { + for (const claim of refClaims) { + await c.query( + "UPDATE messages SET ref = ?, ref_num = ? WHERE session_id = ? AND seq = ? AND ref IS NULL", + [claim.ref, claim.refNum, s.sessionId, claim.seq], + ); + } + } + // JOINED rows keep their stored seq; the rest anchor at + // max(seq)+1 (re-read inside this transaction) — see the + // sqlite lane's renumber note. + let nextSeq = probe.maxSeq + 1; + messages = input.messages.map((m, i) => { + const kept = joinedSeq.get(i); + return kept !== undefined ? { ...m, seq: kept } : { ...m, seq: nextSeq++ }; + }); + const tailRows = messages.slice(0, tailCount); const tail = JSON.parse(s.tailJson) as { tailSeqs?: number[] | null }; const prefixOk = Array.isArray(tail.tailSeqs) && - tail.tailSeqs.length === delta.trustedTailSeqs.length + input.messages.length && + tail.tailSeqs.length === delta.trustedTailSeqs.length + tailRows.length && delta.trustedTailSeqs.every((v, i) => tail.tailSeqs?.[i] === v); if (!prefixOk) return false; s.tailJson = JSON.stringify({ ...tail, - tailSeqs: [...delta.trustedTailSeqs, ...messages.map((m) => m.seq)], + tailSeqs: [...delta.trustedTailSeqs, ...tailRows.map((m) => m.seq)], }); - watermark = messages.length > 0 ? messages[messages.length - 1].seq : delta.trustedSeq; + watermark = tailRows.length > 0 ? tailRows[tailRows.length - 1].seq : delta.trustedSeq; } else { // Remapping mutates tailJson; persist the same session object // it updates, not the caller's separate plan-space snapshot. diff --git a/src/storage/store.ts b/src/storage/store.ts index db1fef5a07..9f8572a18f 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -344,6 +344,13 @@ export interface IngestSessionInput { * absolute seqs before insert — see remapPlanSeqs. */ pluginSnapshotSeqs?: number[] | null; tailSeqs?: number[] | null; + /** #6947 direction C: count of TRAILING `messages` entries that are + * non-tail log rows (the stashed turn-final reply — in the log, never + * in tailSeqs). The full path ignores it (those rows land like any + * other); the delta path keeps its tail-seq math on the view rows only + * and joins them in place when their row already exists (the client + * re-send presents the reply as a normal tail message). */ + nonTailCount?: number; /** Serialized orphan map (Record) — blockContents * entries whose block is absent from state.blocks (#4216/#4242: the reap * shape is REPRESENTABLE; rides the same transaction so the session row, @@ -968,43 +975,80 @@ export class UnifiedStore { return false; } // Strictly-append guard: a delta message whose identity - // already has a row (rewound/re-derived tail) must not be - // silently skipped by the UNIQUE(session_id, msg_id, - // content_hash) constraint — its tailSeqs slot would point - // at nothing and the read view would drop it. One indexed - // probe per NEW message; any hit → full ingest instead. + // already has a row must not be silently skipped by the + // UNIQUE(session_id, msg_id, content_hash) constraint — its + // tailSeqs slot would point at nothing and the read view + // would drop it. One indexed probe per NEW message. + // #6947 direction C narrows "hit" into two classes: + // - hit at/BELOW the trusted tip (seq <= trustedSeq) or a + // retired row → a rewind/redone tail: refuse, full path + // reactivates tombstones and re-dedups (unchanged); + // - hit ABOVE the tip, live → a row this lane stored itself + // without a view slot (the turn-end stash, above the + // watermark): the re-send is ABOUT to present it as a + // normal tail message — JOIN in place (keep the stored + // seq, claim its NULL ref exactly like the full path's + // #6953 backfill). Semantically identical to the full + // path's identity dedup, so the #3133 fast path stays + // green across the turn-end → re-send boundary. + const nonTailCount = input.nonTailCount ?? 0; + const tailCount = input.messages.length - nonTailCount; const existingIdentity = this.db.prepare( - "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ?", + "SELECT seq, ref, ref_num, retired_at FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ?", ); - for (const m of input.messages) { + const joinedSeq = new Map(); + const refClaims: { seq: number; ref: string; refNum: number | null }[] = []; + for (let i = 0; i < input.messages.length; i++) { + const m = input.messages[i]; if (!m.contentHash) return false; - // #4390: join on identity (position-independent). Retired - // rows count as existing — a rewind-redo falls back to the - // full path, which reactivates the tombstone in place. - if (existingIdentity.get(s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content))) return false; + // #4390: join on identity (position-independent). + const hit = existingIdentity.get( + s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content), + ) as { seq: number; retired_at: number | null } | undefined; + if (hit) { + if (Number(hit.seq) <= delta.trustedSeq || hit.retired_at !== null && hit.retired_at !== undefined) return false; + joinedSeq.set(i, Number(hit.seq)); + if (m.ref !== null && m.ref !== "BLOCKED" && m.refNum !== null) { + refClaims.push({ seq: Number(hit.seq), ref: m.ref, refNum: m.refNum }); + } + continue; + } } - // Placeholder→absolute renumber: the plan emitted - // watermark+1.. under the aligned assumption; anchoring at - // the CURRENT max(seq)+1 (re-read inside this transaction) - // is what makes slot collisions structurally impossible — - // identical to the full path's nextSeq discipline, and it - // keeps the delta valid even when a fold left rows above - // the trusted tip. - messages = input.messages.map((m, i) => ({ ...m, seq: probe.maxSeq + 1 + i })); + if (refClaims.length > 0) { + const claim = this.db.prepare( + "UPDATE messages SET ref = ?, ref_num = ? WHERE session_id = ? AND seq = ? AND ref IS NULL", + ); + for (const c of refClaims) claim.run(c.ref, c.refNum, s.sessionId, c.seq); + } + // Placeholder→absolute renumber: JOINED rows keep their stored + // seq (the re-send's tailSeqs slot points at the row the + // turn-end save already wrote); the rest anchor at the + // CURRENT max(seq)+1 (re-read inside this transaction) so slot + // collisions are structurally impossible — identical to the + // full path's nextSeq discipline, and it keeps the delta valid + // even when a fold left rows above the trusted tip. + let nextSeq = probe.maxSeq + 1; + messages = input.messages.map((m, i) => { + const kept = joinedSeq.get(i); + return kept !== undefined ? { ...m, seq: kept } : { ...m, seq: nextSeq++ }; + }); + // tailSeqs math runs on the VIEW rows only — trailing stash + // rows (if any) have no view slot by construction. + const tailRows = messages.slice(0, tailCount); const tail = JSON.parse(s.tailJson) as { tailSeqs?: number[] | null }; // Defense: the plan's tailSeqs prefix must be exactly the // trusted prefix the watermark read produced (same-tick by // construction; disagreement means a stale plan). const prefixOk = Array.isArray(tail.tailSeqs) && - tail.tailSeqs.length === delta.trustedTailSeqs.length + input.messages.length && + tail.tailSeqs.length === delta.trustedTailSeqs.length + tailRows.length && delta.trustedTailSeqs.every((v, i) => tail.tailSeqs?.[i] === v); if (!prefixOk) return false; s.tailJson = JSON.stringify({ ...tail, - tailSeqs: [...delta.trustedTailSeqs, ...messages.map((m) => m.seq)], + tailSeqs: [...delta.trustedTailSeqs, ...tailRows.map((m) => m.seq)], }); - watermark = messages.length > 0 ? messages[messages.length - 1].seq : delta.trustedSeq; + watermark = tailRows.length > 0 ? tailRows[tailRows.length - 1].seq : delta.trustedSeq; } else { const full = this.remapPlanToAbsolute(input); messages = full.messages; diff --git a/tests/issue6947-turn-final-writeback.test.ts b/tests/issue6947-turn-final-writeback.test.ts index 7892d091bd..d068263cd1 100644 --- a/tests/issue6947-turn-final-writeback.test.ts +++ b/tests/issue6947-turn-final-writeback.test.ts @@ -1,25 +1,41 @@ /** - * #6947: the turn-final reply was captured (handoffReply) but only written to - * session.metadata.handoffResponse — consumed solely by the `bili export` CLI. - * The store row existed only after the NEXT request re-sent the conversation - * (client re-send), so an idle session's last reply was missing from - * index.db and the /__bili web UI kept showing the previous reply. + * #6947 (direction C — persistence separated from the session identity + * ledger): the turn-final reply was captured (handoffResponse) but only + * written to the store when the NEXT request re-sent the conversation, so an + * idle session's last reply never reached index.db and the /__bili web UI + * kept showing the previous reply. * - * Fix: handoffReply.save() appends the captured messages back into - * session.lastMessages with codec-derivable ids (the SAME canonical seed the - * codecs feed deriveCanonicalMessageId with, #3642, + cluster numbering), so - * the debounced turn-end save persists them immediately. - * store.ts: when a full re-ingest joins a stored row whose ref is still NULL - * (its ref was minted only once the client re-sent the tagged text), backfill - * it — NULL only, never a rewrite, never the "BLOCKED" byRaw token. + * Fix (direction C): handoffReply.save() must NOT append the reply to + * session.lastMessages — that array is the single source of truth for the + * resident window (digest parity, web view, fork raw/ref check), and the + * reply's REF is minted by the kernel only when a request presents it. + * Pushing the reply there created a window where fork/snapshot saw a + * message with no ref and 409'd (the #6953 regression). Instead: + * + * - the captured reply is stashed as plan-ready rows in + * metadata.handoffReply (persisted → both views digest-equal), with ids + * derived EXACTLY as the next codec pass derives them (SAME canonical + * seed the codecs feed deriveCanonicalMessageId, #3642, + cluster + * numbering); + * - the planners (ingest.ts) turn the stash into trailing NON-TAIL log + * rows — never tailSeqs — so the reconstructed view stays equal to the + * session file's lastMessages; + * - store.ts: the full path joins the re-send onto the stored row by + * msg_id+identity and backfills the NULL ref (#6953, NULL only, never + * BLOCKED); the delta path (this PR's join extension) joins a stored row + * ABOVE the trusted tip in place (keep seq, claim the NULL ref), so the + * #3133 fast path stays green across the turn-end → re-send boundary + * instead of paying a refused delta + O(n) full re-ingest per turn. * * Invariants pinned here: - * 1. write-back ids are byte-identical to what the production codec derives - * on re-send → the store join (`${msg_id}\u0000${identity}`) hits and the - * UNIQUE constraint keeps exactly one row; - * 2. cluster numbering matches a full codec pass (repeated reply text); - * 3. folded snapshots and stale guards are untouched by the write-back; - * 4. ref backfill is idempotent and one-directional (NULL → minted only). + * 1. lastMessages is NEVER touched by the write-back (fork invariant); + * 2. stash ids are byte-identical to what the production codec derives on + * re-send → the store join hits and the UNIQUE constraint keeps exactly + * one row; + * 3. cluster numbering matches a full codec pass (repeated reply text); + * 4. folded snapshots and stale guards suppress the stash too; + * 5. ref backfill is idempotent and one-directional (NULL → minted only); + * 6. the delta fast path stays green across turn-end stash + re-send join. */ import { test } from "node:test"; @@ -32,7 +48,7 @@ import { canonicalToolArgs, deriveCanonicalMessageId, deriveMessageId, openaiToC import { createSession, type Session } from "../src/session.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; import { UnifiedStore } from "../src/storage/store.ts"; -import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; process.env.BILI_PERSIST_ZSTD = "0"; // Same hermeticity pin as the other storage suites. @@ -61,6 +77,18 @@ function capturedIds(session: Session): string[] { return (session.metadata.handoffResponse as { messages?: CoreMessage[] } | undefined)?.messages?.map((m) => m.id) ?? []; } +/** The stashed turn-final reply rows (direction C) decoded back to cores — + * row.content is the exact planner serialization, so parsing recovers the + * core the store will see. */ +function stashCores(session: Session): CoreMessage[] { + const stash = session.metadata.handoffReply as { rows?: Array<{ msgId: string; content: string }> } | undefined; + return (stash?.rows ?? []).map((r) => JSON.parse(r.content) as CoreMessage); +} + +function stashMetadata(session: Session): Record { + return session.metadata.handoffReply === undefined ? {} : { handoffReply: session.metadata.handoffReply }; +} + function withReply(session: Session, text: string): void { session.stats.requests = 1; const reply = createHandoffReply(session); @@ -68,19 +96,23 @@ function withReply(session: Session, text: string): void { reply.save(); } -test("write-back appends the turn-final reply with a codec-derivable id", () => { +test("turn-final reply is stashed for persistence WITHOUT touching lastMessages (fork invariant)", () => { const session = createSession("wb-basic"); - session.lastMessages = convo([{ role: "user", content: "hello" }]); + const tail = convo([{ role: "user", content: "hello" }]); + session.lastMessages = tail; withReply(session, "héllo world"); const messages = session.lastMessages ?? []; - assert.equal(messages.length, 2, "the reply must land in lastMessages, not only in metadata"); - const appended = messages[1]; - assert.ok(appended, "the reply row must exist"); - assert.equal(appended.role, "assistant"); - assert.equal(appended.contentType, "text"); - assert.equal(appended.text, "héllo world"); - assert.ok(appended.id.startsWith("h_"), `codec id shape, got ${appended.id}`); - assert.equal(appended.id, deriveMessageId("assistant", "text", "héllo world", {})); + assert.equal(session.lastMessages, tail, "lastMessages must stay the exact resident window (invariant 1)"); + assert.equal(messages.length, 1, "the reply must NOT land in lastMessages — the view is the ledger's territory"); + const stashed = stashCores(session); + assert.equal(stashed.length, 1, "the reply must be stashed for the store"); + const row = stashed[0]; + assert.ok(row, "the stash row must exist"); + assert.equal(row.role, "assistant"); + assert.equal(row.contentType, "text"); + assert.equal(row.text, "héllo world"); + assert.ok(row.id.startsWith("h_"), `codec id shape, got ${row.id}`); + assert.equal(row.id, deriveMessageId("assistant", "text", "héllo world", {})); // The export path is untouched: metadata still carries the capture's own ids. assert.deepEqual(capturedIds(session), ["reply_0"]); }); @@ -94,7 +126,7 @@ test("a client re-send through the production codec derives the SAME ids (store { role: "assistant", content: "héllo world" }, ]); assert.equal(resent[0].id, session.lastMessages![0].id, "user id stable across passes"); - assert.equal(resent[1].id, session.lastMessages![1].id, "reply id must match the re-send derivation"); + assert.equal(resent[1].id, stashCores(session)[0].id, "stash id must match the re-send derivation"); }); test("cluster numbering matches a full codec pass when reply text repeats", () => { @@ -109,13 +141,13 @@ test("cluster numbering matches a full codec pass when reply text repeats", () = const base = deriveMessageId("assistant", "text", "pong", {}); assert.notEqual(session.lastMessages![1].id, session.lastMessages![3].id, "fixture sanity: codec clusters repeats"); withReply(session, "pong"); - const appended = session.lastMessages![4]; + const stashed = stashCores(session); const expected = convo([...wire, { role: "assistant", content: "pong" }])[4]; - assert.equal(appended.id, expected.id, "third 'pong' must get the same cluster suffix the codec would mint"); - assert.notEqual(appended.id, base, "must not collide with the base (cluster 0) id"); + assert.equal(stashed[0].id, expected.id, "third 'pong' must get the same cluster suffix the codec would mint"); + assert.notEqual(stashed[0].id, base, "must not collide with the base (cluster 0) id"); }); -test("tool calls write back with toolName/toolCallId preserved and id-derivable", () => { +test("tool calls stash with toolName/toolCallId preserved and id-derivable", () => { const session = createSession("wb-tool"); session.lastMessages = convo([{ role: "user", content: "go" }]); session.stats.requests = 1; @@ -123,10 +155,10 @@ test("tool calls write back with toolName/toolCallId preserved and id-derivable" reply.append("0", "text", "looking"); reply.tool("1", "search", "call_1", "{\"q\":1}"); reply.save(); - const messages = session.lastMessages ?? []; - assert.equal(messages.length, 3); - const tool = messages[2]; - assert.equal(tool.contentType, "tool-call"); + assert.equal((session.lastMessages ?? []).length, 1, "no push into lastMessages"); + const stashed = stashCores(session); + assert.equal(stashed.length, 2); + const tool = stashed.find((m) => m.contentType === "tool-call")!; assert.equal(tool.toolName, "search"); assert.equal(tool.toolCallId, "call_1"); // Canonical seed (#3642): toolCallId OUT of the id (transport plumbing), @@ -146,7 +178,7 @@ test("a client re-send of a TOOL-CALL reply through the production codec derives reply.append("0", "text", "looking"); reply.tool("1", "search", "call_1", " { \"q\" : 1 } "); reply.save(); - const stored = session.lastMessages!; + const stashed = stashCores(session); // The client re-sends the turn: assistant text + tool call + tool result. // The codec sees the args as the client stored them (byte-identical — the // client echoes what it received); canonicalToolArgs normalizes both sides. @@ -155,9 +187,9 @@ test("a client re-send of a TOOL-CALL reply through the production codec derives { role: "assistant", content: "looking", tool_calls: [{ id: "call_1", type: "function", function: { name: "search", arguments: " { \"q\" : 1 } " } }] }, { role: "tool", tool_call_id: "call_1", content: "found it" }, ] }).msgs as unknown as CoreMessage[]; - assert.equal(resent[0].id, stored[0].id, "user id stable"); - assert.equal(resent[1].id, stored[1].id, "reply text id must match the re-send derivation"); - assert.equal(resent[2].id, stored[2].id, "tool-call id must match the re-send derivation (canonical join)"); + assert.equal(resent[0].id, session.lastMessages![0].id, "user id stable"); + assert.equal(resent[1].id, stashed[0].id, "reply text id must match the re-send derivation"); + assert.equal(resent[2].id, stashed[1].id, "tool-call id must match the re-send derivation (canonical join)"); }); /** @@ -170,7 +202,7 @@ test("a client re-send of a TOOL-CALL reply through the production codec derives * the same logical message: tombstone churn + the content CAS storing the * reply bytes twice, on every reply of every turn of a long session. */ -test("write-back bytes JOIN the client re-send by identity (openai lane: reasoning sidecar + compacted args)", () => { +test("stash bytes JOIN the client re-send by identity (openai lane: reasoning sidecar + compacted args)", () => { const argsRaw = '{ "command" : "echo hi" }'; // streamed verbatim (host JSON style) const session = createSession("wb-join"); session.lastMessages = convo([{ role: "user", content: "go" }]); @@ -180,10 +212,11 @@ test("write-back bytes JOIN the client re-send by identity (openai lane: reasoni reply.append("0", "text", "looking"); reply.tool("1", "bash", "call_1", argsRaw); reply.save(); - const stored = session.lastMessages!; - const reasoning = stored.find((m) => m.contentType === "reasoning")!; + assert.equal((session.lastMessages ?? []).length, 1, "lastMessages untouched"); + const stashed = stashCores(session); + const reasoning = stashed.find((m) => m.contentType === "reasoning")!; assert.equal((reasoning as { reasoningContent?: string }).reasoningContent, "let me think", "openai reasoning cores must carry the codec's reasoningContent sidecar"); - const tool = stored.find((m) => m.contentType === "tool-call")!; + const tool = stashed.find((m) => m.contentType === "tool-call")!; assert.equal(tool.text, '{"command":"echo hi"}', "tool-call args must be re-serialized the way clients (pi) re-send them: parse + JSON.stringify"); // The re-send: pi re-sends the reasoning via the reasoning_content field @@ -198,21 +231,26 @@ test("write-back bytes JOIN the client re-send by identity (openai lane: reasoni try { const store = UnifiedStore.open(path.join(dir, "index.db")); try { - // Turn-end save: write-back rows land (no refs minted yet). - assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [stored[0], reasoning, stored.find((m) => m.contentType === "text" && m.role === "assistant")!, tool], {}))), true); + const user = session.lastMessages![0]; + // Turn-end save: the view is [user] ONLY; the stash rows land as + // non-tail log rows (never tailSeqs), no refs minted yet. + assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [user], {}, stashMetadata(session)))), true); let rows = store.getMessages("sjoin"); - assert.equal(rows.length, 4); + assert.equal(rows.length, 4, `user + 3 stash rows (got ${rows.length}: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.retiredAt ?? "live"]))})`); + for (const m of stashed) { + const row = rows.find((r) => r.msgId === m.id); + assert.ok(row, `stash row for ${m.contentType} must exist after the turn-end save`); + assert.equal(row.ref, null, "no ref until the re-send (invariant 2)"); + } // Next request: the client re-sends the whole turn. The reply - // cores must JOIN the stored rows — no second rows, seq stable. - assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [...convo([{ role: "user", content: "go" }]), ...resent], { [resent[1].id]: "m00002" }))), true); + // cores must JOIN the stash rows — no second rows, seq stable. + assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [...convo([{ role: "user", content: "go" }]), ...resent], { [resent[1].id]: "m00002" }, stashMetadata(session)))), true); rows = store.getMessages("sjoin"); assert.equal(rows.length, 5, `the re-send must join, not duplicate (got ${rows.length} rows: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.retiredAt ?? "live"]))})`); for (const m of resent) { const storedRow = rows.find((r) => r.msgId === m.id); - assert.ok(storedRow, `re-send core ${m.contentType} must share the write-back row's id`); + assert.ok(storedRow, `re-send core ${m.contentType} must share the stash row's id`); } - const joinedTool = rows.find((r) => r.msgId === tool.id); - assert.ok(joinedTool); const turnEndTool = rows.filter((r) => r.msgId === tool.id); assert.equal(turnEndTool.length, 1, "tool-call: exactly one live row after the join"); } finally { @@ -231,24 +269,25 @@ test("without the protocol hint the openai reasoning/tool-call join is NOT claim reply.append("r", "reasoning", "let me think"); reply.tool("1", "bash", "call_1", '{ "command" : "echo hi" }'); reply.save(); - const stored = session.lastMessages!; - const reasoning = stored.find((m) => m.contentType === "reasoning")!; + const stashed = stashCores(session); + const reasoning = stashed.find((m) => m.contentType === "reasoning")!; assert.equal((reasoning as { reasoningContent?: string }).reasoningContent, undefined, "no sidecar without the protocol hint"); - const tool = stored.find((m) => m.contentType === "tool-call")!; + const tool = stashed.find((m) => m.contentType === "tool-call")!; assert.equal(tool.text, '{ "command" : "echo hi" }', "args stay verbatim without the protocol hint"); }); -test("folded snapshot: metadata is written but lastMessages is not grown", () => { +test("folded snapshot: neither stash nor lastMessages is touched", () => { const session = createSession("wb-folded"); const tail = convo([{ role: "user", content: "hello" }]); session.lastMessages = tail; session.lastMessagesFolded = true; withReply(session, "answer"); assert.equal(session.lastMessages, tail, "folded view must stay untouched"); + assert.equal(session.metadata.handoffReply, undefined, "a folded snapshot is a pruned view — no stash rows from it"); assert.deepEqual(capturedIds(session), ["reply_0"], "metadata capture still lands for export"); }); -test("stale guards (request advance / snapshot swap) suppress the write-back too", () => { +test("stale guards (request advance / snapshot swap) suppress the stash too", () => { for (const advance of [ (session: Session) => { session.stats.requests++; @@ -265,19 +304,20 @@ test("stale guards (request advance / snapshot swap) suppress the write-back too reply.append("0", "text", "old answer"); advance(session); reply.save(); - assert.equal((session.lastMessages ?? []).length, before, "no write-back for a stale response"); + assert.equal((session.lastMessages ?? []).length, before, "no touch for a stale response"); assert.equal(session.metadata.handoffResponse, undefined); + assert.equal(session.metadata.handoffReply, undefined, "stale response must not stash rows either"); } }); -function mkLegacy(id: string, lastMessages: CoreMessage[], byRaw: Record): LegacySessionLike { +function mkLegacy(id: string, lastMessages: CoreMessage[], byRaw: Record, metadata: Record = {}): LegacySessionLike { return { id, createdAt: 1_000, lastSeen: 2_000, meta: { protocol: "openai" }, stats: { requests: 1 }, - metadata: {}, + metadata, state: { blocks: [], messageRefs: { byRaw, byRef: {} }, @@ -290,13 +330,13 @@ function mkLegacy(id: string, lastMessages: CoreMessage[], byRaw: Record } { const session = createSession("wb-store-pair"); session.lastMessages = convo([{ role: "user", content: "install it" }]); withReply(session, "done: installed"); const user = session.lastMessages![0]; - const reply = session.lastMessages![1]; - return { user, reply }; + const reply = stashCores(session)[0]; + return { user, reply, metadata: stashMetadata(session) }; } test("store: turn-final row lands once, ref backfilled on the re-send ingest (NULL only)", () => { @@ -304,18 +344,19 @@ test("store: turn-final row lands once, ref backfilled on the re-send ingest (NU try { const store = UnifiedStore.open(path.join(dir, "index.db")); try { - const { user, reply } = turnFinalPair(); - // Turn-end save: the reply row lands; its ref is not minted yet. - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user, reply], {}))), true); + const { user, reply, metadata } = turnFinalPair(); + // Turn-end save: the view is [user]; the stash row lands, its ref + // is not minted yet, and it is NOT in tailSeqs (view parity). + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user], {}, metadata))), true); let rows = store.getMessages("s6947"); - assert.equal(rows.length, 2); + assert.equal(rows.length, 2, "user row + one stash row"); const first = rows.find((r) => r.msgId === reply.id); assert.ok(first, "the turn-final row must exist after the turn-end save"); assert.equal(first.ref, null); // Next request: the client re-sends the tagged reply; the ref is // minted. A full re-ingest joins the stored row and backfills. - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user, reply], { [reply.id]: "m00002" }))), true); + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user, reply], { [reply.id]: "m00002" }, metadata))), true); rows = store.getMessages("s6947"); assert.equal(rows.length, 2, "re-send must not duplicate the turn-final row"); const joined = rows.find((r) => r.msgId === reply.id); @@ -336,11 +377,11 @@ test("store: backfill never rewrites an already-stored ref and never writes BLOC try { const store = UnifiedStore.open(path.join(dir, "index.db")); try { - const { user, reply } = turnFinalPair(); + const { user, reply, metadata } = turnFinalPair(); // Ref minted at insert time (the common flow). - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00002" }))), true); + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00002" }, metadata))), true); // A later ingest with a different minted ref must NOT rewrite it. - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00009" }))), true); + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00009" }, metadata))), true); let row = store.getMessages("s6947g").find((r) => r.msgId === reply.id); assert.equal(row?.ref, "m00002", "stored ref is frozen (index-time projection)"); @@ -349,9 +390,10 @@ test("store: backfill never rewrites an already-stored ref and never writes BLOC session.lastMessages = convo([{ role: "user", content: "install it" }]); withReply(session, "blocked reply"); const u2 = session.lastMessages![0]; - const r2 = session.lastMessages![1]; - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2, r2], {}))), true); - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2, r2], { [r2.id]: "BLOCKED" }))), true); + const r2 = stashCores(session)[0]; + const md2 = stashMetadata(session); + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2], {}, md2))), true); + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2, r2], { [r2.id]: "BLOCKED" }, md2))), true); row = store.getMessages("s6947b").find((r) => r.msgId === r2.id); assert.equal(row?.ref, null, "BLOCKED must stay out of messages.ref"); } finally { @@ -361,3 +403,61 @@ test("store: backfill never rewrites an already-stored ref and never writes BLOC rmBestEffort(dir); } }); + +/** + * #3133 × #6947: the delta fast path must stay green across the turn-end → + * re-send boundary. Turn end is an equal-length raw delta whose only payload + * is the stash (non-tail rows); the re-send then presents the reply as a + * NORMAL tail message that joins the stored stash row in place (seq kept, + * NULL ref claimed) instead of tripping the strictly-append guard into an + * O(n) full re-ingest on every turn of every long session. + */ +test("store: delta fast path survives the turn-end stash save and the re-send join", () => { + const dir = tmpdir("bili-6947-delta-"); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + const session = createSession("wb-delta"); + const u1 = convo([{ role: "user", content: "install it" }])[0]; + session.lastMessages = [u1]; + withReply(session, "done: installed"); + const reply = stashCores(session)[0]; + const md = stashMetadata(session); + + // Request N (full path primes the watermark at the user row). + assert.equal(store.ingestSession(planIngest(mkLegacy("sd", [u1], {}))), true); + const boundary = store.probeIngestBoundary("sd", 0); + assert.ok(boundary, "watermark probe must find the user row at seq 0"); + + // Turn N end: equal-length raw delta, payload = the stash row. + const turnEnd = planIngestDelta({ ...(mkLegacy("sd", [u1], {}, md) as object) } as LegacySessionLike, 0, [0], { msgId: boundary.msgId, contentHash: boundary.contentHash }, true, null); + assert.ok(turnEnd, "the turn-end save must take the delta fast path (not the full re-ingest)"); + assert.equal(turnEnd.nonTailCount, 1, "the stash row is the only payload and it is non-tail"); + assert.deepEqual(turnEnd.tailSeqs, [0], "the stash row must NOT extend tailSeqs (view parity)"); + assert.equal(store.ingestSession(turnEnd, { trustedSeq: 0, boundaryMsgId: boundary.msgId, boundaryHash: boundary.contentHash, trustedTailSeqs: [0] }), true, "the turn-end delta must commit"); + let rows = store.getMessages("sd"); + assert.equal(rows.length, 2, "user + stash row"); + assert.equal(rows.find((r) => r.msgId === reply.id)?.ref, null, "still no ref at turn end"); + const stashSeq = rows.find((r) => r.msgId === reply.id)!.seq; + + // Request N+1: the client re-sends [u1, reply] plus a new user + // turn. The reply is a TAIL row now and joins the stored stash + // row in place; the new row appends. + const u2 = convo([{ role: "user", content: "next" }])[0]; + const resend = { ...(mkLegacy("sd", [u1, reply, u2], { [reply.id]: "m00002" }, md) as object) } as LegacySessionLike; + const resendPlan = planIngestDelta(resend, 0, [0], { msgId: boundary.msgId, contentHash: boundary.contentHash }, true, null); + assert.ok(resendPlan, "the re-send must take the delta fast path (join in place, no full re-ingest)"); + assert.equal(resendPlan.nonTailCount, 0, "the stash is gated out once the reply is a view row"); + assert.equal(store.ingestSession(resendPlan, { trustedSeq: 0, boundaryMsgId: boundary.msgId, boundaryHash: boundary.contentHash, trustedTailSeqs: [0] }), true, "the re-send delta must commit"); + rows = store.getMessages("sd"); + assert.equal(rows.length, 3, `no duplicate rows after the join (got ${rows.length}: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.ref ?? "null"]))})`); + const joined = rows.find((r) => r.msgId === reply.id)!; + assert.equal(joined.seq, stashSeq, "the joined row keeps its stored seq"); + assert.equal(joined.ref, "m00002", "the NULL ref was claimed on the join"); + } finally { + store.close(); + } + } finally { + rmBestEffort(dir); + } +}); From 80daffc2f5dc5362bfc3645f253237871d3a8bc4 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Tue, 17 Nov 2026 10:58:38 +0800 Subject: [PATCH 9/9] fix(storage): keep the reply stash out of the messages table (direction C1, #6947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first cut of this fix persisted the turn-final reply as non-tail rows in the messages table. That breaks the table's contract: its seq space is the ref-ledger substrate — on the first fold, persist.maybeRestampToLedger aligns every live ref onto the store row's seq+1 (#4316), so a non-view row interleaved in the table desyncs every post-fold ref (golden wire drift: responses/google legs, body #10 onward). C1: the stash lives only in sessions.metadata_json.handoffReply. That column is persisted in both the legacy file and the unified DB, identical in both views (digest-equal), so the reply still lands durably in the database for #6947's 'anything reading the DB loses the turn's final reply' concern — without touching the view, the ledger, or any wire. A client re-send then persists the reply as a normal view row with the kernel-minted ref, exactly as before #6947. - handoff-response.ts: unchanged from the previous cut (metadata-only stash, plan-ready rows with the codec-canonical ids, restampForJoin sidecars) - ingest.ts: keep the planStoredMessageRow/HandoffReplyStashRow exports the capture uses; no planner emission - store.ts / mysql.ts: back to the pre-cut delta/full paths (the join extension is gone with the table rows) - tests: rewrite for C1 (stash never reaches the table; re-send is base behavior; identity join-key parity; golden re-baseline = metadata section only) Local (node 22, clean env): issue6947 12/12, fork 4 files 91/91, lane/ storage/golden 6 files + storage-engine-proofs green at BASE expectations, build OK. --- reference/architecture.md | 2 +- src/storage/ingest.ts | 91 ++----- src/storage/mysql.ts | 58 +---- src/storage/store.ts | 88 ++----- tests/golden/storage-render/anthropic.json | 6 +- tests/golden/storage-render/chat.json | 6 +- tests/golden/storage-render/google.json | 6 +- tests/golden/storage-render/responses.json | 6 +- tests/issue6947-turn-final-writeback.test.ts | 258 +++++++------------ 9 files changed, 160 insertions(+), 361 deletions(-) diff --git a/reference/architecture.md b/reference/architecture.md index 477af01d9e..2d25cd9ee1 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -19,7 +19,7 @@ billion-context/ │ │ ├── context-window.ts # Context-window resolution (launcher model channel, beta headers) │ │ ├── dsh-compaction-guard.ts # Rejects dsh whole-prefix compaction replays (#1729) │ │ ├── handle.ts # Proxy request pipeline (route triage, compat relay, compress orchestration) (#1440 P2 cut 3) -│ │ ├── handoff-response.ts # Retains parsed assistant output for handoff export without changing request snapshots +│ │ ├── handoff-response.ts # Retains parsed assistant output for handoff export without changing request snapshots; turn-final reply stashed in metadata.handoffReply for durable DB persistence (never lastMessages/messages table — the view is the ref-ledger's territory, #6947) │ │ ├── headers.ts # Client-provided id sanitization for dump filenames (#286) │ │ ├── inject.ts # Per-wire tool-injection wrappers + FORCE_TEXT_PROTOCOL switch (#1440) │ │ ├── observability.ts # Body dumps (dumps/req-*.json, raw/*), unrecognized-path stats diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 089a6fadf4..d1ba085308 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -73,13 +73,6 @@ interface IngestPlan extends IngestSessionInput { /** Resolved log seqs for pluginSnapshot entries (null when absent). */ pluginSnapshotSeqs: number[] | null; tailSeqs: number[] | null; - /** #6947 direction C: how many TRAILING messages[] entries are non-tail - * log rows (the stashed turn-final reply — persisted without ever - * entering the resident window). The full path ignores the count (those - * rows land like any other); the delta path uses it to keep the tail-seq - * math on the view rows only and to JOIN the reply row in place when - * the client re-send presents it as a normal tail message. */ - nonTailCount: number; } /** #3133 test instrumentation: per-message JSON serializations performed by @@ -181,12 +174,14 @@ function stringifyForStore(m: CoreMessage): string { } } -/** #6947 direction C: the stashed turn-final reply rows the host parked in - * `metadata.handoffReply` (see handoff-response.ts) — persisted through - * metadata so BOTH views (legacy file, unified replay) see them, keeping - * the #2960 digest pair equal. One row per captured reply message, built - * by planStoredMessageRow with the id the NEXT codec pass will derive, so - * the re-send joins it instead of duplicating it. */ +/** A persisted turn-final reply row. Stored under the session's + * `metadata.handoffReply` (see src/server/handoff-response.ts) — OUTSIDE the + * messages table on purpose: the table's seq space is the ref-ledger + * substrate (persist.maybeRestampToLedger aligns live refs onto row seq+1, + * #4316), so non-view rows would desync every post-fold ref. metadata_json + * is persisted in both the legacy file and the unified DB (identical in + * both views ⇒ digest-equal), so the reply still lands durably in the DB — + * #6947 — without touching the view, the ledger, or any wire. */ export interface HandoffReplyStashRow { msgId: string; content: string; @@ -198,13 +193,13 @@ export interface HandoffReplyStashRow { ts?: number; } -/** Plan-ready message row for a core the store has never seen through a +/** Plan-ready representation of a core the store has never seen through a * request — the turn-final handoff reply. Runs the EXACT serialization * chain the planners use (serializeForStore → splitMediaContent → - * identityOfStoredContent), so the stored row's identity equals the one - * the client re-send will derive: same join key, no duplicate row. - * Exported so the host capture (handoff-response.ts) and the planners - * cannot drift apart. */ + * identityOfStoredContent), so the row's identity equals the one a client + * re-send of the same reply would derive — DB readers can dedup the stash + * against view rows by (msgId, identity) if they ever join. Exported so the + * host capture (handoff-response.ts) and this module cannot drift apart. */ export function planStoredMessageRow(m: CoreMessage): HandoffReplyStashRow { const raw = serializeForStore(m); ingestSerializations.count++; @@ -222,49 +217,6 @@ export function planStoredMessageRow(m: CoreMessage): HandoffReplyStashRow { }; } -/** Emits the handoffReply stash as TRAILING log rows (after every view row) - * and returns how many landed. A stashed row whose msgId is already a VIEW - * row of this save is skipped — the client re-send is planning the reply as - * a normal tail message and the planner's own row is authoritative. Rows - * carry whatever ref the ledger already knows for the id (null in the - * normal case — the ref is minted by the kernel at the re-send, invariant 2 - * of direction C) and NEVER enter tailSeqs, so the reconstructed view stays - * byte-equal to the session file's lastMessages. */ -function emitHandoffStash( - session: LegacySessionLike, - byRawRef: Record, - log: LogEntry[], - viewIds: Set, - seqBase: number, -): number { - const stash = session.metadata["handoffReply"] as { rows?: unknown } | null | undefined; - if (!Array.isArray(stash?.rows)) return 0; - let emitted = 0; - for (const value of stash.rows as unknown[]) { - if (value === null || typeof value !== "object") continue; - const r = value as Partial; - if ( - typeof r.msgId !== "string" || typeof r.content !== "string" || - typeof r.hash !== "string" || typeof r.identity !== "string" - ) continue; - if (viewIds.has(r.msgId)) continue; - log.push({ - seq: seqBase + emitted, - msgId: r.msgId, - content: r.content, - hash: r.hash, - identity: r.identity, - role: typeof r.role === "string" ? r.role : "assistant", - kind: typeof r.kind === "string" ? r.kind : "message", - ref: byRawRef[r.msgId] ?? null, - ...(Array.isArray(r.media) && r.media.length > 0 ? { media: r.media } : {}), - ...(typeof r.ts === "number" ? { ts: r.ts } : {}), - }); - emitted++; - } - return emitted; -} - /** Builds rows + pointer sequences. Pure: no database touched, so tests can * assert the mapping itself. seq assignment: first-seen over the ordered * union (pluginSnapshot first — frozen raw prefix — then fresh tail ids). */ @@ -302,12 +254,7 @@ export function planIngest(session: LegacySessionLike): IngestPlan { const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null; const tailSeqs = session.lastMessages ? session.lastMessages.map(intern) : null; - // #6947 direction C: the turn-final reply lands as trailing non-tail - // rows — persisted, but outside the view the tailSeqs reconstruct. - const viewIds = new Set(log.map((e) => e.msgId)); - const nonTailCount = emitHandoffStash(session, byRawRef, log, viewIds, log.length); - - return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs, nonTailCount); + return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); } /** #3133 fast-path planner: build the ingest plan for a session whose @@ -448,12 +395,7 @@ export function planIngestDelta( // holes included — folding removes log positions from the middle); the // delta path skips remapPlanToAbsolute by construction. const tailSeqs = [...trustedTailSeqs, ...appended]; - // #6947 direction C (delta twin): a turn-end save is exactly an equal- - // length raw delta whose only payload is the stash — viewIds is the FULL - // tail (trusted prefix included) so a re-sent reply is never re-emitted. - const viewIds = new Set(tail.map((m) => m.id)); - const nonTailCount = emitHandoffStash(session, byRawRef, log, viewIds, watermark + 1 + log.length); - return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs, nonTailCount); + return finishPlan(session, byRawRef, log, pluginSnapshotSeqs, tailSeqs); } /** Shared plan tail (everything except the messages log): given the same @@ -467,7 +409,6 @@ function finishPlan( log: LogEntry[], pluginSnapshotSeqs: number[] | null, tailSeqs: number[] | null, - nonTailCount = 0, ): IngestPlan { // Ref ledger: verbatim, ordered by the persisted map's key order. // Ghost ids (dead refs) have rows here and nowhere else — by design. @@ -607,7 +548,7 @@ function finishPlan( ccrJson: session.contentStore ? JSON.stringify(session.contentStore) : null, }; - return { session: sessionRow, messages, blocks, deadRefs, refs, ccrEntries, pluginSnapshotSeqs, tailSeqs, nonTailCount }; + return { session: sessionRow, messages, blocks, deadRefs, refs, ccrEntries, pluginSnapshotSeqs, tailSeqs }; } /** Rebuilds the legacy load view from unified rows alone. */ diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 67d0c65ff5..36ab35b657 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -645,68 +645,28 @@ export class MysqlBackend implements StorageBackend { if (!probe || probe.msgId !== delta.boundaryMsgId || probe.contentHash !== delta.boundaryHash) { return false; } - // Strictly-append guard, #6947 direction C (mirror of the - // sqlite lane): a hit at/BELOW the trusted tip or on a - // retired row refuses (full path re-dedups / reactivates); - // a LIVE hit above the tip is a row this lane stored without - // a view slot (the turn-end stash) — the re-send joins it in - // place: keep the stored seq, claim its NULL ref exactly like - // the full path's #6953 backfill. - const nonTailCount = input.nonTailCount ?? 0; - const tailCount = input.messages.length - nonTailCount; - const joinedSeq = new Map(); - const refClaims: { seq: number; ref: string; refNum: number | null }[] = []; - for (let i = 0; i < input.messages.length; i++) { - const m = input.messages[i]; + for (const m of input.messages) { if (!m.contentHash) return false; - // #4390: identity join (position-independent). + // #4390: identity join; retired rows count as existing + // (rewind-redo falls back to the full path). const [hit] = (await c.query( - "SELECT seq, retired_at FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ? LIMIT 1", + "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ? LIMIT 1", [s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content)], )) as unknown as Row[][]; - if (hit.length > 0) { - const row = hit[0]; - if ( - Number(row.seq) <= delta.trustedSeq || - (row.retired_at !== null && row.retired_at !== undefined) - ) { - return false; - } - joinedSeq.set(i, Number(row.seq)); - if (m.ref !== null && m.ref !== "BLOCKED" && m.refNum !== null) { - refClaims.push({ seq: Number(row.seq), ref: m.ref, refNum: m.refNum }); - } - continue; - } + if (hit.length > 0) return false; } - if (refClaims.length > 0) { - for (const claim of refClaims) { - await c.query( - "UPDATE messages SET ref = ?, ref_num = ? WHERE session_id = ? AND seq = ? AND ref IS NULL", - [claim.ref, claim.refNum, s.sessionId, claim.seq], - ); - } - } - // JOINED rows keep their stored seq; the rest anchor at - // max(seq)+1 (re-read inside this transaction) — see the - // sqlite lane's renumber note. - let nextSeq = probe.maxSeq + 1; - messages = input.messages.map((m, i) => { - const kept = joinedSeq.get(i); - return kept !== undefined ? { ...m, seq: kept } : { ...m, seq: nextSeq++ }; - }); - const tailRows = messages.slice(0, tailCount); + messages = input.messages.map((m, i) => ({ ...m, seq: probe.maxSeq + 1 + i })); const tail = JSON.parse(s.tailJson) as { tailSeqs?: number[] | null }; const prefixOk = Array.isArray(tail.tailSeqs) && - tail.tailSeqs.length === delta.trustedTailSeqs.length + tailRows.length && + tail.tailSeqs.length === delta.trustedTailSeqs.length + input.messages.length && delta.trustedTailSeqs.every((v, i) => tail.tailSeqs?.[i] === v); if (!prefixOk) return false; s.tailJson = JSON.stringify({ ...tail, - tailSeqs: [...delta.trustedTailSeqs, ...tailRows.map((m) => m.seq)], + tailSeqs: [...delta.trustedTailSeqs, ...messages.map((m) => m.seq)], }); - watermark = tailRows.length > 0 ? tailRows[tailRows.length - 1].seq : delta.trustedSeq; + watermark = messages.length > 0 ? messages[messages.length - 1].seq : delta.trustedSeq; } else { // Remapping mutates tailJson; persist the same session object // it updates, not the caller's separate plan-space snapshot. diff --git a/src/storage/store.ts b/src/storage/store.ts index 9f8572a18f..db1fef5a07 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -344,13 +344,6 @@ export interface IngestSessionInput { * absolute seqs before insert — see remapPlanSeqs. */ pluginSnapshotSeqs?: number[] | null; tailSeqs?: number[] | null; - /** #6947 direction C: count of TRAILING `messages` entries that are - * non-tail log rows (the stashed turn-final reply — in the log, never - * in tailSeqs). The full path ignores it (those rows land like any - * other); the delta path keeps its tail-seq math on the view rows only - * and joins them in place when their row already exists (the client - * re-send presents the reply as a normal tail message). */ - nonTailCount?: number; /** Serialized orphan map (Record) — blockContents * entries whose block is absent from state.blocks (#4216/#4242: the reap * shape is REPRESENTABLE; rides the same transaction so the session row, @@ -975,80 +968,43 @@ export class UnifiedStore { return false; } // Strictly-append guard: a delta message whose identity - // already has a row must not be silently skipped by the - // UNIQUE(session_id, msg_id, content_hash) constraint — its - // tailSeqs slot would point at nothing and the read view - // would drop it. One indexed probe per NEW message. - // #6947 direction C narrows "hit" into two classes: - // - hit at/BELOW the trusted tip (seq <= trustedSeq) or a - // retired row → a rewind/redone tail: refuse, full path - // reactivates tombstones and re-dedups (unchanged); - // - hit ABOVE the tip, live → a row this lane stored itself - // without a view slot (the turn-end stash, above the - // watermark): the re-send is ABOUT to present it as a - // normal tail message — JOIN in place (keep the stored - // seq, claim its NULL ref exactly like the full path's - // #6953 backfill). Semantically identical to the full - // path's identity dedup, so the #3133 fast path stays - // green across the turn-end → re-send boundary. - const nonTailCount = input.nonTailCount ?? 0; - const tailCount = input.messages.length - nonTailCount; + // already has a row (rewound/re-derived tail) must not be + // silently skipped by the UNIQUE(session_id, msg_id, + // content_hash) constraint — its tailSeqs slot would point + // at nothing and the read view would drop it. One indexed + // probe per NEW message; any hit → full ingest instead. const existingIdentity = this.db.prepare( - "SELECT seq, ref, ref_num, retired_at FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ?", + "SELECT 1 FROM messages WHERE session_id = ? AND msg_id = ? AND identity = ?", ); - const joinedSeq = new Map(); - const refClaims: { seq: number; ref: string; refNum: number | null }[] = []; - for (let i = 0; i < input.messages.length; i++) { - const m = input.messages[i]; + for (const m of input.messages) { if (!m.contentHash) return false; - // #4390: join on identity (position-independent). - const hit = existingIdentity.get( - s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content), - ) as { seq: number; retired_at: number | null } | undefined; - if (hit) { - if (Number(hit.seq) <= delta.trustedSeq || hit.retired_at !== null && hit.retired_at !== undefined) return false; - joinedSeq.set(i, Number(hit.seq)); - if (m.ref !== null && m.ref !== "BLOCKED" && m.refNum !== null) { - refClaims.push({ seq: Number(hit.seq), ref: m.ref, refNum: m.refNum }); - } - continue; - } + // #4390: join on identity (position-independent). Retired + // rows count as existing — a rewind-redo falls back to the + // full path, which reactivates the tombstone in place. + if (existingIdentity.get(s.sessionId, m.msgId, m.identity ?? identityOfStoredContent(m.content))) return false; } - if (refClaims.length > 0) { - const claim = this.db.prepare( - "UPDATE messages SET ref = ?, ref_num = ? WHERE session_id = ? AND seq = ? AND ref IS NULL", - ); - for (const c of refClaims) claim.run(c.ref, c.refNum, s.sessionId, c.seq); - } - // Placeholder→absolute renumber: JOINED rows keep their stored - // seq (the re-send's tailSeqs slot points at the row the - // turn-end save already wrote); the rest anchor at the - // CURRENT max(seq)+1 (re-read inside this transaction) so slot - // collisions are structurally impossible — identical to the - // full path's nextSeq discipline, and it keeps the delta valid - // even when a fold left rows above the trusted tip. - let nextSeq = probe.maxSeq + 1; - messages = input.messages.map((m, i) => { - const kept = joinedSeq.get(i); - return kept !== undefined ? { ...m, seq: kept } : { ...m, seq: nextSeq++ }; - }); - // tailSeqs math runs on the VIEW rows only — trailing stash - // rows (if any) have no view slot by construction. - const tailRows = messages.slice(0, tailCount); + // Placeholder→absolute renumber: the plan emitted + // watermark+1.. under the aligned assumption; anchoring at + // the CURRENT max(seq)+1 (re-read inside this transaction) + // is what makes slot collisions structurally impossible — + // identical to the full path's nextSeq discipline, and it + // keeps the delta valid even when a fold left rows above + // the trusted tip. + messages = input.messages.map((m, i) => ({ ...m, seq: probe.maxSeq + 1 + i })); const tail = JSON.parse(s.tailJson) as { tailSeqs?: number[] | null }; // Defense: the plan's tailSeqs prefix must be exactly the // trusted prefix the watermark read produced (same-tick by // construction; disagreement means a stale plan). const prefixOk = Array.isArray(tail.tailSeqs) && - tail.tailSeqs.length === delta.trustedTailSeqs.length + tailRows.length && + tail.tailSeqs.length === delta.trustedTailSeqs.length + input.messages.length && delta.trustedTailSeqs.every((v, i) => tail.tailSeqs?.[i] === v); if (!prefixOk) return false; s.tailJson = JSON.stringify({ ...tail, - tailSeqs: [...delta.trustedTailSeqs, ...tailRows.map((m) => m.seq)], + tailSeqs: [...delta.trustedTailSeqs, ...messages.map((m) => m.seq)], }); - watermark = tailRows.length > 0 ? tailRows[tailRows.length - 1].seq : delta.trustedSeq; + watermark = messages.length > 0 ? messages[messages.length - 1].seq : delta.trustedSeq; } else { const full = this.remapPlanToAbsolute(input); messages = full.messages; diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 558829d15d..2ebb2f40a2 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "1f528f4fee4f59026d3b005f99a0489cadc139e54698896b2e5a0dadbc95ee2b", - "stateDigestAligned": "a796f61ccd006f5fa1573e1ef33bf756dc24bf017154fab8bde4c0b369516537", + "stateDigest": "97c49de03bb6c42c99a9d0f03ce7cef247b4cb034ee1da24d5954207225bb341", + "stateDigestAligned": "2f913ebef14626bd7923a11fffc9676e2919664f2a5e1649b456bf03ec09cb29", "sectionDigests": { "state": "56e66f6b72c7f0dd", "blockContents": "dc7d13d250c94bb4", "stats": "e83198a55e6b2c29", - "metadata": "65894bb8839e9947" + "metadata": "5d34f4fe94270d09" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index ae4923dd78..0469c7da94 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "a7b5a67f54a2cbd7978e2a57ad78f522f7af8f94a4e2402898aee4d93479541a", - "stateDigestAligned": "76f701f5ab1631fe1615a3f1cd8fa0fcd0a3fa19c02658cc5d077ae565f4897a", + "stateDigest": "accd7fe100acd1e3adf3b6186b4c709085f4e229ea2d8d553f903cf30f97b42c", + "stateDigestAligned": "4d6b17bddffef0fbaf73bd585a8a702c34a866c105719da5fa16bc90da1be38b", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", "stats": "5c2f0fe203d710e8", - "metadata": "0c7d1819e0d4c0f6" + "metadata": "5e869987612d599b" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 3dde0e8cdc..7278d9ff2d 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "f8d07462fd8881896c5b52864e84ff9a1a79b0cdb73cd2ef87e16d361b5a07d3", - "stateDigestAligned": "219c42a0f17ed7c9061174627ca56e5b6733e7413d8af98953105bf5d60a40b7", + "stateDigest": "a9769a0169874c9b501b27a73cf721dcb57097969e974f050a3fa68a741b17fd", + "stateDigestAligned": "3db9797c6435542e793cfd27d4d5d9a76d49474289601d07c90d65fa9d567112", "sectionDigests": { "state": "c5243ba17364d3ac", "blockContents": "473264d9df8ba338", "stats": "a3c84817f1d0c3dd", - "metadata": "615f125909ed135d" + "metadata": "631282b79ac9672c" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index c9aa385f1a..c1a8cdf618 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -95,13 +95,13 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "aa95de9ed8ddaff40f0b0e2deb1991c85582e33a17149c030687f46cfa34a9a3", - "stateDigestAligned": "5f40c3661d4c43338d855b5a5e79b2915da934affe33d6cb07680a1da39bb996", + "stateDigest": "afe47188cdeafd5b6670fe757632e67d04bb05cb6e7c2a6971f0d498199e86a8", + "stateDigestAligned": "ed7a20e60d65949c92e27963ca373fc3481ef423ee9f6e33f67bcc213f964e50", "sectionDigests": { "state": "e686a14f31d69ce5", "blockContents": "b6701268a0bdcd8a", "stats": "099bde0a9cf64cad", - "metadata": "be12181116331419" + "metadata": "5b0d05c2e6677cac" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", diff --git a/tests/issue6947-turn-final-writeback.test.ts b/tests/issue6947-turn-final-writeback.test.ts index d068263cd1..08b8d8f0de 100644 --- a/tests/issue6947-turn-final-writeback.test.ts +++ b/tests/issue6947-turn-final-writeback.test.ts @@ -1,41 +1,40 @@ /** - * #6947 (direction C — persistence separated from the session identity + * #6947 (direction C1 — persistence separated from the session identity * ledger): the turn-final reply was captured (handoffResponse) but only * written to the store when the NEXT request re-sent the conversation, so an * idle session's last reply never reached index.db and the /__bili web UI * kept showing the previous reply. * - * Fix (direction C): handoffReply.save() must NOT append the reply to - * session.lastMessages — that array is the single source of truth for the - * resident window (digest parity, web view, fork raw/ref check), and the - * reply's REF is minted by the kernel only when a request presents it. - * Pushing the reply there created a window where fork/snapshot saw a - * message with no ref and 409'd (the #6953 regression). Instead: + * Fix (direction C1): the reply is stashed as plan-ready rows in + * `metadata.handoffReply` — OUTSIDE the messages table, on purpose: * - * - the captured reply is stashed as plan-ready rows in - * metadata.handoffReply (persisted → both views digest-equal), with ids - * derived EXACTLY as the next codec pass derives them (SAME canonical - * seed the codecs feed deriveCanonicalMessageId, #3642, + cluster - * numbering); - * - the planners (ingest.ts) turn the stash into trailing NON-TAIL log - * rows — never tailSeqs — so the reconstructed view stays equal to the - * session file's lastMessages; - * - store.ts: the full path joins the re-send onto the stored row by - * msg_id+identity and backfills the NULL ref (#6953, NULL only, never - * BLOCKED); the delta path (this PR's join extension) joins a stored row - * ABOVE the trusted tip in place (keep seq, claim the NULL ref), so the - * #3133 fast path stays green across the turn-end → re-send boundary - * instead of paying a refused delta + O(n) full re-ingest per turn. + * - `session.lastMessages` is the single source of truth for the resident + * window (digest parity, web view, fork raw/ref check) and the reply's + * REF is minted by the kernel only when a request presents it; appending + * the reply there created a window where fork/snapshot saw a message + * with no ref and 409'd (the original #6953 regression). + * - the messages table's seq space is the ref-ledger substrate: on the + * first fold `persist.maybeRestampToLedger` aligns every live ref onto + * the store row's seq+1 (#4316), so a non-view row interleaved in the + * table would desync every post-fold ref (golden wire drift). + * + * metadata_json is persisted in BOTH the legacy file and the unified DB, + * identical in both views ⇒ digest-equal — so the reply still lands + * durably in the database for #6947's "anything reading the DB loses the + * turn's final reply" concern, without touching the view, the ledger, or + * any wire. A client re-send then persists the reply as a normal view row + * (kernel-minted ref) exactly as before #6947. * * Invariants pinned here: * 1. lastMessages is NEVER touched by the write-back (fork invariant); * 2. stash ids are byte-identical to what the production codec derives on - * re-send → the store join hits and the UNIQUE constraint keeps exactly - * one row; - * 3. cluster numbering matches a full codec pass (repeated reply text); - * 4. folded snapshots and stale guards suppress the stash too; - * 5. ref backfill is idempotent and one-directional (NULL → minted only); - * 6. the delta fast path stays green across turn-end stash + re-send join. + * re-send (DB readers can dedup the stash against view rows by id); + * 3. stash row identity equals the planner identity of the re-sent core + * (same join key, should readers ever join them); + * 4. cluster numbering matches a full codec pass (repeated reply text); + * 5. folded snapshots and stale guards suppress the stash too; + * 6. the stash NEVER reaches the messages table: ingests of a session + * carrying the stash produce exactly the view's rows, refs included. */ import { test } from "node:test"; @@ -48,7 +47,7 @@ import { canonicalToolArgs, deriveCanonicalMessageId, deriveMessageId, openaiToC import { createSession, type Session } from "../src/session.ts"; import { createHandoffReply } from "../src/server/handoff-response.ts"; import { UnifiedStore } from "../src/storage/store.ts"; -import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; process.env.BILI_PERSIST_ZSTD = "0"; // Same hermeticity pin as the other storage suites. @@ -77,9 +76,9 @@ function capturedIds(session: Session): string[] { return (session.metadata.handoffResponse as { messages?: CoreMessage[] } | undefined)?.messages?.map((m) => m.id) ?? []; } -/** The stashed turn-final reply rows (direction C) decoded back to cores — +/** The stashed turn-final reply rows (direction C1) decoded back to cores — * row.content is the exact planner serialization, so parsing recovers the - * core the store will see. */ + * core a re-send would present. */ function stashCores(session: Session): CoreMessage[] { const stash = session.metadata.handoffReply as { rows?: Array<{ msgId: string; content: string }> } | undefined; return (stash?.rows ?? []).map((r) => JSON.parse(r.content) as CoreMessage); @@ -105,7 +104,7 @@ test("turn-final reply is stashed for persistence WITHOUT touching lastMessages assert.equal(session.lastMessages, tail, "lastMessages must stay the exact resident window (invariant 1)"); assert.equal(messages.length, 1, "the reply must NOT land in lastMessages — the view is the ledger's territory"); const stashed = stashCores(session); - assert.equal(stashed.length, 1, "the reply must be stashed for the store"); + assert.equal(stashed.length, 1, "the reply must be stashed for the DB"); const row = stashed[0]; assert.ok(row, "the stash row must exist"); assert.equal(row.role, "assistant"); @@ -117,7 +116,7 @@ test("turn-final reply is stashed for persistence WITHOUT touching lastMessages assert.deepEqual(capturedIds(session), ["reply_0"]); }); -test("a client re-send through the production codec derives the SAME ids (store join stability)", () => { +test("a client re-send through the production codec derives the SAME ids (join stability)", () => { const session = createSession("wb-resend"); session.lastMessages = convo([{ role: "user", content: "hello" }]); withReply(session, "héllo world"); @@ -163,14 +162,12 @@ test("tool calls stash with toolName/toolCallId preserved and id-derivable", () assert.equal(tool.toolCallId, "call_1"); // Canonical seed (#3642): toolCallId OUT of the id (transport plumbing), // args canonicalized. Legacy deriveMessageId (toolCallId slot filled, - // raw args) would mint a DIFFERENT id than the re-send codec pass — the - // re-send then joins by (msg_id, identity) and the old row is retired, - // costing a full re-ingest per tool call on every turn. Pinned here so - // the exact-join property cannot regress to that shape. + // raw args) would mint a DIFFERENT id than the re-send codec pass. + // Pinned here so the exact-match property cannot regress to that shape. assert.equal(tool.id, deriveCanonicalMessageId("assistant", "tool-call", canonicalToolArgs("{\"q\":1}"), { toolName: "search" })); }); -test("a client re-send of a TOOL-CALL reply through the production codec derives the SAME ids (store join stability)", () => { +test("a client re-send of a TOOL-CALL reply through the production codec derives the SAME ids (join stability)", () => { const session = createSession("wb-tool-resend"); session.lastMessages = convo([{ role: "user", content: "go" }]); session.stats.requests = 1; @@ -198,11 +195,12 @@ test("a client re-send of a TOOL-CALL reply through the production codec derives * (id/wireIndex stripped). The raw capture lacks the sidecars the client's * re-send codec stamps (openai `reasoningContent` on reasoning cores) and * keeps the streamed args verbatim while clients like pi re-serialize them - * compact — so without restampForJoin the re-send lands a SECOND row for - * the same logical message: tombstone churn + the content CAS storing the - * reply bytes twice, on every reply of every turn of a long session. + * compact — so without restampForJoin a DB reader deduping the stash + * against view rows would miss the match. The stash rows therefore carry + * the same restamped bytes the re-send will present, and their identity + * equals the planner identity of the re-sent cores. */ -test("stash bytes JOIN the client re-send by identity (openai lane: reasoning sidecar + compacted args)", () => { +test("stash rows JOIN the client re-send by identity (openai lane: reasoning sidecar + compacted args)", () => { const argsRaw = '{ "command" : "echo hi" }'; // streamed verbatim (host JSON style) const session = createSession("wb-join"); session.lastMessages = convo([{ role: "user", content: "go" }]); @@ -227,37 +225,27 @@ test("stash bytes JOIN the client re-send by identity (openai lane: reasoning si { role: "tool", tool_call_id: "call_1", content: "hi" }, ] }).msgs as unknown as CoreMessage[]; - const dir = tmpdir("bili-6947-join-"); - try { - const store = UnifiedStore.open(path.join(dir, "index.db")); - try { - const user = session.lastMessages![0]; - // Turn-end save: the view is [user] ONLY; the stash rows land as - // non-tail log rows (never tailSeqs), no refs minted yet. - assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [user], {}, stashMetadata(session)))), true); - let rows = store.getMessages("sjoin"); - assert.equal(rows.length, 4, `user + 3 stash rows (got ${rows.length}: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.retiredAt ?? "live"]))})`); - for (const m of stashed) { - const row = rows.find((r) => r.msgId === m.id); - assert.ok(row, `stash row for ${m.contentType} must exist after the turn-end save`); - assert.equal(row.ref, null, "no ref until the re-send (invariant 2)"); - } - // Next request: the client re-sends the whole turn. The reply - // cores must JOIN the stash rows — no second rows, seq stable. - assert.equal(store.ingestSession(planIngest(mkLegacy("sjoin", [...convo([{ role: "user", content: "go" }]), ...resent], { [resent[1].id]: "m00002" }, stashMetadata(session)))), true); - rows = store.getMessages("sjoin"); - assert.equal(rows.length, 5, `the re-send must join, not duplicate (got ${rows.length} rows: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.retiredAt ?? "live"]))})`); - for (const m of resent) { - const storedRow = rows.find((r) => r.msgId === m.id); - assert.ok(storedRow, `re-send core ${m.contentType} must share the stash row's id`); - } - const turnEndTool = rows.filter((r) => r.msgId === tool.id); - assert.equal(turnEndTool.length, 1, "tool-call: exactly one live row after the join"); - } finally { - store.close(); - } - } finally { - rmBestEffort(dir); + // DB-reader join key: (msgId, identity). The stash must present the SAME + // key the planner derives for the re-sent cores — verified by planning a + // session whose view IS the re-send (no stash involvement in the planner). + const plan = planIngest({ + id: "sjoinplan", + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "openai" }, + stats: { requests: 1 }, + metadata: {}, + state: { blocks: [], messageRefs: { byRaw: { [resent[1].id]: "m00002" }, byRef: {} }, nextBlockId: 1, nextRunId: 1 }, + blockContents: new Map(), + lastMessages: [convo([{ role: "user", content: "go" }])[0], resent[1], resent[2], resent[3]], + lastMessagesFolded: false, + } as unknown as LegacySessionLike); + assert.ok(plan, "planner must accept the re-send view"); + for (const m of stashed) { + const row = plan.messages.find((e) => e.msgId === m.id); + assert.ok(row, `re-send view must contain a row for stash msgId ${m.id}`); + const stashRow = (stashMetadata(session).handoffReply as { rows: { msgId: string; identity: string }[] }).rows.find((r) => r.msgId === m.id)!; + assert.equal(row.identity, stashRow.identity, `stash identity === planner identity for ${m.contentType} (same join key)`); } }); @@ -339,31 +327,26 @@ function turnFinalPair(): { user: CoreMessage; reply: CoreMessage; metadata: Rec return { user, reply, metadata: stashMetadata(session) }; } -test("store: turn-final row lands once, ref backfilled on the re-send ingest (NULL only)", () => { - const dir = tmpdir("bili-6947-backfill-"); +/** + * #6947 × #4316: the messages table's seq space is the ref-ledger substrate + * (on the first fold, live refs are restamped onto row seq+1). The stash + * must therefore NEVER materialize as a table row — a stray row would + * desync every post-fold ref (golden wire drift). What lands in the DB is + * `sessions.metadata_json.handoffReply`; the messages table stays exactly + * the view. + */ +test("store: the stash never reaches the messages table; the view's rows are exactly the view", () => { + const dir = tmpdir("bili-6947-table-"); try { const store = UnifiedStore.open(path.join(dir, "index.db")); try { const { user, reply, metadata } = turnFinalPair(); - // Turn-end save: the view is [user]; the stash row lands, its ref - // is not minted yet, and it is NOT in tailSeqs (view parity). + // Turn end: the session carries the stash in metadata; the + // planner must ignore it — table rows == view rows == [user]. assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user], {}, metadata))), true); let rows = store.getMessages("s6947"); - assert.equal(rows.length, 2, "user row + one stash row"); - const first = rows.find((r) => r.msgId === reply.id); - assert.ok(first, "the turn-final row must exist after the turn-end save"); - assert.equal(first.ref, null); - - // Next request: the client re-sends the tagged reply; the ref is - // minted. A full re-ingest joins the stored row and backfills. - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947", [user, reply], { [reply.id]: "m00002" }, metadata))), true); - rows = store.getMessages("s6947"); - assert.equal(rows.length, 2, "re-send must not duplicate the turn-final row"); - const joined = rows.find((r) => r.msgId === reply.id); - assert.ok(joined); - assert.equal(joined.seq, first.seq, "seq stability (D4-1) across the join"); - assert.equal(joined.ref, "m00002", "minted ref backfilled into the NULL row"); - assert.equal(rows.find((r) => r.msgId === user.id)?.ref, null, "unminted rows stay untouched"); + assert.equal(rows.length, 1, `only the user row may exist (got ${rows.length}: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.ref ?? "null"]))})`); + assert.equal(rows[0].msgId, user.id); } finally { store.close(); } @@ -372,30 +355,20 @@ test("store: turn-final row lands once, ref backfilled on the re-send ingest (NU } }); -test("store: backfill never rewrites an already-stored ref and never writes BLOCKED", () => { - const dir = tmpdir("bili-6947-guard-"); +test("store: the re-send lands as a normal view row with the kernel-minted ref", () => { + const dir = tmpdir("bili-6947-resend-"); try { const store = UnifiedStore.open(path.join(dir, "index.db")); try { const { user, reply, metadata } = turnFinalPair(); - // Ref minted at insert time (the common flow). - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00002" }, metadata))), true); - // A later ingest with a different minted ref must NOT rewrite it. - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [user, reply], { [reply.id]: "m00009" }, metadata))), true); - let row = store.getMessages("s6947g").find((r) => r.msgId === reply.id); - assert.equal(row?.ref, "m00002", "stored ref is frozen (index-time projection)"); - - // BLOCKED is a byRaw-only token: it must not land in messages.ref. - const session = createSession("wb-store-blocked"); - session.lastMessages = convo([{ role: "user", content: "install it" }]); - withReply(session, "blocked reply"); - const u2 = session.lastMessages![0]; - const r2 = stashCores(session)[0]; - const md2 = stashMetadata(session); - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2], {}, md2))), true); - assert.equal(store.ingestSession(planIngest(mkLegacy("s6947b", [u2, r2], { [r2.id]: "BLOCKED" }, md2))), true); - row = store.getMessages("s6947b").find((r) => r.msgId === r2.id); - assert.equal(row?.ref, null, "BLOCKED must stay out of messages.ref"); + // Next request: the client re-sends the reply; it becomes a + // normal view row with the kernel-minted ref — pre-#6947 + // behavior, no stash involvement. + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947r", [user, reply], { [reply.id]: "m00002" }, metadata))), true); + const rows = store.getMessages("s6947r"); + assert.equal(rows.length, 2, `both view rows exist (got ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.ref ?? "null"]))})`); + assert.equal(rows.find((r) => r.msgId === reply.id)?.ref, "m00002", "kernel-minted ref lands at insert time"); + assert.equal(rows.find((r) => r.msgId === user.id)?.seq, 0, "view row seqs stay contiguous (ledger substrate)"); } finally { store.close(); } @@ -404,56 +377,25 @@ test("store: backfill never rewrites an already-stored ref and never writes BLOC } }); -/** - * #3133 × #6947: the delta fast path must stay green across the turn-end → - * re-send boundary. Turn end is an equal-length raw delta whose only payload - * is the stash (non-tail rows); the re-send then presents the reply as a - * NORMAL tail message that joins the stored stash row in place (seq kept, - * NULL ref claimed) instead of tripping the strictly-append guard into an - * O(n) full re-ingest on every turn of every long session. - */ -test("store: delta fast path survives the turn-end stash save and the re-send join", () => { - const dir = tmpdir("bili-6947-delta-"); +test("store: ref backfill never rewrites a stored ref and never writes BLOCKED", () => { + const dir = tmpdir("bili-6947-guard-"); try { const store = UnifiedStore.open(path.join(dir, "index.db")); try { - const session = createSession("wb-delta"); - const u1 = convo([{ role: "user", content: "install it" }])[0]; - session.lastMessages = [u1]; - withReply(session, "done: installed"); - const reply = stashCores(session)[0]; - const md = stashMetadata(session); - - // Request N (full path primes the watermark at the user row). - assert.equal(store.ingestSession(planIngest(mkLegacy("sd", [u1], {}))), true); - const boundary = store.probeIngestBoundary("sd", 0); - assert.ok(boundary, "watermark probe must find the user row at seq 0"); - - // Turn N end: equal-length raw delta, payload = the stash row. - const turnEnd = planIngestDelta({ ...(mkLegacy("sd", [u1], {}, md) as object) } as LegacySessionLike, 0, [0], { msgId: boundary.msgId, contentHash: boundary.contentHash }, true, null); - assert.ok(turnEnd, "the turn-end save must take the delta fast path (not the full re-ingest)"); - assert.equal(turnEnd.nonTailCount, 1, "the stash row is the only payload and it is non-tail"); - assert.deepEqual(turnEnd.tailSeqs, [0], "the stash row must NOT extend tailSeqs (view parity)"); - assert.equal(store.ingestSession(turnEnd, { trustedSeq: 0, boundaryMsgId: boundary.msgId, boundaryHash: boundary.contentHash, trustedTailSeqs: [0] }), true, "the turn-end delta must commit"); - let rows = store.getMessages("sd"); - assert.equal(rows.length, 2, "user + stash row"); - assert.equal(rows.find((r) => r.msgId === reply.id)?.ref, null, "still no ref at turn end"); - const stashSeq = rows.find((r) => r.msgId === reply.id)!.seq; - - // Request N+1: the client re-sends [u1, reply] plus a new user - // turn. The reply is a TAIL row now and joins the stored stash - // row in place; the new row appends. - const u2 = convo([{ role: "user", content: "next" }])[0]; - const resend = { ...(mkLegacy("sd", [u1, reply, u2], { [reply.id]: "m00002" }, md) as object) } as LegacySessionLike; - const resendPlan = planIngestDelta(resend, 0, [0], { msgId: boundary.msgId, contentHash: boundary.contentHash }, true, null); - assert.ok(resendPlan, "the re-send must take the delta fast path (join in place, no full re-ingest)"); - assert.equal(resendPlan.nonTailCount, 0, "the stash is gated out once the reply is a view row"); - assert.equal(store.ingestSession(resendPlan, { trustedSeq: 0, boundaryMsgId: boundary.msgId, boundaryHash: boundary.contentHash, trustedTailSeqs: [0] }), true, "the re-send delta must commit"); - rows = store.getMessages("sd"); - assert.equal(rows.length, 3, `no duplicate rows after the join (got ${rows.length}: ${JSON.stringify(rows.map((r) => [r.seq, r.msgId, r.ref ?? "null"]))})`); - const joined = rows.find((r) => r.msgId === reply.id)!; - assert.equal(joined.seq, stashSeq, "the joined row keeps its stored seq"); - assert.equal(joined.ref, "m00002", "the NULL ref was claimed on the join"); + const session = createSession("wb-store-blocked"); + session.lastMessages = convo([{ role: "user", content: "install it" }]); + withReply(session, "blocked reply"); + const u2 = session.lastMessages![0]; + const r2 = stashCores(session)[0]; + const md2 = stashMetadata(session); + // Ref minted at insert time (the common flow). + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [u2, r2], { [r2.id]: "m00002" }, md2))), true); + // A later ingest presenting the same row under a BLOCKED token + // (or any other ref) must NOT rewrite the stored projection: + // the backfill only claims NULL refs (#6953 guard). + assert.equal(store.ingestSession(planIngest(mkLegacy("s6947g", [u2, r2], { [r2.id]: "BLOCKED" }, md2))), true); + const row = store.getMessages("s6947g").find((r) => r.msgId === r2.id)!; + assert.equal(row.ref, "m00002", "stored ref is frozen (index-time projection); backfill never rewrites"); } finally { store.close(); }