From e4f11847df6272198a0adab58f1f6ad9833b1ce1 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 12 Oct 2026 12:56:19 +0800 Subject: [PATCH 1/4] fix: compute body-stability LCP over real UTF-8 bytes (#2868) --- src/cache-ledger.ts | 70 ++++++---- tests/issue2868-byte-lcp-utf8.test.ts | 188 ++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 25 deletions(-) create mode 100644 tests/issue2868-byte-lcp-utf8.test.ts diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index b89b4299d0..02d6ba5ae6 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -376,12 +376,21 @@ const BODY_FULL_CAP = 8 * 1024 * 1024; * `null` = unknown (wire shape without a message array). `digest` hashes the * FULL body (the capped head cannot distinguish two payloads sharing 512 KiB); * `full` keeps the whole payload while it fits BODY_FULL_CAP so divergent - * pairs can report their first-diverging byte offset. */ + * pairs can report their first-diverging byte offset. #2868: bodies are kept + * as UTF-8 Buffers — every cap, LCP and tolerance in this file is BYTES, and + * the capture-time encoding is reused instead of re-encoded per comparison + * (a code-unit LCP misreported CJK offsets by up to 3×, and slicing a string + * then re-encoding cannot repair it: distinct codepoints share leading + * UTF-8 bytes). #2923: slots live under session.transient so the retained + * copies release with the session. */ export interface SeamSlot { - str: string; + /** Capped head (≤ SEAM_BODY_CAP bytes) — seam forensics + JSON parse side. */ + head: Buffer; + /** Exact outbound body size in bytes — precise cap/full detection. */ + bytes: number; msgs: number | null; digest: string; - full: string | null; + full: Buffer | null; /** #2131: credential fingerprint of the headers that carried THIS body out. * Preserved across byte-identical re-notes (the non-streaming settle * re-notes the same wireBody without headers in hand). */ @@ -530,23 +539,26 @@ export function credentialFingerprint(headers: Record | undefine * the outbound credential fingerprint (#2131) — likewise inherited from the * prior slot on a byte-identical re-note without a fresh capture. */ export function noteForwardedBody(session: Session, body: string, msgs?: number | null, keyFp?: string): void { - // V8 的 substring 可保留整个请求;UTF-16 拷贝独立分配且保留孤立 surrogate。 - const str = Buffer.from(body.slice(0, SEAM_BODY_CAP), "utf16le").toString("utf16le"); + // #2868: encode to UTF-8 ONCE here — digest, capped head and retained full + // body all consume this same buffer, so no later comparison re-encodes. + // Caps are BYTES (the old code-unit caps overshot them up to 3× on CJK). + // The buffer is a fresh allocation and nothing below retains `body` itself + // (#2923: the request string releases once this returns). // #2131: hash the FULL body — the capped head alone cannot tell two payloads // sharing a 512 KiB prefix apart. Re-noting a byte-identical payload (the // non-streaming settle path) reuses the prior slot's count/full instead of // re-deriving them. - const digest = createHash("sha256").update(body, "utf8").digest("hex"); + const buf = Buffer.from(body, "utf8"); + const digest = createHash("sha256").update(buf).digest("hex"); const transient = session.transient ??= {}; const prior = transient.sentBody; const samePayload = prior !== undefined && prior.digest === digest; transient.sentBody = { - str, + head: buf.byteLength > SEAM_BODY_CAP ? Buffer.from(buf.subarray(0, SEAM_BODY_CAP)) : buf, + bytes: buf.byteLength, msgs: msgs ?? (samePayload ? prior.msgs : null), digest, - full: body.length <= SEAM_BODY_CAP ? str - : body.length <= BODY_FULL_CAP ? Buffer.from(body, "utf16le").toString("utf16le") - : samePayload ? prior.full : null, + full: buf.byteLength <= BODY_FULL_CAP ? buf : samePayload ? prior.full : null, keyFp: keyFp ?? (samePayload ? prior.keyFp : undefined), }; // prepare may reuse a measured baseline; this new payload is not measured yet. @@ -622,19 +634,23 @@ export function learnedImageReserve(session: Session, host: string, nImages: num return per * nImages; } -function seamLcp(a: string, b: string): { lcpBytes: number; msgIndex: number | null; prevMsgs: number; curMsgs: number } { +// #2868: compares UTF-8 BYTES — lcpBytes feeds byte thresholds and displays, +// so comparing UTF-16 code units misreported every non-ASCII offset. +function seamLcp(a: Buffer, b: Buffer): { lcpBytes: number; msgIndex: number | null; prevMsgs: number; curMsgs: number } { let lcp = 0; const n = Math.min(a.length, b.length); - while (lcp < n && a.charCodeAt(lcp) === b.charCodeAt(lcp)) lcp++; + while (lcp < n && a[lcp] === b[lcp]) lcp++; // #2339: the message array lives under `messages` (Anthropic/chat), `input` // (Responses) or `contents` (Google native) — same caliber as the send-time // count captures (server.ts / loop/core.ts). Reading only `.messages` made // every Responses/Google pair parse to zero messages: msgIndex degenerated // to a fallback 0 that rendered as a real "message[0]", and the count-based - // classification arms went dead on those wires. - const msgsOf = (s: string): unknown[] => { + // classification arms went dead on those wires. A capped head may end mid + // character; toString("utf8") replaces it and the truncated JSON fails to + // parse — the same outcome as before #2868. + const msgsOf = (s: Buffer): unknown[] => { try { - const p = JSON.parse(s) as { messages?: unknown; input?: unknown; contents?: unknown }; + const p = JSON.parse(s.toString("utf8")) as { messages?: unknown; input?: unknown; contents?: unknown }; const arr = p.messages ?? p.input ?? p.contents; return Array.isArray(arr) ? arr : []; } catch { @@ -654,10 +670,13 @@ function seamLcp(a: string, b: string): { lcpBytes: number; msgIndex: number | n // #2131: byte-LCP WITHOUT the JSON parse seamLcp does — the per-call stability // check runs on every settled pair and must stay cheap on multi-MiB bodies // (message-level detail belongs to detectSeam's forensics path only). -function byteLcp(a: string, b: string): number { +// #2868: real bytes over the capture-time buffers — the old charCodeAt loop +// returned UTF-16 code units, so CJK offsets were short by up to 3× and the +// 1024-byte head threshold / ±64-byte append tolerance misfired on them. +function byteLcp(a: Buffer, b: Buffer): number { let lcp = 0; const n = Math.min(a.length, b.length); - while (lcp < n && a.charCodeAt(lcp) === b.charCodeAt(lcp)) lcp++; + while (lcp < n && a[lcp] === b[lcp]) lcp++; return lcp; } @@ -751,12 +770,13 @@ function detectSeam(session: Session, led: CacheLedger): void { }; let f: ReturnType | undefined; if (cur !== undefined && prev !== undefined) { - f = seamLcp(prev.str, cur.str); + f = seamLcp(prev.head, cur.head); // #2131: a body stored at full length is readable; one clipped at // SEAM_BODY_CAP parses to zero messages, so parse-derived arms below - // only decide pairs where that side is under the cap. - const prevCapped = prev.str.length >= SEAM_BODY_CAP; - const curCapped = cur.str.length >= SEAM_BODY_CAP; + // only decide pairs where that side is under the cap. #2868: capped + // means CLIPPED (bytes > cap) — an exactly-at-cap body is fully stored. + const prevCapped = prev.bytes > SEAM_BODY_CAP; + const curCapped = cur.bytes > SEAM_BODY_CAP; // Client reverted/trimmed history: the miss is the sanctioned // one-time re-bill of the retained prefix (or the gap's TTL). Only // trust the parsed counts when BOTH sides are fully stored — a side @@ -768,7 +788,7 @@ function detectSeam(session: Session, led: CacheLedger): void { agg.rewindMissed += line.tr; return; } - if (cur.str.length < prev.str.length && f.lcpBytes >= cur.str.length - 4) { + if (cur.head.byteLength < prev.head.byteLength && f.lcpBytes >= cur.head.byteLength - 4) { // #2131: the WHOLE current payload survives inside the previous // one — a JSON array only diverges from its own longer superstring // in the closing-bracket region (the 4-byte tolerance), so this is @@ -780,7 +800,7 @@ function detectSeam(session: Session, led: CacheLedger): void { agg.rewindMissed += line.tr; return; } - if (f.lcpBytes >= cur.str.length) { + if (f.lcpBytes >= cur.head.byteLength) { // Wire was byte-stable against the previous request — the // upstream simply did not serve its cache. Provider-side. Above // the cap this covers the identical-recorded-head case: anything @@ -801,7 +821,7 @@ function detectSeam(session: Session, led: CacheLedger): void { bookProvider(); return; } - if (prev.str.length < cur.str.length && f.lcpBytes >= prev.str.length - 4) { + if (prev.head.byteLength < cur.head.byteLength && f.lcpBytes >= prev.head.byteLength - 4) { // #2131: the WHOLE previous payload survives inside the longer // current one (JSON appends diverge only in the previous body's // closing-bracket region — the 4-byte tolerance). Under the cap the @@ -855,7 +875,7 @@ function detectSeam(session: Session, led: CacheLedger): void { led.seamEvents.shift(); } if (cur !== undefined && prev !== undefined) { - const forensics = f ?? seamLcp(prev.str, cur.str); + const forensics = f ?? seamLcp(prev.head, cur.head); const ev: SeamEvent = { seq: line.seq, at: line.at, input: line.input, hitPct: line.hitPct ?? 0, ...forensics, prevMsgs: prev.msgs ?? forensics.prevMsgs, curMsgs: cur.msgs ?? forensics.curMsgs }; (led.seamEvents ?? (led.seamEvents = [])).push(ev); if (agg.seamSuspects === 1) { diff --git a/tests/issue2868-byte-lcp-utf8.test.ts b/tests/issue2868-byte-lcp-utf8.test.ts new file mode 100644 index 0000000000..37e7b6fe34 --- /dev/null +++ b/tests/issue2868-byte-lcp-utf8.test.ts @@ -0,0 +1,188 @@ +// #2868: the body-stability LCP compared UTF-16 code units but every consumer +// treated the result as UTF-8 bytes — LedgerLine.bl, SeamEvent.lcpBytes, the +// 1024-byte head threshold, the ±64-byte append tolerance and the 512 KiB / +// 8 MiB caps. Non-ASCII bodies got wrong offsets (up to 3× short on CJK) and +// misclassified divergence classes. These tests pin the byte-unit contract +// end-to-end: capture (noteForwardedBody) → settle (bl/bs) → seam forensics +// (lcpBytes). The expected values come from an independent reference that +// walks real UTF-8 Buffers — never from the implementation under test. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { noteForwardedBody, settleUsageReport, getCacheLedger } from "../src/cache-ledger.ts"; +import type { Session } from "../src/session.ts"; + +let seq = 0; +function makeSession(): Session { + seq += 1; + return { + id: `byteu8-${seq}`, + metadata: {}, + stats: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} }, + } as unknown as Session; +} + +const T0 = Date.parse("2026-10-12T10:00:00Z"); + +function settle(session: Session, at: number, input: number, cached: number): void { + settleUsageReport(session, { total: input, reportedCached: cached, output: 0, protocol: "openai", upstream: "http://u" }); +} + +function lastLine(s: Session): { be?: number; bl?: number; bs?: string; bd?: string } { + const led = getCacheLedger(s); + return led.lines[led.lines.length - 1]!; +} + +function pair(s: Session, b1: string, b2: string): { be?: number; bl?: number; bs?: string; bd?: string } { + noteForwardedBody(s, b1); + settle(s, T0, 100_000, 99_000); + noteForwardedBody(s, b2); + settle(s, T0 + 1000, 100_000, 90_000); + return lastLine(s); +} + +function utf8Lcp(a: string, b: string): number { + const aa = Buffer.from(a, "utf8"); + const bb = Buffer.from(b, "utf8"); + let n = 0; + while (n < Math.min(aa.length, bb.length) && aa[n] === bb[n]) n++; + return n; +} + +function codeUnitLcp(a: string, b: string): number { + let n = 0; + const lim = Math.min(a.length, b.length); + while (n < lim && a.charCodeAt(n) === b.charCodeAt(n)) n++; + return n; +} + +test("the issue's five synthetic pairs report true UTF-8 byte offsets", () => { + const cases: Array<[string, string, number]> = [ + ["abcX", "abcY", 3], + ["中X", "中Y", 3], + ["\u4e2d", "\u4e2e", 2], + ["\u4e2d".repeat(400) + "A".repeat(200), "\u4e2d".repeat(400) + "B".repeat(200), 1200], + [String.fromCodePoint(0x20000) + "X", String.fromCodePoint(0x20400) + "Y", 2], + ]; + for (const [a, b, want] of cases) { + const s = makeSession(); + const line = pair(s, a, b); + assert.equal(line.be, 0); + assert.equal(line.bl, want, `byte LCP must count UTF-8 bytes (case ${JSON.stringify(a.slice(0, 8))})`); + } +}); + +test("CJK pair with a 1200-byte LCP classifies mid, not head (the #2868 regression)", () => { + const s = makeSession(); + const line = pair(s, "\u4e2d".repeat(400) + "A".repeat(200), "\u4e2d".repeat(400) + "B".repeat(200)); + // Each payload is 1400 bytes; 1200 is past the 1024-byte head threshold and + // not within ±64 of the tail → mid. The old code-unit math read 400 → head. + assert.equal(line.bs, "mid"); +}); + +test("non-ASCII divergence inside the first 1 KiB reports the byte-exact head offset", () => { + const s = makeSession(); + const a = "A".repeat(100) + "\u4e2dX" + "A".repeat(1000); + const b = "A".repeat(100) + "\u4e2dY" + "A".repeat(1000); + const line = pair(s, a, b); + assert.equal(line.bl, 103, "100 ASCII bytes plus the full 3-byte CJK char survive (old code read 101 code units)"); + assert.equal(line.bs, "head"); +}); + +test("JSON request: divergence inside a Chinese message reports the JSON byte offset", () => { + const s = makeSession(); + const b1 = JSON.stringify({ model: "m", messages: [{ role: "user", content: "p".repeat(1200) }, { role: "user", content: "相同前缀甲" }, { role: "user", content: "q".repeat(400) }] }); + const b2 = JSON.stringify({ model: "m", messages: [{ role: "user", content: "p".repeat(1200) }, { role: "user", content: "相同前缀乙" }, { role: "user", content: "q".repeat(400) }] }); + const line = pair(s, b1, b2); + assert.equal(line.be, 0); + assert.equal(line.bl, utf8Lcp(b1, b2)); + assert.ok(line.bl !== undefined && line.bl > 1024 && line.bl < Buffer.byteLength(b1) - 64, "divergence sits mid-body in the JSON text"); + assert.equal(line.bs, "mid"); + assert.equal(line.bd, createHash("sha256").update(b2, "utf8").digest("hex"), "digest stays the ground-truth equality proof"); +}); + +test("append and trim with CJK content keep the append class at byte-exact offsets", () => { + const one = { role: "user", content: "中文内容" }; + const two = { role: "user", content: "追加的消息" }; + const b1 = JSON.stringify({ model: "m", messages: [one] }); + const b2 = JSON.stringify({ model: "m", messages: [one, two] }); + const s = makeSession(); + const line = pair(s, b1, b2); + assert.equal(line.bs, "append"); + assert.equal(line.bl, utf8Lcp(b1, b2)); + assert.ok(line.bl !== undefined && Math.abs(line.bl - Buffer.byteLength(b1)) <= 64, "append diverges only in the closing-bracket region"); + + const s2 = makeSession(); + const trimmed = pair(s2, b2, b1); + assert.equal(trimmed.bs, "append", "trimming back to the shorter payload also appends (whole short side survives)"); + assert.equal(trimmed.bl, utf8Lcp(b1, b2)); +}); + +test("caps are enforced in BYTES: a clipped CJK head still settles via the retained full body", () => { + const s = makeSession(); + const cjk = "\u4e2d".repeat(262_145); // 786_435 bytes > 512 KiB, yet < 512 KiB code units + const b1 = JSON.stringify({ model: "m", messages: [{ role: "user", content: cjk + "AAA" }] }); + const b2 = JSON.stringify({ model: "m", messages: [{ role: "user", content: cjk + "BBB" }] }); + assert.ok(Buffer.byteLength(b1) > 512 * 1024 && b1.length < 512 * 1024, "rig: over the byte cap, under the old code-unit cap"); + const line = pair(s, b1, b2); + assert.equal(line.be, 0); + assert.equal(line.bl, utf8Lcp(b1, b2), "full bodies are retained (< 8 MiB) so the offset stays byte-exact despite the clipped seam head"); + assert.equal(line.bs, "append", "the 3-char suffix difference sits within ±64 of the tail"); +}); + +test("over BODY_FULL_CAP in bytes but under it in code units is digest-only (bs=unknown)", () => { + const s = makeSession(); + const big = "\u4e2d".repeat(4_200_001); // ~12.6 MB > 8 MiB bytes, yet < 8 MiB code units (the old code RETAINED these) + const b1 = JSON.stringify({ model: "m", messages: [{ role: "user", content: big + "A" }] }); + const b2 = JSON.stringify({ model: "m", messages: [{ role: "user", content: big + "B" }] }); + assert.ok(Buffer.byteLength(b1) > 8 * 1024 * 1024 && b1.length < 8 * 1024 * 1024, "rig: over the byte cap, under the old code-unit cap"); + const line = pair(s, b1, b2); + assert.equal(line.be, 0, "inequality proven by digest without retention"); + assert.equal(line.bs, "unknown"); + assert.equal(line.bl, undefined); + assert.equal(line.bd, createHash("sha256").update(b2, "utf8").digest("hex")); +}); + +test("identical CJK bodies stay be=1 with no divergence fields", () => { + const s = makeSession(); + const b = JSON.stringify({ model: "m", messages: [{ role: "user", content: "中文重复" }] }); + noteForwardedBody(s, b); + settle(s, T0, 100_000, 99_000); + noteForwardedBody(s, b); + settle(s, T0 + 1000, 100_000, 99_000); + const line = lastLine(s); + assert.equal(line.be, 1); + assert.equal(line.bl, undefined); + assert.equal(line.bs, undefined); + assert.equal(line.bd, createHash("sha256").update(b, "utf8").digest("hex")); +}); + +test("oversized (16 MiB) bodies stay fast on the digest-only path", () => { + const s = makeSession(); + const huge = "x".repeat(16 * 1024 * 1024); + const b1 = JSON.stringify({ model: "m", messages: [{ role: "user", content: huge }] }); + const b2 = JSON.stringify({ model: "m", messages: [{ role: "user", content: huge + "z" }] }); + const t0 = Date.now(); + const line = pair(s, b1, b2); + const ms = Date.now() - t0; + assert.equal(line.be, 0); + assert.equal(line.bs, "unknown"); + assert.ok(ms < 5000, `single encode + digest must stay cheap on multi-MiB bodies (took ${ms}ms)`); +}); + +test("seam forensics reports byte-exact lcpBytes for CJK bodies", () => { + const s = makeSession(); + const mk = (c1: string, c2: string): string => JSON.stringify({ model: "m", messages: [{ role: "user", content: c1 }, { role: "user", content: c2 }] }); + const b1 = mk("第一条消息的内容", "第二条消息的内容"); + const b2 = mk("第一条消息的内容", "第二条消息被改过"); + noteForwardedBody(s, b1); + settle(s, T0, 100_000, 99_000); + noteForwardedBody(s, b2); + settle(s, T0 + 1000, 100_000, 20_000); + const ev = getCacheLedger(s).seamEvents?.[0]; + assert.ok(ev, "unexplained CJK mid-history break records a seam event"); + assert.equal(ev!.msgIndex, 1); + assert.equal(ev!.lcpBytes, utf8Lcp(b1, b2), "SeamEvent.lcpBytes must be real UTF-8 bytes"); + assert.notEqual(ev!.lcpBytes, codeUnitLcp(b1, b2), "must NOT be the old code-unit value"); +}); From 8cd5b89e80cf1fad7b266eca85aee0cfd6a8d754 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 12 Oct 2026 22:22:42 +0800 Subject: [PATCH 2/4] test: adapt #2923 transient-memory test to byte-unit seam slot (#2868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retention assertions now read the UTF-8 encoding back out of the Buffer slot (byte-equal for valid UTF-8 bodies), and a new test pins that a lone surrogate is retained as U+FFFD — it has no UTF-8 form. Heap-probe thresholds unchanged. --- tests/session-transient-memory.test.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/session-transient-memory.test.ts b/tests/session-transient-memory.test.ts index 4b4c96a9db..f25d6c8bf0 100644 --- a/tests/session-transient-memory.test.ts +++ b/tests/session-transient-memory.test.ts @@ -10,18 +10,29 @@ import { rememberPluginMessages, _rememberedForTest } from "../src/plugin.ts"; import { createSession, getSession, dropSessionForGc, _resetSessionsForTest } from "../src/session.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; -test("#2920 诊断副本保留 UTF-16 内容,结算只转移会话内引用", () => { +test("#2920/#2868 诊断副本保留 UTF-8 内容,结算只转移会话内引用", () => { const session = createSession("transient-seam"); - const body = JSON.stringify({ messages: [{ role: "user", content: "中😀" }] }) + "\ud800"; + const body = JSON.stringify({ messages: [{ role: "user", content: "中😀" }] }); noteForwardedBody(session, body); const sent = session.transient?.sentBody; - assert.equal(sent?.str, body); - assert.equal(sent?.full, body); + // #2868: slots retain the UTF-8 ENCODING of the body (bytes, not code + // units) — byte-equal here because the body is valid UTF-8. + assert.equal(sent?.head.toString("utf8"), body); + assert.equal(sent?.full?.toString("utf8"), body); settleUsageReport(session, { total: 1000, reportedCached: 0, output: 10, protocol: "openai", upstream: "http://synthetic" }); assert.equal(session.transient?.sentBody, undefined); assert.equal(session.transient?.settledBody, sent); }); +test("#2868 孤立代理项无 UTF-8 形式,保留字节记录 U+FFFD", () => { + const session = createSession("surrogate-seam"); + // A lone surrogate has no UTF-8 form: retained bytes carry the wire-visible + // replacement char, never a raw half-surrogate (#2868's unit is bytes). + noteForwardedBody(session, "a\ud800b"); + const slot = session.transient?.sentBody; + assert.equal(slot?.head.toString("utf8"), "a\ufffdb"); +}); + test("#2920 派生缓存不落盘,淘汰后不能重新挂上工具快照", async () => { const dir = mkdtempSync(path.join(tmpdir(), "bili-transient-persist-")); const store = new SessionStore({ dir, enabled: true, debounceMs: 0 }); @@ -136,7 +147,7 @@ test("#2920 完整诊断副本也不保留其截取前的大字符串", (t) => { console.log(JSON.stringify({ retained, released })); `); t.diagnostic(JSON.stringify(result)); - // 完整 UTF-16 副本可能每字符两字节;上界仍远低于原始 96 MiB。 + // #2868 后完整副本是 UTF-8:ASCII hex 语料每字符一字节(原 UTF-16 为二);上界仍远低于原始 96 MiB。 assert.ok(result.retained < 48 * 1024 * 1024, JSON.stringify(result)); assert.ok((result.released ?? 0) > 12 * 1024 * 1024, JSON.stringify(result)); }); From 706508a3d130e3ebacaac69ec093c818a958111a Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 14 Oct 2026 00:23:21 +0800 Subject: [PATCH 3/4] fix: adapt #2982 tail-class scan and #3023 client arm to byte-unit seam slot (#2868) --- src/cache-ledger.ts | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index 02d6ba5ae6..e4f0624a86 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -687,7 +687,14 @@ function byteLcp(a: Buffer, b: Buffer): number { // re-serialized element offsets do not match the original text (whitespace), // and one linear pass keeps multi-MiB bodies cheap. Runs only on pairs that // would otherwise class "mid" — the hot path never pays for it. -function messagesArrayCloseOffset(body: string): number | null { +// #2868: scans the UTF-8 Buffer directly, returning a BYTE offset comparable +// to byteLcp's result. Safe on multi-byte content: every delimiter checked +// here (quotes/backslash/braces/brackets/whitespace ≤0x20) is ASCII, and +// UTF-8 continuation bytes (0x80–0xBF) and lead bytes (0xC0+) can never +// equal an ASCII code point — a multi-byte character cannot hide or fake a +// delimiter. Escapes consume exactly one following byte, matching valid JSON +// where the escaped char after '\' is always ASCII. +function messagesArrayCloseOffset(body: Buffer): number | null { const n = body.length; let depth = 0; // object/array nesting depth let inStr = false; @@ -695,7 +702,7 @@ function messagesArrayCloseOffset(body: string): number | null { let want = -1; // depth at which the tracked array closes (-1 = not tracking) let keyAt = -1; // opening-quote offset of a candidate top-level key for (let i = 0; i < n; i++) { - const c = body.charCodeAt(i); + const c = body[i]; if (inStr) { if (esc) esc = false; else if (c === 92) esc = true; // backslash @@ -703,17 +710,17 @@ function messagesArrayCloseOffset(body: string): number | null { inStr = false; if (keyAt >= 0) { // +1/-1 strip the surrounding quotes from the captured range. - const key = body.slice(keyAt + 1, i); + const key = body.subarray(keyAt + 1, i).toString("utf8"); keyAt = -1; // A top-level string VALUE is also seen at depth 1 — the // following-colon check below is what separates keys from it. if (depth === 1 && (key === "messages" || key === "input" || key === "contents")) { let j = i + 1; - while (j < n && body.charCodeAt(j) <= 32) j++; - if (body.charCodeAt(j) === 58) { // ':' + while (j < n && body[j] <= 32) j++; + if (body[j] === 58) { // ':' j++; - while (j < n && body.charCodeAt(j) <= 32) j++; - if (body.charCodeAt(j) === 91) want = depth + 1; // '[' + while (j < n && body[j] <= 32) j++; + if (body[j] === 91) want = depth + 1; // '[' } } } @@ -857,7 +864,7 @@ function detectSeam(session: Session, led: CacheLedger): void { const m = curIn.meta[k] ?? { role: "?", len: 0, cls: "?" }; if (cur !== undefined && prev !== undefined) { if (led.seamEvents !== undefined && led.seamEvents.length >= SEAM_EVENTS_CAP) led.seamEvents.shift(); - const forensics = f ?? seamLcp(prev.str, cur.str); + const forensics = f ?? seamLcp(prev.head, cur.head); (led.seamEvents ?? (led.seamEvents = [])).push({ seq: line.seq, at: line.at, input: line.input, hitPct: line.hitPct ?? 0, ...forensics, prevMsgs: prev.msgs ?? forensics.prevMsgs, curMsgs: cur.msgs ?? forensics.curMsgs, client: { idx: k, role: m.role, len: m.len, cls: m.cls } }); } if (agg.clientInserts === 1) { From 51881022a8e5f09c9f03a5a9d7119ea1937919fc Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 14 Oct 2026 00:23:21 +0800 Subject: [PATCH 4/4] test: regen storage-render goldens on rebased tree (#2868) --- tests/golden/storage-render/anthropic.json | 4 ++-- tests/golden/storage-render/chat.json | 4 ++-- tests/golden/storage-render/google.json | 4 ++-- tests/golden/storage-render/responses.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index f7b97f7709..719b61ec48 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "f72372b9bcbe7dfbf0b9bc122746719c7c4a10eade36dfb461e6490cb71dbcdc", + "stateDigest": "0585fc694cbddd0ff4fffb2c89bc7a1ca2007e865a0d4575015f95eb5bab5f33", "sectionDigests": { "state": "5db7a736df3a3e5b", "blockContents": "dc7d13d250c94bb4", "stats": "d296e6929cff0970", - "metadata": "f29b72d540702a13" + "metadata": "d9ecadbcc477db25" }, "stateKeyDigests": { "blocks": "1cbe172309e4490d", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 26c700d1c7..7fdf9c3978 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "2d9fd2e429b0b40dd795a1dda2d57212eb196543740c5087c32f8abcc94f2ac9", + "stateDigest": "911a462160bbac3d9a9f44824403c3a1b0ad45b8f14eb1d6bd9bbdbe635f7cec", "sectionDigests": { "state": "bba2f434570acbd2", "blockContents": "94a58f96d7c5cfc3", "stats": "0387e0a7f852af4e", - "metadata": "7fa67cfd26330ecf" + "metadata": "0353a8c4e631df0b" }, "stateKeyDigests": { "blocks": "9aa73e5fb9f93d6c", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 2ef4620abf..bf63a7de42 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "d2963290f33409d9a460b3fade8b24803a286bd146ae11a9ff0f6a8279eddfff", + "stateDigest": "a0ad11b7ba567ea6d182847f2a9a17c9700b1793f241c6e08adfe73c5172a112", "sectionDigests": { "state": "6d2f312d7bc924d7", "blockContents": "473264d9df8ba338", "stats": "5702a6493cdfe768", - "metadata": "76bf7bd95a1e6044" + "metadata": "fe714004e9ae3104" }, "stateKeyDigests": { "blocks": "8e14f0879bf7ebd7", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 9c90234662..dac5c04d24 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "d3044b3993b2a7c959ff94020c0eb7b033e62809320ed3760ee69b057eb215a3", + "stateDigest": "616f038d216fa76fca77d2b96cfc37b3ba3a256b24f433a2acc11e6e30d7eed3", "sectionDigests": { "state": "07823f2d3271c52a", "blockContents": "8460729b2654a816", "stats": "c21b1ef672b5274c", - "metadata": "cf85986f245b7d38" + "metadata": "8a2a9b8cb52d676f" }, "stateKeyDigests": { "blocks": "7fff1dfb54a617cf",