diff --git a/src/acp-cache-diff.ts b/src/acp-cache-diff.ts index 6df7c6b095..15d28b0358 100644 --- a/src/acp-cache-diff.ts +++ b/src/acp-cache-diff.ts @@ -194,6 +194,15 @@ function prefixCompare(a: Buffer, b: Buffer): { rel: "append" | "diverge"; offse return { rel: "diverge", offset: n, extra: b.length - a.length }; } +// Wire offsets are UTF-8 bytes; JSON spans and string slices use UTF-16. +// A differing byte can be inside a multibyte code point, so begin evidence at +// its leading byte instead of decoding a partial prefix as a replacement char. +function textOffsetAtByte(body: Buffer, offset: number): number { + let start = offset; + while (start > 0 && start < body.length && (body[start]! & 0xc0) === 0x80) start--; + return body.subarray(0, start).toString("utf8").length; +} + // #3476: true iff `prev` is fully covered by the common prefix (length `lcp`) // plus some common suffix β€” i.e. its middle residual is EMPTY β€” while `cur` // carries extra bytes in that middle. That is a SEMANTIC append: no prior byte @@ -341,6 +350,7 @@ export function compareWires(aBody: Buffer, bBody: Buffer): WireRel { }; } const O = byteRel.offset; + const textOffset = textOffsetAtByte(bBody, O); const ia = msgArrayInfo(aText); const ib = msgArrayInfo(bText); if (ia && ib && ia.key === ib.key && ia.items.length > 0 && ib.items.length >= ia.items.length) { @@ -354,20 +364,20 @@ export function compareWires(aBody: Buffer, bBody: Buffer): WireRel { } if (sharedOk) { const lastEnd = ib.spans[n - 1]![1]; - if (O >= lastEnd) { + if (textOffset >= lastEnd) { return { rel: "append", byteOffset: O, extra: byteRel.extra, preArray: false, append: { kind: ib.items.length > ia.items.length ? "json-tail" : "tail-fields", countA: ia.items.length, countB: ib.items.length }, }; } - if (O < ib.spans[0]![0]) { + if (textOffset < ib.spans[0]![0]) { return { rel: "diverge", byteOffset: O, extra: byteRel.extra, append: null, preArray: true }; } // O sits in the glue between two shared elements: tolerate // whitespace/comma-only reshaping (pretty vs compact). let gi = -1; for (let i = 0; i < n - 1; i++) { - if (O >= ib.spans[i]![1] && O <= ib.spans[i + 1]![0]) { gi = i; break; } + if (textOffset >= ib.spans[i]![1] && textOffset <= ib.spans[i + 1]![0]) { gi = i; break; } } if (gi >= 0) { const strip = (s: string) => s.replace(/[,\s]/g, ""); @@ -487,8 +497,9 @@ function classifyPair(index: number, tsA: number, tsB: number, aOut: DumpContent : passthroughPair ? "client" : "both"; const bText = bOut.body.toString("utf8"); const info = msgArrayInfo(bText); - const loc = info ? locateMessage(info.items, info.spans, outRel.byteOffset) : { index: null, roleType: null }; - const snip = snippetAround(bText, outRel.byteOffset); + const textOffset = textOffsetAtByte(bOut.body, outRel.byteOffset); + const loc = info ? locateMessage(info.items, info.spans, textOffset) : { index: null, roleType: null }; + const snip = snippetAround(bText, textOffset); divergence = { offset: outRel.byteOffset, messageIndex: loc.index, roleType: loc.roleType, before: snip.before, after: snip.after }; if (outRel.preArray) notes.push("divergence before the message array (field-level change)"); if (inRel && inRel.rel === "diverge") incomingDivergenceOffset = inRel.byteOffset; diff --git a/tests/acp-cache-diff-unicode.test.ts b/tests/acp-cache-diff-unicode.test.ts new file mode 100644 index 0000000000..7600a734c8 --- /dev/null +++ b/tests/acp-cache-diff-unicode.test.ts @@ -0,0 +1,70 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { compareWires, runDiff } from "../src/acp-cache-diff.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const message = { role: "user", content: "same question" }; + +for (const key of ["messages", "input", "contents"]) { + for (const prefix of ["η•Œ".repeat(100), "πŸ˜€".repeat(100)]) { + test(`classifies a Unicode-prefixed field rewrite before ${key} as divergence`, () => { + const before = Buffer.from(JSON.stringify({ system: `${prefix}A`, [key]: [message] })); + const after = Buffer.from(JSON.stringify({ system: `${prefix}B`, [key]: [message] })); + const result = compareWires(before, after); + assert.equal(result.rel, "diverge"); + assert.equal(result.preArray, true); + assert.equal(result.append, null); + assert.equal(result.byteOffset, after.indexOf(Buffer.from("B"))); + }); + } +} + +async function pair(before: string, after: string) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "acp-diff-unicode-")); + try { + for (const [stamp, body] of [[1000, before], [2000, after]] as const) { + for (const kind of ["REQ", "INCOMING"]) { + fs.writeFileSync(path.join(dir, `${stamp}-unicode-${kind}.txt`), `POST /v1/chat/completions\ncontent-type: application/json\n\n${body}`); + } + } + const report = await runDiff(dir, { noLog: true }); + assert.equal(report.totalPairs, 1); + return report.sessions[0]!.pairs[0]!; + } finally { rmrf(dir); } +} + +test("locates rewritten message evidence after a multibyte history prefix while retaining wire byte offsets", async () => { + const make = (text: string) => JSON.stringify({ messages: [{ role: "user", content: "η•Œ".repeat(100) }, { role: "assistant", content: text }] }); + const before = make("OLD answer"), after = make("NEW answer"); + const report = await pair(before, after); + assert.equal(report.category, "mid-stream-rewrite"); + assert.equal(report.attribution, "client"); + assert.equal(report.divergence?.messageIndex, 1); + assert.equal(report.divergence?.offset, Buffer.from(after).indexOf(Buffer.from("NEW"))); + assert.ok(report.divergence?.after.startsWith("NEW answer")); +}); + +test("places evidence at the beginning of a code point when differing bytes are inside it", async () => { + const make = (text: string) => JSON.stringify({ messages: [{ role: "user", content: text }] }); + const report = await pair(make("πŸ˜€ old"), make("😁 old")); + assert.equal(report.divergence?.messageIndex, 0); + assert.ok(report.divergence?.after.startsWith("😁 old"), report.divergence?.after); +}); + +test("retains healthy appends and trailing-field changes after Unicode messages", () => { + const first = { role: "user", content: "η•ŒπŸ˜€".repeat(80) }; + const before = Buffer.from(JSON.stringify({ messages: [first], stream: false })); + const appended = Buffer.from(JSON.stringify({ messages: [first, message], stream: false })); + const trailing = Buffer.from(JSON.stringify({ messages: [first], stream: true })); + assert.equal(compareWires(before, appended).append?.kind, "json-tail"); + assert.equal(compareWires(before, trailing).append?.kind, "tail-fields"); +}); + +test("retains ASCII prefix-field attribution", () => { + const result = compareWires(Buffer.from(JSON.stringify({ model: "old", messages: [message] })), Buffer.from(JSON.stringify({ model: "new", messages: [message] }))); + assert.equal(result.rel, "diverge"); + assert.equal(result.preArray, true); +}); diff --git a/tests/e2e/e2e-update-state.test.ts b/tests/e2e/e2e-update-state.test.ts index 03f2b1db22..e31baf7659 100644 --- a/tests/e2e/e2e-update-state.test.ts +++ b/tests/e2e/e2e-update-state.test.ts @@ -299,6 +299,11 @@ test("self-update keeps live compression state: folds survive the OLDβ†’NEW hop envBase.BILI_UPDATE_POLICY_URL = "http://127.0.0.1:1/no-policy.json"; envBase.BILI_PERSIST_DEBOUNCE_MS = "200"; envBase.BILI_SESSIONS_DIR = sessionsDir; + // Pure-sqlite default (5a80e2b8e, #6480) turns the JSON envelope + // writer OFF, but this lane asserts on the on-disk .json session file + // (state survival across the update). Pin the writer ON so the + // envelope this test was written for is still produced. + envBase.BILI_STORAGE_JSON_WRITER = "on"; const port = await freePort(); proxy = spawnProxy(installDir, port, envBase);