Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions src/acp-cache-diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,15 @@ function prefixCompare(a: Buffer, b: Buffer): { rel: "append" | "diverge"; offse
return { rel: "diverge", offset: n, extra: b.length - a.length };
}

// Wire offsets are UTF-8 bytes; JSON spans and string slices use UTF-16.
// A differing byte can be inside a multibyte code point, so begin evidence at
// its leading byte instead of decoding a partial prefix as a replacement char.
function textOffsetAtByte(body: Buffer, offset: number): number {
let start = offset;
while (start > 0 && start < body.length && (body[start]! & 0xc0) === 0x80) start--;
return body.subarray(0, start).toString("utf8").length;
}

// #3476: true iff `prev` is fully covered by the common prefix (length `lcp`)
// plus some common suffix — i.e. its middle residual is EMPTY — while `cur`
// carries extra bytes in that middle. That is a SEMANTIC append: no prior byte
Expand Down Expand Up @@ -341,6 +350,7 @@ export function compareWires(aBody: Buffer, bBody: Buffer): WireRel {
};
}
const O = byteRel.offset;
const textOffset = textOffsetAtByte(bBody, O);
const ia = msgArrayInfo(aText);
const ib = msgArrayInfo(bText);
if (ia && ib && ia.key === ib.key && ia.items.length > 0 && ib.items.length >= ia.items.length) {
Expand All @@ -354,20 +364,20 @@ export function compareWires(aBody: Buffer, bBody: Buffer): WireRel {
}
if (sharedOk) {
const lastEnd = ib.spans[n - 1]![1];
if (O >= lastEnd) {
if (textOffset >= lastEnd) {
return {
rel: "append", byteOffset: O, extra: byteRel.extra, preArray: false,
append: { kind: ib.items.length > ia.items.length ? "json-tail" : "tail-fields", countA: ia.items.length, countB: ib.items.length },
};
}
if (O < ib.spans[0]![0]) {
if (textOffset < ib.spans[0]![0]) {
return { rel: "diverge", byteOffset: O, extra: byteRel.extra, append: null, preArray: true };
}
// O sits in the glue between two shared elements: tolerate
// whitespace/comma-only reshaping (pretty vs compact).
let gi = -1;
for (let i = 0; i < n - 1; i++) {
if (O >= ib.spans[i]![1] && O <= ib.spans[i + 1]![0]) { gi = i; break; }
if (textOffset >= ib.spans[i]![1] && textOffset <= ib.spans[i + 1]![0]) { gi = i; break; }
}
if (gi >= 0) {
const strip = (s: string) => s.replace(/[,\s]/g, "");
Expand Down Expand Up @@ -487,8 +497,9 @@ function classifyPair(index: number, tsA: number, tsB: number, aOut: DumpContent
: passthroughPair ? "client" : "both";
const bText = bOut.body.toString("utf8");
const info = msgArrayInfo(bText);
const loc = info ? locateMessage(info.items, info.spans, outRel.byteOffset) : { index: null, roleType: null };
const snip = snippetAround(bText, outRel.byteOffset);
const textOffset = textOffsetAtByte(bOut.body, outRel.byteOffset);
const loc = info ? locateMessage(info.items, info.spans, textOffset) : { index: null, roleType: null };
const snip = snippetAround(bText, textOffset);
divergence = { offset: outRel.byteOffset, messageIndex: loc.index, roleType: loc.roleType, before: snip.before, after: snip.after };
if (outRel.preArray) notes.push("divergence before the message array (field-level change)");
if (inRel && inRel.rel === "diverge") incomingDivergenceOffset = inRel.byteOffset;
Expand Down
70 changes: 70 additions & 0 deletions tests/acp-cache-diff-unicode.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { compareWires, runDiff } from "../src/acp-cache-diff.ts";
import { rmrf } from "./tmp-rm.ts";

const message = { role: "user", content: "same question" };

for (const key of ["messages", "input", "contents"]) {
for (const prefix of ["界".repeat(100), "😀".repeat(100)]) {
test(`classifies a Unicode-prefixed field rewrite before ${key} as divergence`, () => {
const before = Buffer.from(JSON.stringify({ system: `${prefix}A`, [key]: [message] }));
const after = Buffer.from(JSON.stringify({ system: `${prefix}B`, [key]: [message] }));
const result = compareWires(before, after);
assert.equal(result.rel, "diverge");
assert.equal(result.preArray, true);
assert.equal(result.append, null);
assert.equal(result.byteOffset, after.indexOf(Buffer.from("B")));
});
}
}

async function pair(before: string, after: string) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "acp-diff-unicode-"));
try {
for (const [stamp, body] of [[1000, before], [2000, after]] as const) {
for (const kind of ["REQ", "INCOMING"]) {
fs.writeFileSync(path.join(dir, `${stamp}-unicode-${kind}.txt`), `POST /v1/chat/completions\ncontent-type: application/json\n\n${body}`);
}
}
const report = await runDiff(dir, { noLog: true });
assert.equal(report.totalPairs, 1);
return report.sessions[0]!.pairs[0]!;
} finally { rmrf(dir); }
}

test("locates rewritten message evidence after a multibyte history prefix while retaining wire byte offsets", async () => {
const make = (text: string) => JSON.stringify({ messages: [{ role: "user", content: "界".repeat(100) }, { role: "assistant", content: text }] });
const before = make("OLD answer"), after = make("NEW answer");
const report = await pair(before, after);
assert.equal(report.category, "mid-stream-rewrite");
assert.equal(report.attribution, "client");
assert.equal(report.divergence?.messageIndex, 1);
assert.equal(report.divergence?.offset, Buffer.from(after).indexOf(Buffer.from("NEW")));
assert.ok(report.divergence?.after.startsWith("NEW answer"));
});

test("places evidence at the beginning of a code point when differing bytes are inside it", async () => {
const make = (text: string) => JSON.stringify({ messages: [{ role: "user", content: text }] });
const report = await pair(make("😀 old"), make("😁 old"));
assert.equal(report.divergence?.messageIndex, 0);
assert.ok(report.divergence?.after.startsWith("😁 old"), report.divergence?.after);
});

test("retains healthy appends and trailing-field changes after Unicode messages", () => {
const first = { role: "user", content: "界😀".repeat(80) };
const before = Buffer.from(JSON.stringify({ messages: [first], stream: false }));
const appended = Buffer.from(JSON.stringify({ messages: [first, message], stream: false }));
const trailing = Buffer.from(JSON.stringify({ messages: [first], stream: true }));
assert.equal(compareWires(before, appended).append?.kind, "json-tail");
assert.equal(compareWires(before, trailing).append?.kind, "tail-fields");
});

test("retains ASCII prefix-field attribution", () => {
const result = compareWires(Buffer.from(JSON.stringify({ model: "old", messages: [message] })), Buffer.from(JSON.stringify({ model: "new", messages: [message] })));
assert.equal(result.rel, "diverge");
assert.equal(result.preArray, true);
});
5 changes: 5 additions & 0 deletions tests/e2e/e2e-update-state.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,11 @@ test("self-update keeps live compression state: folds survive the OLD→NEW hop
envBase.BILI_UPDATE_POLICY_URL = "http://127.0.0.1:1/no-policy.json";
envBase.BILI_PERSIST_DEBOUNCE_MS = "200";
envBase.BILI_SESSIONS_DIR = sessionsDir;
// Pure-sqlite default (5a80e2b8e, #6480) turns the JSON envelope
// writer OFF, but this lane asserts on the on-disk .json session file
// (state survival across the update). Pin the writer ON so the
// envelope this test was written for is still produced.
envBase.BILI_STORAGE_JSON_WRITER = "on";

const port = await freePort();
proxy = spawnProxy(installDir, port, envBase);
Expand Down
Loading