From e23468ac8c0162b646105f9fea135f20aafaa719 Mon Sep 17 00:00:00 2001 From: Afloat Date: Sat, 14 Nov 2026 21:41:26 +0800 Subject: [PATCH] perf: align cache usage in a single ordered pass --- scripts/bench-cache-usage-alignment.ts | 50 +++++++++++++ src/acp-cache-diff.ts | 7 +- tests/acp-cache-usage-alignment.test.ts | 97 +++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 scripts/bench-cache-usage-alignment.ts create mode 100644 tests/acp-cache-usage-alignment.test.ts diff --git a/scripts/bench-cache-usage-alignment.ts b/scripts/bench-cache-usage-alignment.ts new file mode 100644 index 0000000000..6a701176ad --- /dev/null +++ b/scripts/bench-cache-usage-alignment.ts @@ -0,0 +1,50 @@ +// Run at either revision: node --import tsx scripts/bench-cache-usage-alignment.ts [requests] [samples-per-request] +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { runDiff } from "../src/acp-cache-diff.ts"; + +const count = Number(process.argv[2] ?? 10_000); +const perRequest = Number(process.argv[3] ?? 10); +assert.ok(Number.isSafeInteger(count) && count >= 2); +assert.ok(Number.isSafeInteger(perRequest) && perRequest > 0 && perRequest < 900); +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-align-bench-")); +const logFile = path.join(dir, "bili.log"); +const start = Date.parse("2026-11-14T01:00:00.000Z"); +const body = JSON.stringify({ model: "benchmark", messages: [{ role: "user", content: "retained history" }] }); + +try { + const lines: string[] = []; + for (let i = 0; i < count; i++) { + const ts = start + i * 1000; + fs.writeFileSync(path.join(dir, `${ts}-session-REQ.txt`), `POST /v1/chat/completions\ncontent-type: application/json\n\n${body}`); + for (let j = 0; j < perRequest; j++) { + lines.push(`${new Date(ts + 100 + j).toISOString()} [info] [session] [acp-usage] input=${100_000 + i} cached=${99_000 + i}\n`); + } + } + fs.writeFileSync(logFile, lines.join("")); + await runDiff(dir, { logFile }); + const cpuBefore = process.cpuUsage(); + const before = performance.now(); + const report = await runDiff(dir, { logFile }); + const elapsedMs = performance.now() - before; + const cpu = process.cpuUsage(cpuBefore); + assert.equal(report.totalPairs, count - 1); + assert.equal(report.usageSamples, count * perRequest); + for (let i = 0; i < count - 1; i++) { + assert.deepEqual(report.sessions[0]!.pairs[i]!.usage, { + inputA: 100_000 + i, + inputB: 100_001 + i, + cachedB: 99_001 + i, + missedPrefix: 999, + hitPctB: Math.round((99_001 + i) / (100_001 + i) * 100), + }); + } + const normalized = { ...report, dir: "", logSource: "/bili.log" }; + const sha256 = createHash("sha256").update(JSON.stringify(normalized)).digest("hex"); + process.stdout.write(`${JSON.stringify({ requests: count, samples: count * perRequest, elapsedMs, cpuMs: (cpu.user + cpu.system) / 1000, sha256 })}\n`); +} finally { + fs.rmSync(dir, { recursive: true, force: true }); +} diff --git a/src/acp-cache-diff.ts b/src/acp-cache-diff.ts index 023955e941..61a7f4bfc3 100644 --- a/src/acp-cache-diff.ts +++ b/src/acp-cache-diff.ts @@ -461,11 +461,12 @@ const USAGE_SKEW_MS = 50; function alignUsage(outTs: number[], samples: UsageSample[]): UsageSample[][] { // A usage line belongs to the latest request whose dump predates it — // responses (and their usage logs) always land after the request dump. + // Both inputs are sorted by time, so each request need only be visited once. const assigned: UsageSample[][] = outTs.map(() => []); + let request = -1; for (const s of samples) { - for (let i = outTs.length - 1; i >= 0; i--) { - if (outTs[i]! <= s.ts + USAGE_SKEW_MS) { assigned[i]!.push(s); break; } - } + while (request + 1 < outTs.length && outTs[request + 1]! <= s.ts + USAGE_SKEW_MS) request++; + if (request >= 0) assigned[request]!.push(s); } return assigned; } diff --git a/tests/acp-cache-usage-alignment.test.ts b/tests/acp-cache-usage-alignment.test.ts new file mode 100644 index 0000000000..4b252bd70b --- /dev/null +++ b/tests/acp-cache-usage-alignment.test.ts @@ -0,0 +1,97 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { runDiff } from "../src/acp-cache-diff.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const START = Date.parse("2026-11-14T01:00:00.000Z"); +const BODY = JSON.stringify({ model: "test-model", messages: [{ role: "user", content: "retained history" }] }); + +async function reportFor(requests: Array<{ at: number; sid?: string; subdir?: string }>, samples: Array<{ at: number; input: number; cached: number; sid?: string }>, session?: string) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-usage-align-")); + try { + for (const request of requests) { + const folder = path.join(dir, request.subdir ?? ""); + fs.mkdirSync(folder, { recursive: true }); + fs.writeFileSync(path.join(folder, `${START + request.at}-${request.sid ?? "session"}-REQ.txt`), `POST /v1/chat/completions\ncontent-type: application/json\n\n${BODY}`); + } + const logFile = path.join(dir, "bili.log"); + fs.writeFileSync(logFile, samples.map((sample) => `${new Date(START + sample.at).toISOString()} [info] [${sample.sid ?? "session"}] [acp-usage] input=${sample.input} cached=${sample.cached}\n`).join("")); + return await runDiff(dir, { logFile, session }); + } finally { + rmrf(dir); + } +} + +test("cache usage alignment keeps the earliest sample per request after sorting the log", async () => { + const report = await reportFor([{ at: 3000 }, { at: 1000 }, { at: 2000 }], [ + { at: 3200, input: 30_000, cached: 19_000 }, + { at: 2300, input: 21_000, cached: 9000 }, + { at: 900, input: 999_999, cached: 0 }, + { at: 2100, input: 20_000, cached: 10_000 }, + { at: 1200, input: 10_000, cached: 8000 }, + { at: 3100, input: 30_000, cached: 20_000 }, + { at: 99_000, input: 999_999, cached: 0 }, + ]); + assert.equal(report.usageSamples, 7); + assert.deepEqual(report.sessions[0]!.pairs.map((pair) => pair.usage), [ + { inputA: 10_000, inputB: 20_000, cachedB: 10_000, missedPrefix: 0, hitPctB: 50 }, + { inputA: 20_000, inputB: 30_000, cachedB: 20_000, missedPrefix: 0, hitPctB: 67 }, + ]); + assert.deepEqual(report.byCategory, { "pure-append": 2, "mid-stream-rewrite": 0, "prefix-stable-miss": 0 }); +}); + +for (const offset of [-51, -50, 0, 50]) { + test(`cache usage alignment retains the 50 ms admission boundary (${offset} ms)`, async () => { + const report = await reportFor([{ at: 1000 }, { at: 2000 }], [ + { at: 1000 + offset, input: 10_000, cached: 8000 }, + { at: 2000 + offset, input: 20_000, cached: 10_000 }, + ]); + const pair = report.sessions[0]!.pairs[0]!; + assert.equal(pair.usageAvailable, offset >= -50); + assert.deepEqual(pair.usage, offset >= -50 + ? { inputA: 10_000, inputB: 20_000, cachedB: 10_000, missedPrefix: 0, hitPctB: 50 } + : null); + }); +} + +test("cache usage alignment assigns equal-time dumps to the last retained request", async () => { + const report = await reportFor([{ at: 1000 }, { at: 1000, subdir: "raw" }, { at: 2000 }], [ + { at: 1100, input: 10_000, cached: 8000 }, + { at: 2100, input: 20_000, cached: 10_000 }, + ]); + const pairs = report.sessions[0]!.pairs; + assert.equal(pairs.length, 2); + assert.equal(pairs[0]!.usageAvailable, false); + assert.deepEqual(pairs[1]!.usage, { inputA: 10_000, inputB: 20_000, cachedB: 10_000, missedPrefix: 0, hitPctB: 50 }); +}); + +test("cache usage alignment keeps same-time samples stable and sessions separate", async () => { + const report = await reportFor([ + { at: 1000, sid: "first" }, { at: 2000, sid: "first" }, + { at: 1000, sid: "second" }, { at: 2000, sid: "second" }, + ], [ + { at: 2100, input: 20_000, cached: 1000, sid: "first" }, + { at: 1100, input: 900_000, cached: 0, sid: "second" }, + { at: 1100, input: 10_000, cached: 8000, sid: "first" }, + { at: 2100, input: 99_999, cached: 10_000, sid: "first" }, + ], "first"); + assert.equal(report.sessions.length, 1); + assert.equal(report.usageSamples, 4); + const pair = report.sessions[0]!.pairs[0]!; + assert.equal(pair.category, "prefix-stable-miss"); + assert.deepEqual(pair.usage, { inputA: 10_000, inputB: 20_000, cachedB: 1000, missedPrefix: 9000, hitPctB: 5 }); +}); + +test("cache usage alignment leaves requests without samples unavailable", async () => { + const report = await reportFor([{ at: 1000 }, { at: 2000 }, { at: 3000 }], [ + { at: 1100, input: 10_000, cached: 8000 }, + { at: 3100, input: 30_000, cached: 10_000 }, + ]); + assert.deepEqual(report.sessions[0]!.pairs.map((pair) => pair.usage), [null, null]); + const empty = await reportFor([{ at: 1000 }, { at: 2000 }], []); + assert.equal(empty.usageSamples, 0); + assert.equal(empty.sessions[0]!.pairs[0]!.usageAvailable, false); +});