Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions scripts/bench-cache-usage-alignment.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Run at either revision: node --import tsx scripts/bench-cache-usage-alignment.ts [requests] [samples-per-request]
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { runDiff } from "../src/acp-cache-diff.ts";

const count = Number(process.argv[2] ?? 10_000);
const perRequest = Number(process.argv[3] ?? 10);
assert.ok(Number.isSafeInteger(count) && count >= 2);
assert.ok(Number.isSafeInteger(perRequest) && perRequest > 0 && perRequest < 900);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-align-bench-"));
const logFile = path.join(dir, "bili.log");
const start = Date.parse("2026-11-14T01:00:00.000Z");
const body = JSON.stringify({ model: "benchmark", messages: [{ role: "user", content: "retained history" }] });

try {
const lines: string[] = [];
for (let i = 0; i < count; i++) {
const ts = start + i * 1000;
fs.writeFileSync(path.join(dir, `${ts}-session-REQ.txt`), `POST /v1/chat/completions\ncontent-type: application/json\n\n${body}`);
for (let j = 0; j < perRequest; j++) {
lines.push(`${new Date(ts + 100 + j).toISOString()} [info] [session] [acp-usage] input=${100_000 + i} cached=${99_000 + i}\n`);
}
}
fs.writeFileSync(logFile, lines.join(""));
await runDiff(dir, { logFile });
const cpuBefore = process.cpuUsage();
const before = performance.now();
const report = await runDiff(dir, { logFile });
const elapsedMs = performance.now() - before;
const cpu = process.cpuUsage(cpuBefore);
assert.equal(report.totalPairs, count - 1);
assert.equal(report.usageSamples, count * perRequest);
for (let i = 0; i < count - 1; i++) {
assert.deepEqual(report.sessions[0]!.pairs[i]!.usage, {
inputA: 100_000 + i,
inputB: 100_001 + i,
cachedB: 99_001 + i,
missedPrefix: 999,
hitPctB: Math.round((99_001 + i) / (100_001 + i) * 100),
});
}
const normalized = { ...report, dir: "<fixture>", logSource: "<fixture>/bili.log" };
const sha256 = createHash("sha256").update(JSON.stringify(normalized)).digest("hex");
process.stdout.write(`${JSON.stringify({ requests: count, samples: count * perRequest, elapsedMs, cpuMs: (cpu.user + cpu.system) / 1000, sha256 })}\n`);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
7 changes: 4 additions & 3 deletions src/acp-cache-diff.ts
Original file line number Diff line number Diff line change
Expand Up @@ -461,11 +461,12 @@ const USAGE_SKEW_MS = 50;
function alignUsage(outTs: number[], samples: UsageSample[]): UsageSample[][] {
// A usage line belongs to the latest request whose dump predates it —
// responses (and their usage logs) always land after the request dump.
// Both inputs are sorted by time, so each request need only be visited once.
const assigned: UsageSample[][] = outTs.map(() => []);
let request = -1;
for (const s of samples) {
for (let i = outTs.length - 1; i >= 0; i--) {
if (outTs[i]! <= s.ts + USAGE_SKEW_MS) { assigned[i]!.push(s); break; }
}
while (request + 1 < outTs.length && outTs[request + 1]! <= s.ts + USAGE_SKEW_MS) request++;
if (request >= 0) assigned[request]!.push(s);
}
return assigned;
}
Expand Down
97 changes: 97 additions & 0 deletions tests/acp-cache-usage-alignment.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { runDiff } from "../src/acp-cache-diff.ts";
import { rmrf } from "./tmp-rm.ts";

const START = Date.parse("2026-11-14T01:00:00.000Z");
const BODY = JSON.stringify({ model: "test-model", messages: [{ role: "user", content: "retained history" }] });

async function reportFor(requests: Array<{ at: number; sid?: string; subdir?: string }>, samples: Array<{ at: number; input: number; cached: number; sid?: string }>, session?: string) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-usage-align-"));
try {
for (const request of requests) {
const folder = path.join(dir, request.subdir ?? "");
fs.mkdirSync(folder, { recursive: true });
fs.writeFileSync(path.join(folder, `${START + request.at}-${request.sid ?? "session"}-REQ.txt`), `POST /v1/chat/completions\ncontent-type: application/json\n\n${BODY}`);
}
const logFile = path.join(dir, "bili.log");
fs.writeFileSync(logFile, samples.map((sample) => `${new Date(START + sample.at).toISOString()} [info] [${sample.sid ?? "session"}] [acp-usage] input=${sample.input} cached=${sample.cached}\n`).join(""));
return await runDiff(dir, { logFile, session });
} finally {
rmrf(dir);
}
}

test("cache usage alignment keeps the earliest sample per request after sorting the log", async () => {
const report = await reportFor([{ at: 3000 }, { at: 1000 }, { at: 2000 }], [
{ at: 3200, input: 30_000, cached: 19_000 },
{ at: 2300, input: 21_000, cached: 9000 },
{ at: 900, input: 999_999, cached: 0 },
{ at: 2100, input: 20_000, cached: 10_000 },
{ at: 1200, input: 10_000, cached: 8000 },
{ at: 3100, input: 30_000, cached: 20_000 },
{ at: 99_000, input: 999_999, cached: 0 },
]);
assert.equal(report.usageSamples, 7);
assert.deepEqual(report.sessions[0]!.pairs.map((pair) => pair.usage), [
{ inputA: 10_000, inputB: 20_000, cachedB: 10_000, missedPrefix: 0, hitPctB: 50 },
{ inputA: 20_000, inputB: 30_000, cachedB: 20_000, missedPrefix: 0, hitPctB: 67 },
]);
assert.deepEqual(report.byCategory, { "pure-append": 2, "mid-stream-rewrite": 0, "prefix-stable-miss": 0 });
});

for (const offset of [-51, -50, 0, 50]) {
test(`cache usage alignment retains the 50 ms admission boundary (${offset} ms)`, async () => {
const report = await reportFor([{ at: 1000 }, { at: 2000 }], [
{ at: 1000 + offset, input: 10_000, cached: 8000 },
{ at: 2000 + offset, input: 20_000, cached: 10_000 },
]);
const pair = report.sessions[0]!.pairs[0]!;
assert.equal(pair.usageAvailable, offset >= -50);
assert.deepEqual(pair.usage, offset >= -50
? { inputA: 10_000, inputB: 20_000, cachedB: 10_000, missedPrefix: 0, hitPctB: 50 }
: null);
});
}

test("cache usage alignment assigns equal-time dumps to the last retained request", async () => {
const report = await reportFor([{ at: 1000 }, { at: 1000, subdir: "raw" }, { at: 2000 }], [
{ at: 1100, input: 10_000, cached: 8000 },
{ at: 2100, input: 20_000, cached: 10_000 },
]);
const pairs = report.sessions[0]!.pairs;
assert.equal(pairs.length, 2);
assert.equal(pairs[0]!.usageAvailable, false);
assert.deepEqual(pairs[1]!.usage, { inputA: 10_000, inputB: 20_000, cachedB: 10_000, missedPrefix: 0, hitPctB: 50 });
});

test("cache usage alignment keeps same-time samples stable and sessions separate", async () => {
const report = await reportFor([
{ at: 1000, sid: "first" }, { at: 2000, sid: "first" },
{ at: 1000, sid: "second" }, { at: 2000, sid: "second" },
], [
{ at: 2100, input: 20_000, cached: 1000, sid: "first" },
{ at: 1100, input: 900_000, cached: 0, sid: "second" },
{ at: 1100, input: 10_000, cached: 8000, sid: "first" },
{ at: 2100, input: 99_999, cached: 10_000, sid: "first" },
], "first");
assert.equal(report.sessions.length, 1);
assert.equal(report.usageSamples, 4);
const pair = report.sessions[0]!.pairs[0]!;
assert.equal(pair.category, "prefix-stable-miss");
assert.deepEqual(pair.usage, { inputA: 10_000, inputB: 20_000, cachedB: 1000, missedPrefix: 9000, hitPctB: 5 });
});

test("cache usage alignment leaves requests without samples unavailable", async () => {
const report = await reportFor([{ at: 1000 }, { at: 2000 }, { at: 3000 }], [
{ at: 1100, input: 10_000, cached: 8000 },
{ at: 3100, input: 30_000, cached: 10_000 },
]);
assert.deepEqual(report.sessions[0]!.pairs.map((pair) => pair.usage), [null, null]);
const empty = await reportFor([{ at: 1000 }, { at: 2000 }], []);
assert.equal(empty.usageSamples, 0);
assert.equal(empty.sessions[0]!.pairs[0]!.usageAvailable, false);
});
Loading