Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions scripts/bench-handoff-capture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Real HTTP client -> production SSE parser -> saved handoff reply. An optional
// module path selects an unchanged historical createHandoffReply for comparison.
// Build fixtures outside the measured interval; validate complete wire + reply.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { once } from "node:events";
import { createServer } from "node:http";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { performance } from "node:perf_hooks";
import { pipePluginChatWithStrip } from "../src/plugin.ts";
import { createSession, type Session } from "../src/session.ts";
import { createHandoffReply } from "../src/server/handoff-response.ts";

const modulePath = process.argv[2];
const capture: typeof createHandoffReply = modulePath
? (await import(pathToFileURL(path.resolve(modulePath)).href) as typeof import("../src/server/handoff-response.ts")).createHandoffReply
: createHandoffReply;
const scenarios = [
{ name: "unicode-3000", count: 3000, delta: "界😀".repeat(16) },
{ name: "ascii-3000", count: 3000, delta: "abcdefgh".repeat(12) },
{ name: "unicode-8", count: 8, delta: "界😀".repeat(16) },
];
let chunks: Buffer[] = [];
let session: Session;
const server = createServer((_req, res) => {
session = createSession("handoff-capture-benchmark");
const reply = capture(session);
const source = new ReadableStream<Uint8Array>({ start(controller) {
for (const chunk of chunks) controller.enqueue(chunk);
controller.close();
} });
void pipePluginChatWithStrip(source, res, "openai", undefined, undefined, undefined, undefined, undefined, undefined, undefined, reply)
.catch((error: unknown) => res.destroy(error instanceof Error ? error : new Error(String(error))));
});
server.listen(0, "127.0.0.1");
await once(server, "listening");
const address = server.address();
assert(address && typeof address !== "string");
const rows: { name: string; deltas: number; textBytes: number; ms: number; wireBytes: number; outputSha256: string }[] = [];
try {
for (const scenario of scenarios) {
const frame = Buffer.from(`data: ${JSON.stringify({ choices: [{ delta: { content: scenario.delta } }] })}\n\n`);
chunks = Array.from({ length: scenario.count }, () => frame);
chunks.push(Buffer.from(`data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }] })}\n\ndata: [DONE]\n\n`));
const expected = scenario.delta.repeat(scenario.count);
let warmupHash = "";
for (let run = 0; run < 2; run++) {
const start = performance.now();
const response = await fetch(`http://127.0.0.1:${address.port}`);
const wire = Buffer.from(await response.arrayBuffer());
const ms = performance.now() - start;
const saved = session!.metadata.handoffResponse;
assert.deepEqual(saved, { request: 0, messages: [{ id: "reply_text_1", role: "assistant", contentType: "text", text: expected }] });
const outputSha256 = createHash("sha256").update(wire).update(JSON.stringify(saved)).digest("hex");
if (run === 0) warmupHash = outputSha256;
else {
assert.equal(outputSha256, warmupHash);
rows.push({ name: scenario.name, deltas: scenario.count, textBytes: Buffer.byteLength(expected), ms, wireBytes: wire.length, outputSha256 });
}
}
}
} finally {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
console.log(JSON.stringify({ node: process.version, warmupsPerScenario: 1, measuredPerScenario: 1, rows }, null, 2));
24 changes: 17 additions & 7 deletions src/server/handoff-response.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,32 @@ export type HandoffReply = ReturnType<typeof createHandoffReply>;
export function createHandoffReply(session: Session, canSave: () => boolean = () => true) {
const request = session.stats.requests;
const snapshot = session.lastMessages;
const messages = new Map<string, CoreMessage>();
const messages = new Map<string, { message: CoreMessage; bytes: number; textBytes: number; lastCodeUnit: number }>();
let bytes = 0;
let discarded = false;
const discard = () => { discarded = true; messages.clear(); };
const put = (key: string, message: CoreMessage) => {
const put = (key: string, message: CoreMessage, textBytes?: number, lastCodeUnit?: number) => {
if (discarded) return;
const previous = messages.get(key);
const size = (m: CoreMessage) => Buffer.byteLength(m.text ?? "") + Buffer.byteLength(m.toolName ?? "") + Buffer.byteLength(m.toolCallId ?? "") + 128;
bytes += size(message) - (previous ? size(previous) : 0);
const text = message.text ?? "";
const measuredTextBytes = textBytes ?? Buffer.byteLength(text);
const size = measuredTextBytes + Buffer.byteLength(message.toolName ?? "") + Buffer.byteLength(message.toolCallId ?? "") + 128;
bytes += size - (previous?.bytes ?? 0);
if (bytes > HANDOFF_REPLY_MAX_BYTES) { discard(); return; }
messages.set(key, message);
messages.set(key, { message, bytes: size, textBytes: measuredTextBytes, lastCodeUnit: lastCodeUnit ?? text.charCodeAt(text.length - 1) });
};
return {
append(key: string, contentType: "text" | "reasoning", text: string) {
if (!text || discarded) return;
put(key, { id: `reply_${key}`, role: "assistant", contentType, text: (messages.get(key)?.text ?? "") + text });
const previous = messages.get(key);
const last = previous?.lastCodeUnit ?? 0;
const first = text.charCodeAt(0);
// Two separately encoded lone surrogates use six bytes; joined
// across deltas they form one four-byte code point. Keep only the
// trailing code unit so metering never rescans the growing prefix.
const joinsPair = last >= 0xd800 && last <= 0xdbff && first >= 0xdc00 && first <= 0xdfff;
const textBytes = (previous?.textBytes ?? 0) + Buffer.byteLength(text) - (joinsPair ? 2 : 0);
put(key, { id: `reply_${key}`, role: "assistant", contentType, text: (previous?.message.text ?? "") + text }, textBytes, text.charCodeAt(text.length - 1));
},
tool(key: string, toolName: string, toolCallId: string, text: string) {
put(key, { id: `reply_${key}`, role: "assistant", contentType: "tool-call", toolName, toolCallId, text });
Expand All @@ -38,7 +48,7 @@ export function createHandoffReply(session: Session, canSave: () => boolean = ()
discard,
save() {
if (discarded || !messages.size || session.stats.requests !== request || session.lastMessages !== snapshot || !canSave()) return;
session.metadata.handoffResponse = { request, messages: [...messages.values()] };
session.metadata.handoffResponse = { request, messages: [...messages.values()].map((entry) => entry.message) };
markDirty(session);
},
};
Expand Down
162 changes: 162 additions & 0 deletions tests/handoff-capture-work.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
import assert from "node:assert/strict";
import { once } from "node:events";
import { createServer } from "node:http";
import test from "node:test";
import type { CoreMessage } from "acp-kernel";
import { pipePluginChatWithStrip } from "../src/plugin.ts";
import { createSession, type Session } from "../src/session.ts";
import { createHandoffReply, HANDOFF_REPLY_MAX_BYTES } from "../src/server/handoff-response.ts";

function captured(session: Session): CoreMessage[] {
return (session.metadata.handoffResponse as { messages?: CoreMessage[] } | undefined)?.messages ?? [];
}

for (const kind of ["text", "reasoning"] as const) {
test(`handoff ${kind} metering visits appended text once rather than every accumulated prefix`, (t) => {
const session = createSession(`handoff-work-${kind}`);
const reply = createHandoffReply(session);
const delta = "界😀".repeat(8);
const count = 2400;
const byteLength = Buffer.byteLength;
let measuredChars = 0;
t.mock.method(Buffer, "byteLength", (...args: Parameters<typeof Buffer.byteLength>) => {
if (typeof args[0] === "string") measuredChars += args[0].length;
return byteLength(...args);
});
for (let i = 0; i < count; i++) reply.append("one", kind, delta);
reply.save();
t.mock.restoreAll();
assert.equal(captured(session)[0]?.text, delta.repeat(count));
assert.equal(captured(session)[0]?.contentType, kind);
assert.ok(measuredChars <= delta.length * count * 2, `${measuredChars} characters scanned for ${delta.length * count} input characters`);
});
}

test("production HTTP streaming capture meters work proportional to received frames", async (t) => {
const delta = "界😀".repeat(8);
const count = 1000;
const frame = `data: ${JSON.stringify({ choices: [{ delta: { content: delta } }] })}\n\n`;
const ending = `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }] })}\n\ndata: [DONE]\n\n`;
const chunks = Array.from({ length: count }, () => Buffer.from(frame));
chunks.push(Buffer.from(ending));
const session = createSession("handoff-http-work");
let record = true;
const server = createServer((_req, res) => {
const source = new ReadableStream<Uint8Array>({ start(controller) {
for (const chunk of chunks) controller.enqueue(chunk);
controller.close();
} });
const reply = record ? createHandoffReply(session) : undefined;
void pipePluginChatWithStrip(source, res, "openai", undefined, undefined, undefined, undefined, undefined, undefined, undefined, reply)
.catch((error: unknown) => res.destroy(error instanceof Error ? error : new Error(String(error))));
});
server.listen(0, "127.0.0.1");
await once(server, "listening");
const address = server.address();
assert(address && typeof address !== "string");
const byteLength = Buffer.byteLength;
let measuredChars = 0;
try {
t.mock.method(Buffer, "byteLength", (...args: Parameters<typeof Buffer.byteLength>) => {
if (typeof args[0] === "string") measuredChars += args[0].length;
return byteLength(...args);
});
const response = await fetch(`http://127.0.0.1:${address.port}`);
const wire = Buffer.from(await response.arrayBuffer());
t.mock.restoreAll();
assert.equal(captured(session)[0]?.text, delta.repeat(count));
record = false;
const baseline = await fetch(`http://127.0.0.1:${address.port}`);
assert.deepEqual(wire, Buffer.from(await baseline.arrayBuffer()), "recording leaves forwarded bytes unchanged");
const inputChars = count * frame.length + ending.length;
assert.ok(measuredChars <= inputChars * 10, `${measuredChars} characters scanned for ${inputChars} wire characters`);
} finally {
t.mock.restoreAll();
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});

const TEXT_BUDGET = HANDOFF_REPLY_MAX_BYTES - 128;

for (const [name, parts] of [
["split surrogate pair", ["x".repeat(TEXT_BUDGET - 4) + "\ud83d", "\ude00"]],
["lone high surrogate", ["x".repeat(TEXT_BUDGET - 4), "\ud83d", "x"]],
["lone low surrogate", ["x".repeat(TEXT_BUDGET - 4), "\ude00", "x"]],
["complete emoji", ["x".repeat(TEXT_BUDGET - 4), "😀"]],
["BMP multibyte", ["x".repeat(TEXT_BUDGET - 3), "界"]],
] as const) {
test(`keeps the exact UTF-8 budget at ${name}`, () => {
const session = createSession(`handoff-budget-${name}`);
const reply = createHandoffReply(session);
for (const part of parts) reply.append("one", "text", part);
reply.save();
assert.equal(Buffer.byteLength(parts.join("")), TEXT_BUDGET);
assert.equal(captured(session)[0]?.text, parts.join(""));
delete session.metadata.handoffResponse;
reply.append("one", "text", "a");
reply.save();
assert.equal(session.metadata.handoffResponse, undefined, "one byte over the cap still discards the entire capture");
});
}

test("meters independent entries against the shared reply budget", () => {
const session = createSession("handoff-shared-budget");
const reply = createHandoffReply(session);
reply.append("thinking", "reasoning", "思考");
reply.append("answer", "text", "a".repeat(HANDOFF_REPLY_MAX_BYTES - 256 - Buffer.byteLength("思考")));
reply.save();
assert.equal(captured(session).length, 2);
delete session.metadata.handoffResponse;
reply.append("thinking", "reasoning", "!");
reply.save();
assert.equal(session.metadata.handoffResponse, undefined);
});

test("tool replacement releases its previous budget before a new entry is appended", () => {
const session = createSession("handoff-tool-replacement");
const reply = createHandoffReply(session);
reply.tool("tool", "look", "call", "a".repeat(TEXT_BUDGET - 8));
reply.tool("tool", "x", "y", "{}");
reply.append("answer", "text", "a".repeat(HANDOFF_REPLY_MAX_BYTES - 256 - 4));
reply.save();
assert.equal(captured(session).length, 2);
assert.equal(captured(session)[0]?.text, "{}");
assert.equal(captured(session)[0]?.toolName, "x");
});

test("appending to a replaced tool entry counts text but drops its former tool fields", () => {
const session = createSession("handoff-tool-to-text");
const reply = createHandoffReply(session);
reply.tool("one", "long-tool-name", "long-call-id", "界");
reply.append("one", "reasoning", "a".repeat(TEXT_BUDGET - 3));
reply.save();
assert.equal(captured(session)[0]?.contentType, "reasoning");
assert.equal(captured(session)[0]?.toolName, undefined);
assert.equal(captured(session)[0]?.toolCallId, undefined);
assert.equal(Buffer.byteLength(captured(session)[0]?.text ?? ""), TEXT_BUDGET);
});

test("whole-response replacement resets entry budgets and supports later appends", () => {
const session = createSession("handoff-replace-budget");
const reply = createHandoffReply(session);
reply.append("old", "text", "a".repeat(TEXT_BUDGET));
reply.replace([{ id: "replacement", role: "assistant", contentType: "text", text: "\ud83d" }]);
reply.append("0", "text", "\ude00");
reply.append("0", "text", "a".repeat(TEXT_BUDGET - 4));
reply.save();
assert.equal(captured(session).length, 1);
assert.equal(captured(session)[0]?.text, "😀" + "a".repeat(TEXT_BUDGET - 4));
});

test("a saved response remains immutable when the same capture accepts another delta", () => {
const session = createSession("handoff-save-boundary");
const reply = createHandoffReply(session);
reply.append("one", "text", "before");
reply.save();
const before = captured(session);
reply.append("one", "text", " after");
reply.save();
assert.equal(before[0]?.text, "before");
assert.equal(captured(session)[0]?.text, "before after");
});
Loading