diff --git a/scripts/bench-json-rewrite-history.ts b/scripts/bench-json-rewrite-history.ts new file mode 100644 index 0000000000..f0f9a2dd88 --- /dev/null +++ b/scripts/bench-json-rewrite-history.ts @@ -0,0 +1,51 @@ +import { createHash } from "node:crypto"; +import { performance } from "node:perf_hooks"; +import { createCore, defaultConfig, type CoreMessage } from "acp-kernel"; +import { createSession } from "../src/session.ts"; +import { rewriteJsonResponseAsync, type RewriteCtx } from "../src/stream.ts"; +import { rewriteOpenaiJsonResponseAsync } from "../src/stream-openai.ts"; +import { rewriteResponsesJsonResponseAsync } from "../src/stream-responses.ts"; +import { rewriteGoogleJsonResponseAsync } from "../src/stream-google.ts"; + +// Run in a fresh process for each revision and size, with the test env preload: +// node --import ./tests/preload-test-env.mjs --import tsx scripts/bench-json-rewrite-history.ts 8388608 +const bytes = Number(process.argv[2] ?? 8 * 1024 * 1024); +if (!Number.isSafeInteger(bytes) || bytes < 1) throw new Error("Expected a positive history byte count"); +const messages: CoreMessage[] = []; +for (let remaining = bytes; remaining > 0;) { + const length = Math.min(8192, remaining); + messages.push({ id: `history-${messages.length}`, role: "assistant", contentType: "text", text: "x".repeat(length) }); + remaining -= length; +} +const ctx: RewriteCtx = { + core: createCore(), config: defaultConfig(4_000_000), messages, + session: createSession("json-rewrite-benchmark"), log: () => {}, +}; +const text = "The requested change is complete. Tests passed."; +const lanes = [ + { name: "anthropic", rewrite: rewriteJsonResponseAsync, body: { content: [{ type: "text", text }], stop_reason: "end_turn" } }, + { name: "openai", rewrite: rewriteOpenaiJsonResponseAsync, body: { choices: [{ message: { role: "assistant", content: text }, finish_reason: "stop" }] } }, + { name: "responses", rewrite: rewriteResponsesJsonResponseAsync, body: { output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text }] }], status: "completed" } }, + { name: "google", rewrite: rewriteGoogleJsonResponseAsync, body: { candidates: [{ content: { role: "model", parts: [{ text }] }, finishReason: "STOP" }] } }, +]; +const results = []; +for (const armed of [false, true]) { + ctx.config.absorb!.enabled = armed; + for (const lane of lanes) { + const times: number[] = []; + const digests = new Set(); + for (let run = -2; run < 5; run++) { + const body = structuredClone(lane.body); + const started = performance.now(); + const result = await lane.rewrite(body, ctx); + const elapsed = performance.now() - started; + if (run >= 0) { + times.push(elapsed); + digests.add(createHash("sha256").update(JSON.stringify(result)).digest("hex")); + } + } + if (digests.size !== 1) throw new Error("Output changed between samples"); + results.push({ protocol: lane.name, absorb: armed, timings_ms: times, median_ms: [...times].sort((a, b) => a - b)[2], digest: [...digests][0] }); + } +} +console.log(JSON.stringify({ node: process.version, bytes, messages: messages.length, warmups: 2, samples: 5, results, max_rss_kib: process.resourceUsage().maxRSS })); diff --git a/src/stream-google.ts b/src/stream-google.ts index 54dc5dc18a..fa511a2151 100644 --- a/src/stream-google.ts +++ b/src/stream-google.ts @@ -27,7 +27,11 @@ function* rewriteGoogleJsonSteps(body: unknown, ctx: RewriteCtx): Generator containsRenderTagText(text) || containsMarkerLineText(text) || containsBiliInternalText(text) || (absorbArmed && containsToolCallEmissionText(text)); + // Capture request evidence before a yielded proxy tool can change the context. + const requestText = absorbArmed && parts.some(part => part && typeof part === "object" + && (part as { thought?: boolean }).thought !== true && typeof part.text === "string" && containsEcho(part.text)) + ? JSON.stringify(ctx.messages) : undefined; for (const part of parts) { if (!part || typeof part !== "object") continue; const fc = part.functionCall as { name?: unknown; args?: unknown; id?: unknown } | undefined; @@ -41,7 +45,7 @@ function* rewriteGoogleJsonSteps(body: unknown, ctx: RewriteCtx): Generator {} }; +} + +const lanes = [ + { name: "anthropic", rewrite: rewriteJsonResponseAsync, body: (text: string) => ({ content: [{ type: "text", text }], stop_reason: "end_turn" }) }, + { name: "openai", rewrite: rewriteOpenaiJsonResponseAsync, body: (text: string) => ({ choices: [{ message: { role: "assistant", content: text }, finish_reason: "stop" }] }) }, + { name: "responses", rewrite: rewriteResponsesJsonResponseAsync, body: (text: string) => ({ output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text }] }], status: "completed" }) }, + { name: "google", rewrite: rewriteGoogleJsonResponseAsync, body: (text: string) => ({ candidates: [{ content: { role: "model", parts: [{ text }] }, finishReason: "STOP" }] }) }, +]; + +for (const lane of lanes) { + for (const scenario of [ + { name: "ordinary text, absorb disabled", armed: false, request: "Answer normally", text: "你好 🌏 — complete.", expected: "你好 🌏 — complete." }, + { name: "ordinary text, absorb enabled", armed: true, request: "Answer normally", text: "你好 🌏 — complete.", expected: "你好 🌏 — complete." }, + { name: "user-quoted emission", armed: true, request: `Output exactly this fragment:\n${emission.trim()}`, text: emission, expected: emission }, + { name: "model-invented emission", armed: true, request: emission.replace("m00608", "m00999"), text: emission, expected: "" }, + { name: "unarmed emission", armed: false, request: "Answer normally", text: emission, expected: emission }, + { name: "render tags without absorb", armed: false, request: "Answer normally", text: `before ${renderTag} after`, expected: "before after" }, + ]) { + test(`${lane.name}: preserves ${scenario.name} behavior`, async () => { + const ctx = context(scenario.armed, scenario.request); + const history = structuredClone(ctx.messages); + assert.deepEqual(await lane.rewrite(lane.body(scenario.text), ctx), lane.body(scenario.expected)); + assert.deepEqual(ctx.messages, history, "reply filtering never changes request history"); + }); + } +} + +test("anthropic: each prose field sees the same quoted-request evidence", async () => { + const ctx = context(true, `Output exactly:\n${emission.trim()}`); + const body = { content: [{ type: "text", text: renderTag }, { type: "text", text: emission }, { type: "text", text: emission }] }; + assert.deepEqual(await rewriteJsonResponseAsync(body, ctx), { content: [{ type: "text", text: "" }, { type: "text", text: emission }, { type: "text", text: emission }] }); +}); + +test("google: quoted prose is preserved without touching signed thoughts or tool arguments", async () => { + const ctx = context(true, `Output exactly:\n${emission.trim()}`); + const thought = { text: emission + renderTag, thought: true, thoughtSignature: "signed-thought" }; + const call = { functionCall: { name: "write_file", args: { content: emission + renderTag } } }; + const body = { candidates: [{ content: { role: "model", parts: [thought, { text: renderTag }, call, { text: emission }, { text: emission }] } }] }; + assert.deepEqual(await rewriteGoogleJsonResponseAsync(body, ctx), { candidates: [{ content: { role: "model", parts: [thought, { text: "" }, call, { text: emission }, { text: emission }] } }] }); +});