Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions scripts/bench-json-rewrite-history.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import { createHash } from "node:crypto";
import { performance } from "node:perf_hooks";
import { createCore, defaultConfig, type CoreMessage } from "acp-kernel";
import { createSession } from "../src/session.ts";
import { rewriteJsonResponseAsync, type RewriteCtx } from "../src/stream.ts";
import { rewriteOpenaiJsonResponseAsync } from "../src/stream-openai.ts";
import { rewriteResponsesJsonResponseAsync } from "../src/stream-responses.ts";
import { rewriteGoogleJsonResponseAsync } from "../src/stream-google.ts";

// Run in a fresh process for each revision and size, with the test env preload:
// node --import ./tests/preload-test-env.mjs --import tsx scripts/bench-json-rewrite-history.ts 8388608
const bytes = Number(process.argv[2] ?? 8 * 1024 * 1024);
if (!Number.isSafeInteger(bytes) || bytes < 1) throw new Error("Expected a positive history byte count");
const messages: CoreMessage[] = [];
for (let remaining = bytes; remaining > 0;) {
const length = Math.min(8192, remaining);
messages.push({ id: `history-${messages.length}`, role: "assistant", contentType: "text", text: "x".repeat(length) });
remaining -= length;
}
const ctx: RewriteCtx = {
core: createCore(), config: defaultConfig(4_000_000), messages,
session: createSession("json-rewrite-benchmark"), log: () => {},
};
const text = "The requested change is complete. Tests passed.";
const lanes = [
{ name: "anthropic", rewrite: rewriteJsonResponseAsync, body: { content: [{ type: "text", text }], stop_reason: "end_turn" } },
{ name: "openai", rewrite: rewriteOpenaiJsonResponseAsync, body: { choices: [{ message: { role: "assistant", content: text }, finish_reason: "stop" }] } },
{ name: "responses", rewrite: rewriteResponsesJsonResponseAsync, body: { output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text }] }], status: "completed" } },
{ name: "google", rewrite: rewriteGoogleJsonResponseAsync, body: { candidates: [{ content: { role: "model", parts: [{ text }] }, finishReason: "STOP" }] } },
];
const results = [];
for (const armed of [false, true]) {
ctx.config.absorb!.enabled = armed;
for (const lane of lanes) {
const times: number[] = [];
const digests = new Set<string>();
for (let run = -2; run < 5; run++) {
const body = structuredClone(lane.body);
const started = performance.now();
const result = await lane.rewrite(body, ctx);
const elapsed = performance.now() - started;
if (run >= 0) {
times.push(elapsed);
digests.add(createHash("sha256").update(JSON.stringify(result)).digest("hex"));
}
}
if (digests.size !== 1) throw new Error("Output changed between samples");
results.push({ protocol: lane.name, absorb: armed, timings_ms: times, median_ms: [...times].sort((a, b) => a - b)[2], digest: [...digests][0] });
}
}
console.log(JSON.stringify({ node: process.version, bytes, messages: messages.length, warmups: 2, samples: 5, results, max_rss_kib: process.resourceUsage().maxRSS }));
8 changes: 6 additions & 2 deletions src/stream-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@ function* rewriteGoogleJsonSteps(body: unknown, ctx: RewriteCtx): Generator<Json
// Absorb signature in whole text (m00885): drop only when the request
// actually instructed the model about absorb.
const absorbArmed = effectiveAbsorbConfig(ctx.session, ctx.config)?.enabled === true;
const requestText = JSON.stringify(ctx.messages);
const containsEcho = (text: string): boolean => containsRenderTagText(text) || containsMarkerLineText(text) || containsBiliInternalText(text) || (absorbArmed && containsToolCallEmissionText(text));
// Capture request evidence before a yielded proxy tool can change the context.
const requestText = absorbArmed && parts.some(part => part && typeof part === "object"
&& (part as { thought?: boolean }).thought !== true && typeof part.text === "string" && containsEcho(part.text))
? JSON.stringify(ctx.messages) : undefined;
for (const part of parts) {
if (!part || typeof part !== "object") continue;
const fc = part.functionCall as { name?: unknown; args?: unknown; id?: unknown } | undefined;
Expand All @@ -41,7 +45,7 @@ function* rewriteGoogleJsonSteps(body: unknown, ctx: RewriteCtx): Generator<Json
// #1960/KDD#10: a thought part carries a thoughtSignature the client
// echoes back verbatim — rewriting its text desyncs the signature and
// bricks replay. Strip prose only from non-thought parts.
if ((part as { thought?: boolean }).thought !== true && typeof part.text === "string" && (containsRenderTagText(part.text) || containsMarkerLineText(part.text) || containsBiliInternalText(part.text) || (absorbArmed && containsToolCallEmissionText(part.text)))) {
if ((part as { thought?: boolean }).thought !== true && typeof part.text === "string" && containsEcho(part.text)) {
ctx.log(`[tag-echo] stripped: non-stream google output ACP echo (render tags/markers/internal artifacts): ${part.text.slice(0, 120).replace(/\n/g, " ")}`);
part.text = stripAcpTags(part.text, absorbArmed, requestText);
}
Expand Down
2 changes: 1 addition & 1 deletion src/stream-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ function* rewriteOpenaiJsonSteps(body: unknown, ctx: RewriteCtx): Generator<Json
// actually instructed the model about absorb — the shape check alone
// cannot tell an emission from a user-quoted fragment.
const absorbArmed = effectiveAbsorbConfig(ctx.session, ctx.config)?.enabled === true;
const requestText = JSON.stringify(ctx.messages);
if (existingText && (containsRenderTagText(existingText) || containsMarkerLineText(existingText) || containsBiliInternalText(existingText) || (absorbArmed && containsToolCallEmissionText(existingText)))) {
ctx.log(`[tag-echo] stripped: non-stream openai output ACP echo (render tags/markers/internal artifacts): ${existingText.slice(0, 120).replace(/\n/g, " ")}`);
const requestText = absorbArmed ? JSON.stringify(ctx.messages) : undefined;
existingText = stripAcpTags(existingText, absorbArmed, requestText);
msg.content = existingText;
}
Expand Down
2 changes: 1 addition & 1 deletion src/stream-responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ function* rewriteResponsesJsonSteps(body: unknown, ctx: RewriteCtx): Generator<J
// Absorb signature in whole text (m00885): drop only when the request
// actually instructed the model about absorb.
const absorbArmed = effectiveAbsorbConfig(ctx.session, ctx.config)?.enabled === true;
const requestText = JSON.stringify(ctx.messages);
if (containsRenderTagText(probe) || containsMarkerLineText(probe) || containsBiliInternalText(probe) || (absorbArmed && containsToolCallEmissionText(probe))) {
ctx.log(`[tag-echo] stripped: non-stream responses output ACP echo (render tags/markers/internal artifacts): ${probe.slice(0, 120).replace(/\n/g, " ")}`);
const requestText = absorbArmed ? JSON.stringify(ctx.messages) : undefined;
stripResponsesText(b, absorbArmed, requestText);
}
let converted = false;
Expand Down
4 changes: 2 additions & 2 deletions src/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1761,12 +1761,12 @@ function* rewriteJsonSteps(body: unknown, ctx: RewriteCtx): Generator<JsonToolCa
// actually instructed the model about absorb — the shape check alone
// cannot tell an emission from a user-quoted fragment.
const absorbArmed = effectiveAbsorbConfig(ctx.session, ctx.config)?.enabled === true;
const requestText = JSON.stringify(ctx.messages);
let requestText: string | undefined;
for (const blk of newContent) {
const t = (blk as { type?: string; text?: string }).text;
if (typeof t === "string" && (containsRenderTagText(t) || containsMarkerLineText(t) || containsBiliInternalText(t) || (absorbArmed && containsToolCallEmissionText(t)))) {
ctx.log(`[tag-echo] stripped: non-stream model output ACP echo (render tags/markers/internal artifacts): ${t.slice(0, 120).replace(/\n/g, " ")}`);
(blk as { text?: string }).text = stripAcpTags(t, absorbArmed, requestText);
(blk as { text?: string }).text = stripAcpTags(t, absorbArmed, absorbArmed ? (requestText ??= JSON.stringify(ctx.messages)) : undefined);
}
}
return body;
Expand Down
57 changes: 57 additions & 0 deletions tests/json-rewrite-history.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createCore, defaultConfig, type CoreMessage } from "acp-kernel";
import { createSession } from "../src/session.ts";
import { rewriteJsonResponseAsync, type RewriteCtx } from "../src/stream.ts";
import { rewriteOpenaiJsonResponseAsync } from "../src/stream-openai.ts";
import { rewriteResponsesJsonResponseAsync } from "../src/stream-responses.ts";
import { rewriteGoogleJsonResponseAsync } from "../src/stream-google.ts";

const emission = "\n<parameter=ref>\nm00608\n</parameter>\n<parameter=summary>\nconsume the read output\n</parameter>\n</function>\n</function_calls>";
const renderTag = '<acp tokens="177" type="text">m00155</acp>';

function context(armed: boolean, requestText: string): RewriteCtx {
const config = defaultConfig(200_000);
config.absorb!.enabled = armed;
const messages: CoreMessage[] = [{ id: "request", role: "user", contentType: "text", text: requestText }];
return { core: createCore(), config, messages, session: createSession("json-history-test"), log: () => {} };
}

const lanes = [
{ name: "anthropic", rewrite: rewriteJsonResponseAsync, body: (text: string) => ({ content: [{ type: "text", text }], stop_reason: "end_turn" }) },
{ name: "openai", rewrite: rewriteOpenaiJsonResponseAsync, body: (text: string) => ({ choices: [{ message: { role: "assistant", content: text }, finish_reason: "stop" }] }) },
{ name: "responses", rewrite: rewriteResponsesJsonResponseAsync, body: (text: string) => ({ output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text }] }], status: "completed" }) },
{ name: "google", rewrite: rewriteGoogleJsonResponseAsync, body: (text: string) => ({ candidates: [{ content: { role: "model", parts: [{ text }] }, finishReason: "STOP" }] }) },
];

for (const lane of lanes) {
for (const scenario of [
{ name: "ordinary text, absorb disabled", armed: false, request: "Answer normally", text: "你好 🌏 — complete.", expected: "你好 🌏 — complete." },
{ name: "ordinary text, absorb enabled", armed: true, request: "Answer normally", text: "你好 🌏 — complete.", expected: "你好 🌏 — complete." },
{ name: "user-quoted emission", armed: true, request: `Output exactly this fragment:\n${emission.trim()}`, text: emission, expected: emission },
{ name: "model-invented emission", armed: true, request: emission.replace("m00608", "m00999"), text: emission, expected: "" },
{ name: "unarmed emission", armed: false, request: "Answer normally", text: emission, expected: emission },
{ name: "render tags without absorb", armed: false, request: "Answer normally", text: `before ${renderTag} after`, expected: "before after" },
]) {
test(`${lane.name}: preserves ${scenario.name} behavior`, async () => {
const ctx = context(scenario.armed, scenario.request);
const history = structuredClone(ctx.messages);
assert.deepEqual(await lane.rewrite(lane.body(scenario.text), ctx), lane.body(scenario.expected));
assert.deepEqual(ctx.messages, history, "reply filtering never changes request history");
});
}
}

test("anthropic: each prose field sees the same quoted-request evidence", async () => {
const ctx = context(true, `Output exactly:\n${emission.trim()}`);
const body = { content: [{ type: "text", text: renderTag }, { type: "text", text: emission }, { type: "text", text: emission }] };
assert.deepEqual(await rewriteJsonResponseAsync(body, ctx), { content: [{ type: "text", text: "" }, { type: "text", text: emission }, { type: "text", text: emission }] });
});

test("google: quoted prose is preserved without touching signed thoughts or tool arguments", async () => {
const ctx = context(true, `Output exactly:\n${emission.trim()}`);
const thought = { text: emission + renderTag, thought: true, thoughtSignature: "signed-thought" };
const call = { functionCall: { name: "write_file", args: { content: emission + renderTag } } };
const body = { candidates: [{ content: { role: "model", parts: [thought, { text: renderTag }, call, { text: emission }, { text: emission }] } }] };
assert.deepEqual(await rewriteGoogleJsonResponseAsync(body, ctx), { candidates: [{ content: { role: "model", parts: [thought, { text: "" }, call, { text: emission }, { text: emission }] } }] });
});
Loading