Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 30 additions & 30 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1737,6 +1737,33 @@ function diagNudge(turn: { nudge?: { shouldInject: boolean; reason: string; cont
return `[${sessionId}] nudge ${inject}: usage=${pct} (${tokenCount}/${limit}), growth=${growth}/${floor} (ref=${ref}, interval=${interval}), pendingT1=${pendingT1}/${interval}${modelTag}, reason="${n.reason.slice(0, 120)}"`;
}

// #590: hosts that get the #408 uncompressed-baseline usage backfill. pi's
// plugin cancels pi's own auto-compaction (billion-context-pi
// wireCompactionDisable: "ACP owns compression"), so the baseline drives
// nothing there — reporting it only put a >100% footer on the host that
// mismatches the folded request actually forwarded. Hosts whose native
// compaction stays live (omp anchored-rewrite accounting, codex native-compact
// interception, plain proxy clients) keep the #408 behavior.
function armHostUsageCredit(
session: Session,
originalMessages: CoreMessage[],
processedMessages: CoreMessage[],
log: (level: string, msg: string) => void,
): void {
session.hostCreditTokens = 0;
if (session.metadata.pluginAgent === "pi") return;
// #408: tokens folded out of the forwarded view vs the host's own (unfolded)
// view — added back into the usage reported to the host so its anchor
// reflects the uncompressed baseline. Same estimator both sides, so
// systematic error cancels in the difference.
session.hostCreditTokens = processedMessages.length > 0
? Math.max(0, estimateCoreMessages(originalMessages) - estimateCoreMessages(processedMessages))
: 0;
if (session.hostCreditTokens > 0) {
log("info", `[${session.id}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`);
}
}

function prepareAnthropic(
parsed: AnthropicRequestBody,
req: http.IncomingMessage,
Expand Down Expand Up @@ -1846,16 +1873,7 @@ function prepareAnthropic(
// identity chain (#268), not part of the Anthropic Messages API — strip it
// so the real upstream never sees a field it doesn't know.
delete (rebuilt as Record<string, unknown>).prompt_cache_key;
// #408: tokens folded out of the forwarded view vs the host's own (unfolded)
// view — added back into the usage reported to the host so its anchor
// reflects the uncompressed baseline. Same estimator both sides, so
// systematic error cancels in the difference.
session.hostCreditTokens = processedMessages.length > 0
? Math.max(0, estimateCoreMessages(originalMessages) - estimateCoreMessages(processedMessages))
: 0;
if (session.hostCreditTokens > 0) {
log("info", `[${sessionId}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`);
}
armHostUsageCredit(session, originalMessages, processedMessages, log);
return { body: JSON.stringify(rebuilt), session, processedMessages, originalMessages, anthropicSystem: parsed.system, protocol: "anthropic", stream, compressInjected: injectTools, pluginMode, nudge, prompts, renderTags: "text-only" } as Prepared;
}

Expand Down Expand Up @@ -2099,16 +2117,7 @@ function prepareOpenai(
if (stream && (rebuilt as Record<string, unknown>).stream_options === undefined) {
(rebuilt as Record<string, unknown>).stream_options = { include_usage: true };
}
// #408: tokens folded out of the forwarded view vs the host's own (unfolded)
// view — added back into the usage reported to the host so its anchor
// reflects the uncompressed baseline. Same estimator both sides, so
// systematic error cancels in the difference.
session.hostCreditTokens = processedMessages.length > 0
? Math.max(0, estimateCoreMessages(originalMessages) - estimateCoreMessages(processedMessages))
: 0;
if (session.hostCreditTokens > 0) {
log("info", `[${sessionId}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`);
}
armHostUsageCredit(session, originalMessages, processedMessages, log);
// #532: title-gen side requests carry their own tiny system and would
// clobber the conversation's measured overhead — skip them.
if (!isTitleGen && openaiOutboundSystem !== undefined) {
Expand Down Expand Up @@ -2353,16 +2362,7 @@ function prepareResponses(
});
log("info", `[${sessionId}] responses forward tools=[${fwdTools.join(",")}] injectTool=${injectTools}${pluginMode ? " (plugin mode: wire injection suppressed)" : ""} NO_INJECT_TOOL=${!!process.env.ACP_NO_INJECT_TOOL} NO_COMPRESS_PROMPT=${!!process.env.ACP_NO_COMPRESS_PROMPT}`);
}
// #408: tokens folded out of the forwarded view vs the host's own (unfolded)
// view — added back into the usage reported to the host so its anchor
// reflects the uncompressed baseline. Same estimator both sides, so
// systematic error cancels in the difference.
session.hostCreditTokens = processedMessages.length > 0
? Math.max(0, estimateCoreMessages(originalMessages) - estimateCoreMessages(processedMessages))
: 0;
if (session.hostCreditTokens > 0) {
log("info", `[${sessionId}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`);
}
armHostUsageCredit(session, originalMessages, processedMessages, log);
// #532: measure the outbound developer(system)+tools overhead for the panel.
// On this wire the system rides the injected developer message outside the
// fold space, so counting devContent + tools does not double-count the
Expand Down
119 changes: 119 additions & 0 deletions tests/host-usage-backfill.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -513,3 +513,122 @@ test("#408: prepareOpenai arms the credit — host sees backfilled usage after a
await new Promise<void>((resolve, reject) => relay.close((e) => (e ? reject(e) : resolve())));
}
});

test("#590: pi plugin mode reports folded usage — host backfill suppressed", async () => {
_setStoreForTest(new SessionStore({ enabled: false }));
setRegistryForTest({});
_resetPluginStateForTest();
const upstreamBodies: string[] = [];
const anthropicSse = (event: string, data: unknown): string => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
const relay = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
const body = Buffer.concat(chunks).toString("utf8");
upstreamBodies.push(body);
res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
res.write(anthropicSse("message_start", { type: "message_start", message: { id: "msg_pi_1", role: "assistant", usage: { input_tokens: 100, output_tokens: 3 } } }));
res.write(anthropicSse("content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }));
res.write(anthropicSse("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } }));
res.write(anthropicSse("content_block_stop", { type: "content_block_stop", index: 0 }));
res.write(anthropicSse("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 3 } }));
res.write(anthropicSse("message_stop", { type: "message_stop" }));
res.end();
});
});
relay.listen(0, "127.0.0.1");
await once(relay, "listening");
const relayPort = (relay.address() as { port: number }).port;
const opts: ProxyOptions = {
port: 0,
host: "127.0.0.1",
upstream: "http://127.0.0.1",
routes: { [`http://127.0.0.1:${relayPort}`]: { models: { "claude-test": { context: 400_000 } } } } as ProxyOptions["routes"],
modelContextLimit: 400_000,
kernelConfig: defaultConfig(400_000),
compress: { injectTool: true, injectNudge: false },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
log: false,
debug: false,
passthrough: false,
autoUpdate: false,
mitm: { enabled: false, domains: [] },
};
const proxy = await startServer(opts);
await once(proxy, "listening");
const proxyPort = (proxy.address() as { port: number }).port;
const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${relayPort}/v1/messages`;
// Sizing copied from plugin-protocol.test.ts: the compressed head
// (m00001..m00002) exceeds minCompressibleChars while the protected-zone
// walk exhausts itself on the tail — so the fold is real and large
// enough that an ungated backfill would be plainly visible.
const headFiller = "lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ".repeat(28);
const tailFiller = "enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat duis aute irure dolor in reprehenderit in voluptate. ".repeat(28);
type AnthropicMessage = { role: string; content: string | Array<Record<string, unknown>> };
const history: AnthropicMessage[] = [];
for (let i = 1; i <= 2; i++) {
history.push({ role: "user", content: `turn-${i}-marker question: ${headFiller}` });
history.push({ role: "assistant", content: `turn-${i}-marker ${i === 1 ? "SENTINEL_FOLD_GONE " : ""}answer: ${headFiller}` });
}
for (let i = 3; i <= 5; i++) {
history.push({ role: "user", content: `turn-${i} padding question: ${tailFiller}` });
history.push({ role: "assistant", content: `turn-${i} padding answer: ${tailFiller}` });
}
const conv = "pi-folded-usage-1";
const post = async (messages: AnthropicMessage[]): Promise<string> => {
const res = await fetch(url, {
method: "POST",
headers: {
"content-type": "application/json",
"x-acp-session": conv,
"x-bili-plugin": "pi",
"x-bili-plugin-conversation": conv,
},
body: JSON.stringify({ model: "claude-test", max_tokens: 1024, stream: true, system: "You are a test assistant.", messages }),
});
if (!res.ok) {
const text = await res.text();
assert.fail(`HTTP ${res.status}: ${text}`);
}
let raw = "";
for await (const chunk of res.body!) raw += Buffer.from(chunk).toString("utf8");
return raw;
};
const inputTokensOf = (raw: string): number => {
const m = raw.match(/"input_tokens":(\d+)/);
assert.ok(m, `message_start usage missing: ${raw.slice(0, 400)}`);
return Number(m[1]);
};
try {
const r1 = await post(history);
assert.equal(inputTokensOf(r1), 100, "pre-fold turn must pass the raw usage through");

const toolResp = await fetch(`http://127.0.0.1:${proxyPort}/__bili/plugin/tool`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
conversationId: conv,
tool: "compress",
args: { content: [{ topic: "pi-folded-usage", startId: "m00001", endId: "m00002", summary: "pi-side compress of the two early turns covering the lorem-ipsum questions and answers" }] },
}),
});
assert.equal(toolResp.status, 200);
const toolJson = (await toolResp.json()) as { ok: boolean; result: string };
assert.equal(toolJson.ok, true, `compress tool reported failure: ${toolJson.result}`);

const r2 = await post([
...history,
{ role: "assistant", content: [{ type: "tool_use", id: "toolu_pi_1", name: "compress", input: { startId: "m00001", endId: "m00002", topic: "pi-folded-usage", summary: "pi-side compress of the two early turns covering the lorem-ipsum questions and answers" } }] },
{ role: "user", content: [{ type: "tool_result", tool_use_id: "toolu_pi_1", content: toolJson.result }] },
]);
// The fold must have happened upstream (sentinel gone) — otherwise
// this would pass vacuously with no credit to suppress.
assert.equal(upstreamBodies.length, 2);
assert.ok(!upstreamBodies[1]!.includes("SENTINEL_FOLD_GONE"), "post-fold upstream body must not carry the folded head content");
assert.equal(inputTokensOf(r2), 100, "pi plugin mode must report the folded request's own usage — no uncompressed-baseline backfill (#590)");
} finally {
await new Promise<void>((resolve, reject) => proxy.close((e) => (e ? reject(e) : resolve())));
await new Promise<void>((resolve, reject) => relay.close((e) => (e ? reject(e) : resolve())));
}
});
Loading