From 294f7c5a2660c9f2ab462c7605c2f2a78d44f799 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sun, 6 Sep 2026 14:58:07 +0800 Subject: [PATCH] fix: report uncompressed baseline in host usage + clamp negative persisted token stats (#408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - prepare* (anthropic/openai/responses) arm session.hostCreditTokens = est(original) - est(processed) each request; every host-facing usage outlet adds the credit back (adapters' raw SSE chunks, loop completion, plugin passthrough SSE/JSON, server JSON path) so the host anchors on the uncompressed baseline instead of the post-fold value - /acp panel contextTokens switches to hostContextTokens; pi agent panel shows the backfill line - persist: buildSession clamps negative lastInputTokens/contextTokens at restore; loaders rewrite stale files carrying negatives (one-time migration, logged) - openai split-semantics upstreams (prompt_tokens < cached_tokens) now add the cached segment back in usageTotals/recordUsage/applyUsageSample via shared promptInputTotal — fixes the ~0% context reading on cache-heavy anthropic-family upstreams served over an openai wire --- src/agent/pi.ts | 4 + src/loop/adapter-anthropic.ts | 22 +- src/loop/adapter-openai.ts | 28 +- src/loop/core.ts | 25 +- src/loop/index.ts | 5 +- src/persist.ts | 44 ++- src/plugin.ts | 91 ++++-- src/server.ts | 44 ++- src/session.ts | 15 + src/util.ts | 53 ++- tests/host-usage-backfill.test.ts | 515 ++++++++++++++++++++++++++++++ 11 files changed, 801 insertions(+), 45 deletions(-) create mode 100644 tests/host-usage-backfill.test.ts diff --git a/src/agent/pi.ts b/src/agent/pi.ts index 7d2c5e3b3f..6c65b6336b 100644 --- a/src/agent/pi.ts +++ b/src/agent/pi.ts @@ -122,6 +122,10 @@ function renderAcpStatus(s: Record): string { const pct = contextLimit !== null && contextLimit > 0 ? ` (${((contextTokens / contextLimit) * 100).toFixed(1)}%)` : ""; lines.push(` context: ${fmtTok(contextTokens)}${contextLimit !== null ? ` / ${fmtTok(contextLimit)}` : ""}${pct}`); } + const hostCredit = num(s.hostCredit); + if (hostCredit !== null && hostCredit > 0) { + lines.push(` host baseline: uncompressed (proxy backfilled +${fmtTok(hostCredit)} tok)`); + } if (inputTokens !== null || outputTokens !== null || cachedTokens !== null) { lines.push(` in/out/cached: ${fmtTok(inputTokens ?? 0)} / ${fmtTok(outputTokens ?? 0)} / ${fmtTok(cachedTokens ?? 0)}`); } diff --git a/src/loop/adapter-anthropic.ts b/src/loop/adapter-anthropic.ts index 7b4d50c9f6..0ac26ceaa3 100644 --- a/src/loop/adapter-anthropic.ts +++ b/src/loop/adapter-anthropic.ts @@ -117,7 +117,7 @@ function buildTextDeltaEvent(index: number, text: string): Buffer { ); } -export function createAnthropicAdapter(requestBody: Record, originalSystem?: AnthropicRequestBody["system"]): CompressLoopAdapter { +export function createAnthropicAdapter(requestBody: Record, originalSystem?: AnthropicRequestBody["system"], hostCredit = 0): CompressLoopAdapter { const model = (requestBody.model as string) ?? undefined; let messageId: string | undefined; let clientIndex = 0; @@ -236,8 +236,26 @@ export function createAnthropicAdapter(requestBody: Record, ori if (typeof u.input_tokens === "number") roundInput = u.input_tokens; if (typeof u.cache_read_input_tokens === "number") roundCached = u.cache_read_input_tokens; if (round === 1) { + // #408: this raw message_start (post-fold input_tokens) + // reaches the host verbatim — add the prepare-time + // credit back so the host anchors on the uncompressed + // baseline. + let chunk = rawBuf; + if (hostCredit > 0 && typeof u.input_tokens === "number") { + const patched = structuredClone(data); + const pmsg = patched["message"] as Record | undefined; + const pu = (pmsg?.["usage"] ?? {}) as Record; + if (typeof pu.input_tokens === "number") { + pu.input_tokens += hostCredit; + const out = eventStr + .split("\n") + .map((l) => (l.startsWith("data:") ? `data: ${JSON.stringify(patched)}` : l)) + .join("\n"); + chunk = Buffer.from(out + "\n\n", "utf8"); + } + } messageStartForwarded = true; - yield { kind: "meta", chunk: rawBuf, firstRoundOnly: true } as ParsedStreamEvent; + yield { kind: "meta", chunk, firstRoundOnly: true } as ParsedStreamEvent; } } else if (type === "ping") { yield { kind: "meta", chunk: rawBuf } as ParsedStreamEvent; diff --git a/src/loop/adapter-openai.ts b/src/loop/adapter-openai.ts index a86d47cb66..64a5ed08d0 100644 --- a/src/loop/adapter-openai.ts +++ b/src/loop/adapter-openai.ts @@ -115,7 +115,7 @@ function stripFinishReasonChunk(buf: Buffer): Buffer { } } -export function createOpenaiAdapter(requestBody: Record, clientSystem?: string): CompressLoopAdapter { +export function createOpenaiAdapter(requestBody: Record, clientSystem?: string, hostCredit = 0): CompressLoopAdapter { const model = (requestBody.model as string) ?? "unknown"; let responseId = `chatcmpl-proxy-${Date.now()}`; let toolIndex = 0; @@ -330,11 +330,35 @@ export function createOpenaiAdapter(requestBody: Record, client cachedTokens: typeof pd?.cached_tokens === "number" ? pd.cached_tokens : undefined, } as ParsedStreamEvent; if (sawRealToolCall) { + // #408: this raw finish chunk (with the provider's + // post-fold usage) reaches the host verbatim — add the + // prepare-time credit back so the host anchors on the + // uncompressed baseline. + let chunk = rawBuf; + if (hostCredit > 0 && u) { + const pu = typeof u.prompt_tokens === "number" ? u.prompt_tokens : undefined; + const tu = typeof u.total_tokens === "number" ? u.total_tokens : undefined; + if (pu !== undefined || tu !== undefined) { + const patched = { + ...parsed, + usage: { + ...u, + ...(pu !== undefined ? { prompt_tokens: pu + hostCredit } : {}), + ...(tu !== undefined ? { total_tokens: tu + hostCredit } : {}), + }, + }; + const out = eventStr + .split("\n") + .map((l) => (l.startsWith("data:") ? `data: ${JSON.stringify(patched)}` : l)) + .join("\n"); + chunk = Buffer.from(out + "\n\n", "utf8"); + } + } // This verbatim chunk IS the round's authoritative completion // (suppressCompletion); write it once and never fall through // to the text/reasoning branches (which would re-emit the same // bytes after the finish reason). - yield { kind: "meta", chunk: rawBuf } as ParsedStreamEvent; + yield { kind: "meta", chunk } as ParsedStreamEvent; yield { kind: "done", finishReason, suppressCompletion: true } as ParsedStreamEvent; continue; } else { diff --git a/src/loop/core.ts b/src/loop/core.ts index 8e6ba6be55..3f03f931ae 100644 --- a/src/loop/core.ts +++ b/src/loop/core.ts @@ -22,7 +22,7 @@ import { proxyDispatcher } from "../upstream-proxy.js"; import { noteWeakOverflow } from "../weak-overflow.js"; import { warnCacheCollapse } from "../cache-warn.js"; import { log as loggerLog } from "../logger.js"; -import type { WireProtocol } from "../util.js"; +import { promptInputTotal, type WireProtocol } from "../util.js"; export const MAX_LOOP_ROUNDS = 10; @@ -200,21 +200,15 @@ function recordUsage( const prompt = usage.inputTokens; const cached = usage.cachedTokens; const out = usage.outputTokens; - // Adapters emit `inputTokens` in protocol-native units: Anthropic's - // `input_tokens` is the NEW (uncached) portion only (cached reported - // separately), while OpenAI/Responses report the TOTAL (cached already - // included). Add `cached` back in ONLY when it is not already part of - // `prompt` — otherwise the cached portion is double-counted, inflating the - // context size (→ premature compression) and deflating the hit rate. - const includesCached = ctx.protocol === "openai" || ctx.protocol === "responses"; - const total = - (typeof prompt === "number" ? prompt : 0) + - (!includesCached && typeof cached === "number" ? cached : 0); + const total = promptInputTotal(ctx.protocol, prompt, cached); if (total > 0) ctx.session.stats.inputTokens += total; // Net out this turn's compress credit: the post-compress re-request // re-sends the unfolded history, so its usage report over-reports the // context the NEXT request will actually carry (see stream.ts applyRanges). ctx.session.stats.lastInputTokens = Math.max(0, total - (ctx.session.stats.compressCreditTokens ?? 0)); + // #408: remember the input-side total reported to the host AFTER the + // prepare-time fold backfill (uncompressed baseline), for the /acp panel. + ctx.session.hostContextTokens = total + (ctx.session.hostCreditTokens ?? 0); if (typeof cached === "number") { ctx.session.stats.cachedTokens += cached; ctx.session.stats.cacheSamples += 1; @@ -395,6 +389,15 @@ export async function* runCompressLoop( ) { recordUsage(ctx, usage, round); } + // #408: the provider measured the FOLDED (post-compress) view; add + // the prepare-time credit back so the completion event the host + // anchors on reports the uncompressed baseline. recordUsage above + // already ran on the un-backfilled numbers (internal ledger stays + // post-fold). + const hostCredit = ctx.session.hostCreditTokens ?? 0; + if (hostCredit > 0 && typeof usage.inputTokens === "number") { + usage.inputTokens += hostCredit; + } let resolvedText = assistantText; let allCalls = calls; diff --git a/src/loop/index.ts b/src/loop/index.ts index 9619e7cc18..ed3662c21a 100644 --- a/src/loop/index.ts +++ b/src/loop/index.ts @@ -25,9 +25,10 @@ export function pickAdapter( responsesProjection?: ResponsesProjection, anthropicSystem?: AnthropicRequestBody["system"], openaiSystem?: string, + hostCredit = 0, ): CompressLoopAdapter { if (protocol === "responses") return createResponsesAdapter(textProtocol, responsesProjection); - if (protocol === "openai") return createOpenaiAdapter(requestBody, openaiSystem); - if (protocol === "anthropic") return createAnthropicAdapter(requestBody, anthropicSystem); + if (protocol === "openai") return createOpenaiAdapter(requestBody, openaiSystem, hostCredit); + if (protocol === "anthropic") return createAnthropicAdapter(requestBody, anthropicSystem, hostCredit); throw new Error(`[acp-loop] unknown protocol: ${protocol}`); } diff --git a/src/persist.ts b/src/persist.ts index 84210ae05c..d96e0badf9 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -207,8 +207,20 @@ export class SessionStore { async loadAll(): Promise> { const out = new Map(); if (!this.enabled) return out; + let clamped = 0; for (const [id, envelope] of await this.store.loadAll()) { - out.set(id, buildSession(envelope.payload)); + const session = buildSession(envelope.payload); + if (hasNegativePersistedTokens(envelope.payload)) { + // #408 one-time migration: the in-memory value is already + // clamped by buildSession — rewrite the stale file so the + // negative value is gone from disk. + await this.store.writeNow(id, () => buildRecord(session)); + clamped++; + } + out.set(id, session); + } + if (clamped > 0) { + loggerLog("info", `[persist] one-time migration (#408): clamped negative lastInputTokens/contextTokens in ${clamped} session(s) to 0`); } return out; } @@ -349,7 +361,16 @@ export class SessionStore { meta?.protocol ? this.store.loadSync(id, relPathFor(id)) : null, ]; for (const envelope of envelopes) { - if (envelope) return buildSession(envelope.payload); + if (envelope) { + const session = buildSession(envelope.payload); + if (hasNegativePersistedTokens(envelope.payload)) { + // #408: sync context — debounce the stale-file rewrite + // (buildSession already clamped the in-memory value). + this.scheduleSave(session); + loggerLog("info", `[persist] clamped negative token stats on reload for ${id} (#408)`); + } + return session; + } } return null; } @@ -449,10 +470,14 @@ function buildSession(parsed: PersistedSession): Session { cachedTokens: stats.cachedTokens ?? parsed.cachedTokens ?? 0, outputTokens: stats.outputTokens ?? parsed.outputTokens ?? 0, cacheSamples: stats.cacheSamples ?? parsed.cacheSamples ?? 0, - lastInputTokens: stats.lastInputTokens ?? parsed.lastInputTokens ?? 0, + // #408: clamp at restore — pre-clamp versions persisted negative + // values (lastInputTokens = total − credit before the Math.max + // guard existed) which would otherwise revive after upgrade and + // feed the /acp panel + web stats as negative percentages. + lastInputTokens: Math.max(0, stats.lastInputTokens ?? parsed.lastInputTokens ?? 0), // In-memory only — a fresh process has no pending compress fold. compressCreditTokens: 0, - contextTokens: stats.contextTokens ?? parsed.contextTokens ?? 0, + contextTokens: Math.max(0, stats.contextTokens ?? parsed.contextTokens ?? 0), }, metadata: parsed.metadata ?? {}, state: mergeState(parsed.state), @@ -477,6 +502,17 @@ function isValidRecord(parsed: unknown): parsed is PersistedSession { return typeof r.id === "string" && typeof r.state === "object" && r.state !== null && Array.isArray(r.state.blocks); } +/** #408: true when a persisted record (grouped v2+ or flat v1) carries a + * negative lastInputTokens/contextTokens — only possible in files written by + * pre-clamp versions. buildSession clamps on read; this lets the loaders + * rewrite the stale file once so the negative value is gone from disk. */ +function hasNegativePersistedTokens(parsed: PersistedSession): boolean { + const stats = parsed.stats ?? {}; + const last = stats.lastInputTokens ?? parsed.lastInputTokens; + const ctx = stats.contextTokens ?? parsed.contextTokens; + return (typeof last === "number" && last < 0) || (typeof ctx === "number" && ctx < 0); +} + function defaultDir(): string { return sessionsDir(); } diff --git a/src/plugin.ts b/src/plugin.ts index 4a4b6ad69c..b8d9aa942c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -11,7 +11,7 @@ import { containsRenderTagText, createTagEchoFilter, mayStartRenderTag, stripAnt import { log as loggerLog } from "./logger.js"; import { noteWeakOverflow } from "./weak-overflow.js"; import { warnCacheCollapse } from "./cache-warn.js"; -import type { WireProtocol } from "./util.js"; +import { backfillHostUsage, promptInputTotal, type WireProtocol } from "./util.js"; import { stateDir } from "./paths.js"; // The proxy's own version, read from package.json at runtime (works in both dev @@ -389,7 +389,7 @@ export function handlePluginStatus(conversationId: string, res: import("node:htt try { panel = buildStatusPanel({ version: `billion-context@${PROXY_VERSION}`, - tokenCount: session.stats.lastInputTokens, + tokenCount: session.hostContextTokens ?? session.stats.lastInputTokens, systemPromptTokens: 0, state: session.state, nudge: mem?.nudge, @@ -409,7 +409,8 @@ export function handlePluginStatus(conversationId: string, res: import("node:htt label: session.meta.label ?? null, pluginAgent: session.metadata.pluginAgent ?? null, contextLimit: typeof limit === "number" ? limit : null, - contextTokens: session.stats.lastInputTokens, + contextTokens: session.hostContextTokens ?? session.stats.lastInputTokens, + hostCredit: session.hostCreditTokens ?? 0, inputTokens: session.stats.inputTokens, outputTokens: session.stats.outputTokens, cachedTokens: session.stats.cachedTokens, @@ -537,21 +538,22 @@ function usageFromSseEvent(obj: Record): UsageSample | undefine function applyUsageSample(session: Session, sample: UsageSample, protocol?: WireProtocol): void { // inputTokens is protocol-native: Anthropic reports it NEW-only (cached // separate); OpenAI/Responses report the TOTAL (cached already included). - // Add cached back in only when it is not already part of inputTokens. - const includesCached = protocol === "openai" || protocol === "responses"; + // promptInputTotal adds the cached segment back when it is not part of + // inputTokens — including the #408 split-semantics violation on OpenAI + // wires (prompt_tokens < cached_tokens → cached is NOT included). if (sample.cachedTokens !== undefined) { session.stats.cachedTokens += sample.cachedTokens; session.stats.cacheSamples += 1; } if (sample.inputTokens !== undefined) { - const total = - sample.inputTokens + - (!includesCached && sample.cachedTokens !== undefined ? sample.cachedTokens : 0); + const total = promptInputTotal(protocol, sample.inputTokens, sample.cachedTokens); session.stats.inputTokens += total; // Net out pending compress savings (see stream.ts applyRanges): plugin // compress tool results shrink the next request, not this report. session.stats.lastInputTokens = Math.max(0, total - (session.stats.compressCreditTokens ?? 0)); warnCacheCollapse(session, total, sample.cachedTokens ?? 0); + // #408: host-facing baseline = this report + prepare-time fold credit. + session.hostContextTokens = total + (session.hostCreditTokens ?? 0); } if (sample.outputTokens !== undefined) session.stats.outputTokens += sample.outputTokens; } @@ -592,6 +594,7 @@ export async function pipePluginChatWithStrip( const decoder = new TextDecoder("utf-8"); let buf = ""; const acc: UsageSample = {}; + const credit = session?.hostCreditTokens ?? 0; const onDrop = (snippet: string) => { loggerLog("warn", `[tag-echo] stripped model-emitted render tag (plugin passthrough): ${snippet.slice(0, 80).replace(/\n/g, " ")}`); log?.(`[tag-echo] stripped model-emitted render tag from plugin passthrough text`); @@ -757,14 +760,38 @@ export async function pipePluginChatWithStrip( if (ev["type"] === "message_stop") sawTerminal = true; const sample = usageFromSseEvent(ev); if (sample) mergeUsageSample(acc, sample); + // #408: backfill the input-side usage so the host anchors on + // the uncompressed baseline. Patch `ev` BEFORE the tag-echo + // processors run and only rebuild when they return the event + // verbatim — otherwise their render-tag stripping is lost. + // message_delta input_tokens is normally absent (or a 0 echo) + // — only patch a real echo. + let backfilled = false; + if (credit > 0 && protocol) { + const usage = + ev["type"] === "message_start" + ? ((ev["message"] as Record | undefined)?.["usage"] as Record | undefined) + : (ev["usage"] as Record | undefined); + const deltaEcho = ev["type"] === "message_delta" && (num(usage?.["input_tokens"]) ?? 0) <= 0; + if (usage && !deltaEcho && backfillHostUsage(protocol, usage, credit)) backfilled = true; + } const out = protocol === "anthropic" ? processAnthropic(ev, rawEvent) : processOpenai(ev, rawEvent); - if (out.length > 0) await write(out); + if (backfilled && out === rawEvent + "\n\n") { + await write(rebuildEvent(rawEvent, ev)); + } else if (out.length > 0) { + await write(out); + } } } if (res.destroyed || res.writableEnded) break; } + if (buf.length > 0 && !res.destroyed && !res.writableEnded) await write(buf); const rest = flushTails(); if (rest.length > 0 && !res.destroyed && !res.writableEnded) await write(rest); + // Settle BEFORE res.end() in the finally below: the client can issue + // its next request (e.g. /__bili/plugin/status, or the follow-up turn + // that reads lastInputTokens for the nudge decision) the moment the + // stream completes, and those must already see this usage. settleUsage(); maybeNoteTruncated(); } catch (e) { @@ -825,10 +852,11 @@ export async function pipePluginResponsesWithStrip( loggerLog("warn", `[tag-echo] stripped model-emitted render tag (plugin passthrough): ${snippet.slice(0, 80).replace(/\n/g, " ")}`); log?.(`[tag-echo] stripped model-emitted render tag from plugin passthrough text`); }); - const write = (s: string) => { + const write = (s: string): Promise => { if (!res.write(Buffer.from(s, "utf8"))) { return new Promise((r) => res.once("drain", () => r())); } + return Promise.resolve(); }; // #411: keep the usage sniffed before an abort (see // pipePluginChatWithStrip). @@ -896,7 +924,21 @@ export async function pipePluginResponsesWithStrip( ) { if (type === "response.completed" || type === "response.failed" || type === "response.incomplete") sawTerminal = true; // done-family events also carry full text payloads — strip those too. - const out = containsRenderTagText(jsonStr) ? rebuildEvent(rawEvent, stripResponsesText(ev)) : rawEvent + "\n\n"; + let evOut = ev; + let rebuild = containsRenderTagText(jsonStr); + if (rebuild) evOut = stripResponsesText(ev); + if (type === "response.completed") { + // #408: acc already holds the pre-backfill sample + // (usageFromSseEvent ran above) — the internal + // ledger stays post-fold, the host gets the + // uncompressed baseline. + const credit = session?.hostCreditTokens ?? 0; + const usage = (evOut["response"] as Record | undefined)?.["usage"] as Record | undefined; + if (credit > 0 && usage && backfillHostUsage("responses", usage, credit)) { + rebuild = true; + } + } + const out = rebuild ? rebuildEvent(rawEvent, evOut) : rawEvent + "\n\n"; await write(flushTail(out)); continue; } @@ -992,8 +1034,10 @@ export async function pipePluginJson( } reader.releaseLock(); const text = Buffer.concat(chunks).toString("utf8"); + let json: Record | undefined; + let mutated = false; try { - const json = JSON.parse(text) as Record; + json = JSON.parse(text) as Record; const usage = json["usage"] as Record | undefined; if (session && usage) { const input = num(usage["prompt_tokens"]) ?? num(usage["input_tokens"]); @@ -1007,19 +1051,26 @@ export async function pipePluginJson( num(usage["cache_read_input_tokens"]), }, protocol); markDirty(session); + // #408: backfill mutates json.usage in place — reserialize below. + const credit = session.hostCreditTokens ?? 0; + if (credit > 0 && protocol && backfillHostUsage(protocol, usage, credit)) { + mutated = true; + } } } } catch { /* non-JSON body — forward verbatim */ } - if (containsRenderTagText(text)) { + if (json && containsRenderTagText(text)) { // #206 parity for the non-streaming plugin path: the compress loop's // JSON branch strips render tags from every round; a verbatim plugin - // JSON response would re-feed the model's tag echoes. - try { - const json = JSON.parse(text) as Record; - const stripped = protocol === "responses" ? stripResponsesText(json) : protocol === "anthropic" ? stripAnthropicText(json) : stripOpenaiChatText(json); - res.end(Buffer.from(JSON.stringify(stripped), "utf8")); - return; - } catch { /* fall through to verbatim */ } + // JSON response would re-feed the model's tag echoes. Strips mutate in + // place, so this composes with the #408 usage backfill above — one + // parse, one reserialize. + json = protocol === "responses" ? stripResponsesText(json) : protocol === "anthropic" ? stripAnthropicText(json) : stripOpenaiChatText(json); + mutated = true; + } + if (mutated && json) { + res.end(Buffer.from(JSON.stringify(json), "utf8")); + return; } res.end(text); } diff --git a/src/server.ts b/src/server.ts index 1d1e19c518..e28936a802 100644 --- a/src/server.ts +++ b/src/server.ts @@ -70,7 +70,7 @@ import { flushPrefixAffinity, hydratePrefixAffinity, scheduleAffinityPersist } f import { consumePluginRegisterFor, flushConversations, handlePluginCompact, handlePluginManifest, handlePluginRegister, handlePluginStatus, handlePluginTool, loadConversations, pipePluginChatWithStrip, pipePluginJson, pipePluginResponsesWithStrip, pluginAgentHeader, pluginConversationHeader, pluginReportedContextWindow, recordPluginSession, rememberPluginMessages, takePendingPluginRegister } from "./plugin.js"; import { setupMitm, readMitmUpstream } from "./mitm.js"; import type { BiliMessage } from "acp-kernel/wire"; -import { systemToUser, isLoopbackAddress, inspectContextOverflow, reserveOutputHeadroom, shouldReserveOutputHeadroom, usageTotals, type WireProtocol } from "./util.js"; +import { backfillHostUsage, isLoopbackAddress, inspectContextOverflow, reserveOutputHeadroom, shouldReserveOutputHeadroom, systemToUser, usageTotals, type WireProtocol } from "./util.js"; import { BILI_TUNNEL_HEADER, checkTunnelDestination, tunnelAllowlistFromEnv } from "./tunnel-guard.js"; import { decodeRequestBody } from "./content-encoding.js"; @@ -1629,6 +1629,7 @@ function prepareAnthropic( const sessionId = session.id; const stream = parsed.stream === true; ++session.stats.requests; + session.hostCreditTokens = 0; const injectTools = opts.compress.injectTool && !pluginMode; if (isAutoModeClassifier(parsed)) { @@ -1719,6 +1720,16 @@ function prepareAnthropic( // identity chain (#268), not part of the Anthropic Messages API — strip it // so the real upstream never sees a field it doesn't know. delete (rebuilt as Record).prompt_cache_key; + // #408: tokens folded out of the forwarded view vs the host's own (unfolded) + // view — added back into the usage reported to the host so its anchor + // reflects the uncompressed baseline. Same estimator both sides, so + // systematic error cancels in the difference. + session.hostCreditTokens = processedMessages.length > 0 + ? Math.max(0, estimateCoreMessages(originalMessages) - estimateCoreMessages(processedMessages)) + : 0; + if (session.hostCreditTokens > 0) { + log("info", `[${sessionId}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`); + } return { body: JSON.stringify(rebuilt), session, processedMessages, originalMessages, anthropicSystem: parsed.system, protocol: "anthropic", stream, compressInjected: injectTools, pluginMode, nudge, prompts, renderTags: "text-only" } as Prepared; } @@ -1803,6 +1814,7 @@ function prepareOpenai( const sessionId = session.id; const stream = parsed.stream === true; ++session.stats.requests; + session.hostCreditTokens = 0; let openaiSystemText = ""; let processedMessages: CoreMessage[] = []; let originalMessages: CoreMessage[] = []; @@ -1913,6 +1925,16 @@ function prepareOpenai( if (stream && (rebuilt as Record).stream_options === undefined) { (rebuilt as Record).stream_options = { include_usage: true }; } + // #408: tokens folded out of the forwarded view vs the host's own (unfolded) + // view — added back into the usage reported to the host so its anchor + // reflects the uncompressed baseline. Same estimator both sides, so + // systematic error cancels in the difference. + session.hostCreditTokens = processedMessages.length > 0 + ? Math.max(0, estimateCoreMessages(originalMessages) - estimateCoreMessages(processedMessages)) + : 0; + if (session.hostCreditTokens > 0) { + log("info", `[${sessionId}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`); + } snapshotMessages(session, originalMessages); markDirty(session); return { body: JSON.stringify(rebuilt), session, processedMessages, originalMessages, protocol: "openai", stream, compressInjected: injectTools, pluginMode, nudge, prompts, openaiSystemText, renderTags: "text-only" } as Prepared; @@ -1935,6 +1957,7 @@ function prepareResponses( const sessionId = session.id; const stream = parsed.stream === true; ++session.stats.requests; + session.hostCreditTokens = 0; if (reconcileNativeCompactionBoundary(session)) { log("info", `[${sessionId}] reconciled ACP state after native Responses compact boundary`); } @@ -2148,6 +2171,16 @@ function prepareResponses( }); log("info", `[${sessionId}] responses forward tools=[${fwdTools.join(",")}] injectTool=${injectTools}${pluginMode ? " (plugin mode: wire injection suppressed)" : ""} NO_INJECT_TOOL=${!!process.env.ACP_NO_INJECT_TOOL} NO_COMPRESS_PROMPT=${!!process.env.ACP_NO_COMPRESS_PROMPT}`); } + // #408: tokens folded out of the forwarded view vs the host's own (unfolded) + // view — added back into the usage reported to the host so its anchor + // reflects the uncompressed baseline. Same estimator both sides, so + // systematic error cancels in the difference. + session.hostCreditTokens = processedMessages.length > 0 + ? Math.max(0, estimateCoreMessages(originalMessages) - estimateCoreMessages(processedMessages)) + : 0; + if (session.hostCreditTokens > 0) { + log("info", `[${sessionId}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`); + } snapshotMessages(session, originalMessages); markDirty(session); return { @@ -3070,7 +3103,7 @@ async function forward( const reqHeaders = buildForwardHeaders(headers); const textProtocol = prepared.protocol === "responses" && !!prepared.responsesTextProtocol; const systemPrompt = textProtocol ? buildCompressHybridSystemPrompt(prepared.prompts ?? defaultPrompts) : buildCompressSystemPrompt(prepared.prompts ?? defaultPrompts); - const adapter = pickAdapter(prepared.protocol, parsedReq, textProtocol, prepared.responsesProjection, prepared.anthropicSystem, prepared.openaiSystemText); + const adapter = pickAdapter(prepared.protocol, parsedReq, textProtocol, prepared.responsesProjection, prepared.anthropicSystem, prepared.openaiSystemText, prepared.session.hostCreditTokens ?? 0); const refreshFolded = (current: CoreMessage[]): CoreMessage[] => { // #422: mirror the prepare's fold with the post-compress state so // the re-request shows the compression the model just performed. @@ -3178,6 +3211,13 @@ async function forward( const out = u.completion_tokens ?? u.output_tokens; if (typeof out === "number") prepared.session.stats.outputTokens += out; } + // #408: the provider measured the folded view — add the + // prepare-time credit back so the host anchors on the + // uncompressed baseline. + const credit = prepared.session.hostCreditTokens ?? 0; + if (credit > 0 && backfillHostUsage(prepared.protocol, u, credit)) { + prepared.session.hostContextTokens = (typeof total === "number" ? total : 0) + credit; + } if (prepared.protocol === "openai") { rewriteOpenaiJsonResponse(json, ctx); } else if (prepared.protocol === "responses") { diff --git a/src/session.ts b/src/session.ts index f9d6144f2d..cae8a8fc8b 100644 --- a/src/session.ts +++ b/src/session.ts @@ -135,6 +135,19 @@ export type Session = { * retry callbacks to correlate a transient upstream rejection with the * rewrite that preceded it (#189). A fresh process has none. */ lastCompress?: LastCompressInfo; + /** In-memory only (NOT persisted): tokens folded out of THIS request's + * forwarded view vs the host's own (unfolded) view, computed in prepare* + * as est(originalMessages) − est(processedMessages). Usage recorders add + * this back into the input-side usage field before forwarding to the host, + * so the host's usage anchor carries the uncompressed baseline instead of + * the post-fold value (#408). Overwritten each prepare(); 0 when nothing + * was folded this request. */ + hostCreditTokens?: number; + /** In-memory only (NOT persisted): last input-side usage total reported to + * the host AFTER the hostCreditTokens backfill (uncompressed baseline). + * Feeds the /acp panel tokenCount so it matches what the host footer + * shows (#408). 0/undefined until the first backfilled usage lands. */ + hostContextTokens?: number; /** Promise chain for per-session serialization. Two concurrent requests * sharing a session id would interleave processTurn / stream-rewriter * mutations on session.state, corrupting it. withSessionLock chains each @@ -287,6 +300,8 @@ export function resetSessionCompression(session: Session): void { session.blockContents.clear(); session.stats.lastInputTokens = 0; session.stats.contextTokens = 0; + session.hostCreditTokens = 0; + session.hostContextTokens = 0; session.metadata.nativeCompactionAt = Date.now(); markDirty(session); } diff --git a/src/util.ts b/src/util.ts index 44f5066229..1f3f203e6c 100644 --- a/src/util.ts +++ b/src/util.ts @@ -78,9 +78,11 @@ export function usageTotals( }; } if (protocol === "openai") { + const prompt = num(usage["prompt_tokens"]); + const cached = num((usage["prompt_tokens_details"] as Record | undefined)?.["cached_tokens"]); return { - total: num(usage["prompt_tokens"]), - cached: num((usage["prompt_tokens_details"] as Record | undefined)?.["cached_tokens"]), + total: prompt !== undefined ? promptInputTotal("openai", prompt, cached) : undefined, + cached, }; } // responses @@ -90,6 +92,53 @@ export function usageTotals( }; } +/** #408: input-side total from a protocol-native input/cached pair. + * OpenAI/Responses report the TOTAL (cached already included); Anthropic + * reports fresh-only. Some OpenAI-wire upstreams violate that and report + * Anthropic-style split semantics (prompt_tokens = fresh-only, cached + * separate) — under true OpenAI semantics prompt_tokens >= cached_tokens + * always holds, so a violation proves the cached segment is NOT part of + * prompt_tokens and must be added back (e.g. {prompt_tokens:6, + * cached_tokens:26278} is a real ~26284-token prompt, not 6). */ +export function promptInputTotal( + protocol: WireProtocol | undefined, + input: number | undefined, + cached: number | undefined, +): number { + if (input === undefined) return 0; + const includesCached = protocol === "openai" || protocol === "responses"; + const splitSemantics = !includesCached || (typeof cached === "number" && input < cached); + return input + (splitSemantics && typeof cached === "number" ? cached : 0); +} + +/** #408: add the prepare-time fold credit back into a usage object's + * input-side fields, in place, so the host's usage anchor reports the + * uncompressed baseline instead of the post-fold value the provider measured. + * Field names per protocol: Anthropic/Responses `input_tokens` (TOTAL there), + * OpenAI `prompt_tokens` (+ `total_tokens` to keep the sum consistent). + * Returns true when anything was patched. */ +export function backfillHostUsage( + protocol: WireProtocol, + usage: Record, + credit: number, +): boolean { + if (!Number.isFinite(credit) || credit <= 0) return false; + let patched = false; + const add = (key: string): void => { + if (typeof usage[key] === "number" && Number.isFinite(usage[key])) { + usage[key] = (usage[key] as number) + credit; + patched = true; + } + }; + if (protocol === "openai") { + add("prompt_tokens"); + add("total_tokens"); + } else { + add("input_tokens"); + } + return patched; +} + /** Result of inspecting an upstream response for a "context too long" error. */ export interface ContextOverflowInfo { /** True if the response looks like an upstream context-overflow error. */ diff --git a/tests/host-usage-backfill.test.ts b/tests/host-usage-backfill.test.ts new file mode 100644 index 0000000000..af02c0c7b9 --- /dev/null +++ b/tests/host-usage-backfill.test.ts @@ -0,0 +1,515 @@ +import { test, before } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import http from "node:http"; +import { once } from "node:events"; +import type { Config, CoreMessage } from "acp-kernel"; +import { createCore, createInitialState, defaultConfig } from "acp-kernel"; +import type { Session } from "../src/session.ts"; +import { runCompressLoop, createResponsesAdapter, createOpenaiAdapter, createAnthropicAdapter } from "../src/loop/index.ts"; +import { backfillHostUsage, promptInputTotal, usageTotals } from "../src/util.ts"; +import { pipePluginChatWithStrip, pipePluginResponsesWithStrip, pipePluginJson, _resetPluginStateForTest } from "../src/plugin.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; + +function makeSession(id: string, hostCreditTokens?: number): Session { + return { + id, + meta: {}, + stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0 }, + metadata: {}, + state: createInitialState(), + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + ...(hostCreditTokens !== undefined ? { hostCreditTokens } : {}), + }; +} + +function makeCtx(id: string, messages: CoreMessage[], hostCreditTokens?: number): { + core: ReturnType; + config: Config; + messages: CoreMessage[]; + session: Session; + log: (m: string) => void; +} { + return { + core: createCore(), + config: defaultConfig(200000), + messages, + session: makeSession(id, hostCreditTokens), + log: () => {}, + }; +} + +function textMsg(id: string, role: "user" | "assistant", text: string): CoreMessage { + return { id, role, contentType: "text", text }; +} + +function streamOf(events: string[]): ReadableStream { + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < events.length) { + controller.enqueue(Buffer.from(events[i++], "utf8")); + } else { + controller.close(); + } + }, + }); +} + +function makeRes(chunks: Buffer[]) { + return { + write: (b: Buffer | string) => { + chunks.push(Buffer.from(b as string)); + return true; + }, + end: (b?: Buffer | string) => { + if (b !== undefined) chunks.push(Buffer.from(b as string)); + }, + once: () => {}, + destroyed: false, + writableEnded: false, + } as unknown as import("node:http").ServerResponse; +} + +async function withTempStore(name: string, fn: (dir: string, store: SessionStore) => Promise): Promise { + const dir = mkdtempSync(join(tmpdir(), `bili-host-usage-${name}-`)); + const store = new SessionStore({ dir, debounceMs: 5, enabled: true }); + try { + await fn(dir, store); + } finally { + store.cancelAll(); + rmSync(dir, { recursive: true, force: true }); + } +} + +function jsonFilesUnder(dir: string): string[] { + const out: string[] = []; + const walk = (d: string): void => { + for (const entry of readdirSync(d, { withFileTypes: true })) { + const p = join(d, entry.name); + if (entry.isDirectory()) { + walk(p); + } else if (entry.name.endsWith(".json") && !entry.name.includes(".tmp-")) { + out.push(p); + } + } + }; + walk(dir); + return out; +} + +test("backfillHostUsage: openai patches prompt_tokens + total_tokens", () => { + const u: Record = { prompt_tokens: 60000, completion_tokens: 5, total_tokens: 60005 }; + assert.equal(backfillHostUsage("openai", u, 40000), true); + assert.equal(u.prompt_tokens, 100000); + assert.equal(u.total_tokens, 100005); + assert.equal(u.completion_tokens, 5); +}); + +test("backfillHostUsage: openai prompt_tokens only (no total_tokens invented)", () => { + const u: Record = { prompt_tokens: 60000 }; + assert.equal(backfillHostUsage("openai", u, 40000), true); + assert.equal(u.prompt_tokens, 100000); + assert.equal("total_tokens" in u, false); +}); + +test("backfillHostUsage: anthropic + responses patch input_tokens", () => { + const a: Record = { input_tokens: 60000, output_tokens: 5 }; + assert.equal(backfillHostUsage("anthropic", a, 40000), true); + assert.equal(a.input_tokens, 100000); + assert.equal(a.output_tokens, 5); + const r: Record = { input_tokens: 60000 }; + assert.equal(backfillHostUsage("responses", r, 40000), true); + assert.equal(r.input_tokens, 100000); +}); + +test("backfillHostUsage: credit <= 0 or missing input field is a no-op", () => { + const u: Record = { prompt_tokens: 60000 }; + assert.equal(backfillHostUsage("openai", u, 0), false); + assert.equal(u.prompt_tokens, 60000); + const v: Record = { completion_tokens: 5 }; + assert.equal(backfillHostUsage("openai", v, 40000), false); + assert.equal(v.completion_tokens, 5); +}); + +test("#408: responses loop — host completion carries uncompressed baseline, internal ledger stays post-fold", async () => { + const ctx = makeCtx("loop-resp", [textMsg("raw_1", "user", "hello")], 40000); + const sse = (type: string, data: unknown): string => `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; + const body = + sse("response.created", { type: "response.created", response: { id: "resp_1", status: "in_progress" } }) + + sse("response.completed", { + type: "response.completed", + response: { id: "resp_1", status: "completed", output: [], usage: { input_tokens: 60000, output_tokens: 5 } }, + }); + const chunks: Buffer[] = []; + for await (const chunk of runCompressLoop( + streamOf([body]), + { ...ctx, protocol: "responses" }, + { model: "m", input: [] }, + { url: "https://upstream.test/v1/responses", headers: { authorization: "Bearer t" } }, + createResponsesAdapter(), + "", + )) { + chunks.push(chunk); + } + const out = Buffer.concat(chunks).toString("utf8"); + assert.ok(out.includes('"input_tokens":100000'), `expected backfilled input_tokens in completed event, got: ${out}`); + assert.ok(!out.includes('"input_tokens":60000'), "raw folded input_tokens must not reach the host"); + assert.equal(ctx.session.hostContextTokens, 100000); + assert.equal(ctx.session.stats.lastInputTokens, 60000); + assert.equal(ctx.session.stats.inputTokens, 60000); +}); + +test("#408: responses loop — no credit leaves usage untouched (control)", async () => { + const ctx = makeCtx("loop-resp-ctrl", [textMsg("raw_1", "user", "hello")]); + const sse = (type: string, data: unknown): string => `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; + const body = + sse("response.created", { type: "response.created", response: { id: "resp_1", status: "in_progress" } }) + + sse("response.completed", { + type: "response.completed", + response: { id: "resp_1", status: "completed", output: [], usage: { input_tokens: 60000, output_tokens: 5 } }, + }); + const chunks: Buffer[] = []; + for await (const chunk of runCompressLoop( + streamOf([body]), + { ...ctx, protocol: "responses" }, + { model: "m", input: [] }, + { url: "https://upstream.test/v1/responses", headers: { authorization: "Bearer t" } }, + createResponsesAdapter(), + "", + )) { + chunks.push(chunk); + } + const out = Buffer.concat(chunks).toString("utf8"); + assert.ok(out.includes('"input_tokens":60000'), out); + assert.equal(ctx.session.hostContextTokens, 60000); + assert.equal(ctx.session.stats.lastInputTokens, 60000); +}); + +test("#408: openai adapter — raw finish chunk with usage carries the backfill on real tool calls", async () => { + const adapter = createOpenaiAdapter({ model: "m" }, undefined, 40000); + const chunk = (o: unknown): string => `data: ${JSON.stringify(o)}\n\n`; + const stream = streamOf([ + chunk({ id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "get_weather", arguments: "" } }] } }] }), + chunk({ id: "c1", object: "chat.completion.chunk", created: 1, model: "m", choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 50000, completion_tokens: 10, total_tokens: 50010 } }), + "data: [DONE]\n\n", + ]); + let meta = ""; + for await (const ev of adapter.parseStream(stream, 1)) { + if (ev.kind === "meta") meta += ev.chunk.toString("utf8"); + } + assert.ok(meta.includes('"prompt_tokens":90000'), `expected backfilled prompt_tokens in raw finish chunk: ${meta}`); + assert.ok(meta.includes('"total_tokens":90010'), meta); +}); + +test("#408: anthropic adapter — first-round message_start meta carries backfilled input_tokens", async () => { + const adapter = createAnthropicAdapter({ model: "m" }, undefined, 40000); + const stream = streamOf([ + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { input_tokens: 60000, output_tokens: 1 } } })}\n\n`, + `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`, + ]); + let meta = ""; + for await (const ev of adapter.parseStream(stream, 1)) { + if (ev.kind === "meta") meta += ev.chunk.toString("utf8"); + } + assert.ok(meta.includes('"input_tokens":100000'), `expected backfilled input_tokens in message_start: ${meta}`); +}); + +before(_resetPluginStateForTest); + +test("#408: pipePluginChatWithStrip — openai final usage chunk backfilled, ledger post-fold", async () => { + await withTempStore("pipe-openai", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("pipe-oai", 40000); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + `data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: "Hi" } }] })}\n\n`, + `data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 60000, completion_tokens: 5, total_tokens: 60005 } })}\n\n`, + "data: [DONE]\n\n", + ]); + await pipePluginChatWithStrip(stream, res, "openai", session); + const out = chunks.join(""); + assert.ok(out.includes('"prompt_tokens":100000'), out); + assert.ok(out.includes('"total_tokens":100005'), out); + assert.equal(session.stats.lastInputTokens, 60000); + assert.equal(session.hostContextTokens, 100000); + }); +}); + +test("#408: pipePluginChatWithStrip — anthropic message_start backfilled, zero message_delta untouched", async () => { + await withTempStore("pipe-anthropic", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("pipe-ant", 40000); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "msg_1", usage: { input_tokens: 60000, cache_read_input_tokens: 1000 } } })}\n\n`, + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", usage: { input_tokens: 0, output_tokens: 10 } })}\n\n`, + ]); + await pipePluginChatWithStrip(stream, res, "anthropic", session); + const out = chunks.join(""); + assert.ok(out.includes('"input_tokens":100000'), out); + assert.ok(out.includes('"input_tokens":0'), "zero message_delta input_tokens must stay 0"); + assert.equal(session.stats.lastInputTokens, 61000); + assert.equal(session.hostContextTokens, 101000); + }); +}); + +test("#408: pipePluginChatWithStrip — no credit leaves bytes verbatim (control)", async () => { + await withTempStore("pipe-ctrl", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("pipe-ctrl"); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + `data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 60000, completion_tokens: 5, total_tokens: 60005 } })}\n\n`, + "data: [DONE]\n\n", + ]); + await pipePluginChatWithStrip(stream, res, "openai", session); + const out = chunks.join(""); + assert.ok(out.includes('"prompt_tokens":60000'), out); + assert.ok(!out.includes('"prompt_tokens":100000'), out); + assert.equal(session.hostContextTokens, 60000); + }); +}); + +test("#408: promptInputTotal — protocol matrix incl. openai split-semantics violation", () => { + // anthropic: fresh-only input, cached separate → sum + assert.equal(promptInputTotal("anthropic", 1000, 40000), 41000); + // openai true semantics: prompt_tokens is the total, cached ⊆ prompt + assert.equal(promptInputTotal("openai", 41000, 40000), 41000); + assert.equal(promptInputTotal("openai", 41000, undefined), 41000); + // openai split-semantics violation (prompt < cached → cached NOT included) + assert.equal(promptInputTotal("openai", 6, 26278), 26284); + assert.equal(promptInputTotal("responses", 6, 26278), 26284); + assert.equal(promptInputTotal("responses", 41000, 40000), 41000); + // undefined protocol behaves like anthropic (fresh-only) + assert.equal(promptInputTotal(undefined, 1000, 40000), 41000); + // missing input → 0 + assert.equal(promptInputTotal("openai", undefined, 26278), 0); +}); + +test("#408: usageTotals — openai split-semantics upstream adds the cached segment back", () => { + const { total, cached } = usageTotals("openai", { + prompt_tokens: 6, + completion_tokens: 197, + total_tokens: 203, + prompt_tokens_details: { cached_tokens: 26278 }, + }); + assert.equal(total, 26284); + assert.equal(cached, 26278); + // true OpenAI semantics unchanged + const ok = usageTotals("openai", { prompt_tokens: 30000, prompt_tokens_details: { cached_tokens: 29999 } }); + assert.equal(ok.total, 30000); +}); + +test("#408: pipePluginChatWithStrip — split-semantics openai usage keeps lastInputTokens at the real prompt size", async () => { + await withTempStore("pipe-split", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("pipe-split"); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + `data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: "Hi" } }] })}\n\n`, + `data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 6, completion_tokens: 197, total_tokens: 203, prompt_tokens_details: { cached_tokens: 26278 } } })}\n\n`, + "data: [DONE]\n\n", + ]); + await pipePluginChatWithStrip(stream, res, "openai", session); + assert.equal(session.stats.lastInputTokens, 26284); + assert.equal(session.hostContextTokens, 26284); + assert.equal(session.stats.cachedTokens, 26278); + }); +}); + +test("#408: pipePluginResponsesWithStrip — response.completed usage backfilled", async () => { + await withTempStore("pipe-resp", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("pipe-resp", 40000); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: { id: "resp_1", status: "completed", usage: { input_tokens: 60000, output_tokens: 5 } } })}\n\n`, + ]); + await pipePluginResponsesWithStrip(stream, res, session); + const out = chunks.join(""); + assert.ok(out.includes('"input_tokens":100000'), out); + assert.equal(session.stats.lastInputTokens, 60000); + assert.equal(session.hostContextTokens, 100000); + }); +}); + +test("#408: pipePluginJson — openai JSON usage backfilled", async () => { + await withTempStore("pipe-json", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("pipe-json", 40000); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const body = JSON.stringify({ + id: "c1", + object: "chat.completion", + choices: [{ message: { role: "assistant", content: "Hi" }, finish_reason: "stop" }], + usage: { prompt_tokens: 60000, completion_tokens: 5, total_tokens: 60005 }, + }); + await pipePluginJson(streamOf([body]), res, session, "openai"); + const out = chunks.join(""); + const json = JSON.parse(out) as { usage: Record }; + assert.equal(json.usage.prompt_tokens, 100000); + assert.equal(json.usage.total_tokens, 100005); + assert.equal(session.stats.lastInputTokens, 60000); + assert.equal(session.hostContextTokens, 100000); + }); +}); + +test("#408: persist — negative lastInputTokens/contextTokens clamp to 0 on load and the stale file is migrated", async () => { + await withTempStore("neg-stats", async (dir, store) => { + const s = makeSession("neg1"); + s.stats.lastInputTokens = -54119; + s.stats.contextTokens = -5; + await store.writeNow(s); + const store2 = new SessionStore({ dir, debounceMs: 5, enabled: true }); + const loaded = await store2.loadAll(); + const got = loaded.get("neg1"); + assert.ok(got); + assert.equal(got.stats.lastInputTokens, 0); + assert.equal(got.stats.contextTokens, 0); + const files = jsonFilesUnder(dir); + assert.equal(files.length, 1); + const env = JSON.parse(readFileSync(files[0], "utf8")) as { payload: { stats?: { lastInputTokens?: number; contextTokens?: number } } }; + assert.equal(env.payload.stats?.lastInputTokens, 0); + assert.equal(env.payload.stats?.contextTokens, 0); + store2.cancelAll(); + }); +}); + +test("#408: persist — flat v1 negative lastInputTokens clamps to 0 on load", async () => { + await withTempStore("neg-flat", async (dir, store) => { + const s = makeSession("neg2"); + await store.writeNow(s); + const files = jsonFilesUnder(dir); + assert.equal(files.length, 1); + const env = JSON.parse(readFileSync(files[0], "utf8")) as { payload: unknown }; + env.payload = { id: "neg2", state: createInitialState(), lastInputTokens: -54119 }; + writeFileSync(files[0], JSON.stringify(env)); + const store2 = new SessionStore({ dir, debounceMs: 5, enabled: true }); + const loaded = await store2.loadAll(); + const got = loaded.get("neg2"); + assert.ok(got); + assert.equal(got.stats.lastInputTokens, 0); + store2.cancelAll(); + }); +}); + +test("#408: prepareOpenai arms the credit — host sees backfilled usage after a compress fold", async () => { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const upstreamBodies: string[] = []; + const relay = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const body = Buffer.concat(chunks).toString("utf8"); + upstreamBodies.push(body); + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + const sse = (o: unknown): string => `data: ${JSON.stringify(o)}\n\n`; + if (upstreamBodies.length === 5) { + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: null, tool_calls: [{ index: 0, id: "call_c1", type: "function", function: { name: "compress", arguments: JSON.stringify({ content: [{ startId: "m00003", endId: "m00004", topic: "arm", summary: "summary of turns two and three: long-form filler dialogue about incremental context growth; key facts preserved for later reference" }] }) } }] }, finish_reason: null }] })); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], usage: { prompt_tokens: 100, completion_tokens: 2 } })); + } else { + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "ok" } }] })); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 100, completion_tokens: 5 } })); + } + res.write("data: [DONE]\n\n"); + res.end(); + }); + }); + relay.listen(0, "127.0.0.1"); + await once(relay, "listening"); + const relayPort = (relay.address() as { port: number }).port; + const opts: ProxyOptions = { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${relayPort}`]: { models: { "gpt-test": { context: 1_000_000 } }, compressProtocol: "marker" } } as ProxyOptions["routes"], + modelContextLimit: 1_000_000, + kernelConfig: defaultConfig(1_000_000, { + // preserveRecentTokens defaults to 5000 — the whole small fixture + // conversation would sit inside the token window and every range + // would be protected. Disable it; the last-5-messages window stays. + preserveRecentMessages: 5, + preserveRecentTokens: 0, + compress: { minCompressRange: 1000, maxSummaryLength: 20000, minSummaryLength: 50 }, + }), + compress: { injectTool: true, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + }; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${relayPort}/v1/chat/completions`; + const big = "the quick brown fox jumps over the lazy dog. ".repeat(120); + const history: Array<{ role: string; content: string }> = []; + const post = async (): Promise => { + const res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "host-usage-arm-1" }, + body: JSON.stringify({ model: "gpt-test", stream: true, messages: history }), + duplex: "half", + } as RequestInit); + if (!res.ok) { + const text = await res.text(); + assert.fail(`HTTP ${res.status}: ${text}`); + } + let raw = ""; + for await (const chunk of res.body!) raw += Buffer.from(chunk).toString("utf8"); + return raw; + }; + try { + for (let i = 1; i <= 4; i++) { + // u2 carries a sentinel: the fold must remove it from the forwarded view + history.push({ role: "user", content: i === 2 ? `t2 SENTINEL_FOLD_GONE ${big}` : `t${i} ${big}` }); + const r = await post(); + assert.ok(r.includes('"prompt_tokens":100'), `turn ${i} must report the raw usage (no fold yet): ${r}`); + history.push({ role: "assistant", content: "ok" }); + } + // turn 5: the model folds m00003..m00004 (u2 + a2, outside the kernel's + // protected zone of the last 5 messages). The range must NOT include + // m00001 — the kernel never prunes the first user message, so folding + // it would leave the big content in the forwarded view and the credit + // (est(original) − est(processed)) would stay ~0. + history.push({ role: "user", content: "t5" }); + const r2 = await post(); + assert.ok(!r2.includes('"name":"compress"'), `compress tool call must be suppressed from the host: ${r2}`); + assert.ok(!r2.includes("Compression FAILED"), `compress must succeed: ${r2}`); + + // turn 6: the forwarded view is folded; provider reports post-fold 100 + history.push({ role: "assistant", content: "ok" }); + history.push({ role: "user", content: "t6" }); + const r3 = await post(); + const m = r3.match(/"prompt_tokens":(\d+)/); + assert.ok(m, `turn 6 usage chunk missing: ${r3}`); + const prompt = Number(m[1]); + assert.ok(prompt > 100, `host-facing prompt_tokens must be backfilled above the post-fold 100 (got ${prompt})`); + assert.ok(prompt >= 100 + 200, `backfill must carry a meaningful share of the folded ~1400-token range (got ${prompt})`); + assert.ok(upstreamBodies.length >= 7, `expected 7 upstream requests (turn5 has a compress round-trip), got ${upstreamBodies.length}`); + assert.ok(!upstreamBodies[6]!.includes("SENTINEL_FOLD_GONE"), "turn-6 upstream body must not carry the folded u2 content — fold must have happened"); + } finally { + await new Promise((resolve, reject) => proxy.close((e) => (e ? reject(e) : resolve()))); + await new Promise((resolve, reject) => relay.close((e) => (e ? reject(e) : resolve()))); + } +});