From b1061ec605f3fbf49b381b503a56d4eb3689c917 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 15 Sep 2026 09:48:02 +0800 Subject: [PATCH] fix: count Anthropic cache-write/read segments in plugin + loop usage accounting (#790) --- src/loop/adapter-anthropic.ts | 11 +- src/loop/core.ts | 9 +- src/plugin.ts | 44 +++++-- src/util.ts | 11 +- tests/host-usage-postfold.test.ts | 189 ++++++++++++++++++++++++++++++ 5 files changed, 246 insertions(+), 18 deletions(-) diff --git a/src/loop/adapter-anthropic.ts b/src/loop/adapter-anthropic.ts index 4566e81d38..c0aebf2b68 100644 --- a/src/loop/adapter-anthropic.ts +++ b/src/loop/adapter-anthropic.ts @@ -210,6 +210,7 @@ export function createAnthropicAdapter(requestBody: Record, ori const pending = new Map(); let roundInput: number | undefined; let roundCached: number | undefined; + let roundCreation: number | undefined; let roundOutput: number | undefined; let stopReason: string | undefined; let usageYielded = false; @@ -252,6 +253,7 @@ export function createAnthropicAdapter(requestBody: Record, ori const u = (msg.usage ?? {}) as Record; if (typeof u.input_tokens === "number") roundInput = u.input_tokens; if (typeof u.cache_read_input_tokens === "number") roundCached = u.cache_read_input_tokens; + if (typeof u.cache_creation_input_tokens === "number") roundCreation = u.cache_creation_input_tokens; if (round === 1) { // The raw message_start (with the provider's measured // usage) reaches the host verbatim — no rewriting. @@ -347,6 +349,7 @@ export function createAnthropicAdapter(requestBody: Record, ori if (typeof u.output_tokens === "number") roundOutput = u.output_tokens; const input = typeof u.input_tokens === "number" ? u.input_tokens : undefined; const cached = typeof u.cache_read_input_tokens === "number" ? u.cache_read_input_tokens : undefined; + const creation = typeof u.cache_creation_input_tokens === "number" ? u.cache_creation_input_tokens : undefined; if (input !== undefined && input > 0 && cached !== undefined) { // Complete authoritative usage object — e.g. the synthetic // terminal of a stitched multi-round stream (round1 @@ -355,9 +358,12 @@ export function createAnthropicAdapter(requestBody: Record, ori // compress re-request — overwrites the stale value carried by // an earlier round's message_start. Per-field merging would // double-count (new input + old cache) and trip false - // EMERGENCY nudges (issue #299). + // EMERGENCY nudges (issue #299). The cache-write segment + // (#790) is optional even in a complete object — adopt it + // when present, keep the prior value when absent. roundInput = input; roundCached = cached; + if (creation !== undefined) roundCreation = creation; } else { // Incomplete usage object: the input context is FIXED within // a turn, so message_start is authoritative for input/cache @@ -369,6 +375,7 @@ export function createAnthropicAdapter(requestBody: Record, ori // legitimate update, so adopt only > 0. if (input !== undefined && input > 0) roundInput = input; if (cached !== undefined && cached > 0) roundCached = cached; + if (creation !== undefined && creation > 0) roundCreation = creation; } const d = (data.delta ?? {}) as Record; if (typeof d.stop_reason === "string") stopReason = d.stop_reason; @@ -379,6 +386,7 @@ export function createAnthropicAdapter(requestBody: Record, ori inputTokens: roundInput, outputTokens: roundOutput, cachedTokens: roundCached, + creationTokens: roundCreation, } as ParsedStreamEvent; } maybeWarnDegenerate(stopReason); @@ -398,6 +406,7 @@ export function createAnthropicAdapter(requestBody: Record, ori inputTokens: roundInput, outputTokens: roundOutput, cachedTokens: roundCached, + creationTokens: roundCreation, } as ParsedStreamEvent; } maybeWarnDegenerate(stopReason); diff --git a/src/loop/core.ts b/src/loop/core.ts index c6d2203d59..a361b49449 100644 --- a/src/loop/core.ts +++ b/src/loop/core.ts @@ -99,7 +99,7 @@ export type ParsedStreamEvent = | { kind: "text"; delta: string; raw?: Buffer } | { kind: "reasoning"; delta: string; raw?: Buffer; signature?: string; blockEnd?: boolean } | { kind: "tool_call"; name: string; callId: string; arguments: string; passthrough?: boolean } - | { kind: "usage"; inputTokens?: number; outputTokens?: number; cachedTokens?: number } + | { kind: "usage"; inputTokens?: number; outputTokens?: number; cachedTokens?: number; creationTokens?: number } | { kind: "done"; finishReason?: string; suppressCompletion?: boolean; truncated?: boolean; thinking?: boolean } | { kind: "meta"; chunk: Buffer; firstRoundOnly?: boolean }; @@ -163,13 +163,13 @@ export function executeProxyTool( function recordUsage( ctx: LoopCtx, - usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number }, + usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number; creationTokens?: number }, round: number, ): void { const prompt = usage.inputTokens; const cached = usage.cachedTokens; const out = usage.outputTokens; - const total = promptInputTotal(ctx.protocol, prompt, cached); + const total = promptInputTotal(ctx.protocol, prompt, cached, usage.creationTokens); if (total > 0) ctx.session.stats.inputTokens += total; // Net out this turn's compress credit: the post-compress re-request // re-sends the unfolded history, so its usage report over-reports the @@ -249,7 +249,7 @@ export async function* runCompressLoop( const reasoningSegments: { text: string; signature: string }[] = []; let reasoningSealed = true; const calls: ToolCallEmit[] = []; - let usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number } = {}; + let usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number; creationTokens?: number } = {}; let finishReason: string | undefined; let sawDone = false; let suppressCompletion = false; @@ -309,6 +309,7 @@ export async function* runCompressLoop( inputTokens: ev.inputTokens, outputTokens: ev.outputTokens, cachedTokens: ev.cachedTokens, + creationTokens: ev.creationTokens, }; } else if (ev.kind === "done") { sawDone = true; diff --git a/src/plugin.ts b/src/plugin.ts index 380b34fdcb..f18c23f1bf 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -563,7 +563,9 @@ export async function handlePluginTool( res.end(JSON.stringify({ ok: true, tool, conversationId, result })); } -type UsageSample = { inputTokens?: number; cachedTokens?: number; outputTokens?: number }; +// creationTokens = Anthropic cache-write segment (cache_creation_input_tokens): +// part of the context size, but NOT a cache hit (#790). +type UsageSample = { inputTokens?: number; cachedTokens?: number; outputTokens?: number; creationTokens?: number }; function num(v: unknown): number | undefined { return typeof v === "number" && Number.isFinite(v) ? v : undefined; @@ -574,9 +576,17 @@ function usageFromSseEvent(obj: Record): UsageSample | undefine if (type === "message_start") { const usage = (obj["message"] as Record | undefined)?.["usage"] as Record | undefined; if (!usage) return undefined; + // Per-field snapshot (#790): a zero is a real value here (a fully + // cache-hit turn reports input_tokens: 0), so record whatever is + // present and let later events overwrite field by field. + const sample: UsageSample = {}; const input = num(usage["input_tokens"]); - if (input === undefined) return undefined; - return { inputTokens: input, cachedTokens: num(usage["cache_read_input_tokens"]) }; + if (input !== undefined) sample.inputTokens = input; + const read = num(usage["cache_read_input_tokens"]); + if (read !== undefined) sample.cachedTokens = read; + const creation = num(usage["cache_creation_input_tokens"]); + if (creation !== undefined) sample.creationTokens = creation; + return Object.keys(sample).length > 0 ? sample : undefined; } if (type === "message_delta") { const usage = obj["usage"] as Record | undefined; @@ -584,10 +594,20 @@ function usageFromSseEvent(obj: Record): UsageSample | undefine // Some relays echo `input_tokens: 0` in message_delta (the field is // normally absent — message_start is authoritative for the input size, // which is fixed within a turn). A 0 here is never a legitimate new - // value; merging it would zero acc.inputTokens (set by message_start) - // and collapse lastInputTokens to the cached portion only. + // value; merging it would zero out acc.inputTokens (set by message_start) + // and collapse lastInputTokens to the cached portion only. The same + // guard covers the cache segments (#790): a zeroed echo must not + // clobber real values carried from message_start. const input = num(usage["input_tokens"]); - return { inputTokens: input && input > 0 ? input : undefined, outputTokens: num(usage["output_tokens"]) }; + const read = num(usage["cache_read_input_tokens"]); + const creation = num(usage["cache_creation_input_tokens"]); + const sample: UsageSample = {}; + if (input !== undefined && input > 0) sample.inputTokens = input; + if (read !== undefined && read > 0) sample.cachedTokens = read; + if (creation !== undefined && creation > 0) sample.creationTokens = creation; + const output = num(usage["output_tokens"]); + if (output !== undefined) sample.outputTokens = output; + return Object.keys(sample).length > 0 ? sample : undefined; } if (type === "response.completed") { const usage = (obj["response"] as Record | undefined)?.["usage"] as Record | undefined; @@ -614,15 +634,15 @@ function usageFromSseEvent(obj: Record): UsageSample | undefine export function applyUsageSample(session: Session, sample: UsageSample, protocol?: WireProtocol): void { // inputTokens is protocol-native: Anthropic reports it NEW-only (cached // separate); OpenAI/Responses report the TOTAL (cached already included). - // promptInputTotal adds the cached segment back when it is not part of - // inputTokens — including the #408 split-semantics violation on OpenAI - // wires (prompt_tokens < cached_tokens → cached is NOT included). + // promptInputTotal adds back every segment not part of inputTokens — + // cached, plus the Anthropic cache-write segment under split semantics + // (#408/#790). Cache writes count toward context size, never toward hits. if (sample.cachedTokens !== undefined) { session.stats.cachedTokens += sample.cachedTokens; session.stats.cacheSamples += 1; } if (sample.inputTokens !== undefined) { - const total = promptInputTotal(protocol, sample.inputTokens, sample.cachedTokens); + const total = promptInputTotal(protocol, sample.inputTokens, sample.cachedTokens, sample.creationTokens); session.stats.inputTokens += total; // Net out pending compress savings (see stream.ts applyRanges): plugin // compress tool results shrink the next request, not this report. @@ -646,6 +666,7 @@ export function applyUsageSample(session: Session, sample: UsageSample, protocol function mergeUsageSample(acc: UsageSample, sample: UsageSample): void { if (sample.inputTokens !== undefined) acc.inputTokens = sample.inputTokens; if (sample.cachedTokens !== undefined) acc.cachedTokens = sample.cachedTokens; + if (sample.creationTokens !== undefined) acc.creationTokens = sample.creationTokens; if (sample.outputTokens !== undefined) acc.outputTokens = sample.outputTokens; } @@ -732,7 +753,7 @@ export async function pipePluginChatWithStrip( // before any prose, and dropping it froze lastInputTokens at the previous // turn's value, corrupting every later nudge decision. const settleUsage = () => { - if (session && (acc.inputTokens !== undefined || acc.outputTokens !== undefined || acc.cachedTokens !== undefined)) { + if (session && (acc.inputTokens !== undefined || acc.outputTokens !== undefined || acc.cachedTokens !== undefined || acc.creationTokens !== undefined)) { applyUsageSample(session, acc, protocol); markDirty(session); } @@ -1247,6 +1268,7 @@ export async function pipePluginJson( num(usage["cache_read_input_tokens"]) ?? // #779: DeepSeek-style top-level field (openai wire) num(usage["prompt_cache_hit_tokens"]), + creationTokens: num(usage["cache_creation_input_tokens"]), }, protocol); markDirty(session); } diff --git a/src/util.ts b/src/util.ts index 4079fa794f..1871ded647 100644 --- a/src/util.ts +++ b/src/util.ts @@ -103,16 +103,23 @@ export function usageTotals( * separate) — under true OpenAI semantics prompt_tokens >= cached_tokens * always holds, so a violation proves the cached segment is NOT part of * prompt_tokens and must be added back (e.g. {prompt_tokens:6, - * cached_tokens:26278} is a real ~26284-token prompt, not 6). */ + * cached_tokens:26278} is a real ~26284-token prompt, not 6). + * #790: under split semantics the Anthropic cache-WRITE segment + * (`cache_creation_input_tokens`) is a third additive piece — part of the + * context size, but NOT a cache hit. */ export function promptInputTotal( protocol: WireProtocol | undefined, input: number | undefined, cached: number | undefined, + creation?: number, ): number { if (input === undefined) return 0; const includesCached = protocol === "openai" || protocol === "responses"; const splitSemantics = !includesCached || (typeof cached === "number" && input < cached); - return input + (splitSemantics && typeof cached === "number" ? cached : 0); + const additive = + (splitSemantics && typeof cached === "number" ? cached : 0) + + (splitSemantics && typeof creation === "number" ? creation : 0); + return input + additive; } /** Result of inspecting an upstream response for a "context too long" error. */ diff --git a/tests/host-usage-postfold.test.ts b/tests/host-usage-postfold.test.ts index 6909348c12..9c45263b18 100644 --- a/tests/host-usage-postfold.test.ts +++ b/tests/host-usage-postfold.test.ts @@ -950,3 +950,192 @@ test("#645/#660: codex UA client (responses wire) reports folded usage verbatim" assert.equal(completedUsageOf(raw).input_tokens, 1000, "codex (UA) must report the folded request's own provider-measured usage — no baseline backfill (#645/#660)"); }); }); + +// #790: Anthropic input context = fresh(input_tokens) + read(cache_read_input_tokens) +// + write(cache_creation_input_tokens); total context size is the sum, but the +// cache HIT is the read segment only. The plugin SSE/JSON paths and the loop +// adapter dropped the read segment from message_delta and never counted the +// write segment, so a turn whose full usage arrived in message_delta was +// reported as input=344 cached=0 instead of input=56040 cached=53696. + +test("#790: promptInputTotal — anthropic cache-write segment is additive under split semantics", () => { + assert.equal(promptInputTotal("anthropic", 344, 53696, 2000), 56040); + assert.equal(promptInputTotal("anthropic", 0, 56000, 0), 56000); + assert.equal(promptInputTotal("anthropic", 1000, undefined, 2000), 3000); + assert.equal(promptInputTotal(undefined, 1000, 40000, 500), 41500); + // openai/responses report TOTAL-includes-cached → creation must NOT be added back + assert.equal(promptInputTotal("openai", 41000, 40000, undefined), 41000); + assert.equal(promptInputTotal("responses", 41000, 40000, 500), 41000); +}); + +const A_SSE = (type: string, data: unknown): string => `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; +const A_TEXT_BLOCK = [ + A_SSE("content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }), + A_SSE("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } }), + A_SSE("content_block_stop", { type: "content_block_stop", index: 0 }), +].join(""); + +test("#790: plugin SSE — zero message_start + full usage in message_delta counts all three segments", async () => { + await withTempStore("p790-repro", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("p790-repro"); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + A_SSE("message_start", { type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { input_tokens: 0, output_tokens: 0 } } }), + A_TEXT_BLOCK, + A_SSE("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { input_tokens: 344, cache_read_input_tokens: 53696, cache_creation_input_tokens: 2000, output_tokens: 5 } }), + A_SSE("message_stop", { type: "message_stop" }), + ]); + await pipePluginChatWithStrip(stream, res, "anthropic", session); + const out = chunks.join(""); + // 344 fresh + 53696 read + 2000 write = 56040; hit = read only + assert.equal(session.stats.lastInputTokens, 56040); + assert.equal(session.stats.inputTokens, 56040); + assert.equal(session.stats.cachedTokens, 53696); + assert.equal(session.stats.outputTokens, 5); + assert.ok(out.includes('"cache_read_input_tokens":53696'), `forwarded bytes must keep the original usage frame: ${out}`); + assert.ok(out.includes('"cache_creation_input_tokens":2000'), out); + }); +}); + +test("#790: plugin SSE — real shape (start carries all three, delta carries output only)", async () => { + await withTempStore("p790-real", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("p790-real"); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + A_SSE("message_start", { type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { input_tokens: 2344, cache_read_input_tokens: 50000, cache_creation_input_tokens: 2000, output_tokens: 1 } } }), + A_TEXT_BLOCK, + A_SSE("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 5 } }), + A_SSE("message_stop", { type: "message_stop" }), + ]); + await pipePluginChatWithStrip(stream, res, "anthropic", session); + assert.equal(session.stats.lastInputTokens, 54344, "2344 + 50000 + 2000"); + assert.equal(session.stats.cachedTokens, 50000); + }); +}); + +test("#790: plugin SSE — repeated identical snapshots do not accumulate", async () => { + await withTempStore("p790-repeat", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("p790-repeat"); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const usage = { input_tokens: 100, cache_read_input_tokens: 50, cache_creation_input_tokens: 10 }; + const stream = streamOf([ + A_SSE("message_start", { type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { ...usage, output_tokens: 1 } } }), + A_TEXT_BLOCK, + A_SSE("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { ...usage, output_tokens: 5 } }), + A_SSE("message_stop", { type: "message_stop" }), + ]); + await pipePluginChatWithStrip(stream, res, "anthropic", session); + assert.equal(session.stats.lastInputTokens, 160, "per-field overwrite: counted once, not 320"); + assert.equal(session.stats.inputTokens, 160); + }); +}); + +test("#790: plugin SSE — relay zero-echo in message_delta must not clobber start values", async () => { + await withTempStore("p790-zeroecho", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("p790-zeroecho"); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + A_SSE("message_start", { type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { input_tokens: 100, cache_read_input_tokens: 50, cache_creation_input_tokens: 10, output_tokens: 1 } } }), + A_TEXT_BLOCK, + A_SSE("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { input_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, output_tokens: 5 } }), + A_SSE("message_stop", { type: "message_stop" }), + ]); + await pipePluginChatWithStrip(stream, res, "anthropic", session); + assert.equal(session.stats.lastInputTokens, 160); + assert.equal(session.stats.cachedTokens, 50); + }); +}); + +test("#790: plugin SSE — fully cache-hit turn reports the read segment as its size", async () => { + await withTempStore("p790-hit", async (_dir, store) => { + _setStoreForTest(store); + const session = makeSession("p790-hit"); + const chunks: Buffer[] = []; + const res = makeRes(chunks); + const stream = streamOf([ + A_SSE("message_start", { type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { input_tokens: 0, cache_read_input_tokens: 56000, output_tokens: 1 } } }), + A_TEXT_BLOCK, + A_SSE("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 5 } }), + A_SSE("message_stop", { type: "message_stop" }), + ]); + await pipePluginChatWithStrip(stream, res, "anthropic", session); + assert.equal(session.stats.lastInputTokens, 56000, "input_tokens:0 is a real value at start and must not drop the sample"); + assert.equal(session.stats.cachedTokens, 56000); + }); +}); + +test("#790: plugin JSON — anthropic body counts all three segments; openai total semantics unchanged", async () => { + await withTempStore("p790-json", async (_dir, store) => { + _setStoreForTest(store); + const anthro = makeSession("p790-json-a"); + const chunksA: Buffer[] = []; + const resA = makeRes(chunksA); + const bodyA = JSON.stringify({ + id: "msg_1", type: "message", role: "assistant", model: "m", content: [{ type: "text", text: "hi" }], stop_reason: "end_turn", + usage: { input_tokens: 344, cache_read_input_tokens: 53696, cache_creation_input_tokens: 2000, output_tokens: 5 }, + }); + await pipePluginJson(streamOf([bodyA]), resA, anthro, "anthropic"); + assert.equal(anthro.stats.lastInputTokens, 56040); + assert.equal(anthro.stats.cachedTokens, 53696); + assert.ok(chunksA.join("").includes('"cache_creation_input_tokens":2000')); + + const openai = makeSession("p790-json-o"); + const chunksO: Buffer[] = []; + const resO = makeRes(chunksO); + const bodyO = JSON.stringify({ + id: "c1", object: "chat.completion", created: 1, model: "m", choices: [{ index: 0, message: { role: "assistant", content: "hi" }, finish_reason: "stop" }], + usage: { prompt_tokens: 41000, completion_tokens: 5, total_tokens: 41005, prompt_tokens_details: { cached_tokens: 40000 } }, + }); + await pipePluginJson(streamOf([bodyO]), resO, openai, "openai"); + assert.equal(openai.stats.lastInputTokens, 41000, "prompt_tokens is already the total — no double count"); + assert.equal(openai.stats.cachedTokens, 40000); + }); +}); + +async function drainAnthropicLoop(body: string, id: string): Promise> { + const ctx = makeCtx(id, [textMsg("raw_1", "user", "hello")]); + for await (const chunk of runCompressLoop( + streamOf([body]), + { ...ctx, protocol: "anthropic" }, + { model: "m", input: [] }, + { url: "https://upstream.test/v1/messages", headers: {} }, + createAnthropicAdapter({ model: "m" }), + "", + )) { + void chunk; + } + return ctx; +} + +test("#790: loop (proxy mode) — anthropic adapter counts all three segments from message_start", async () => { + const body = + A_SSE("message_start", { type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { input_tokens: 344, cache_read_input_tokens: 53696, cache_creation_input_tokens: 2000, output_tokens: 1 } } }) + + A_TEXT_BLOCK + + A_SSE("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 5 } }) + + A_SSE("message_stop", { type: "message_stop" }); + const ctx = await drainAnthropicLoop(body, "loop-790-start"); + assert.equal(ctx.session.stats.lastInputTokens, 56040); + assert.equal(ctx.session.stats.inputTokens, 56040); + assert.equal(ctx.session.stats.cachedTokens, 53696); + assert.equal(ctx.session.stats.cacheSamples, 1); + assert.equal(ctx.session.stats.outputTokens, 5); +}); + +test("#790: loop (proxy mode) — complete usage in message_delta adopts all three atomically", async () => { + const body = + A_SSE("message_start", { type: "message_start", message: { id: "msg_1", type: "message", role: "assistant", content: [], usage: { input_tokens: 0, output_tokens: 1 } } }) + + A_TEXT_BLOCK + + A_SSE("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { input_tokens: 344, cache_read_input_tokens: 53696, cache_creation_input_tokens: 2000, output_tokens: 5 } }) + + A_SSE("message_stop", { type: "message_stop" }); + const ctx = await drainAnthropicLoop(body, "loop-790-delta"); + assert.equal(ctx.session.stats.lastInputTokens, 56040); + assert.equal(ctx.session.stats.cachedTokens, 53696); +});