Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion src/loop/adapter-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ export function createAnthropicAdapter(requestBody: Record<string, unknown>, ori
const pending = new Map<number, ToolUseBuffer>();
let roundInput: number | undefined;
let roundCached: number | undefined;
let roundCreation: number | undefined;
let roundOutput: number | undefined;
let stopReason: string | undefined;
let usageYielded = false;
Expand Down Expand Up @@ -252,6 +253,7 @@ export function createAnthropicAdapter(requestBody: Record<string, unknown>, ori
const u = (msg.usage ?? {}) as Record<string, unknown>;
if (typeof u.input_tokens === "number") roundInput = u.input_tokens;
if (typeof u.cache_read_input_tokens === "number") roundCached = u.cache_read_input_tokens;
if (typeof u.cache_creation_input_tokens === "number") roundCreation = u.cache_creation_input_tokens;
if (round === 1) {
// The raw message_start (with the provider's measured
// usage) reaches the host verbatim — no rewriting.
Expand Down Expand Up @@ -347,6 +349,7 @@ export function createAnthropicAdapter(requestBody: Record<string, unknown>, ori
if (typeof u.output_tokens === "number") roundOutput = u.output_tokens;
const input = typeof u.input_tokens === "number" ? u.input_tokens : undefined;
const cached = typeof u.cache_read_input_tokens === "number" ? u.cache_read_input_tokens : undefined;
const creation = typeof u.cache_creation_input_tokens === "number" ? u.cache_creation_input_tokens : undefined;
if (input !== undefined && input > 0 && cached !== undefined) {
// Complete authoritative usage object — e.g. the synthetic
// terminal of a stitched multi-round stream (round1
Expand All @@ -355,9 +358,12 @@ export function createAnthropicAdapter(requestBody: Record<string, unknown>, ori
// compress re-request — overwrites the stale value carried by
// an earlier round's message_start. Per-field merging would
// double-count (new input + old cache) and trip false
// EMERGENCY nudges (issue #299).
// EMERGENCY nudges (issue #299). The cache-write segment
// (#790) is optional even in a complete object — adopt it
// when present, keep the prior value when absent.
roundInput = input;
roundCached = cached;
if (creation !== undefined) roundCreation = creation;
} else {
// Incomplete usage object: the input context is FIXED within
// a turn, so message_start is authoritative for input/cache
Expand All @@ -369,6 +375,7 @@ export function createAnthropicAdapter(requestBody: Record<string, unknown>, ori
// legitimate update, so adopt only > 0.
if (input !== undefined && input > 0) roundInput = input;
if (cached !== undefined && cached > 0) roundCached = cached;
if (creation !== undefined && creation > 0) roundCreation = creation;
}
const d = (data.delta ?? {}) as Record<string, unknown>;
if (typeof d.stop_reason === "string") stopReason = d.stop_reason;
Expand All @@ -379,6 +386,7 @@ export function createAnthropicAdapter(requestBody: Record<string, unknown>, ori
inputTokens: roundInput,
outputTokens: roundOutput,
cachedTokens: roundCached,
creationTokens: roundCreation,
} as ParsedStreamEvent;
}
maybeWarnDegenerate(stopReason);
Expand All @@ -398,6 +406,7 @@ export function createAnthropicAdapter(requestBody: Record<string, unknown>, ori
inputTokens: roundInput,
outputTokens: roundOutput,
cachedTokens: roundCached,
creationTokens: roundCreation,
} as ParsedStreamEvent;
}
maybeWarnDegenerate(stopReason);
Expand Down
9 changes: 5 additions & 4 deletions src/loop/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ export type ParsedStreamEvent =
| { kind: "text"; delta: string; raw?: Buffer }
| { kind: "reasoning"; delta: string; raw?: Buffer; signature?: string; blockEnd?: boolean }
| { kind: "tool_call"; name: string; callId: string; arguments: string; passthrough?: boolean }
| { kind: "usage"; inputTokens?: number; outputTokens?: number; cachedTokens?: number }
| { kind: "usage"; inputTokens?: number; outputTokens?: number; cachedTokens?: number; creationTokens?: number }
| { kind: "done"; finishReason?: string; suppressCompletion?: boolean; truncated?: boolean; thinking?: boolean }
| { kind: "meta"; chunk: Buffer; firstRoundOnly?: boolean };

Expand Down Expand Up @@ -163,13 +163,13 @@ export function executeProxyTool(

function recordUsage(
ctx: LoopCtx,
usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number },
usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number; creationTokens?: number },
round: number,
): void {
const prompt = usage.inputTokens;
const cached = usage.cachedTokens;
const out = usage.outputTokens;
const total = promptInputTotal(ctx.protocol, prompt, cached);
const total = promptInputTotal(ctx.protocol, prompt, cached, usage.creationTokens);
if (total > 0) ctx.session.stats.inputTokens += total;
// Net out this turn's compress credit: the post-compress re-request
// re-sends the unfolded history, so its usage report over-reports the
Expand Down Expand Up @@ -249,7 +249,7 @@ export async function* runCompressLoop(
const reasoningSegments: { text: string; signature: string }[] = [];
let reasoningSealed = true;
const calls: ToolCallEmit[] = [];
let usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number } = {};
let usage: { inputTokens?: number; outputTokens?: number; cachedTokens?: number; creationTokens?: number } = {};
let finishReason: string | undefined;
let sawDone = false;
let suppressCompletion = false;
Expand Down Expand Up @@ -309,6 +309,7 @@ export async function* runCompressLoop(
inputTokens: ev.inputTokens,
outputTokens: ev.outputTokens,
cachedTokens: ev.cachedTokens,
creationTokens: ev.creationTokens,
};
} else if (ev.kind === "done") {
sawDone = true;
Expand Down
44 changes: 33 additions & 11 deletions src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -563,7 +563,9 @@ export async function handlePluginTool(
res.end(JSON.stringify({ ok: true, tool, conversationId, result }));
}

type UsageSample = { inputTokens?: number; cachedTokens?: number; outputTokens?: number };
// creationTokens = Anthropic cache-write segment (cache_creation_input_tokens):
// part of the context size, but NOT a cache hit (#790).
type UsageSample = { inputTokens?: number; cachedTokens?: number; outputTokens?: number; creationTokens?: number };

function num(v: unknown): number | undefined {
return typeof v === "number" && Number.isFinite(v) ? v : undefined;
Expand All @@ -574,20 +576,38 @@ function usageFromSseEvent(obj: Record<string, unknown>): UsageSample | undefine
if (type === "message_start") {
const usage = (obj["message"] as Record<string, unknown> | undefined)?.["usage"] as Record<string, unknown> | undefined;
if (!usage) return undefined;
// Per-field snapshot (#790): a zero is a real value here (a fully
// cache-hit turn reports input_tokens: 0), so record whatever is
// present and let later events overwrite field by field.
const sample: UsageSample = {};
const input = num(usage["input_tokens"]);
if (input === undefined) return undefined;
return { inputTokens: input, cachedTokens: num(usage["cache_read_input_tokens"]) };
if (input !== undefined) sample.inputTokens = input;
const read = num(usage["cache_read_input_tokens"]);
if (read !== undefined) sample.cachedTokens = read;
const creation = num(usage["cache_creation_input_tokens"]);
if (creation !== undefined) sample.creationTokens = creation;
return Object.keys(sample).length > 0 ? sample : undefined;
}
if (type === "message_delta") {
const usage = obj["usage"] as Record<string, unknown> | undefined;
if (!usage) return undefined;
// Some relays echo `input_tokens: 0` in message_delta (the field is
// normally absent — message_start is authoritative for the input size,
// which is fixed within a turn). A 0 here is never a legitimate new
// value; merging it would zero acc.inputTokens (set by message_start)
// and collapse lastInputTokens to the cached portion only.
// value; merging it would zero out acc.inputTokens (set by message_start)
// and collapse lastInputTokens to the cached portion only. The same
// guard covers the cache segments (#790): a zeroed echo must not
// clobber real values carried from message_start.
const input = num(usage["input_tokens"]);
return { inputTokens: input && input > 0 ? input : undefined, outputTokens: num(usage["output_tokens"]) };
const read = num(usage["cache_read_input_tokens"]);
const creation = num(usage["cache_creation_input_tokens"]);
const sample: UsageSample = {};
if (input !== undefined && input > 0) sample.inputTokens = input;
if (read !== undefined && read > 0) sample.cachedTokens = read;
if (creation !== undefined && creation > 0) sample.creationTokens = creation;
const output = num(usage["output_tokens"]);
if (output !== undefined) sample.outputTokens = output;
return Object.keys(sample).length > 0 ? sample : undefined;
}
if (type === "response.completed") {
const usage = (obj["response"] as Record<string, unknown> | undefined)?.["usage"] as Record<string, unknown> | undefined;
Expand All @@ -614,15 +634,15 @@ function usageFromSseEvent(obj: Record<string, unknown>): UsageSample | undefine
export function applyUsageSample(session: Session, sample: UsageSample, protocol?: WireProtocol): void {
// inputTokens is protocol-native: Anthropic reports it NEW-only (cached
// separate); OpenAI/Responses report the TOTAL (cached already included).
// promptInputTotal adds the cached segment back when it is not part of
// inputTokens — including the #408 split-semantics violation on OpenAI
// wires (prompt_tokens < cached_tokens → cached is NOT included).
// promptInputTotal adds back every segment not part of inputTokens —
// cached, plus the Anthropic cache-write segment under split semantics
// (#408/#790). Cache writes count toward context size, never toward hits.
if (sample.cachedTokens !== undefined) {
session.stats.cachedTokens += sample.cachedTokens;
session.stats.cacheSamples += 1;
}
if (sample.inputTokens !== undefined) {
const total = promptInputTotal(protocol, sample.inputTokens, sample.cachedTokens);
const total = promptInputTotal(protocol, sample.inputTokens, sample.cachedTokens, sample.creationTokens);
session.stats.inputTokens += total;
// Net out pending compress savings (see stream.ts applyRanges): plugin
// compress tool results shrink the next request, not this report.
Expand All @@ -646,6 +666,7 @@ export function applyUsageSample(session: Session, sample: UsageSample, protocol
function mergeUsageSample(acc: UsageSample, sample: UsageSample): void {
if (sample.inputTokens !== undefined) acc.inputTokens = sample.inputTokens;
if (sample.cachedTokens !== undefined) acc.cachedTokens = sample.cachedTokens;
if (sample.creationTokens !== undefined) acc.creationTokens = sample.creationTokens;
if (sample.outputTokens !== undefined) acc.outputTokens = sample.outputTokens;
}

Expand Down Expand Up @@ -732,7 +753,7 @@ export async function pipePluginChatWithStrip(
// before any prose, and dropping it froze lastInputTokens at the previous
// turn's value, corrupting every later nudge decision.
const settleUsage = () => {
if (session && (acc.inputTokens !== undefined || acc.outputTokens !== undefined || acc.cachedTokens !== undefined)) {
if (session && (acc.inputTokens !== undefined || acc.outputTokens !== undefined || acc.cachedTokens !== undefined || acc.creationTokens !== undefined)) {
applyUsageSample(session, acc, protocol);
markDirty(session);
}
Expand Down Expand Up @@ -1247,6 +1268,7 @@ export async function pipePluginJson(
num(usage["cache_read_input_tokens"]) ??
// #779: DeepSeek-style top-level field (openai wire)
num(usage["prompt_cache_hit_tokens"]),
creationTokens: num(usage["cache_creation_input_tokens"]),
}, protocol);
markDirty(session);
}
Expand Down
11 changes: 9 additions & 2 deletions src/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,16 +103,23 @@ export function usageTotals(
* separate) — under true OpenAI semantics prompt_tokens >= cached_tokens
* always holds, so a violation proves the cached segment is NOT part of
* prompt_tokens and must be added back (e.g. {prompt_tokens:6,
* cached_tokens:26278} is a real ~26284-token prompt, not 6). */
* cached_tokens:26278} is a real ~26284-token prompt, not 6).
* #790: under split semantics the Anthropic cache-WRITE segment
* (`cache_creation_input_tokens`) is a third additive piece — part of the
* context size, but NOT a cache hit. */
export function promptInputTotal(
protocol: WireProtocol | undefined,
input: number | undefined,
cached: number | undefined,
creation?: number,
): number {
if (input === undefined) return 0;
const includesCached = protocol === "openai" || protocol === "responses";
const splitSemantics = !includesCached || (typeof cached === "number" && input < cached);
return input + (splitSemantics && typeof cached === "number" ? cached : 0);
const additive =
(splitSemantics && typeof cached === "number" ? cached : 0) +
(splitSemantics && typeof creation === "number" ? creation : 0);
return input + additive;
}

/** Result of inspecting an upstream response for a "context too long" error. */
Expand Down
Loading
Loading