From a1fb74dc24f069dde29342da78c7bdb792a6003d Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 16 Sep 2026 13:46:06 +0000 Subject: [PATCH 1/2] fix(openai): count compatible cache reads Amp-Thread-ID: https://ampcode.com/threads/T-01a0aa02-b6de-7763-97b8-1650ea4b46da --- .../__tests__/openai-usage-tracking.spec.ts | 48 +++++++++++++++++++ src/api/providers/openai.ts | 4 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index e9c2f5e4fd..151defd3ac 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -181,6 +181,31 @@ describe("OpenAiHandler with usage tracking fix", () => { }) }) + it("should report OpenAI-compatible cached prompt tokens", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + choices: [{ delta: { content: "Cached response" }, index: 0 }], + usage: { + prompt_tokens: 5_053, + completion_tokens: 16, + total_tokens: 5_069, + prompt_tokens_details: { cached_tokens: 4_864 }, + }, + }, + ]), + ) + + const chunks = await collectStream(handler.createMessage(systemPrompt, messages)) + + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 5_053, + outputTokens: 16, + cacheReadTokens: 4_864, + }) + }) + it("should handle case where no usage is provided", async () => { // Override the mock for this specific test mockCreate.mockImplementationOnce(async (options) => { @@ -212,4 +237,27 @@ describe("OpenAiHandler with usage tracking fix", () => { expect(usageChunks).toHaveLength(0) }) }) + + it("should report cached prompt tokens from a non-streaming response", async () => { + const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false }) + mockCreate.mockImplementationOnce(async () => ({ + id: "test-completion", + choices: [{ message: { role: "assistant", content: "Cached response" } }], + usage: { + prompt_tokens: 4_621, + completion_tokens: 16, + total_tokens: 4_637, + prompt_tokens_details: { cached_tokens: 4_608 }, + }, + })) + + const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 4_621, + outputTokens: 16, + cacheReadTokens: 4_608, + }) + }) }) diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index 04b12f233d..ae7405b346 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -282,12 +282,14 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl } protected processUsageMetrics(usage: any, _modelInfo?: ModelInfo): ApiStreamUsageChunk { + const cacheReadTokens = usage?.cache_read_input_tokens ?? usage?.prompt_tokens_details?.cached_tokens + return { type: "usage", inputTokens: usage?.prompt_tokens || 0, outputTokens: usage?.completion_tokens || 0, cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, - cacheReadTokens: usage?.cache_read_input_tokens || undefined, + cacheReadTokens: cacheReadTokens || undefined, } } From b173f6357a3a79a37e7c282c549a51ab1ee787d6 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Fri, 18 Sep 2026 15:26:59 +0200 Subject: [PATCH 2/2] fix(api): validate compatible cache usage --- .../__tests__/openai-usage-tracking.spec.ts | 74 +++++++++++++++++++ src/api/providers/openai.ts | 33 ++++++--- src/eslint-suppressions.json | 2 +- 3 files changed, 99 insertions(+), 10 deletions(-) diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index 151defd3ac..b1f21b304d 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -260,4 +260,78 @@ describe("OpenAiHandler with usage tracking fix", () => { cacheReadTokens: 4_608, }) }) + + it("reports cached prompt tokens for a streaming O3 response", async () => { + const o3Handler = new OpenAiHandler({ ...mockOptions, openAiModelId: "o3-mini" }) + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + choices: [{ delta: { content: "Cached response" }, index: 0 }], + usage: { + prompt_tokens: 5_053, + completion_tokens: 16, + total_tokens: 5_069, + prompt_tokens_details: { cached_tokens: 4_864 }, + }, + }, + ]), + ) + + const chunks = await collectStream(o3Handler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 5_053, + outputTokens: 16, + cacheReadTokens: 4_864, + }) + }) + + it.each([ + ["string", "10"], + ["object", { tokens: 10 }], + ["negative", -1], + ["non-finite", Number.POSITIVE_INFINITY], + ["greater than prompt tokens", 101], + ])("ignores invalid %s cached prompt tokens in streaming responses", async (_name, cachedTokens) => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + choices: [{ delta: { content: "Response" }, index: 0 }], + usage: { + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: cachedTokens }, + }, + }, + ]), + ) + + const chunks = await collectStream(handler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 5 }) + }) + + it.each([ + ["string", "10"], + ["object", { tokens: 10 }], + ["negative", -1], + ["non-finite", Number.POSITIVE_INFINITY], + ["greater than prompt tokens", 101], + ])("ignores invalid %s cached prompt tokens in non-streaming responses", async (_name, cachedTokens) => { + const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false }) + mockCreate.mockImplementationOnce(async () => ({ + id: "test-completion", + choices: [{ message: { role: "assistant", content: "Response" } }], + usage: { + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: cachedTokens }, + }, + })) + + const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 5 }) + }) }) diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index d1f098e63d..a07ef121bf 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -28,6 +28,17 @@ import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, Complete import { handleOpenAIError } from "./utils/error-handler" import { extractReasoningFromDelta } from "./utils/extract-reasoning" +type OpenAiUsage = + | { + prompt_tokens?: number + completion_tokens?: number + cache_creation_input_tokens?: number + cache_read_input_tokens?: unknown + prompt_tokens_details?: { cached_tokens?: unknown } | null + } + | null + | undefined + // TODO: Rename this to OpenAICompatibleHandler. Also, I think the // `OpenAINativeHandler` can subclass from this, since it's obviously // compatible with the OpenAI API. We can also rename it to `OpenAIHandler`. @@ -281,15 +292,23 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl } } - protected processUsageMetrics(usage: any, _modelInfo?: ModelInfo): ApiStreamUsageChunk { - const cacheReadTokens = usage?.cache_read_input_tokens ?? usage?.prompt_tokens_details?.cached_tokens + protected processUsageMetrics(usage: OpenAiUsage, _modelInfo?: ModelInfo): ApiStreamUsageChunk { + const inputTokens = usage?.prompt_tokens || 0 + const reportedCacheReadTokens = usage?.cache_read_input_tokens ?? usage?.prompt_tokens_details?.cached_tokens + const cacheReadTokens = + typeof reportedCacheReadTokens === "number" && + Number.isFinite(reportedCacheReadTokens) && + reportedCacheReadTokens >= 0 && + reportedCacheReadTokens <= inputTokens + ? reportedCacheReadTokens || undefined + : undefined return { type: "usage", - inputTokens: usage?.prompt_tokens || 0, + inputTokens, outputTokens: usage?.completion_tokens || 0, cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, - cacheReadTokens: cacheReadTokens || undefined, + cacheReadTokens, } } @@ -473,11 +492,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl } if (chunk.usage) { - yield { - type: "usage", - inputTokens: chunk.usage.prompt_tokens || 0, - outputTokens: chunk.usage.completion_tokens || 0, - } + yield this.processUsageMetrics(chunk.usage) } } } diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index 93741e9174..bcb5947288 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -401,7 +401,7 @@ }, "api/providers/openai.ts": { "@typescript-eslint/no-explicit-any": { - "count": 3 + "count": 2 } }, "api/providers/openrouter.ts": {