From d7bf014c2340e3b5d899204a0a78ab3c8229acce Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Sun, 20 Sep 2026 07:45:54 +0900 Subject: [PATCH 1/3] fix(anthropic): gate prompt caching on supportsPromptCache instead of model-id list --- src/api/providers/__tests__/anthropic.spec.ts | 64 +++++ src/api/providers/anthropic.ts | 227 +++++++----------- 2 files changed, 156 insertions(+), 135 deletions(-) diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 7d54116a38..31df952133 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -501,6 +501,70 @@ describe("AnthropicHandler", () => { expect(requestBody?.model).toBe("claude-sonnet-5-bf") expect(requestBody?.thinking).toEqual({ type: "adaptive" }) }) + + it("should attach cache breakpoints and the prompt-caching beta header for a custom model whose resolved info supports prompt caching", async () => { + const customHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-5-bf", + }) + + // Not in the model registry; capabilities are guessed from the Sonnet 5 family. + expect(customHandler.getModel().info.supportsPromptCache).toBe(true) + + const stream = customHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "First message" }], + }, + { + role: "assistant", + content: [{ type: "text" as const, text: "Response" }], + }, + { + role: "user", + content: [{ type: "text" as const, text: "Second message" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] + expect(requestBody?.system?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.messages?.[0]?.content?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.messages?.[1]?.content?.[0]).not.toHaveProperty("cache_control") + expect(requestBody?.messages?.[2]?.content?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31") + }) + + it("should not attach cache breakpoints or the prompt-caching beta header when the model info does not support prompt caching", async () => { + const noCacheHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-3-5-sonnet-20241022", + }) + + // No registry model disables prompt caching, so override the resolved info. + const realModel = noCacheHandler.getModel() + vitest.spyOn(noCacheHandler, "getModel").mockReturnValue({ + ...realModel, + info: { ...realModel.info, supportsPromptCache: false }, + }) + + const stream = noCacheHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] + expect(requestBody?.system).toEqual([{ text: systemPrompt, type: "text" }]) + expect(requestBody?.messages?.[0]?.content?.[0]).not.toHaveProperty("cache_control") + expect(requestOptions).toBeUndefined() + }) }) describe("completePrompt", () => { diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 2ef70b78ea..e1986c3b38 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -113,146 +113,103 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa tool_choice: toolChoice, } - switch (modelId) { - case "claude-sonnet-5": - case "claude-sonnet-4-6": - case "claude-sonnet-4-5": - case "claude-sonnet-4-20250514": - case "claude-opus-4-6": - case "claude-opus-4-7": - case "claude-opus-4-8": - case "claude-opus-5": - case "claude-fable-5-1": - case "claude-fable-5": - case "claude-opus-4-5-20251101": - case "claude-opus-4-1-20250805": - case "claude-opus-4-20250514": - case "claude-3-7-sonnet-20250219": - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-haiku-4-5-20251001": - case "claude-3-haiku-20240307": { - /** - * The latest message will be the new user message, one before - * will be the assistant message from a previous request, and - * the user message before that will be a previously cached user - * message. So we need to mark the latest user message as - * ephemeral to cache it for the next request, and mark the - * second to last user message as ephemeral to let the server - * know the last message to retrieve from the cache for the - * current request. - */ - const userMsgIndices = sanitizedMessages.reduce( - (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), - [] as number[], - ) + if (info.supportsPromptCache) { + /** + * The latest message will be the new user message, one before + * will be the assistant message from a previous request, and + * the user message before that will be a previously cached user + * message. So we need to mark the latest user message as + * ephemeral to cache it for the next request, and mark the + * second to last user message as ephemeral to let the server + * know the last message to retrieve from the cache for the + * current request. + */ + const userMsgIndices = sanitizedMessages.reduce( + (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), + [] as number[], + ) - const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 - const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 - - try { - const requestParams = { - model: modelId, - max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, - temperature, - thinking, - // Setting cache breakpoint for system prompt so new tasks can reuse it. - system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], - messages: sanitizedMessages.map((message, index) => { - if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { - return { - ...message, - content: - typeof message.content === "string" - ? [{ type: "text", text: message.content, cache_control: cacheControl }] - : message.content.map((content, contentIndex) => - contentIndex === message.content.length - 1 - ? { ...content, cache_control: cacheControl } - : content, - ), - } + const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 + const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 + + try { + const requestParams = { + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + temperature, + thinking, + // Setting cache breakpoint for system prompt so new tasks can reuse it. + system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], + messages: sanitizedMessages.map((message, index) => { + if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { + return { + ...message, + content: + typeof message.content === "string" + ? [{ type: "text", text: message.content, cache_control: cacheControl }] + : message.content.map((content, contentIndex) => + contentIndex === message.content.length - 1 + ? { ...content, cache_control: cacheControl } + : content, + ), } - return message - }), - stream: true, - ...nativeToolParams, - } - stream = await this.client.messages.create( - requestParams as Anthropic.Messages.MessageCreateParamsStreaming, - (() => { - // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 - // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers - // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 - - // Then check for models that support prompt caching - switch (modelId) { - case "claude-sonnet-5": - case "claude-sonnet-4-6": - case "claude-sonnet-4-5": - case "claude-sonnet-4-20250514": - case "claude-opus-4-6": - case "claude-opus-4-7": - case "claude-opus-4-8": - case "claude-opus-5": - case "claude-fable-5-1": - case "claude-fable-5": - case "claude-opus-4-5-20251101": - case "claude-opus-4-1-20250805": - case "claude-opus-4-20250514": - case "claude-3-7-sonnet-20250219": - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-haiku-4-5-20251001": - case "claude-3-haiku-20240307": - betas.push("prompt-caching-2024-07-31") - return { headers: { "anthropic-beta": betas.join(",") } } - default: - return undefined - } - })(), - ) - } catch (error) { - TelemetryService.instance.captureException( - new ApiProviderError( - error instanceof Error ? error.message : String(error), - this.providerName, - modelId, - "createMessage", - ), - ) - throw error + } + return message + }), + stream: true, + ...nativeToolParams, } - break + stream = await this.client.messages.create( + requestParams as Anthropic.Messages.MessageCreateParamsStreaming, + (() => { + // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 + // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers + // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 + + // Then check for models that support prompt caching + if (info.supportsPromptCache) { + betas.push("prompt-caching-2024-07-31") + return { headers: { "anthropic-beta": betas.join(",") } } + } + return undefined + })(), + ) + } catch (error) { + TelemetryService.instance.captureException( + new ApiProviderError( + error instanceof Error ? error.message : String(error), + this.providerName, + modelId, + "createMessage", + ), + ) + throw error } - default: { - try { - const requestParams = { - model: modelId, - max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, - temperature, - thinking, - system: [{ text: systemPrompt, type: "text" }], - messages: sanitizedMessages, - stream: true, - ...nativeToolParams, - } - stream = (await this.client.messages.create( - requestParams as Anthropic.Messages.MessageCreateParamsStreaming, - )) as any - } catch (error) { - TelemetryService.instance.captureException( - new ApiProviderError( - error instanceof Error ? error.message : String(error), - this.providerName, - modelId, - "createMessage", - ), - ) - throw error + } else { + try { + const requestParams = { + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + temperature, + thinking, + system: [{ text: systemPrompt, type: "text" }], + messages: sanitizedMessages, + stream: true, + ...nativeToolParams, } - break + stream = (await this.client.messages.create( + requestParams as Anthropic.Messages.MessageCreateParamsStreaming, + )) as any + } catch (error) { + TelemetryService.instance.captureException( + new ApiProviderError( + error instanceof Error ? error.message : String(error), + this.providerName, + modelId, + "createMessage", + ), + ) + throw error } } From 201c7e2244a7c31224dae2cda0e12f3ac1edac83 Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Mon, 21 Sep 2026 00:54:06 +0900 Subject: [PATCH 2/3] test(anthropic): cover error paths and fallback branches for cache gating --- src/api/providers/__tests__/anthropic.spec.ts | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 31df952133..b26b41fbca 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -73,6 +73,8 @@ vitest.mock("@anthropic-ai/sdk", () => { // Import after mock import { Anthropic } from "@anthropic-ai/sdk" +import { TelemetryService } from "@roo-code/telemetry" +import { ApiProviderError } from "@roo-code/types" const mockAnthropicConstructor = vitest.mocked(Anthropic) @@ -547,6 +549,7 @@ describe("AnthropicHandler", () => { const realModel = noCacheHandler.getModel() vitest.spyOn(noCacheHandler, "getModel").mockReturnValue({ ...realModel, + maxTokens: undefined, info: { ...realModel.info, supportsPromptCache: false }, }) @@ -562,9 +565,135 @@ describe("AnthropicHandler", () => { const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] expect(requestBody?.system).toEqual([{ text: systemPrompt, type: "text" }]) + expect(requestBody?.max_tokens).toBe(8192) expect(requestBody?.messages?.[0]?.content?.[0]).not.toHaveProperty("cache_control") expect(requestOptions).toBeUndefined() }) + + it("should attach cache breakpoints without the prompt-caching beta header when cache support is gone at header build time", async () => { + const customHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-3-5-sonnet-20241022", + }) + const realModel = customHandler.getModel() + + // The capability is consulted twice: when selecting the caching request + // branch and again inside the beta-header options. Model a model that + // reports no cache support on the second consult. + let cacheSupported = true + const info = { ...realModel.info } + Object.defineProperty(info, "supportsPromptCache", { + get: () => { + const current = cacheSupported + cacheSupported = false + return current + }, + }) + vitest.spyOn(customHandler, "getModel").mockReturnValue({ ...realModel, info, maxTokens: undefined }) + + const stream = customHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] + expect(requestBody?.system?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.max_tokens).toBe(8192) + expect(requestOptions).toBeUndefined() + }) + + it("should attach cache control only to the last content block of a cached user message", async () => { + const stream = handler.createMessage(systemPrompt, [ + { + role: "user", + content: [ + { type: "text" as const, text: "First block" }, + { type: "text" as const, text: "Second block" }, + ], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const content = requestBody?.messages?.[0]?.content + expect(content?.[0]).not.toHaveProperty("cache_control") + expect(content?.[1]?.cache_control).toEqual({ type: "ephemeral" }) + }) + + it("should cache the system prompt but no message when the request has no user message", async () => { + const stream = handler.createMessage(systemPrompt, [ + { + role: "assistant", + content: [{ type: "text" as const, text: "Previous response" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(requestBody?.system?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.messages?.[0]?.content?.[0]).not.toHaveProperty("cache_control") + }) + + it.each([ + ["an Error", new Error("Anthropic API error")], + ["a non-Error rejection", "Anthropic API error"], + ])( + "should capture telemetry and rethrow when the request fails with %s for a cacheable model", + async (_, rejection) => { + mockCreate.mockRejectedValueOnce(rejection) + + const stream = handler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await expect(collectStream(stream)).rejects.toBe(rejection) + expect(TelemetryService.instance.captureException).toHaveBeenCalledTimes(1) + expect(TelemetryService.instance.captureException).toHaveBeenCalledWith(expect.any(ApiProviderError)) + }, + ) + + it.each([ + ["an Error", new Error("Anthropic API error")], + ["a non-Error rejection", "Anthropic API error"], + ])( + "should capture telemetry and rethrow when the request fails with %s for a model without prompt cache support", + async (_, rejection) => { + const noCacheHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-3-5-sonnet-20241022", + }) + + // No registry model disables prompt caching, so override the resolved info. + const realModel = noCacheHandler.getModel() + vitest.spyOn(noCacheHandler, "getModel").mockReturnValue({ + ...realModel, + info: { ...realModel.info, supportsPromptCache: false }, + }) + + mockCreate.mockRejectedValueOnce(rejection) + + const stream = noCacheHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await expect(collectStream(stream)).rejects.toBe(rejection) + expect(TelemetryService.instance.captureException).toHaveBeenCalledTimes(1) + expect(TelemetryService.instance.captureException).toHaveBeenCalledWith(expect.any(ApiProviderError)) + }, + ) }) describe("completePrompt", () => { From 9eb05b2ee7f7bf73885f101579777057b28968f9 Mon Sep 17 00:00:00 2001 From: "Zoo (VP)" Date: Mon, 21 Sep 2026 02:23:43 +0900 Subject: [PATCH 3/3] refactor(anthropic): read supportsPromptCache once per request --- src/api/providers/__tests__/anthropic.spec.ts | 39 ++++++++++--------- src/api/providers/anthropic.ts | 13 +++---- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index b26b41fbca..f97ea00f90 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -570,26 +570,13 @@ describe("AnthropicHandler", () => { expect(requestOptions).toBeUndefined() }) - it("should attach cache breakpoints without the prompt-caching beta header when cache support is gone at header build time", async () => { + it("should send the prompt-caching beta header and fall back to default max tokens when maxTokens is undefined", async () => { const customHandler = new AnthropicHandler({ apiKey: "test-api-key", apiModelId: "claude-3-5-sonnet-20241022", }) const realModel = customHandler.getModel() - - // The capability is consulted twice: when selecting the caching request - // branch and again inside the beta-header options. Model a model that - // reports no cache support on the second consult. - let cacheSupported = true - const info = { ...realModel.info } - Object.defineProperty(info, "supportsPromptCache", { - get: () => { - const current = cacheSupported - cacheSupported = false - return current - }, - }) - vitest.spyOn(customHandler, "getModel").mockReturnValue({ ...realModel, info, maxTokens: undefined }) + vitest.spyOn(customHandler, "getModel").mockReturnValue({ ...realModel, maxTokens: undefined }) const stream = customHandler.createMessage(systemPrompt, [ { @@ -604,7 +591,7 @@ describe("AnthropicHandler", () => { const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] expect(requestBody?.system?.[0]?.cache_control).toEqual({ type: "ephemeral" }) expect(requestBody?.max_tokens).toBe(8192) - expect(requestOptions).toBeUndefined() + expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31") }) it("should attach cache control only to the last content block of a cached user message", async () => { @@ -658,7 +645,15 @@ describe("AnthropicHandler", () => { await expect(collectStream(stream)).rejects.toBe(rejection) expect(TelemetryService.instance.captureException).toHaveBeenCalledTimes(1) - expect(TelemetryService.instance.captureException).toHaveBeenCalledWith(expect.any(ApiProviderError)) + const capturedError = vitest.mocked(TelemetryService.instance.captureException).mock.calls[0]?.[0] + expect(capturedError).toBeInstanceOf(ApiProviderError) + if (!(capturedError instanceof ApiProviderError)) { + throw new Error("expected captureException to receive an ApiProviderError") + } + expect(capturedError.message).toBe("Anthropic API error") + expect(capturedError.provider).toBe("Anthropic") + expect(capturedError.modelId).toBe("claude-3-5-sonnet-20241022") + expect(capturedError.operation).toBe("createMessage") }, ) @@ -691,7 +686,15 @@ describe("AnthropicHandler", () => { await expect(collectStream(stream)).rejects.toBe(rejection) expect(TelemetryService.instance.captureException).toHaveBeenCalledTimes(1) - expect(TelemetryService.instance.captureException).toHaveBeenCalledWith(expect.any(ApiProviderError)) + const capturedError = vitest.mocked(TelemetryService.instance.captureException).mock.calls[0]?.[0] + expect(capturedError).toBeInstanceOf(ApiProviderError) + if (!(capturedError instanceof ApiProviderError)) { + throw new Error("expected captureException to receive an ApiProviderError") + } + expect(capturedError.message).toBe("Anthropic API error") + expect(capturedError.provider).toBe("Anthropic") + expect(capturedError.modelId).toBe("claude-3-5-sonnet-20241022") + expect(capturedError.operation).toBe("createMessage") }, ) }) diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index e1986c3b38..fc0671061c 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -73,6 +73,8 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa info, reasoningBudget, } = this.getModel() + // Read once so the cache breakpoints and the beta header always agree. + const supportsPromptCache = info.supportsPromptCache const thinking = getAnthropicProviderReasoning({ model: info, reasoningBudget, @@ -113,7 +115,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa tool_choice: toolChoice, } - if (info.supportsPromptCache) { + if (supportsPromptCache) { /** * The latest message will be the new user message, one before * will be the assistant message from a previous request, and @@ -165,13 +167,8 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 - - // Then check for models that support prompt caching - if (info.supportsPromptCache) { - betas.push("prompt-caching-2024-07-31") - return { headers: { "anthropic-beta": betas.join(",") } } - } - return undefined + betas.push("prompt-caching-2024-07-31") + return { headers: { "anthropic-beta": betas.join(",") } } })(), ) } catch (error) {