diff --git a/src/api/providers/__tests__/anthropic.spec.ts b/src/api/providers/__tests__/anthropic.spec.ts index 7d54116a38..f97ea00f90 100644 --- a/src/api/providers/__tests__/anthropic.spec.ts +++ b/src/api/providers/__tests__/anthropic.spec.ts @@ -73,6 +73,8 @@ vitest.mock("@anthropic-ai/sdk", () => { // Import after mock import { Anthropic } from "@anthropic-ai/sdk" +import { TelemetryService } from "@roo-code/telemetry" +import { ApiProviderError } from "@roo-code/types" const mockAnthropicConstructor = vitest.mocked(Anthropic) @@ -501,6 +503,200 @@ describe("AnthropicHandler", () => { expect(requestBody?.model).toBe("claude-sonnet-5-bf") expect(requestBody?.thinking).toEqual({ type: "adaptive" }) }) + + it("should attach cache breakpoints and the prompt-caching beta header for a custom model whose resolved info supports prompt caching", async () => { + const customHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-sonnet-5-bf", + }) + + // Not in the model registry; capabilities are guessed from the Sonnet 5 family. + expect(customHandler.getModel().info.supportsPromptCache).toBe(true) + + const stream = customHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "First message" }], + }, + { + role: "assistant", + content: [{ type: "text" as const, text: "Response" }], + }, + { + role: "user", + content: [{ type: "text" as const, text: "Second message" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] + expect(requestBody?.system?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.messages?.[0]?.content?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.messages?.[1]?.content?.[0]).not.toHaveProperty("cache_control") + expect(requestBody?.messages?.[2]?.content?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31") + }) + + it("should not attach cache breakpoints or the prompt-caching beta header when the model info does not support prompt caching", async () => { + const noCacheHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-3-5-sonnet-20241022", + }) + + // No registry model disables prompt caching, so override the resolved info. + const realModel = noCacheHandler.getModel() + vitest.spyOn(noCacheHandler, "getModel").mockReturnValue({ + ...realModel, + maxTokens: undefined, + info: { ...realModel.info, supportsPromptCache: false }, + }) + + const stream = noCacheHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] + expect(requestBody?.system).toEqual([{ text: systemPrompt, type: "text" }]) + expect(requestBody?.max_tokens).toBe(8192) + expect(requestBody?.messages?.[0]?.content?.[0]).not.toHaveProperty("cache_control") + expect(requestOptions).toBeUndefined() + }) + + it("should send the prompt-caching beta header and fall back to default max tokens when maxTokens is undefined", async () => { + const customHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-3-5-sonnet-20241022", + }) + const realModel = customHandler.getModel() + vitest.spyOn(customHandler, "getModel").mockReturnValue({ ...realModel, maxTokens: undefined }) + + const stream = customHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const requestOptions = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[1] + expect(requestBody?.system?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.max_tokens).toBe(8192) + expect(requestOptions?.headers?.["anthropic-beta"]).toContain("prompt-caching-2024-07-31") + }) + + it("should attach cache control only to the last content block of a cached user message", async () => { + const stream = handler.createMessage(systemPrompt, [ + { + role: "user", + content: [ + { type: "text" as const, text: "First block" }, + { type: "text" as const, text: "Second block" }, + ], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + const content = requestBody?.messages?.[0]?.content + expect(content?.[0]).not.toHaveProperty("cache_control") + expect(content?.[1]?.cache_control).toEqual({ type: "ephemeral" }) + }) + + it("should cache the system prompt but no message when the request has no user message", async () => { + const stream = handler.createMessage(systemPrompt, [ + { + role: "assistant", + content: [{ type: "text" as const, text: "Previous response" }], + }, + ]) + + await collectStream(stream) + + const requestBody = mockCreate.mock.calls[mockCreate.mock.calls.length - 1]?.[0] + expect(requestBody?.system?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(requestBody?.messages?.[0]?.content?.[0]).not.toHaveProperty("cache_control") + }) + + it.each([ + ["an Error", new Error("Anthropic API error")], + ["a non-Error rejection", "Anthropic API error"], + ])( + "should capture telemetry and rethrow when the request fails with %s for a cacheable model", + async (_, rejection) => { + mockCreate.mockRejectedValueOnce(rejection) + + const stream = handler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await expect(collectStream(stream)).rejects.toBe(rejection) + expect(TelemetryService.instance.captureException).toHaveBeenCalledTimes(1) + const capturedError = vitest.mocked(TelemetryService.instance.captureException).mock.calls[0]?.[0] + expect(capturedError).toBeInstanceOf(ApiProviderError) + if (!(capturedError instanceof ApiProviderError)) { + throw new Error("expected captureException to receive an ApiProviderError") + } + expect(capturedError.message).toBe("Anthropic API error") + expect(capturedError.provider).toBe("Anthropic") + expect(capturedError.modelId).toBe("claude-3-5-sonnet-20241022") + expect(capturedError.operation).toBe("createMessage") + }, + ) + + it.each([ + ["an Error", new Error("Anthropic API error")], + ["a non-Error rejection", "Anthropic API error"], + ])( + "should capture telemetry and rethrow when the request fails with %s for a model without prompt cache support", + async (_, rejection) => { + const noCacheHandler = new AnthropicHandler({ + apiKey: "test-api-key", + apiModelId: "claude-3-5-sonnet-20241022", + }) + + // No registry model disables prompt caching, so override the resolved info. + const realModel = noCacheHandler.getModel() + vitest.spyOn(noCacheHandler, "getModel").mockReturnValue({ + ...realModel, + info: { ...realModel.info, supportsPromptCache: false }, + }) + + mockCreate.mockRejectedValueOnce(rejection) + + const stream = noCacheHandler.createMessage(systemPrompt, [ + { + role: "user", + content: [{ type: "text" as const, text: "Hello" }], + }, + ]) + + await expect(collectStream(stream)).rejects.toBe(rejection) + expect(TelemetryService.instance.captureException).toHaveBeenCalledTimes(1) + const capturedError = vitest.mocked(TelemetryService.instance.captureException).mock.calls[0]?.[0] + expect(capturedError).toBeInstanceOf(ApiProviderError) + if (!(capturedError instanceof ApiProviderError)) { + throw new Error("expected captureException to receive an ApiProviderError") + } + expect(capturedError.message).toBe("Anthropic API error") + expect(capturedError.provider).toBe("Anthropic") + expect(capturedError.modelId).toBe("claude-3-5-sonnet-20241022") + expect(capturedError.operation).toBe("createMessage") + }, + ) }) describe("completePrompt", () => { diff --git a/src/api/providers/anthropic.ts b/src/api/providers/anthropic.ts index 2ef70b78ea..fc0671061c 100644 --- a/src/api/providers/anthropic.ts +++ b/src/api/providers/anthropic.ts @@ -73,6 +73,8 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa info, reasoningBudget, } = this.getModel() + // Read once so the cache breakpoints and the beta header always agree. + const supportsPromptCache = info.supportsPromptCache const thinking = getAnthropicProviderReasoning({ model: info, reasoningBudget, @@ -113,146 +115,98 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa tool_choice: toolChoice, } - switch (modelId) { - case "claude-sonnet-5": - case "claude-sonnet-4-6": - case "claude-sonnet-4-5": - case "claude-sonnet-4-20250514": - case "claude-opus-4-6": - case "claude-opus-4-7": - case "claude-opus-4-8": - case "claude-opus-5": - case "claude-fable-5-1": - case "claude-fable-5": - case "claude-opus-4-5-20251101": - case "claude-opus-4-1-20250805": - case "claude-opus-4-20250514": - case "claude-3-7-sonnet-20250219": - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-haiku-4-5-20251001": - case "claude-3-haiku-20240307": { - /** - * The latest message will be the new user message, one before - * will be the assistant message from a previous request, and - * the user message before that will be a previously cached user - * message. So we need to mark the latest user message as - * ephemeral to cache it for the next request, and mark the - * second to last user message as ephemeral to let the server - * know the last message to retrieve from the cache for the - * current request. - */ - const userMsgIndices = sanitizedMessages.reduce( - (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), - [] as number[], - ) + if (supportsPromptCache) { + /** + * The latest message will be the new user message, one before + * will be the assistant message from a previous request, and + * the user message before that will be a previously cached user + * message. So we need to mark the latest user message as + * ephemeral to cache it for the next request, and mark the + * second to last user message as ephemeral to let the server + * know the last message to retrieve from the cache for the + * current request. + */ + const userMsgIndices = sanitizedMessages.reduce( + (acc, msg, index) => (msg.role === "user" ? [...acc, index] : acc), + [] as number[], + ) - const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 - const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 - - try { - const requestParams = { - model: modelId, - max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, - temperature, - thinking, - // Setting cache breakpoint for system prompt so new tasks can reuse it. - system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], - messages: sanitizedMessages.map((message, index) => { - if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { - return { - ...message, - content: - typeof message.content === "string" - ? [{ type: "text", text: message.content, cache_control: cacheControl }] - : message.content.map((content, contentIndex) => - contentIndex === message.content.length - 1 - ? { ...content, cache_control: cacheControl } - : content, - ), - } + const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1 + const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1 + + try { + const requestParams = { + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + temperature, + thinking, + // Setting cache breakpoint for system prompt so new tasks can reuse it. + system: [{ text: systemPrompt, type: "text", cache_control: cacheControl }], + messages: sanitizedMessages.map((message, index) => { + if (index === lastUserMsgIndex || index === secondLastMsgUserIndex) { + return { + ...message, + content: + typeof message.content === "string" + ? [{ type: "text", text: message.content, cache_control: cacheControl }] + : message.content.map((content, contentIndex) => + contentIndex === message.content.length - 1 + ? { ...content, cache_control: cacheControl } + : content, + ), } - return message - }), - stream: true, - ...nativeToolParams, - } - stream = await this.client.messages.create( - requestParams as Anthropic.Messages.MessageCreateParamsStreaming, - (() => { - // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 - // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers - // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 - - // Then check for models that support prompt caching - switch (modelId) { - case "claude-sonnet-5": - case "claude-sonnet-4-6": - case "claude-sonnet-4-5": - case "claude-sonnet-4-20250514": - case "claude-opus-4-6": - case "claude-opus-4-7": - case "claude-opus-4-8": - case "claude-opus-5": - case "claude-fable-5-1": - case "claude-fable-5": - case "claude-opus-4-5-20251101": - case "claude-opus-4-1-20250805": - case "claude-opus-4-20250514": - case "claude-3-7-sonnet-20250219": - case "claude-3-5-sonnet-20241022": - case "claude-3-5-haiku-20241022": - case "claude-3-opus-20240229": - case "claude-haiku-4-5-20251001": - case "claude-3-haiku-20240307": - betas.push("prompt-caching-2024-07-31") - return { headers: { "anthropic-beta": betas.join(",") } } - default: - return undefined - } - })(), - ) - } catch (error) { - TelemetryService.instance.captureException( - new ApiProviderError( - error instanceof Error ? error.message : String(error), - this.providerName, - modelId, - "createMessage", - ), - ) - throw error + } + return message + }), + stream: true, + ...nativeToolParams, } - break + stream = await this.client.messages.create( + requestParams as Anthropic.Messages.MessageCreateParamsStreaming, + (() => { + // prompt caching: https://x.com/alexalbert__/status/1823751995901272068 + // https://github.com/anthropics/anthropic-sdk-typescript?tab=readme-ov-file#default-headers + // https://github.com/anthropics/anthropic-sdk-typescript/commit/c920b77fc67bd839bfeb6716ceab9d7c9bbe7393 + betas.push("prompt-caching-2024-07-31") + return { headers: { "anthropic-beta": betas.join(",") } } + })(), + ) + } catch (error) { + TelemetryService.instance.captureException( + new ApiProviderError( + error instanceof Error ? error.message : String(error), + this.providerName, + modelId, + "createMessage", + ), + ) + throw error } - default: { - try { - const requestParams = { - model: modelId, - max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, - temperature, - thinking, - system: [{ text: systemPrompt, type: "text" }], - messages: sanitizedMessages, - stream: true, - ...nativeToolParams, - } - stream = (await this.client.messages.create( - requestParams as Anthropic.Messages.MessageCreateParamsStreaming, - )) as any - } catch (error) { - TelemetryService.instance.captureException( - new ApiProviderError( - error instanceof Error ? error.message : String(error), - this.providerName, - modelId, - "createMessage", - ), - ) - throw error + } else { + try { + const requestParams = { + model: modelId, + max_tokens: maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, + temperature, + thinking, + system: [{ text: systemPrompt, type: "text" }], + messages: sanitizedMessages, + stream: true, + ...nativeToolParams, } - break + stream = (await this.client.messages.create( + requestParams as Anthropic.Messages.MessageCreateParamsStreaming, + )) as any + } catch (error) { + TelemetryService.instance.captureException( + new ApiProviderError( + error instanceof Error ? error.message : String(error), + this.providerName, + modelId, + "createMessage", + ), + ) + throw error } }