diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index 423f119f14..d3d9b52042 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -272,6 +272,27 @@ describe("VsCodeLmHandler", () => { }) }) + describe("system prompt sanitization", () => { + it("sanitizes lone surrogates in the system prompt", async () => { + mockLanguageModelChat.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + yield new vscode.LanguageModelTextPart("ok") + return + })(), + text: (async function* () { + yield "ok" + return + })(), + }) + const stream = handler.createMessage("sys\uD800tem", [{ role: "user" as const, content: "hi" }]) + for await (const _chunk of stream) { + // drain + } + + expect(vscode.LanguageModelChatMessage.Assistant).toHaveBeenCalledWith("sys\uFFFDtem") + }) + }) + it("should handle native tool calls when tools are provided", async () => { const systemPrompt = "You are a helpful assistant" const messages: Anthropic.Messages.MessageParam[] = [ diff --git a/src/api/providers/vscode-lm.ts b/src/api/providers/vscode-lm.ts index 62bcbf0c27..c014cce2df 100644 --- a/src/api/providers/vscode-lm.ts +++ b/src/api/providers/vscode-lm.ts @@ -15,7 +15,7 @@ import { SELECTOR_SEPARATOR, stringifyVsCodeLmModelSelector } from "../../shared import { normalizeToolSchema } from "../../utils/json-schema" import { ApiStream } from "../transform/stream" -import { convertToVsCodeLmMessages, extractTextCountFromMessage } from "../transform/vscode-lm-format" +import { convertToVsCodeLmMessages, extractTextCountFromMessage, sanitizeSurrogates } from "../transform/vscode-lm-format" import { BaseProvider } from "./base-provider" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, CompletePromptOptions } from "../index" @@ -391,7 +391,7 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan // Convert Anthropic messages to VS Code LM messages const vsCodeLmMessages: vscode.LanguageModelChatMessage[] = [ - vscode.LanguageModelChatMessage.Assistant(systemPrompt), + vscode.LanguageModelChatMessage.Assistant(sanitizeSurrogates(systemPrompt)), ...convertToVsCodeLmMessages(cleanedMessages), ] diff --git a/src/api/transform/__tests__/vscode-lm-format.spec.ts b/src/api/transform/__tests__/vscode-lm-format.spec.ts index 3265f2745b..3299fedf4b 100644 --- a/src/api/transform/__tests__/vscode-lm-format.spec.ts +++ b/src/api/transform/__tests__/vscode-lm-format.spec.ts @@ -3,7 +3,12 @@ import { Anthropic } from "@anthropic-ai/sdk" import * as vscode from "vscode" -import { convertToVsCodeLmMessages, convertToAnthropicRole, extractTextCountFromMessage } from "../vscode-lm-format" +import { + convertToVsCodeLmMessages, + convertToAnthropicRole, + extractTextCountFromMessage, + sanitizeSurrogates, +} from "../vscode-lm-format" // Mock crypto using Vitest vitest.stubGlobal("crypto", { @@ -325,6 +330,97 @@ describe("convertToVsCodeLmMessages", () => { }) }) +describe("sanitizeSurrogates", () => { + it("leaves plain ASCII unchanged", () => { + expect(sanitizeSurrogates("hello world")).toBe("hello world") + }) + + it("leaves valid surrogate pairs unchanged", () => { + // 😀 U+1F600 and 𐀀 U+10000 are astral-plane code points encoded as surrogate pairs. + expect(sanitizeSurrogates("a\uD83D\uDE00b\uD800\uDC00c")).toBe("a\uD83D\uDE00b\uD800\uDC00c") + }) + + it("replaces a lone high surrogate with U+FFFD", () => { + expect(sanitizeSurrogates("a\uD800b")).toBe("a\uFFFDb") + }) + + it("replaces a lone low surrogate with U+FFFD", () => { + expect(sanitizeSurrogates("a\uDC00b")).toBe("a\uFFFDb") + }) + + it("replaces a trailing lone high surrogate", () => { + expect(sanitizeSurrogates("abc\uD800")).toBe("abc\uFFFD") + }) + + it("replaces a reversed (low-then-high) pair as two lone surrogates", () => { + expect(sanitizeSurrogates("\uDC00\uD800")).toBe("\uFFFD\uFFFD") + }) + + it("returns empty input unchanged", () => { + expect(sanitizeSurrogates("")).toBe("") + }) +}) + +describe("convertToVsCodeLmMessages surrogate sanitization", () => { + const lone = "bad\uD800end" + const sanitized = "bad\uFFFDend" + + const textValues = (message: { content: unknown }) => + (message.content as MockLanguageModelTextPart[]).map((part) => part.value) + + it("sanitizes a simple string message", () => { + const result = convertToVsCodeLmMessages([{ role: "user", content: lone }]) + expect(textValues(result[0])).toEqual([sanitized]) + }) + + it("sanitizes string tool_result content", () => { + const result = convertToVsCodeLmMessages([ + { role: "user", content: [{ type: "tool_result", tool_use_id: "tool-1", content: lone }] }, + ]) + const toolResult = result[0].content[0] as MockLanguageModelToolResultPart + expect(toolResult.content[0].value).toBe(sanitized) + }) + + it("sanitizes tool_result text blocks", () => { + const result = convertToVsCodeLmMessages([ + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tool-1", content: [{ type: "text", text: lone }] }], + }, + ]) + const toolResult = result[0].content[0] as MockLanguageModelToolResultPart + expect(toolResult.content[0].value).toBe(sanitized) + }) + + it("sanitizes user text blocks", () => { + const result = convertToVsCodeLmMessages([{ role: "user", content: [{ type: "text", text: lone }] }]) + expect(textValues(result[0])).toContain(sanitized) + }) + + it("sanitizes strings nested in tool_use input", () => { + const result = convertToVsCodeLmMessages([ + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool-1", + name: "read_file", + input: { path: lone, nested: { list: [lone] } }, + }, + ], + }, + ]) + const toolCall = result[0].content[0] as MockLanguageModelToolCallPart + expect(toolCall.input).toEqual({ path: sanitized, nested: { list: [sanitized] } }) + }) + + it("sanitizes assistant text blocks", () => { + const result = convertToVsCodeLmMessages([{ role: "assistant", content: [{ type: "text", text: lone }] }]) + expect(textValues(result[0])).toContain(sanitized) + }) +}) + describe("convertToAnthropicRole", () => { it("should convert assistant role correctly", () => { const result = convertToAnthropicRole(vscode.LanguageModelChatMessageRole.Assistant) diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index 7ac51e024f..e516aaa9ae 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -28,6 +28,46 @@ function asObjectSafe(value: unknown): object { } } +/** + * Replaces unpaired UTF-16 surrogate code units with the Unicode replacement character (U+FFFD). + * + * The VS Code LM backend forwards requests to model APIs that require valid UTF-8. A lone surrogate + * — e.g. left behind when some upstream step slices a string through an astral-plane character + * (emoji, CJK extension, etc.) — cannot be encoded as UTF-8, so the backend rejects the entire + * request with a 400 ("string contains an unpaired UTF-16 surrogate code point and cannot be + * encoded as valid UTF-8"). Valid surrogate pairs are matched by the lookahead/lookbehind and left + * untouched. The regex intentionally omits the `u` flag so it operates on UTF-16 code units. + */ +export function sanitizeSurrogates(text: string): string { + if (!text) { + return text + } + return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?).map(([key, nested]) => [ + sanitizeSurrogates(key), + sanitizeSurrogatesDeep(nested), + ]), + ) + } + return value +} + export function convertToVsCodeLmMessages( anthropicMessages: Anthropic.Messages.MessageParam[], ): vscode.LanguageModelChatMessage[] { @@ -36,10 +76,11 @@ export function convertToVsCodeLmMessages( for (const anthropicMessage of anthropicMessages) { // Handle simple string messages if (typeof anthropicMessage.content === "string") { + const safeContent = sanitizeSurrogates(anthropicMessage.content) vsCodeLmMessages.push( anthropicMessage.role === "assistant" - ? vscode.LanguageModelChatMessage.Assistant(anthropicMessage.content) - : vscode.LanguageModelChatMessage.User(anthropicMessage.content), + ? vscode.LanguageModelChatMessage.Assistant(safeContent) + : vscode.LanguageModelChatMessage.User(safeContent), ) continue } @@ -69,7 +110,7 @@ export function convertToVsCodeLmMessages( // Process tool result content into TextParts const toolContentParts: vscode.LanguageModelTextPart[] = typeof toolMessage.content === "string" - ? [new vscode.LanguageModelTextPart(toolMessage.content)] + ? [new vscode.LanguageModelTextPart(sanitizeSurrogates(toolMessage.content))] : (toolMessage.content?.map((part) => { if (part.type === "image") { if (part.source.type === "base64") { @@ -82,7 +123,7 @@ export function convertToVsCodeLmMessages( ) } if (part.type === "text") { - return new vscode.LanguageModelTextPart(part.text) + return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text)) } return new vscode.LanguageModelTextPart("") }) ?? [new vscode.LanguageModelTextPart("")]) @@ -102,7 +143,7 @@ export function convertToVsCodeLmMessages( `[Image (${part.source.type}): not supported by VSCode LM API]`, ) } - return new vscode.LanguageModelTextPart(part.text) + return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text)) }), ] @@ -135,7 +176,7 @@ export function convertToVsCodeLmMessages( if (part.type === "image") { return new vscode.LanguageModelTextPart("[Image generation not supported by VSCode LM API]") } - return new vscode.LanguageModelTextPart(part.text) + return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text)) }), // Convert tool messages to ToolCallParts after text @@ -144,7 +185,7 @@ export function convertToVsCodeLmMessages( new vscode.LanguageModelToolCallPart( toolMessage.id, toolMessage.name, - asObjectSafe(toolMessage.input), + sanitizeSurrogatesDeep(asObjectSafe(toolMessage.input)) as object, ), ), ]