fix(vscode-lm): sanitize lone UTF-16 surrogates in text and tool input

src/api/providers/__tests__/vscode-lm.spec.ts

@@ -272,6 +272,27 @@ describe("VsCodeLmHandler", () => {
 			})
 		})
 
+		describe("system prompt sanitization", () => {
+			it("sanitizes lone surrogates in the system prompt", async () => {
+				mockLanguageModelChat.sendRequest.mockResolvedValueOnce({
+					stream: (async function* () {
+						yield new vscode.LanguageModelTextPart("ok")
+						return
+					})(),
+					text: (async function* () {
+						yield "ok"
+						return
+					})(),
+				})
+				const stream = handler.createMessage("sys\uD800tem", [{ role: "user" as const, content: "hi" }])
+				for await (const _chunk of stream) {
+					// drain
+				}
+
+				expect(vscode.LanguageModelChatMessage.Assistant).toHaveBeenCalledWith("sys\uFFFDtem")
+			})
+		})
+
 		it("should handle native tool calls when tools are provided", async () => {
 			const systemPrompt = "You are a helpful assistant"
 			const messages: Anthropic.Messages.MessageParam[] = [

src/api/providers/vscode-lm.ts

@@ -15,7 +15,7 @@ import { SELECTOR_SEPARATOR, stringifyVsCodeLmModelSelector } from "../../shared
 import { normalizeToolSchema } from "../../utils/json-schema"
 
 import { ApiStream } from "../transform/stream"
-import { convertToVsCodeLmMessages, extractTextCountFromMessage } from "../transform/vscode-lm-format"
+import { convertToVsCodeLmMessages, extractTextCountFromMessage, sanitizeSurrogates } from "../transform/vscode-lm-format"
 
 import { BaseProvider } from "./base-provider"
 import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, CompletePromptOptions } from "../index"
@@ -391,7 +391,7 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan
 
 		// Convert Anthropic messages to VS Code LM messages
 		const vsCodeLmMessages: vscode.LanguageModelChatMessage[] = [
-			vscode.LanguageModelChatMessage.Assistant(systemPrompt),
+			vscode.LanguageModelChatMessage.Assistant(sanitizeSurrogates(systemPrompt)),
 			...convertToVsCodeLmMessages(cleanedMessages),
 		]
 

src/api/transform/__tests__/vscode-lm-format.spec.ts

@@ -3,7 +3,12 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import * as vscode from "vscode"
 
-import { convertToVsCodeLmMessages, convertToAnthropicRole, extractTextCountFromMessage } from "../vscode-lm-format"
+import {
+	convertToVsCodeLmMessages,
+	convertToAnthropicRole,
+	extractTextCountFromMessage,
+	sanitizeSurrogates,
+} from "../vscode-lm-format"
 
 // Mock crypto using Vitest
 vitest.stubGlobal("crypto", {
@@ -325,6 +330,97 @@ describe("convertToVsCodeLmMessages", () => {
 	})
 })
 
+describe("sanitizeSurrogates", () => {
+	it("leaves plain ASCII unchanged", () => {
+		expect(sanitizeSurrogates("hello world")).toBe("hello world")
+	})
+
+	it("leaves valid surrogate pairs unchanged", () => {
+		// 😀 U+1F600 and 𐀀 U+10000 are astral-plane code points encoded as surrogate pairs.
+		expect(sanitizeSurrogates("a\uD83D\uDE00b\uD800\uDC00c")).toBe("a\uD83D\uDE00b\uD800\uDC00c")
+	})
+
+	it("replaces a lone high surrogate with U+FFFD", () => {
+		expect(sanitizeSurrogates("a\uD800b")).toBe("a\uFFFDb")
+	})
+
+	it("replaces a lone low surrogate with U+FFFD", () => {
+		expect(sanitizeSurrogates("a\uDC00b")).toBe("a\uFFFDb")
+	})
+
+	it("replaces a trailing lone high surrogate", () => {
+		expect(sanitizeSurrogates("abc\uD800")).toBe("abc\uFFFD")
+	})
+
+	it("replaces a reversed (low-then-high) pair as two lone surrogates", () => {
+		expect(sanitizeSurrogates("\uDC00\uD800")).toBe("\uFFFD\uFFFD")
+	})
+
+	it("returns empty input unchanged", () => {
+		expect(sanitizeSurrogates("")).toBe("")
+	})
+})
+
+describe("convertToVsCodeLmMessages surrogate sanitization", () => {
+	const lone = "bad\uD800end"
+	const sanitized = "bad\uFFFDend"
+
+	const textValues = (message: { content: unknown }) =>
+		(message.content as MockLanguageModelTextPart[]).map((part) => part.value)
+
+	it("sanitizes a simple string message", () => {
+		const result = convertToVsCodeLmMessages([{ role: "user", content: lone }])
+		expect(textValues(result[0])).toEqual([sanitized])
+	})
+
+	it("sanitizes string tool_result content", () => {
+		const result = convertToVsCodeLmMessages([
+			{ role: "user", content: [{ type: "tool_result", tool_use_id: "tool-1", content: lone }] },
+		])
+		const toolResult = result[0].content[0] as MockLanguageModelToolResultPart
+		expect(toolResult.content[0].value).toBe(sanitized)
+	})
+
+	it("sanitizes tool_result text blocks", () => {
+		const result = convertToVsCodeLmMessages([
+			{
+				role: "user",
+				content: [{ type: "tool_result", tool_use_id: "tool-1", content: [{ type: "text", text: lone }] }],
+			},
+		])
+		const toolResult = result[0].content[0] as MockLanguageModelToolResultPart
+		expect(toolResult.content[0].value).toBe(sanitized)
+	})
+
+	it("sanitizes user text blocks", () => {
+		const result = convertToVsCodeLmMessages([{ role: "user", content: [{ type: "text", text: lone }] }])
+		expect(textValues(result[0])).toContain(sanitized)
+	})
+
+	it("sanitizes strings nested in tool_use input", () => {
+		const result = convertToVsCodeLmMessages([
+			{
+				role: "assistant",
+				content: [
+					{
+						type: "tool_use",
+						id: "tool-1",
+						name: "read_file",
+						input: { path: lone, nested: { list: [lone] } },
+					},
+				],
+			},
+		])
+		const toolCall = result[0].content[0] as MockLanguageModelToolCallPart
+		expect(toolCall.input).toEqual({ path: sanitized, nested: { list: [sanitized] } })
+	})
+
+	it("sanitizes assistant text blocks", () => {
+		const result = convertToVsCodeLmMessages([{ role: "assistant", content: [{ type: "text", text: lone }] }])
+		expect(textValues(result[0])).toContain(sanitized)
+	})
+})
+
 describe("convertToAnthropicRole", () => {
 	it("should convert assistant role correctly", () => {
 		const result = convertToAnthropicRole(vscode.LanguageModelChatMessageRole.Assistant)

src/api/transform/vscode-lm-format.ts

@@ -28,6 +28,46 @@
 	}
 }
 
+/**
+ * Replaces unpaired UTF-16 surrogate code units with the Unicode replacement character (U+FFFD).
+ *
+ * The VS Code LM backend forwards requests to model APIs that require valid UTF-8. A lone surrogate
+ * — e.g. left behind when some upstream step slices a string through an astral-plane character
+ * (emoji, CJK extension, etc.) — cannot be encoded as UTF-8, so the backend rejects the entire
+ * request with a 400 ("string contains an unpaired UTF-16 surrogate code point and cannot be
+ * encoded as valid UTF-8"). Valid surrogate pairs are matched by the lookahead/lookbehind and left
+ * untouched. The regex intentionally omits the `u` flag so it operates on UTF-16 code units.
+ */
+export function sanitizeSurrogates(text: string): string {
+	if (!text) {
+		return text
+	}
+	return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "\uFFFD")
+}
+
+/**
+ * Applies {@link sanitizeSurrogates} to every string nested in a tool-call argument object. The
+ * backend rejects the whole request for a lone surrogate anywhere in the JSON payload, so a tool
+ * argument carrying a sliced astral character fails the request just as message text would.
+ */
+function sanitizeSurrogatesDeep(value: unknown): unknown {
+	if (typeof value === "string") {
+		return sanitizeSurrogates(value)
+	}
+	if (Array.isArray(value)) {
+		return value.map(sanitizeSurrogatesDeep)
+	}
+	if (value && typeof value === "object") {
+		return Object.fromEntries(
+			Object.entries(value as Record<string, unknown>).map(([key, nested]) => [
+				sanitizeSurrogates(key),
+				sanitizeSurrogatesDeep(nested),
+			]),
+		)
+	}
+	return value
+}
+
 export function convertToVsCodeLmMessages(
 	anthropicMessages: Anthropic.Messages.MessageParam[],
 ): vscode.LanguageModelChatMessage[] {
@@ -36,10 +76,11 @@
 	for (const anthropicMessage of anthropicMessages) {
 		// Handle simple string messages
 		if (typeof anthropicMessage.content === "string") {
+			const safeContent = sanitizeSurrogates(anthropicMessage.content)
 			vsCodeLmMessages.push(
 				anthropicMessage.role === "assistant"
-					? vscode.LanguageModelChatMessage.Assistant(anthropicMessage.content)
-					: vscode.LanguageModelChatMessage.User(anthropicMessage.content),
+					? vscode.LanguageModelChatMessage.Assistant(safeContent)
+					: vscode.LanguageModelChatMessage.User(safeContent),
 			)
 			continue
 		}
@@ -69,7 +110,7 @@
 						// Process tool result content into TextParts
 						const toolContentParts: vscode.LanguageModelTextPart[] =
 							typeof toolMessage.content === "string"
-								? [new vscode.LanguageModelTextPart(toolMessage.content)]
+								? [new vscode.LanguageModelTextPart(sanitizeSurrogates(toolMessage.content))]
 								: (toolMessage.content?.map((part) => {
 										if (part.type === "image") {
 											if (part.source.type === "base64") {
@@ -82,7 +123,7 @@
 											)
 										}
 										if (part.type === "text") {
-											return new vscode.LanguageModelTextPart(part.text)
+											return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
 										}
 										return new vscode.LanguageModelTextPart("")
 									}) ?? [new vscode.LanguageModelTextPart("")])
@@ -102,7 +143,7 @@
 								`[Image (${part.source.type}): not supported by VSCode LM API]`,
 							)
 						}
-						return new vscode.LanguageModelTextPart(part.text)
+						return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
 					}),
 				]
 
@@ -135,7 +176,7 @@
 						if (part.type === "image") {
 							return new vscode.LanguageModelTextPart("[Image generation not supported by VSCode LM API]")
 						}
-						return new vscode.LanguageModelTextPart(part.text)
+						return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
 					}),
 
 					// Convert tool messages to ToolCallParts after text
@@ -144,7 +185,7 @@
 							new vscode.LanguageModelToolCallPart(
 								toolMessage.id,
 								toolMessage.name,
-								asObjectSafe(toolMessage.input),
+								sanitizeSurrogatesDeep(asObjectSafe(toolMessage.input)) as object,
 							),
 					),
 				]