[Fix] DeepSeek/OpenAI requests fail permanently when task history contains a lone UTF-16 surrogate

src/api/providers/__tests__/base-provider.spec.ts

@@ -267,6 +267,50 @@ describe("BaseProvider", () => {
 			expect(result?.[0].function.parameters.additionalProperties).toBeUndefined()
 		})
 
+		it("should sanitize lone UTF-16 surrogates in name, description, and parameters (#461)", () => {
+			const tools = [
+				{
+					type: "function",
+					function: {
+						name: "read_file",
+						description: "bad\uD800end",
+						parameters: {
+							type: "object",
+							properties: {
+								path: { type: "string", description: "bad\uDC00end" },
+							},
+						},
+					},
+				},
+			]
+
+			const result = provider.testConvertToolsForOpenAI(tools)
+
+			expect(result?.[0].function.description).toBe("bad\uFFFDend")
+			expect(result?.[0].function.parameters.properties.path.description).toBe("bad\uFFFDend")
+			// The serialized body must not contain a lone surrogate anywhere.
+			expect(JSON.stringify(result)).not.toMatch(
+				/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/,
+			)
+		})
+
+		it("should sanitize lone UTF-16 surrogates in tool names (#461)", () => {
+			const tools = [
+				{
+					type: "function",
+					function: {
+						name: "read\uD800file",
+						description: "Read a file",
+						parameters: { type: "object", properties: {} },
+					},
+				},
+			]
+
+			const result = provider.testConvertToolsForOpenAI(tools)
+
+			expect(result?.[0].function.name).toBe("read\uFFFDfile")
+		})
+
 		it("should preserve non-function tools unchanged", () => {
 			const tools = [
 				{

src/api/providers/__tests__/deepseek.spec.ts

@@ -824,3 +824,70 @@ describe("DeepSeekHandler", () => {
 		})
 	})
 })
+
+describe("DeepSeekHandler lone surrogate sanitization (#461)", () => {
+	// Matches any unpaired UTF-16 code unit; the serialized request body must never contain one.
+	const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/
+
+	beforeEach(() => {
+		mockCreate.mockClear()
+	})
+
+	const surrogateOptions: ApiHandlerOptions = {
+		deepSeekApiKey: "test-api-key",
+		apiModelId: "deepseek-v4-flash",
+		deepSeekBaseUrl: "https://api.deepseek.com",
+	}
+
+	it("sends a request body free of lone surrogates, keeping tool call/result pairing intact", async () => {
+		const lone = "bad\uD800end"
+		const sanitized = "bad\uFFFDend"
+		const handler = new DeepSeekHandler(surrogateOptions)
+		const messages: Anthropic.Messages.MessageParam[] = [
+			{ role: "user", content: lone },
+			{
+				role: "assistant",
+				content: [{ type: "tool_use", id: "call-\uD800", name: "read_file", input: { path: lone } }],
+			},
+			{ role: "user", content: [{ type: "tool_result", tool_use_id: "call-\uD800", content: lone }] },
+		]
+
+		await collectStream(
+			handler.createMessage("system \uD800 prompt", messages, {
+				taskId: "task-1",
+				tools: [
+					{
+						type: "function",
+						function: {
+							name: "read_file",
+							description: lone,
+							parameters: {
+								type: "object",
+								properties: { path: { type: "string", description: lone } },
+							},
+						},
+					},
+				],
+			}),
+		)
+
+		expect(mockCreate).toHaveBeenCalledOnce()
+		const request = mockCreate.mock.calls[0][0]
+
+		// The whole body (messages + tools) must serialize without a lone surrogate.
+		expect(JSON.stringify(request)).not.toMatch(LONE_SURROGATE)
+
+		// The tool call and its result stay paired after injective id sanitization.
+		const assistantMessage = request.messages.find(
+			(message: { role: string }) => message.role === "assistant",
+		) as OpenAI.Chat.ChatCompletionAssistantMessageParam
+		const toolMessage = request.messages.find(
+			(message: { role: string }) => message.role === "tool",
+		) as OpenAI.Chat.ChatCompletionToolMessageParam
+		const toolCalls = assistantMessage.tool_calls as OpenAI.Chat.ChatCompletionMessageFunctionToolCall[]
+		expect(toolMessage.tool_call_id).toBe(toolCalls[0].id)
+		expect(toolMessage.content).toBe(sanitized)
+		expect(JSON.parse(toolCalls[0].function.arguments)).toEqual({ path: sanitized })
+		expect(request.tools[0].function.description).toBe(sanitized)
+	})
+})

src/api/providers/__tests__/openai.spec.ts

@@ -1877,3 +1877,72 @@ describe("getOpenAiModels", () => {
 		expect(result).toEqual(["gpt-4", "gpt-3.5-turbo"])
 	})
 })
+
+describe("OpenAiHandler lone surrogate sanitization (#461)", () => {
+	// Matches any unpaired UTF-16 code unit; the serialized request body must never contain one.
+	const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/
+
+	beforeEach(() => {
+		mockCreate.mockClear()
+	})
+
+	it("sends a request body free of lone surrogates, keeping tool call/result pairing intact", async () => {
+		const lone = "bad\uD800end"
+		const sanitized = "bad\uFFFDend"
+		const handler = new OpenAiHandler(
+			makeApiHandlerOptions({
+				openAiApiKey: "test-api-key",
+				openAiModelId: "gpt-4",
+				openAiBaseUrl: "https://api.openai.com/v1",
+			}),
+		)
+		const messages: Anthropic.Messages.MessageParam[] = [
+			{ role: "user", content: lone },
+			{
+				role: "assistant",
+				content: [{ type: "tool_use", id: "call-\uD800", name: "read_file", input: { path: lone } }],
+			},
+			{ role: "user", content: [{ type: "tool_result", tool_use_id: "call-\uD800", content: lone }] },
+		]
+
+		await collectStream(
+			handler.createMessage("system \uD800 prompt", messages, {
+				taskId: "task-1",
+				tools: [
+					{
+						type: "function",
+						function: {
+							name: "read_file",
+							description: lone,
+							parameters: {
+								type: "object",
+								properties: { path: { type: "string", description: lone } },
+							},
+						},
+					},
+				],
+			}),
+		)
+
+		expect(mockCreate).toHaveBeenCalledOnce()
+		const request = mockCreate.mock.calls[0][0]
+
+		// The whole body (system prompt, messages, tools) must serialize without a lone surrogate.
+		expect(JSON.stringify(request)).not.toMatch(LONE_SURROGATE)
+
+		expect(request.messages[0]).toEqual({ role: "system", content: "system \uFFFD prompt" })
+
+		// The tool call and its result stay paired after injective id sanitization.
+		const assistantMessage = request.messages.find(
+			(message: { role: string }) => message.role === "assistant",
+		) as OpenAI.Chat.ChatCompletionAssistantMessageParam
+		const toolMessage = request.messages.find(
+			(message: { role: string }) => message.role === "tool",
+		) as OpenAI.Chat.ChatCompletionToolMessageParam
+		const toolCalls = assistantMessage.tool_calls as OpenAI.Chat.ChatCompletionMessageFunctionToolCall[]
+		expect(toolMessage.tool_call_id).toBe(toolCalls[0].id)
+		expect(toolMessage.content).toBe(sanitized)
+		expect(JSON.parse(toolCalls[0].function.arguments)).toEqual({ path: sanitized })
+		expect(request.tools[0].function.description).toBe(sanitized)
+	})
+})

src/api/providers/base-provider.ts

@@ -4,6 +4,7 @@ import type { ModelInfo } from "@roo-code/types"
 
 import type { ApiHandler, ApiHandlerCreateMessageMetadata } from "../index"
 import { ApiStream } from "../transform/stream"
+import { sanitizeSurrogates, sanitizeSurrogatesDeep } from "../transform/sanitize-surrogates"
 import { countTokens } from "../../utils/countTokens"
 import { isMcpTool } from "../../utils/mcp-name"
 import { getApiRequestTimeout } from "./utils/timeout-config"
@@ -45,10 +46,14 @@ export abstract class BaseProvider implements ApiHandler {
 				...tool,
 				function: {
 					...tool.function,
+					// Sanitize lone UTF-16 surrogates: providers validating the JSON body
+					// (e.g. DeepSeek) reject the whole request otherwise. See #461.
+					name: sanitizeSurrogates(tool.function.name),
+					description: sanitizeSurrogates(tool.function.description),
 					strict: !isMcp,
-					parameters: isMcp
-						? tool.function.parameters
-						: this.convertToolSchemaForOpenAI(tool.function.parameters),
+					parameters: sanitizeSurrogatesDeep(
+						isMcp ? tool.function.parameters : this.convertToolSchemaForOpenAI(tool.function.parameters),
+					),
 				},
 			}
 		})

src/api/providers/openai.ts

@@ -18,6 +18,7 @@ import type { ApiHandlerOptions } from "../../shared/api"
 import { TagMatcher } from "../../utils/tag-matcher"
 
 import { convertToOpenAiMessages } from "../transform/openai-format"
+import { sanitizeSurrogates } from "../transform/sanitize-surrogates"
 import { convertToR1Format } from "../transform/r1-format"
 import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import { getModelParams } from "../transform/model-params"
@@ -101,7 +102,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
 
 		let systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = {
 			role: "system",
-			content: systemPrompt,
+			// Sanitize lone UTF-16 surrogates: providers validating the JSON body
+			// (e.g. DeepSeek) reject the whole request otherwise. See #461.
+			content: sanitizeSurrogates(systemPrompt),
 		}
 
 		if (this.options.openAiStreamingEnabled ?? true) {
@@ -116,7 +119,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
 						content: [
 							{
 								type: "text",
-								text: systemPrompt,
+								text: sanitizeSurrogates(systemPrompt),
 								// @ts-ignore-next-line
 								cache_control: { type: "ephemeral" },
 							},
@@ -365,7 +368,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
 				messages: [
 					{
 						role: "developer",
-						content: `Formatting re-enabled\n${systemPrompt}`,
+						content: sanitizeSurrogates(`Formatting re-enabled\n${systemPrompt}`),
 					},
 					...convertToOpenAiMessages(messages),
 				],
@@ -402,7 +405,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
 				messages: [
 					{
 						role: "developer",
-						content: `Formatting re-enabled\n${systemPrompt}`,
+						content: sanitizeSurrogates(`Formatting re-enabled\n${systemPrompt}`),
 					},
 					...convertToOpenAiMessages(messages),
 				],

src/api/transform/__tests__/openai-format.spec.ts

@@ -1571,3 +1571,131 @@ describe("sanitizeGeminiMessages", () => {
 		expect(result).toEqual(messages)
 	})
 })
+
+describe("convertToOpenAiMessages lone surrogate sanitization (#461)", () => {
+	const lone = "bad\uD800end"
+	const sanitized = "bad\uFFFDend"
+
+	it("sanitizes a simple string message", () => {
+		const result = convertToOpenAiMessages([{ role: "user", content: lone }])
+		expect(result[0]).toEqual({ role: "user", content: sanitized })
+	})
+
+	it("sanitizes user text blocks", () => {
+		const result = convertToOpenAiMessages([{ role: "user", content: [{ type: "text", text: lone }] }])
+		expect(result[0]).toEqual({ role: "user", content: [{ type: "text", text: sanitized }] })
+	})
+
+	it("sanitizes string tool_result content", () => {
+		const result = convertToOpenAiMessages([
+			{ role: "user", content: [{ type: "tool_result", tool_use_id: "tool-1", content: lone }] },
+		])
+		expect(result[0]).toEqual({ role: "tool", tool_call_id: "tool-1", content: sanitized })
+	})
+
+	it("sanitizes tool_result text blocks", () => {
+		const result = convertToOpenAiMessages([
+			{
+				role: "user",
+				content: [{ type: "tool_result", tool_use_id: "tool-1", content: [{ type: "text", text: lone }] }],
+			},
+		])
+		expect(result[0]).toEqual({ role: "tool", tool_call_id: "tool-1", content: sanitized })
+	})
+
+	it("sanitizes assistant text blocks", () => {
+		const result = convertToOpenAiMessages([{ role: "assistant", content: [{ type: "text", text: lone }] }])
+		expect(result[0]).toMatchObject({ role: "assistant", content: sanitized })
+	})
+
+	it("sanitizes strings nested in tool_use input", () => {
+		const result = convertToOpenAiMessages([
+			{
+				role: "assistant",
+				content: [
+					{
+						type: "tool_use",
+						id: "tool-1",
+						name: "read_file",
+						input: { path: lone, nested: { list: [lone] } },
+					},
+				],
+			},
+		])
+		const toolCalls = (result[0] as OpenAI.Chat.ChatCompletionAssistantMessageParam)
+			.tool_calls as OpenAI.Chat.ChatCompletionMessageFunctionToolCall[]
+		expect(JSON.parse(toolCalls[0].function.arguments)).toEqual({ path: sanitized, nested: { list: [sanitized] } })
+	})
+
+	it("sanitizes tool_use name", () => {
+		const result = convertToOpenAiMessages([
+			{ role: "assistant", content: [{ type: "tool_use", id: "tool-1", name: `read${lone}`, input: {} }] },
+		])
+		const toolCalls = (result[0] as OpenAI.Chat.ChatCompletionAssistantMessageParam)
+			.tool_calls as OpenAI.Chat.ChatCompletionMessageFunctionToolCall[]
+		expect(toolCalls[0].function.name).toBe(`read${sanitized}`)
+	})
+
+	it("sanitizes tool_use ids injectively and keeps them paired with tool_result ids", () => {
+		const loneIdA = "call-\uD800"
+		const loneIdB = "call-\uD801"
+		const result = convertToOpenAiMessages([
+			{
+				role: "assistant",
+				content: [
+					{ type: "tool_use", id: loneIdA, name: "read_file", input: {} },
+					{ type: "tool_use", id: loneIdB, name: "read_file", input: {} },
+				],
+			},
+			{
+				role: "user",
+				content: [
+					{ type: "tool_result", tool_use_id: loneIdA, content: "ok" },
+					{ type: "tool_result", tool_use_id: loneIdB, content: "ok" },
+				],
+			},
+		])
+		const toolCalls = (result[0] as OpenAI.Chat.ChatCompletionAssistantMessageParam)
+			.tool_calls as OpenAI.Chat.ChatCompletionMessageFunctionToolCall[]
+		const toolMessages = result.slice(1) as OpenAI.Chat.ChatCompletionToolMessageParam[]
+		// Injective: the two ids must not collapse onto the same replacement.
+		expect(toolCalls[0].id).not.toBe(toolCalls[1].id)
+		// Pairing: each tool result addresses its own call after sanitization.
+		expect(toolMessages[0].tool_call_id).toBe(toolCalls[0].id)
+		expect(toolMessages[1].tool_call_id).toBe(toolCalls[1].id)
+	})
+
+	it("composes id sanitization with normalizeToolCallId", () => {
+		const result = convertToOpenAiMessages(
+			[{ role: "user", content: [{ type: "tool_result", tool_use_id: "call-\uD800", content: "ok" }] }],
+			{ normalizeToolCallId: (id) => id.toUpperCase() },
+		)
+		expect((result[0] as OpenAI.Chat.ChatCompletionToolMessageParam).tool_call_id).toBe("CALL-\uFFFD" + "D800")
+	})
+
+	it("sanitizes reasoning_content pass-through", () => {
+		const result = convertToOpenAiMessages([
+			Object.assign({ role: "assistant" as const, content: lone }, { reasoning_content: lone }),
+		])
+		expect(result[0]).toMatchObject({ content: sanitized, reasoning_content: sanitized })
+	})
+
+	it("leaves valid surrogate pairs untouched", () => {
+		const pair = "emoji \uD83D\uDE00 done"
+		const result = convertToOpenAiMessages([{ role: "user", content: pair }])
+		expect(result[0]).toEqual({ role: "user", content: pair })
+	})
+
+	it("produces a JSON body free of lone surrogates", () => {
+		const result = convertToOpenAiMessages([
+			{ role: "user", content: lone },
+			{
+				role: "assistant",
+				content: [{ type: "tool_use", id: "call-\uD800", name: "read_file", input: { path: lone } }],
+			},
+			{ role: "user", content: [{ type: "tool_result", tool_use_id: "call-\uD800", content: lone }] },
+		])
+		const body = JSON.stringify(result)
+		expect(body).not.toMatch(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/)
+	})
+})