Skip to content

fix: never return an empty assistant turn for tool-declaring clients #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
serrade wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: never return an empty assistant turn for tool-declaring clients #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 61 additions & 11 deletions macos/CursorAPI/Sources/CursorAPICore/LocalAPIServer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,10 @@ public final class LocalAPIServer: @unchecked Sendable {
usage: usage
))
}
let output = try await harness.complete(prepared: prepared, settings: settings, authorization: request.header("authorization"))
var output = try await harness.complete(prepared: prepared, settings: settings, authorization: request.header("authorization"))
if let repaired = try? await repairedOutput(for: output, prepared: prepared, settings: settings, authorization: request.header("authorization")) {
output = repaired
}
return try .response(withCORS(HTTPResponse.json(OpenAICompatibility.chatCompletionResponse(id: id, created: created, prepared: prepared, output: output))))
}
if method == "POST", path == "/v1/responses" {
Expand Down Expand Up @@ -585,7 +588,17 @@ public final class LocalAPIServer: @unchecked Sendable {
let bufferTextUntilToolDecision = shouldBufferTextUntilToolDecision(prepared)
var emittedText = ""
var emittedToolCalls: [CursorToolCall] = []
var emittedToolCallChunks = 0
var finalOutput: CursorSDKOutput?
// A tool call that cannot be mapped to a client tool produces
// no chunk. Counting it as emitted would close the stream with
// finish_reason tool_calls and nothing in it.
let yieldToolCall: (CursorToolCall, Int) -> Bool = { toolCall, index in
let chunk = OpenAICompatibility.chatCompletionStreamToolCall(id: id, created: created, prepared: prepared, toolCall: toolCall, index: index)
guard !chunk.isEmpty else { return false }
continuation.yield(chunk)
return true
}

for try await event in harness.stream(prepared: prepared, settings: settings, authorization: authorization) {
switch event {
Expand All @@ -595,35 +608,49 @@ public final class LocalAPIServer: @unchecked Sendable {
continuation.yield(OpenAICompatibility.chatCompletionStreamText(id: id, created: created, model: prepared.model, delta: delta))
}
case .toolCall(let toolCall):
let index = emittedToolCalls.count
emittedToolCalls.append(toolCall)
continuation.yield(OpenAICompatibility.chatCompletionStreamToolCall(id: id, created: created, prepared: prepared, toolCall: toolCall, index: index))
if yieldToolCall(toolCall, emittedToolCallChunks) {
emittedToolCallChunks += 1
}
case .done(let output):
finalOutput = output
}
}

let output = resolvedOutput(finalOutput: finalOutput, emittedText: emittedText, emittedToolCalls: emittedToolCalls)
var output = resolvedOutput(finalOutput: finalOutput, emittedText: emittedText, emittedToolCalls: emittedToolCalls)
if emittedToolCallChunks == 0,
let repaired = try? await repairedOutput(for: output, prepared: prepared, settings: settings, authorization: authorization) {
output = repaired
}
usage?.set(Self.usage(fromObject: OpenAICompatibility.chatCompletionResponse(id: id, created: created, prepared: prepared, output: output)))
let shouldEmitText = output.toolCalls.isEmpty
if output.toolCalls.count > emittedToolCalls.count {
for toolCall in output.toolCalls.dropFirst(emittedToolCalls.count) {
if yieldToolCall(toolCall, emittedToolCallChunks) {
emittedToolCallChunks += 1
}
}
}
let shouldEmitText = emittedToolCallChunks == 0
if shouldEmitText, output.text.count > emittedText.count, output.text.hasPrefix(emittedText) {
let suffix = String(output.text.dropFirst(emittedText.count))
continuation.yield(OpenAICompatibility.chatCompletionStreamText(id: id, created: created, model: prepared.model, delta: suffix))
emittedText += suffix
} else if shouldEmitText, (bufferTextUntilToolDecision || emittedText.isEmpty), !output.text.isEmpty {
continuation.yield(OpenAICompatibility.chatCompletionStreamText(id: id, created: created, model: prepared.model, delta: output.text))
emittedText = output.text
}
if output.toolCalls.count > emittedToolCalls.count {
for (offset, toolCall) in output.toolCalls.dropFirst(emittedToolCalls.count).enumerated() {
let index = emittedToolCalls.count + offset
continuation.yield(OpenAICompatibility.chatCompletionStreamToolCall(id: id, created: created, prepared: prepared, toolCall: toolCall, index: index))
}
// Last line of defence: never close a stream that carries
// neither content nor a usable tool call.
if emittedToolCallChunks == 0, emittedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
let fallback = OpenAICompatibility.nonEmptyAssistantText(prepared: prepared, output: output)
continuation.yield(OpenAICompatibility.chatCompletionStreamText(id: id, created: created, model: prepared.model, delta: fallback))
emittedText = fallback
}
continuation.yield(OpenAICompatibility.chatCompletionStreamFinish(
id: id,
created: created,
model: prepared.model,
emittedToolCallCount: max(emittedToolCalls.count, output.toolCalls.count)
emittedToolCallCount: emittedToolCallChunks
))
if prepared.streamIncludeUsage {
continuation.yield(OpenAICompatibility.chatCompletionStreamUsage(id: id, created: created, prepared: prepared, output: output))
Expand Down Expand Up @@ -757,6 +784,29 @@ public final class LocalAPIServer: @unchecked Sendable {
}
}

/// One extra upstream run when the model called a tool the client cannot
/// execute, telling it which tools actually exist. Returns nil when a repair
/// is not warranted or did not improve on the original output.
private func repairedOutput(
for output: CursorSDKOutput,
prepared: PreparedChatRequest,
settings: CursorAPISettings,
authorization: String?
) async throws -> CursorSDKOutput? {
guard OpenAICompatibility.needsToolCallRepair(output, prepared: prepared) else { return nil }
let attempted = OpenAICompatibility.unresolvableToolCallNames(output.toolCalls, prepared: prepared)
guard let repairedRequest = OpenAICompatibility.repairedRequest(prepared, attemptedToolNames: attempted) else { return nil }
let retried = try await harness.complete(prepared: repairedRequest, settings: settings, authorization: authorization)
if !retried.toolCalls.isEmpty,
OpenAICompatibility.unresolvableToolCallNames(retried.toolCalls, prepared: prepared).count < retried.toolCalls.count {
return retried
}
if retried.toolCalls.isEmpty, !retried.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return retried
}
return nil
}

private func resolvedOutput(
finalOutput: CursorSDKOutput?,
emittedText: String,
Expand Down
57 changes: 56 additions & 1 deletion macos/CursorAPI/Sources/CursorAPICore/OpenAICompatibility.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -627,14 +627,69 @@ public enum OpenAICompatibility {
]
}

/// Names of SDK tool calls that cannot be expressed as one of the client's
/// declared tools. Forwarding them as `tool_calls` would be rejected and
/// dropping them silently leaves an empty assistant turn, so callers use
/// this to decide between a repair run and a text fallback.
public static func unresolvableToolCallNames(_ toolCalls: [CursorToolCall], prepared: PreparedChatRequest) -> [String] {
toolCalls.compactMap { toolCall in
guard resolveToolCall(toolCall, tools: prepared.tools, context: prepared.toolContext) == nil else { return nil }
return normalizeSDKToolCall(toolCall).name
}
}

/// True when every tool call the model produced was unmappable, which is the
/// only case worth spending a second upstream run on.
public static func needsToolCallRepair(_ output: CursorSDKOutput, prepared: PreparedChatRequest) -> Bool {
guard !output.toolCalls.isEmpty else { return false }
return unresolvableToolCallNames(output.toolCalls, prepared: prepared).count == output.toolCalls.count
}

/// A copy of the request with a hint naming the client's actual tools, for
/// one retry after the model reached for a tool that does not exist here.
public static func repairedRequest(_ prepared: PreparedChatRequest, attemptedToolNames: [String]) -> PreparedChatRequest? {
guard !prepared.tools.isEmpty else { return nil }
let attempted = attemptedToolNames.filter { !$0.isEmpty }
let attemptedList = attempted.isEmpty ? "a tool" : attempted.map { "\"\($0)\"" }.joined(separator: ", ")
let hint = [
"",
"TOOL CALL NOT AVAILABLE:",
"Your last turn called \(attemptedList), which the outer client cannot execute.",
"Call one of these client tools instead, by exact name: \(prepared.tools.map(\.name).joined(separator: ", ")).",
"Emit exactly one client tool call with arguments matching that tool's schema, and no prose."
].joined(separator: "\n")
var repaired = prepared
repaired.prompt += hint
// The bridge reuses `incrementalPrompt` whenever it already has a warm
// agent for this session, so the hint has to be present in both.
repaired.incrementalPrompt = (prepared.incrementalPrompt ?? prepared.prompt) + hint
repaired.promptCharacters = repaired.prompt.count
return repaired
}

/// Assistant text that is never empty. Clients treat a turn with no content
/// and no usable tool call as a provider failure, so the model's narration is
/// used when present and an explanation otherwise.
public static func nonEmptyAssistantText(prepared: PreparedChatRequest, output: CursorSDKOutput) -> String {
if !output.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return output.text
}
let attempted = unresolvableToolCallNames(output.toolCalls, prepared: prepared).filter { !$0.isEmpty }
if !attempted.isEmpty {
let names = attempted.map { "`\($0)`" }.joined(separator: ", ")
return "I tried to use \(names), which is not one of the tools available here, so nothing was run. Tell me to retry and I will use an available tool."
}
return "The upstream model returned no content for this turn. Nothing was run. Retrying usually resolves it."
}

public static func chatCompletionResponse(
id: String,
created: Int,
prepared: PreparedChatRequest,
output: CursorSDKOutput
) -> [String: Any] {
let toolCalls = toOpenAIToolCalls(output.toolCalls, tools: prepared.tools, responseID: id, context: prepared.toolContext)
let content: Any = toolCalls.isEmpty ? output.text : NSNull()
let content: Any = toolCalls.isEmpty ? nonEmptyAssistantText(prepared: prepared, output: output) : NSNull()
return [
"id": id,
"object": "chat.completion",
Expand Down
Loading