bugfix: fix OpenAI multi-turn tool calls and Content-Length FATAL crash.

xllm_service/chat_template/jinja_chat_template.cpp

@@ -184,12 +184,37 @@ std::optional<std::string> JinjaChatTemplate::apply(
     if (message.tool_calls.has_value()) {
       nlohmann::ordered_json tool_calls_json = nlohmann::json::array();
       for (const auto& tool_call : *message.tool_calls) {
+        // Tool-call arguments arrive as a JSON string (OpenAI wire format), but
+        // chat templates such as GLM iterate `arguments.items()`, which only
+        // works on a JSON object. A non-object value (truncated/malformed JSON,
+        // or a JSON array/scalar) makes minja throw "Unknown method: items",
+        // which would abort the whole process. Always hand the template an
+        // object: parse when possible and fall back to an empty object
+        // otherwise, so rendering can never throw on this field.
+        nlohmann::ordered_json arguments_json =
+            nlohmann::ordered_json::object();
+        if (!tool_call.function.arguments.empty()) {
+          try {
+            auto parsed = nlohmann::json::parse(tool_call.function.arguments);
+            if (parsed.is_object()) {
+              arguments_json = std::move(parsed);
+            } else {
+              LOG(WARNING) << "Tool-call arguments are not a JSON object, "
+                              "rendering empty args: "
+                           << tool_call.function.arguments;
+            }
+          } catch (const nlohmann::json::exception& e) {
+            LOG(WARNING) << "Failed to parse tool-call arguments, rendering "
+                            "empty args: "
+                         << e.what();
+          }
+        }
         tool_calls_json.emplace_back(nlohmann::ordered_json{
             {"id", tool_call.id},
             {"type", tool_call.type},
             {"function",
              {{"name", tool_call.function.name},
-              {"arguments", tool_call.function.arguments}}}});
+              {"arguments", std::move(arguments_json)}}}});
       }
       message_json["tool_calls"] = std::move(tool_calls_json);
     }
@@ -249,7 +274,18 @@ std::optional<std::string> JinjaChatTemplate::apply(
   input.extra_context = chat_template_kwargs;
   minja::chat_template_options options;
 
-  return template_->apply(input, options);
+  // minja throws std::runtime_error on any rendering failure (e.g. calling
+  // `.items()` on a non-object value). The OpenAI HTTP path does not wrap the
+  // scheduler call in a try/catch, so an uncaught exception here would escape
+  // the brpc handler and terminate the whole process. Contain it and degrade
+  // to an empty result, letting callers report a normal "failed to construct
+  // prompt" error instead of crashing.
+  try {
+    return template_->apply(input, options);
+  } catch (const std::exception& e) {
+    LOG(ERROR) << "Failed to apply chat template: " << e.what();
+    return std::nullopt;
+  }
 }
 
 nlohmann::ordered_json JinjaChatTemplate::get_mm_content(

xllm_service/common/call_data.h

@@ -97,7 +97,7 @@ class StreamCallData : public CallData {
 
     } else {
       controller_->http_response().SetHeader("Content-Type",
-                                             "text/javascript; charset=utf-8");
+                                             "application/json; charset=utf-8");
     }
 
     json_options_.bytes_to_base64 = false;

xllm_service/http_service/service.cpp

@@ -299,18 +299,28 @@ constexpr char kInferContentLength[] = "Infer-Content-Length";
 constexpr char kContentLength[] = "Content-Length";
 
 size_t GetJsonContentLength(const brpc::Controller* ctrl) {
+  const auto parse_length = [](const std::string& value) -> size_t {
+    try {
+      return std::stoul(value);
+    } catch (const std::exception& e) {
+      LOG(ERROR) << "Invalid Content-Length value: " << value
+                 << ", error: " << e.what();
+      return (size_t)-1L;
+    }
+  };
+
   const auto infer_content_len =
       ctrl->http_request().GetHeader(kInferContentLength);
   if (infer_content_len != nullptr) {
-    return std::stoul(*infer_content_len);
+    return parse_length(*infer_content_len);
   }
 
   const auto content_len = ctrl->http_request().GetHeader(kContentLength);
   if (content_len != nullptr) {
-    return std::stoul(*content_len);
+    return parse_length(*content_len);
   }
 
-  LOG(FATAL) << "Content-Length header is missing.";
+  LOG(ERROR) << "Content-Length header is missing.";
   return (size_t)-1L;
 }
 
@@ -569,6 +579,10 @@ void XllmHttpServiceImpl::ChatCompletions(
           arena);
 
   auto content_len = GetJsonContentLength(cntl);
+  if (content_len == (size_t)-1L) {
+    cntl->SetFailed("Content-Length header is missing or invalid.");
+    return;
+  }
   std::string attachment;
   cntl->request_attachment().copy_to(&attachment, content_len, 0);
 
@@ -594,7 +608,28 @@ void XllmHttpServiceImpl::ChatCompletions(
   if (req_pb->messages_size() > 0) {
     service_request->messages.reserve(req_pb->messages_size());
     for (const auto& message : req_pb->messages()) {
-      service_request->messages.emplace_back(message.role(), message.content());
+      Message msg(message.role(), message.content());
+      if (message.has_reasoning_content()) {
+        msg.reasoning_content = message.reasoning_content();
+      }
+      if (!message.tool_call_id().empty()) {
+        msg.tool_call_id = message.tool_call_id();
+      }
+      if (message.tool_calls_size() > 0) {
+        Message::ToolCallVec tool_calls;
+        tool_calls.reserve(message.tool_calls_size());
+        for (const auto& tool_call : message.tool_calls()) {
+          Message::ToolCall parsed_tool_call;
+          parsed_tool_call.id = tool_call.id();
+          parsed_tool_call.type = tool_call.type();
+          parsed_tool_call.function.name = tool_call.function().name();
+          parsed_tool_call.function.arguments =
+              tool_call.function().arguments();
+          tool_calls.emplace_back(std::move(parsed_tool_call));
+        }
+        msg.tool_calls = std::move(tool_calls);
+      }
+      service_request->messages.emplace_back(std::move(msg));
     }
     if (req_pb->has_chat_template_kwargs()) {
       service_request->chat_template_kwargs =
@@ -670,6 +705,10 @@ void XllmHttpServiceImpl::AnthropicMessages(
       ::xllm::proto::AnthropicMessagesResponse>(arena);
 
   auto content_len = GetJsonContentLength(cntl);
+  if (content_len == (size_t)-1L) {
+    cntl->SetFailed("Content-Length header is missing or invalid.");
+    return;
+  }
   std::string attachment;
   cntl->request_attachment().copy_to(&attachment, content_len, 0);
 

xllm_service/scheduler/response_handler.cpp

@@ -49,6 +49,31 @@ size_t find_tool_start(const std::string& text) {
   return pos;
 }
 
+// GLM-family control markers that may leak into the streamed output when
+// skip_special_tokens is disabled for tool calling (e.g. the model emits
+// <|observation|> right after </tool_call>). They carry no visible content.
+// The non-stream path already drops trailing text after tool calls, so this
+// keeps the streaming behavior consistent and prevents these markers from
+// reaching the client.
+void strip_special_control_tokens(std::string* text) {
+  if (!text || text->empty()) {
+    return;
+  }
+  static const std::vector<std::string> kControlTokens = {
+      "<|observation|>",
+      "<|user|>",
+      "<|assistant|>",
+      "<|system|>",
+      "<|endoftext|>",
+  };
+  for (const auto& token : kControlTokens) {
+    size_t pos = 0;
+    while ((pos = text->find(token, pos)) != std::string::npos) {
+      text->erase(pos, token.length());
+    }
+  }
+}
+
 AnthropicTracer make_anthropic_tracer(
     const std::shared_ptr<AnthropicCallData>& call_data) {
   return AnthropicTracer(
@@ -89,7 +114,9 @@ bool send_normal_text_chunk(std::shared_ptr<ChatCallData> call_data,
                             const std::string& request_id,
                             int64_t created_time,
                             const std::string& model) {
-  if (content.empty()) {
+  std::string filtered_content = content;
+  strip_special_control_tokens(&filtered_content);
+  if (filtered_content.empty()) {
     return true;
   }
 
@@ -102,7 +129,7 @@ bool send_normal_text_chunk(std::shared_ptr<ChatCallData> call_data,
   auto* choice = response.add_choices();
   choice->set_index(index);
   auto* delta = choice->mutable_delta();
-  delta->set_content(content);
+  delta->set_content(filtered_content);
   return call_data->write(response);
 }
 
@@ -303,18 +330,21 @@ bool ResponseHandler::send_delta_to_client(
           return false;
         }
       } else {
-        response.Clear();
-        response.set_object("chat.completion.chunk");
-        response.set_id(request_id);
-        response.set_created(created_time);
-        response.set_model(model);
-        auto* choice = response.add_choices();
-        choice->set_index(index);
-        set_logprobs(choice, seq_output.logprobs);
-        auto* message = choice->mutable_delta();
-        message->set_content(cur_text);
-        if (!call_data->write(response)) {
-          return false;
+        strip_special_control_tokens(&cur_text);
+        if (!cur_text.empty()) {
+          response.Clear();
+          response.set_object("chat.completion.chunk");
+          response.set_id(request_id);
+          response.set_created(created_time);
+          response.set_model(model);
+          auto* choice = response.add_choices();
+          choice->set_index(index);
+          set_logprobs(choice, seq_output.logprobs);
+          auto* message = choice->mutable_delta();
+          message->set_content(cur_text);
+          if (!call_data->write(response)) {
+            return false;
+          }
         }
       }
     }
@@ -515,7 +545,12 @@ bool ResponseHandler::send_delta_to_client(
     };
 
     auto emit_text = [&](const std::string& text) -> bool {
-      auto result = encoder.on_text(output, text, &sse_events);
+      std::string filtered = text;
+      strip_special_control_tokens(&filtered);
+      if (filtered.empty()) {
+        return true;
+      }
+      auto result = encoder.on_text(output, filtered, &sse_events);
       if (!result.ok) {
         return call_data->finish_with_error(result.error);
       }
@@ -701,6 +736,7 @@ bool ResponseHandler::send_result_to_client(
                                       reasoning_parser,
                                       force_reasoning,
                                       response.GetArena());
+      strip_special_control_tokens(&result.text);
       message->set_content(result.text);
       if (result.reasoning_content.has_value()) {
         message->set_reasoning_content(result.reasoning_content.value());