feat: add request trace dump with runtime toggle.

xllm/api_service/api_service.cpp

@@ -29,6 +29,7 @@ limitations under the License.
 #include "call.h"
 #include "chat.pb.h"
 #include "common.pb.h"
+#include "common/global_flags.h"
 #include "completion.pb.h"
 #include "core/common/constants.h"
 #include "core/common/metrics.h"
@@ -197,6 +198,11 @@ void APIService::CompletionsHttp(::google::protobuf::RpcController* controller,
 
   auto ctrl = reinterpret_cast<brpc::Controller*>(controller);
 
+  std::string raw_body;
+  if (FLAGS_enable_request_trace) {
+    raw_body = ctrl->request_attachment().to_string();
+  }
+
   auto [preprocess_status, processed_json] =
       preprocess_completion_prompt(ctrl->request_attachment().to_string());
   if (!preprocess_status.ok()) {
@@ -218,6 +224,8 @@ void APIService::CompletionsHttp(::google::protobuf::RpcController* controller,
 
   std::shared_ptr<Call> call = std::make_shared<CompletionCall>(
       ctrl, done_guard.release(), req_pb, resp_pb, arena != nullptr);
+  call->set_raw_request_body(std::move(raw_body));
+  call->set_request_endpoint("/v1/completions");
   if (completion_service_impl_) {
     completion_service_impl_->process_async(call);
   } else if (rec_completion_service_impl_) {
@@ -340,6 +348,11 @@ void chat_completions_http_impl(std::unique_ptr<Service>& service,
   std::string attachment;
   ctrl->request_attachment().copy_to(&attachment, content_len, 0);
 
+  std::string raw_body;
+  if (FLAGS_enable_request_trace) {
+    raw_body = attachment;
+  }
+
   auto [preprocess_status, processed_json] =
       chat_json_parser.preprocess(std::move(attachment));
   if (!preprocess_status.ok()) {
@@ -361,6 +374,8 @@ void chat_completions_http_impl(std::unique_ptr<Service>& service,
 
   auto call = std::make_shared<ChatCall>(
       ctrl, guard.release(), req_pb, resp_pb, arena != nullptr /*use_arena*/);
+  call->set_raw_request_body(std::move(raw_body));
+  call->set_request_endpoint("/v1/chat/completions");
   service->process_async(call);
 }
 

xllm/api_service/call.h

@@ -32,6 +32,15 @@ class Call {
   std::string take_request_payload() { return std::move(request_payload_); }
   void init_request_payload();
 
+  void set_raw_request_body(std::string body) {
+    raw_request_body_ = std::move(body);
+  }
+  void set_request_endpoint(std::string path) {
+    request_endpoint_ = std::move(path);
+  }
+  const std::string& raw_request_body() const { return raw_request_body_; }
+  const std::string& request_endpoint() const { return request_endpoint_; }
+
   virtual bool is_disconnected() const = 0;
 
  protected:
@@ -44,6 +53,8 @@ class Call {
   std::string x_request_time_;
 
   std::string request_payload_;
+  std::string raw_request_body_;
+  std::string request_endpoint_;
 };
 
 }  // namespace xllm

xllm/core/common/global_flags.h

@@ -387,3 +387,9 @@ DECLARE_bool(enable_aclnn_swiglu);
 DECLARE_bool(use_cpp_chat_template);
 
 DECLARE_int32(health_check_interval_ms);
+
+DECLARE_bool(enable_request_trace);
+
+DECLARE_string(request_trace_path);
+
+DECLARE_bool(request_trace_per_file);

xllm/core/framework/config/service_config.cpp

@@ -55,6 +55,41 @@ DEFINE_int32(health_check_interval_ms,
              3000,
              "Worker health check interval in milliseconds.");
 
+// --- request input/output trace(dump) config ---
+DEFINE_bool(enable_request_trace,
+            false,
+            "Whether to dump the input and output of each request to a file. "
+            "For streaming requests, the complete output is saved after all "
+            "the streaming outputs are finished.");
+
+DEFINE_string(request_trace_path,
+              "./request_trace.jsonl",
+              "The path to dump the request input/output trace. When "
+              "request_trace_per_file is false, this is a file that JSON "
+              "records are appended to (one record per line). When "
+              "request_trace_per_file is true, this is a directory and each "
+              "request is written to <dir>/<request_id>.json. Only used when "
+              "enable_request_trace is true.");
+
+DEFINE_bool(request_trace_per_file,
+            false,
+            "Whether to dump each request to a separate file named "
+            "<request_trace_path>/<request_id>.json instead of appending all "
+            "requests to a single JSON-lines file.");
+
+namespace {
+// brpc's /flags endpoint refuses to modify a gflag at runtime unless it has a
+// registered validator (it returns "A reloadable gflag must have validator").
+// Register a no-op validator so enable_request_trace can be toggled without
+// restarting the service, as documented for the request trace feature.
+bool validate_enable_request_trace(const char* /*unused*/, bool /*unused*/) {
+  return true;
+}
+const bool kEnableRequestTraceValidatorDummy =
+    google::RegisterFlagValidator(&FLAGS_enable_request_trace,
+                                  &validate_enable_request_trace);
+}  // namespace
+
 namespace xllm {
 
 void ServiceConfig::from_flags() {
@@ -68,6 +103,9 @@ void ServiceConfig::from_flags() {
   XLLM_CONFIG_ASSIGN_FROM_FLAG(num_request_handling_threads);
   XLLM_CONFIG_ASSIGN_FROM_FLAG(num_response_handling_threads);
   XLLM_CONFIG_ASSIGN_FROM_FLAG(health_check_interval_ms);
+  XLLM_CONFIG_ASSIGN_FROM_FLAG(enable_request_trace);
+  XLLM_CONFIG_ASSIGN_FROM_FLAG(request_trace_path);
+  XLLM_CONFIG_ASSIGN_FROM_FLAG(request_trace_per_file);
 }
 
 void ServiceConfig::from_json(const JsonReader& json) {
@@ -82,6 +120,9 @@ void ServiceConfig::from_json(const JsonReader& json) {
   XLLM_CONFIG_ASSIGN_FROM_JSON(num_request_handling_threads);
   XLLM_CONFIG_ASSIGN_FROM_JSON(num_response_handling_threads);
   XLLM_CONFIG_ASSIGN_FROM_JSON(health_check_interval_ms);
+  XLLM_CONFIG_ASSIGN_FROM_JSON(enable_request_trace);
+  XLLM_CONFIG_ASSIGN_FROM_JSON(request_trace_path);
+  XLLM_CONFIG_ASSIGN_FROM_JSON(request_trace_per_file);
 }
 
 void ServiceConfig::append_config_json(
@@ -106,6 +147,12 @@ void ServiceConfig::append_config_json(
       config_json, default_config, num_response_handling_threads);
   APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
       config_json, default_config, health_check_interval_ms);
+  APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
+      config_json, default_config, enable_request_trace);
+  APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
+      config_json, default_config, request_trace_path);
+  APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(
+      config_json, default_config, request_trace_per_file);
 }
 
 ServiceConfig& ServiceConfig::get_instance() {

xllm/core/framework/config/service_config.h

@@ -50,7 +50,10 @@ class ServiceConfig final {
          "max_concurrent_requests",
          "num_request_handling_threads",
          "num_response_handling_threads",
-         "health_check_interval_ms"}};
+         "health_check_interval_ms",
+         "enable_request_trace",
+         "request_trace_path",
+         "request_trace_per_file"}};
     return kOptionCategory;
   }
 
@@ -73,6 +76,12 @@ class ServiceConfig final {
   PROPERTY(int32_t, num_response_handling_threads) = 4;
 
   PROPERTY(int32_t, health_check_interval_ms) = 3000;
+
+  PROPERTY(bool, enable_request_trace) = false;
+
+  PROPERTY(std::string, request_trace_path) = "./request_trace.jsonl";
+
+  PROPERTY(bool, request_trace_per_file) = false;
 };
 
 }  // namespace xllm

xllm/core/framework/request/CMakeLists.txt

@@ -16,6 +16,7 @@ cc_library(
     dit_request_output.h
     dit_request_params.h
     request_params.h
+    request_tracer.h
     sample_slot.h
     sequence.h
     sequence_logprob_state.h
@@ -34,6 +35,7 @@ cc_library(
     request_output.cpp
     dit_request_output.cpp
     request_params.cpp
+    request_tracer.cpp
     sample_slot.cpp
     dit_request_params.cpp
     sequence.cpp
@@ -55,5 +57,6 @@ cc_library(
     absl::strings
     absl::time
     proto::xllm_proto
+    nlohmann_json::nlohmann_json
     torch
 )