feat(serve): override the frontend chat template via --chat-template FILE

docs/serving.md

@@ -29,6 +29,12 @@ With `C=2` and two extra Device checkpoint slots, the process owns two active St
 plus a global pool of two Device-resident checkpoints. Eight pinned Host State slots and 8 GiB of
 pinned Host KV retain inactive continuations under Device pressure. Active request capacity is two.
 
+Pass `--chat-template FILE` to serve an operator-managed chat template in place of the artifact's
+`frontend/chat_template.jinja`. The file replaces the template resource at startup and must be
+byte-identical to a template the target recognizes; validation and acceptance are exact, so any
+later edit to the file is rejected at startup. `qwen3.8-froggeric-v22.5`
+(froggeric/Qwen-Fixed-Chat-Templates) is an accepted reasoning-effort template.
+
 Other artifacts use the same command shape with their own path. For 35B-A3B DFlash, replace the MTP
 selection with `--spec dflash --draft-tokens 7 --lm-head-draft`. It may remain combined with
 `--vision`.

include/ninfer/types.h

@@ -148,6 +148,9 @@ struct ContextCostOptions {
 
 struct EngineOptions {
     std::filesystem::path artifact_path;
+    // Optional startup replacement for the artifact's frontend/chat_template.jinja.
+    // The source must resolve to a template semantics the target accepts.
+    std::filesystem::path chat_template_path;
     EnginePurpose purpose              = EnginePurpose::Generation;
     int device                         = 0;
     std::uint32_t max_context          = 2048; // Logical ceiling of one request or score window.

src/serve/generation_service.cpp

@@ -234,6 +234,7 @@ GenerationService::GenerationService(ServeOptions options, StartupObserver start
     : options_(std::move(options)) {
     ninfer::EngineOptions engine_options;
     engine_options.artifact_path            = options_.artifact_path;
+    engine_options.chat_template_path       = options_.chat_template_path;
     engine_options.device                   = options_.device;
     engine_options.max_context              = options_.max_context;
     engine_options.kv_capacity              = options_.kv_capacity;

src/serve/serve_options.cpp

@@ -80,7 +80,7 @@ std::string serve_usage_text(const char* argv0) {
            "[--response-store-max-records N] [--response-store-max-mib N] "
            "[--kv-dtype bf16|int8|fp8|nvfp4|k8v4] [--spec mtp|dflash --draft-tokens N] "
            "[--default-max-tokens N] [--default-thinking-budget N] "
-           "[--vision] [--no-cuda-graph] [--no-prefix-reuse] "
+           "[--vision] [--no-cuda-graph] [--no-prefix-reuse] [--chat-template FILE] "
            "[--lm-head-draft] [--no-thinking] [--preserve-thinking] [--cors] "
            "[--temperature F] [--top-p F] [--top-k N] [--min-p F] [--presence-penalty F] "
            "[--frequency-penalty F] [--seed N] [--greedy]\n"
@@ -110,6 +110,8 @@ std::string serve_usage_text(const char* argv0) {
            "       --default-thinking-budget caps model-origin thinking for enabled requests; "
            "control tokens count toward the request output limit\n"
            "       --preserve-thinking retains closed-turn assistant reasoning in later prompts\n"
+           "       --chat-template FILE replaces the artifact frontend chat template at startup;\n"
+           "       the file must be byte-identical to a template the target accepts\n"
            "       sampler defaults come from the loaded model and resolved thinking mode; "
            "server flags and request fields override individual values.\n"
            "       --greedy forces temperature 0 (exact argmax).\n";
@@ -154,6 +156,11 @@ ServeOptions parse_serve_options(int argc, char** argv) {
             if (options.model_id_override->empty()) {
                 throw std::invalid_argument("--model-id must not be empty");
             }
+        } else if (arg == "--chat-template") {
+            options.chat_template_path = require_value("--chat-template");
+            if (options.chat_template_path.empty()) {
+                throw std::invalid_argument("--chat-template must not be empty");
+            }
         } else if (arg == "--max-context") {
             options.max_context = static_cast<std::uint32_t>(
                 parse_nonnegative_int(require_value("--max-context"), "max-context"));

src/serve/serve_options.h

@@ -26,6 +26,8 @@ struct ServeOptions {
     int port         = 8080;
     std::string api_key;                          // empty => no auth
     std::optional<std::string> model_id_override; // unset => artifact identity.model_id
+    // Optional startup override for the artifact's frontend/chat_template.jinja.
+    std::filesystem::path chat_template_path;
     std::string request_log_jsonl;                // empty => structured request logging disabled
     std::uint32_t max_context          = 8192;
     KvCapacityPolicy kv_capacity       = KvCapacityPolicy::explicit_capacity(8192);

src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h

@@ -21,6 +21,9 @@ struct FrontendOptions {
     std::size_t media_cache_bytes          = kDefaultMediaCacheBytes;
     std::size_t media_live_bytes           = kDefaultMediaLiveBytes;
     std::uint32_t media_preprocess_threads = 0;
+    // Optional startup replacement for resources.chat_template_jinja. An empty path keeps
+    // the artifact template. The replacement must resolve to an accepted template semantics.
+    std::filesystem::path chat_template_path;
 };
 
 struct FrontendResources;

src/targets/qwen3_6/impl/frontend/chat_template.cpp

@@ -27,6 +27,18 @@ constexpr Sha256Digest kReasoningEffortTemplateDigest{
     0xd3, 0xe2, 0xa7, 0x25, 0xb6, 0xc2, 0x58, 0x6a, 0xaa, 0x3a, 0x8a, 0xf9, 0xd7, 0xa8, 0x10, 0x41,
 };
 
+// froggeric/Qwen-Fixed-Chat-Templates v22.5 (template_version "qwen3.8-froggeric-v22.5").
+// Renders the same ChatML surface as the reasoning-effort template above. The fixed C++
+// renderer does not implement this family's in-message control markers (<|im_start|>think_on|>,
+// <|im_start|>think_off|>, <|im_start|>think_low|> embedded in message content), the
+// auto_disable_thinking_with_tools knob, or max_tool_arg_chars/max_tool_response_chars
+// truncation; requests relying on those render with the semantics above. Acceptance is
+// byte-exact; any edit to the template file changes the digest and is rejected.
+constexpr Sha256Digest kReasoningEffortTemplateV225Digest{
+    0xe5, 0x76, 0x84, 0xba, 0xe4, 0x15, 0x62, 0x11, 0xa5, 0x54, 0x73, 0xc5, 0xa6, 0x3b, 0xe9, 0x76,
+    0xa4, 0x05, 0xa3, 0x7a, 0xb5, 0xbe, 0x5a, 0xe0, 0xe5, 0xab, 0xf1, 0xdf, 0x53, 0x49, 0xc4, 0xb2,
+};
+
 constexpr std::string_view kLowReasoningInstructions =
     "Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to "
     "the conclusion without unnecessary elaboration.";
@@ -419,6 +431,9 @@ CompiledChatTemplate CompiledChatTemplate::resolve(std::string_view source) {
     if (digest == kReasoningEffortTemplateDigest) {
         return CompiledChatTemplate(ChatTemplateSemantics::ReasoningEffort);
     }
+    if (digest == kReasoningEffortTemplateV225Digest) {
+        return CompiledChatTemplate(ChatTemplateSemantics::ReasoningEffort);
+    }
     throw std::invalid_argument("unsupported frontend/chat_template.jinja (sha256 " +
                                 sha256_hex(digest) + ")");
 }

src/targets/qwen3_6/impl/frontend/frontend.cpp

@@ -19,9 +19,11 @@
 #include <cctype>
 #include <cstddef>
 #include <cstdint>
+#include <fstream>
 #include <limits>
 #include <memory>
 #include <optional>
+#include <sstream>
 #include <span>
 #include <stdexcept>
 #include <string>
@@ -230,6 +232,38 @@ fi::CompiledChatTemplate compile_chat_template(const FrontendResources& resource
     return fi::CompiledChatTemplate::resolve(resources.chat_template_jinja);
 }
 
+// Replaces the artifact chat template with an operator-provided source and keeps
+// tokenizer_config.json.chat_template byte-consistent with it, so the existing
+// validation and template-resolution gates run against the override unchanged.
+void apply_chat_template_override(FrontendResources& resources,
+                                  const std::filesystem::path& chat_template_path) {
+    std::ifstream stream(chat_template_path, std::ios::binary);
+    if (!stream) {
+        throw std::invalid_argument("chat template file is not readable: " +
+                                    chat_template_path.string());
+    }
+    std::ostringstream buffer;
+    buffer << stream.rdbuf();
+    std::string source = buffer.str();
+    if (source.size() > (10U << 20)) {
+        throw std::invalid_argument("chat template file exceeds 10 MiB: " +
+                                    chat_template_path.string());
+    }
+    // A single trailing newline is not template content: the jinja source parser
+    // drops it, and the acceptance digest is defined over the content without it.
+    if (!source.empty() && source.back() == '\n') { source.pop_back(); }
+    resources.chat_template_jinja = std::move(source);
+    Json tokenizer_config =
+        parse_resource_json(resources.tokenizer_config_json, "tokenizer_config.json");
+    tokenizer_config["chat_template"] = resources.chat_template_jinja;
+    try {
+        resources.tokenizer_config_json = tokenizer_config.dump();
+    } catch (const nlohmann::json::exception&) {
+        throw std::invalid_argument("chat template file is not valid UTF-8 text: " +
+                                    chat_template_path.string());
+    }
+}
+
 [[noreturn]] void throw_processor_error(const fi::ProcessorError& error) {
     switch (error.kind()) {
     case fi::ProcessorErrorKind::BudgetExceeded:
@@ -1355,6 +1389,11 @@ Frontend& Frontend::operator=(Frontend&&) noexcept = default;
 Frontend::~Frontend()                              = default;
 
 Frontend make_frontend(const FrontendResources& resources, FrontendOptions options) {
+    if (!options.chat_template_path.empty()) {
+        FrontendResources overridden = resources;
+        apply_chat_template_override(overridden, options.chat_template_path);
+        return Frontend(std::make_shared<const Frontend::Impl>(overridden, true, options));
+    }
     return Frontend(std::make_shared<const Frontend::Impl>(resources, true, options));
 }
 
@@ -1368,6 +1407,11 @@ Frontend FrontendTestAccess::create_component(const FrontendResources& resources
 
 Frontend FrontendTestAccess::create_component(const FrontendResources& resources,
                                               FrontendOptions options) {
+    if (!options.chat_template_path.empty()) {
+        FrontendResources overridden = resources;
+        apply_chat_template_override(overridden, options.chat_template_path);
+        return Frontend(std::make_shared<const Frontend::Impl>(overridden, false, options));
+    }
     return Frontend(std::make_shared<const Frontend::Impl>(resources, false, options));
 }
 

src/targets/qwen3_6_27b/impl/package.cpp

@@ -124,6 +124,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO
                                       .media_cache_bytes        = options.media_cache_bytes,
                                       .media_live_bytes         = options.media_live_bytes,
                                       .media_preprocess_threads = options.media_preprocess_threads,
+                                      .chat_template_path       = options.chat_template_path,
                                   });
 }
 

src/targets/qwen3_6_35b_a3b/impl/package.cpp

@@ -96,6 +96,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO
                                       .media_cache_bytes        = options.media_cache_bytes,
                                       .media_live_bytes         = options.media_live_bytes,
                                       .media_preprocess_threads = options.media_preprocess_threads,
+                                      .chat_template_path       = options.chat_template_path,
                                   });
 }