diff --git a/docs/serving.md b/docs/serving.md index d8a232b8e7..009e2ad9cd 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -29,6 +29,12 @@ With `C=2` and two extra Device checkpoint slots, the process owns two active St plus a global pool of two Device-resident checkpoints. Eight pinned Host State slots and 8 GiB of pinned Host KV retain inactive continuations under Device pressure. Active request capacity is two. +Pass `--chat-template FILE` to serve an operator-managed chat template in place of the artifact's +`frontend/chat_template.jinja`. The file replaces the template resource at startup and must be +byte-identical to a template the target recognizes; validation and acceptance are exact, so any +later edit to the file is rejected at startup. `qwen3.8-froggeric-v22.5` +(froggeric/Qwen-Fixed-Chat-Templates) is an accepted reasoning-effort template. + Other artifacts use the same command shape with their own path. For 35B-A3B DFlash, replace the MTP selection with `--spec dflash --draft-tokens 7 --lm-head-draft`. It may remain combined with `--vision`. diff --git a/include/ninfer/types.h b/include/ninfer/types.h index d8398774a8..8ddece07fd 100644 --- a/include/ninfer/types.h +++ b/include/ninfer/types.h @@ -148,6 +148,9 @@ struct ContextCostOptions { struct EngineOptions { std::filesystem::path artifact_path; + // Optional startup replacement for the artifact's frontend/chat_template.jinja. + // The source must resolve to a template semantics the target accepts. + std::filesystem::path chat_template_path; EnginePurpose purpose = EnginePurpose::Generation; int device = 0; std::uint32_t max_context = 2048; // Logical ceiling of one request or score window. diff --git a/src/serve/generation_service.cpp b/src/serve/generation_service.cpp index 686992f101..6155f438de 100644 --- a/src/serve/generation_service.cpp +++ b/src/serve/generation_service.cpp @@ -234,6 +234,7 @@ GenerationService::GenerationService(ServeOptions options, StartupObserver start : options_(std::move(options)) { ninfer::EngineOptions engine_options; engine_options.artifact_path = options_.artifact_path; + engine_options.chat_template_path = options_.chat_template_path; engine_options.device = options_.device; engine_options.max_context = options_.max_context; engine_options.kv_capacity = options_.kv_capacity; diff --git a/src/serve/serve_options.cpp b/src/serve/serve_options.cpp index 73c43a55a1..6c5d3309d8 100644 --- a/src/serve/serve_options.cpp +++ b/src/serve/serve_options.cpp @@ -80,7 +80,7 @@ std::string serve_usage_text(const char* argv0) { "[--response-store-max-records N] [--response-store-max-mib N] " "[--kv-dtype bf16|int8|fp8|nvfp4|k8v4] [--spec mtp|dflash --draft-tokens N] " "[--default-max-tokens N] [--default-thinking-budget N] " - "[--vision] [--no-cuda-graph] [--no-prefix-reuse] " + "[--vision] [--no-cuda-graph] [--no-prefix-reuse] [--chat-template FILE] " "[--lm-head-draft] [--no-thinking] [--preserve-thinking] [--cors] " "[--temperature F] [--top-p F] [--top-k N] [--min-p F] [--presence-penalty F] " "[--frequency-penalty F] [--seed N] [--greedy]\n" @@ -110,6 +110,8 @@ std::string serve_usage_text(const char* argv0) { " --default-thinking-budget caps model-origin thinking for enabled requests; " "control tokens count toward the request output limit\n" " --preserve-thinking retains closed-turn assistant reasoning in later prompts\n" + " --chat-template FILE replaces the artifact frontend chat template at startup;\n" + " the file must be byte-identical to a template the target accepts\n" " sampler defaults come from the loaded model and resolved thinking mode; " "server flags and request fields override individual values.\n" " --greedy forces temperature 0 (exact argmax).\n"; @@ -154,6 +156,11 @@ ServeOptions parse_serve_options(int argc, char** argv) { if (options.model_id_override->empty()) { throw std::invalid_argument("--model-id must not be empty"); } + } else if (arg == "--chat-template") { + options.chat_template_path = require_value("--chat-template"); + if (options.chat_template_path.empty()) { + throw std::invalid_argument("--chat-template must not be empty"); + } } else if (arg == "--max-context") { options.max_context = static_cast( parse_nonnegative_int(require_value("--max-context"), "max-context")); diff --git a/src/serve/serve_options.h b/src/serve/serve_options.h index e147539590..7e5b425150 100644 --- a/src/serve/serve_options.h +++ b/src/serve/serve_options.h @@ -26,6 +26,8 @@ struct ServeOptions { int port = 8080; std::string api_key; // empty => no auth std::optional model_id_override; // unset => artifact identity.model_id + // Optional startup override for the artifact's frontend/chat_template.jinja. + std::filesystem::path chat_template_path; std::string request_log_jsonl; // empty => structured request logging disabled std::uint32_t max_context = 8192; KvCapacityPolicy kv_capacity = KvCapacityPolicy::explicit_capacity(8192); diff --git a/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h b/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h index eb829645cc..fdd27ece9b 100644 --- a/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h +++ b/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h @@ -21,6 +21,9 @@ struct FrontendOptions { std::size_t media_cache_bytes = kDefaultMediaCacheBytes; std::size_t media_live_bytes = kDefaultMediaLiveBytes; std::uint32_t media_preprocess_threads = 0; + // Optional startup replacement for resources.chat_template_jinja. An empty path keeps + // the artifact template. The replacement must resolve to an accepted template semantics. + std::filesystem::path chat_template_path; }; struct FrontendResources; diff --git a/src/targets/qwen3_6/impl/frontend/chat_template.cpp b/src/targets/qwen3_6/impl/frontend/chat_template.cpp index 9fba17ab44..966da8724f 100644 --- a/src/targets/qwen3_6/impl/frontend/chat_template.cpp +++ b/src/targets/qwen3_6/impl/frontend/chat_template.cpp @@ -27,6 +27,18 @@ constexpr Sha256Digest kReasoningEffortTemplateDigest{ 0xd3, 0xe2, 0xa7, 0x25, 0xb6, 0xc2, 0x58, 0x6a, 0xaa, 0x3a, 0x8a, 0xf9, 0xd7, 0xa8, 0x10, 0x41, }; +// froggeric/Qwen-Fixed-Chat-Templates v22.5 (template_version "qwen3.8-froggeric-v22.5"). +// Renders the same ChatML surface as the reasoning-effort template above. The fixed C++ +// renderer does not implement this family's in-message control markers (<|im_start|>think_on|>, +// <|im_start|>think_off|>, <|im_start|>think_low|> embedded in message content), the +// auto_disable_thinking_with_tools knob, or max_tool_arg_chars/max_tool_response_chars +// truncation; requests relying on those render with the semantics above. Acceptance is +// byte-exact; any edit to the template file changes the digest and is rejected. +constexpr Sha256Digest kReasoningEffortTemplateV225Digest{ + 0xe5, 0x76, 0x84, 0xba, 0xe4, 0x15, 0x62, 0x11, 0xa5, 0x54, 0x73, 0xc5, 0xa6, 0x3b, 0xe9, 0x76, + 0xa4, 0x05, 0xa3, 0x7a, 0xb5, 0xbe, 0x5a, 0xe0, 0xe5, 0xab, 0xf1, 0xdf, 0x53, 0x49, 0xc4, 0xb2, +}; + constexpr std::string_view kLowReasoningInstructions = "Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to " "the conclusion without unnecessary elaboration."; @@ -419,6 +431,9 @@ CompiledChatTemplate CompiledChatTemplate::resolve(std::string_view source) { if (digest == kReasoningEffortTemplateDigest) { return CompiledChatTemplate(ChatTemplateSemantics::ReasoningEffort); } + if (digest == kReasoningEffortTemplateV225Digest) { + return CompiledChatTemplate(ChatTemplateSemantics::ReasoningEffort); + } throw std::invalid_argument("unsupported frontend/chat_template.jinja (sha256 " + sha256_hex(digest) + ")"); } diff --git a/src/targets/qwen3_6/impl/frontend/frontend.cpp b/src/targets/qwen3_6/impl/frontend/frontend.cpp index 75c7138318..c1134829a4 100644 --- a/src/targets/qwen3_6/impl/frontend/frontend.cpp +++ b/src/targets/qwen3_6/impl/frontend/frontend.cpp @@ -19,9 +19,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -230,6 +232,38 @@ fi::CompiledChatTemplate compile_chat_template(const FrontendResources& resource return fi::CompiledChatTemplate::resolve(resources.chat_template_jinja); } +// Replaces the artifact chat template with an operator-provided source and keeps +// tokenizer_config.json.chat_template byte-consistent with it, so the existing +// validation and template-resolution gates run against the override unchanged. +void apply_chat_template_override(FrontendResources& resources, + const std::filesystem::path& chat_template_path) { + std::ifstream stream(chat_template_path, std::ios::binary); + if (!stream) { + throw std::invalid_argument("chat template file is not readable: " + + chat_template_path.string()); + } + std::ostringstream buffer; + buffer << stream.rdbuf(); + std::string source = buffer.str(); + if (source.size() > (10U << 20)) { + throw std::invalid_argument("chat template file exceeds 10 MiB: " + + chat_template_path.string()); + } + // A single trailing newline is not template content: the jinja source parser + // drops it, and the acceptance digest is defined over the content without it. + if (!source.empty() && source.back() == '\n') { source.pop_back(); } + resources.chat_template_jinja = std::move(source); + Json tokenizer_config = + parse_resource_json(resources.tokenizer_config_json, "tokenizer_config.json"); + tokenizer_config["chat_template"] = resources.chat_template_jinja; + try { + resources.tokenizer_config_json = tokenizer_config.dump(); + } catch (const nlohmann::json::exception&) { + throw std::invalid_argument("chat template file is not valid UTF-8 text: " + + chat_template_path.string()); + } +} + [[noreturn]] void throw_processor_error(const fi::ProcessorError& error) { switch (error.kind()) { case fi::ProcessorErrorKind::BudgetExceeded: @@ -1355,6 +1389,11 @@ Frontend& Frontend::operator=(Frontend&&) noexcept = default; Frontend::~Frontend() = default; Frontend make_frontend(const FrontendResources& resources, FrontendOptions options) { + if (!options.chat_template_path.empty()) { + FrontendResources overridden = resources; + apply_chat_template_override(overridden, options.chat_template_path); + return Frontend(std::make_shared(overridden, true, options)); + } return Frontend(std::make_shared(resources, true, options)); } @@ -1368,6 +1407,11 @@ Frontend FrontendTestAccess::create_component(const FrontendResources& resources Frontend FrontendTestAccess::create_component(const FrontendResources& resources, FrontendOptions options) { + if (!options.chat_template_path.empty()) { + FrontendResources overridden = resources; + apply_chat_template_override(overridden, options.chat_template_path); + return Frontend(std::make_shared(overridden, false, options)); + } return Frontend(std::make_shared(resources, false, options)); } diff --git a/src/targets/qwen3_6_27b/impl/package.cpp b/src/targets/qwen3_6_27b/impl/package.cpp index c844d21eda..5aedc9dca3 100644 --- a/src/targets/qwen3_6_27b/impl/package.cpp +++ b/src/targets/qwen3_6_27b/impl/package.cpp @@ -124,6 +124,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO .media_cache_bytes = options.media_cache_bytes, .media_live_bytes = options.media_live_bytes, .media_preprocess_threads = options.media_preprocess_threads, + .chat_template_path = options.chat_template_path, }); } diff --git a/src/targets/qwen3_6_35b_a3b/impl/package.cpp b/src/targets/qwen3_6_35b_a3b/impl/package.cpp index 15e55730e0..4bcbfcce12 100644 --- a/src/targets/qwen3_6_35b_a3b/impl/package.cpp +++ b/src/targets/qwen3_6_35b_a3b/impl/package.cpp @@ -96,6 +96,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO .media_cache_bytes = options.media_cache_bytes, .media_live_bytes = options.media_live_bytes, .media_preprocess_threads = options.media_preprocess_threads, + .chat_template_path = options.chat_template_path, }); } diff --git a/tests/fixtures/frontend/reasoning_effort_froggeric_v225_chat_template.jinja b/tests/fixtures/frontend/reasoning_effort_froggeric_v225_chat_template.jinja new file mode 100644 index 0000000000..570ed0798e --- /dev/null +++ b/tests/fixtures/frontend/reasoning_effort_froggeric_v225_chat_template.jinja @@ -0,0 +1,460 @@ +{%- set template_version = "qwen3.8-froggeric-v22.5" %} +{%- set _tool_format = tool_call_format if tool_call_format is defined else 'xml' %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} +{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %} +{%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined else false %} +{%- if preserve_reasoning is defined and preserve_reasoning is not none %} + {%- set _preserve_thinking = preserve_reasoning %} +{%- elif preserve_thinking is defined and preserve_thinking is not none %} + {%- set _preserve_thinking = preserve_thinking %} +{%- else %} + {%- set _preserve_thinking = true %} +{%- endif %} +{%- set max_tool_arg_chars = max_tool_arg_chars if max_tool_arg_chars is defined else 0 %} +{%- set max_tool_response_chars = max_tool_response_chars if max_tool_response_chars is defined else 0 %} +{%- set _default_reasoning_effort = 'medium' %} +{%- set _has_tools = (tools is defined and tools and tools is iterable and tools is not mapping) %} +{%- set _effort_raw = (reasoning_effort | string | lower) if reasoning_effort is defined and reasoning_effort is not none else _default_reasoning_effort %} +{%- set _initial_thinking = enable_thinking %} +{%- if _effort_raw in ('none', 'off') %} + {%- set _initial_thinking = false %} + {%- set _initial_effort = 'medium' %} +{%- elif _effort_raw in ('minimal', 'low') %} + {%- set _initial_effort = 'low' %} +{%- elif _effort_raw in ('high', 'xhigh', 'max', 'ultracode', 'extreme') %} + {%- set _initial_effort = 'xhigh' %} +{%- else %} + {%- set _initial_effort = 'medium' %} +{%- endif %} +{%- set ns_state = namespace(thinking=_initial_thinking, effort=_initial_effort) %} +{%- if auto_disable_thinking_with_tools and _has_tools %} + {%- set ns_state.thinking = false %} +{%- endif %} +{%- for msg in messages %} + {%- if msg.role == 'system' or msg.role == 'developer' or msg.role == 'user' %} + {%- if msg.content is string %} + {%- if '<|think_off|>' in msg.content %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in msg.content or '<|think_high|>' in msg.content or '<|think_ultracode|>' in msg.content or '<|think_extreme|>' in msg.content or '<|think_max|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in msg.content or '<|think_minimal|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in msg.content %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- elif msg.content is iterable and msg.content is not mapping %} + {%- for item in msg.content %} + {%- if item is string %} + {%- set _item_text = item %} + {%- elif item is mapping and 'text' in item and item.text is string %} + {%- set _item_text = item.text %} + {%- else %} + {%- set _item_text = '' %} + {%- endif %} + {%- if _item_text %} + {%- if '<|think_off|>' in _item_text %} + {%- set ns_state.thinking = false %} + {%- elif '<|think_on|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- elif '<|think_xhigh|>' in _item_text or '<|think_high|>' in _item_text or '<|think_ultracode|>' in _item_text or '<|think_extreme|>' in _item_text or '<|think_max|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'xhigh' %} + {%- elif '<|think_low|>' in _item_text or '<|think_minimal|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'low' %} + {%- elif '<|think_medium|>' in _item_text %} + {%- set ns_state.thinking = true %} + {%- set ns_state.effort = 'medium' %} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set reasoning_instructions = '' %} +{%- if ns_state.thinking %} + {%- if ns_state.effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif ns_state.effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if item is mapping %} + {%- if item.type == 'image' or 'image' in item or 'image_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif item.type == 'video' or 'video' in item or 'video_url' in item %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- else %} + {{- item | string }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set head = namespace(count=0, seen_non_system=false) %} +{%- for message in messages %} + {%- set _is_sys = (message.role == 'system' or message.role == 'developer') %} + {%- if _is_sys and not head.seen_non_system %} + {%- set head.count = head.count + 1 %} + {%- else %} + {%- set head.seen_non_system = true %} + {%- endif %} +{%- endfor %} +{%- set sys_state = namespace(content='') %} +{%- for message in messages[:head.count] %} + {%- set _part = render_content(message.content, false, true) | trim %} + {%- if '<|think_off|>' in _part %}{%- set _part = _part.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in _part %}{%- set _part = _part.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in _part %}{%- set _part = _part.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in _part %}{%- set _part = _part.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_ultracode|>' in _part %}{%- set _part = _part.split('<|think_ultracode|>') | join('') | trim %}{%- endif %} + {%- if '<|think_extreme|>' in _part %}{%- set _part = _part.split('<|think_extreme|>') | join('') | trim %}{%- endif %} + {%- if '<|think_max|>' in _part %}{%- set _part = _part.split('<|think_max|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in _part %}{%- set _part = _part.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in _part %}{%- set _part = _part.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in _part %}{%- set _part = _part.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- if _part %} + {%- if sys_state.content %} + {%- set sys_state.content = sys_state.content ~ '\n\n' ~ _part %} + {%- else %} + {%- set sys_state.content = _part %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- set _sc = sys_state.content %} +{%- set _msgs = messages[head.count:] %} +{%- if _has_tools %} + {{- '<|im_start|>system\n' }} + {%- if reasoning_instructions %} + {{- reasoning_instructions + '\n\n' }} + {%- endif %} + {{- '# Tools\n\nYou have access to the following functions:\n\n' }} + {%- for tool in tools %} + {{- '\n' }} + {{- tool | tojson }} + {%- endfor %} + {{- '\n' }} + {%- if _tool_format == 'json' %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }} + {%- if ns_state.thinking %} + {{- '\nBrief explanation of tool call\n\n' }} + {%- endif %} + {{- '\n{"name": "example_function_name", "arguments": {"example_parameter_1": "value_1", "example_parameter_2": "This is the value for the second parameter"}}\n\n\n\nReminder:\n' }} + {%- if ns_state.thinking %} + {{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }} + {%- endif %} + {{- '- Function calls MUST follow the specified format: a single JSON object with "name" and "arguments" keys inside XML tags.\n' }} + {%- if ns_state.thinking %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n' }} + {%- else %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY, with NO conversational text before it.\n' }} + {%- endif %} + {{- '- The tag MUST be at the very beginning of a new line, with NO spaces or indentation before it.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- else %} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n' }} + {%- if ns_state.thinking %} + {{- '\nBrief explanation of tool call\n\n' }} + {%- endif %} + {{- '\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n' }} + {%- if ns_state.thinking %} + {{- '- You can use the block to plan your next tool call OR to synthesize data and formulate your final response to the user.\n- ALL explanation and reasoning MUST be placed strictly inside the block.\n' }} + {%- endif %} + {{- '- Function calls MUST follow the specified format: an inner block must be nested within XML tags.\n' }} + {%- if ns_state.thinking %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY after thinking, with NO conversational text before it.\n' }} + {%- else %} + {{- '- If you choose to call a tool, you MUST output the block IMMEDIATELY, with NO conversational text before it.\n' }} + {%- endif %} + {{- '- The and tags MUST be at the very beginning of a new line, with NO spaces or indentation before them.\n- To call multiple functions, output a separate, completely closed block for EACH function. Do NOT nest blocks.\n- If you have all necessary data, provide your final answer directly to the user without any tool call.\n' }} + {%- endif %} + {%- if _sc %} + {{- '\n\n' + _sc }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if _sc %} + {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + _sc + '<|im_end|>\n' }} + {%- elif reasoning_instructions %} + {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set _last_idx = _msgs | length - 1 %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=_last_idx) %} +{%- for message in _msgs[::-1] %} + {%- set index = (_msgs | length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == 'user' %} + {%- set _rc = render_content(message.content, false) | trim %} + {%- if not (_rc.startswith('') and _rc.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {%- if _last_idx > 50 %} + {%- set ns.last_query_index = _last_idx %} + {%- else %} + {%- set ns.last_query_index = 0 %} + {%- endif %} +{%- endif %} +{%- set ns2 = namespace(prev_role='', consecutive_failures=0) %} +{%- for message in _msgs %} + {%- set is_system = (message.role == "system" or message.role == "developer") %} + {%- set content = render_content(message.content, true, is_system) | trim %} + {%- if is_system or message.role == 'user' %} + {%- if '<|think_off|>' in content %}{%- set content = content.split('<|think_off|>') | join('') | trim %}{%- endif %} + {%- if '<|think_on|>' in content %}{%- set content = content.split('<|think_on|>') | join('') | trim %}{%- endif %} + {%- if '<|think_xhigh|>' in content %}{%- set content = content.split('<|think_xhigh|>') | join('') | trim %}{%- endif %} + {%- if '<|think_high|>' in content %}{%- set content = content.split('<|think_high|>') | join('') | trim %}{%- endif %} + {%- if '<|think_ultracode|>' in content %}{%- set content = content.split('<|think_ultracode|>') | join('') | trim %}{%- endif %} + {%- if '<|think_extreme|>' in content %}{%- set content = content.split('<|think_extreme|>') | join('') | trim %}{%- endif %} + {%- if '<|think_max|>' in content %}{%- set content = content.split('<|think_max|>') | join('') | trim %}{%- endif %} + {%- if '<|think_medium|>' in content %}{%- set content = content.split('<|think_medium|>') | join('') | trim %}{%- endif %} + {%- if '<|think_low|>' in content %}{%- set content = content.split('<|think_low|>') | join('') | trim %}{%- endif %} + {%- if '<|think_minimal|>' in content %}{%- set content = content.split('<|think_minimal|>') | join('') | trim %}{%- endif %} + {%- endif %} + {%- if is_system %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'user' %} + {%- set ns2.consecutive_failures = 0 %} + {{- '<|im_start|>user\n' + content + '<|im_end|>\n' }} + {%- elif message.role == 'assistant' %} + {%- set reasoning_content = '' %} + {%- set _explicit_reasoning = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- if message.reasoning_content is string %} + {%- set _explicit_reasoning = message.reasoning_content %} + {%- else %} + {%- set _explicit_reasoning = message.reasoning_content | string %} + {%- endif %} + {%- elif message.thinking is defined and message.thinking is not none %} + {%- if message.thinking is string %} + {%- set _explicit_reasoning = message.thinking %} + {%- else %} + {%- set _explicit_reasoning = message.thinking | string %} + {%- endif %} + {%- elif message.reasoning is defined and message.reasoning is not none %} + {%- if message.reasoning is string %} + {%- set _explicit_reasoning = message.reasoning %} + {%- else %} + {%- set _explicit_reasoning = message.reasoning | string %} + {%- endif %} + {%- endif %} + {%- if _explicit_reasoning %} + {%- set _lead_end = '' %} + {%- if content.startswith('') and '' in content %} + {%- set _lead_end = '' %} + {%- elif content.startswith('') and '' in content %} + {%- set _lead_end = '' %} + {%- elif content.startswith('') %} + {%- set _lead_end = '' %} + {%- elif content.startswith('') %} + {%- set _lead_end = '' %} + {%- endif %} + {%- if _lead_end %} + {%- set content = content.split(_lead_end)[-1].lstrip('\n') %} + {%- endif %} + {%- set reasoning_content = _explicit_reasoning %} + {%- else %} + {%- set _think_end = '' %} + {%- if content.startswith('') %} + {%- set _think_end = '' %} + {%- elif content.startswith('') %} + {%- set _think_end = '' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif '\n' in content %} + {%- set _think_end = '\n' %} + {%- elif content.startswith('') and '' in content %} + {%- set _think_end = '' %} + {%- elif content.startswith('') and '' in content %} + {%- set _think_end = '' %} + {%- endif %} + {%- if _think_end %} + {%- if 'thinking' in _think_end %} + {%- set _think_start = '' %} + {%- else %} + {%- set _think_start = '' %} + {%- endif %} + {%- set reasoning_content = content.split(_think_end)[0].rstrip('\n') %} + {%- if _think_start in reasoning_content %} + {%- set reasoning_content = reasoning_content.split(_think_start)[-1].lstrip('\n') %} + {%- endif %} + {%- set content = content.split(_think_end)[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content | trim %} + {%- if (_preserve_thinking or loop.index0 > ns.last_query_index) %} + {{- '<|im_start|>assistant\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>assistant\n' + content }} + {%- endif %} + {%- if message.tool_calls is defined and message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined and tool_call.function is not none %} + {%- set tc = tool_call.function %} + {%- else %} + {%- set tc = tool_call %} + {%- endif %} + {%- set tc_name = tc.name if (tc.name is defined and tc.name is not none) else '' %} + {%- if _tool_format == 'json' %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n' }} + {%- endif %} + {%- set _args = '{}' %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- set _args = tc.arguments | tojson %} + {%- elif tc.arguments is string %} + {%- if tc.arguments %} + {%- set _args = tc.arguments %} + {%- endif %} + {%- else %} + {%- set _args = tc.arguments | tojson %} + {%- endif %} + {%- endif %} + {{- '\n{"name": ' }}{{- tc_name | tojson }}{{- ', "arguments": ' }}{{- _args }}{{- '}\n' }} + {%- else %} + {%- if loop.first %} + {%- if content | trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tc.arguments is defined and tc.arguments is not none %} + {%- if tc.arguments is mapping %} + {%- for args_name, args_value in tc.arguments.items() %} + {{- '\n' }} + {%- if args_value is string %} + {%- set _av = args_value %} + {%- else %} + {%- set _av = args_value | tojson %} + {%- endif %} + {%- if max_tool_arg_chars > 0 and _av | length > max_tool_arg_chars %} + {{- _av[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_av | length | string) ~ ' chars]' }} + {%- else %} + {{- _av }} + {%- endif %} + {{- '\n\n' }} + {%- endfor %} + {%- else %} + {%- if tc.arguments is string %} + {%- set _raw_args = tc.arguments %} + {%- else %} + {%- set _raw_args = tc.arguments | tojson %} + {%- endif %} + {%- if _raw_args %} + {%- if max_tool_arg_chars > 0 and _raw_args | length > max_tool_arg_chars %} + {{- _raw_args[:max_tool_arg_chars] + '\n[TRUNCATED - original length ' ~ (_raw_args | length | string) ~ ' chars]' }} + {%- else %} + {{- _raw_args }} + {%- endif %} + {%- endif %} + {%- endif %} + {%- endif %} + {{- '\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == 'tool' %} + {%- set _content_lower = content | lower %} + {%- set _content_head = _content_lower[:120] %} + {%- set _is_code_or_grep = ('throw new ' in _content_lower or 'throw error' in _content_lower or 'console.error' in _content_lower or 'logger.error' in _content_lower or 'logging.error' in _content_lower or 'import ' in _content_head or 'def ' in _content_head or 'function ' in _content_head) %} + {%- set _exit_code_zero = ('exit code: 0' in _content_head or 'process exited with code 0' in _content_head) %} + {%- set _error_field_ok = ('"error": null' in _content_head or '"error":null' in _content_head or '"error": false' in _content_head or '"error":false' in _content_head or '"error": ""' in _content_head or '"error":""' in _content_head) %} + {%- set _strong_error = (('"error":' in _content_head and not _error_field_ok) or '"status": "error"' in _content_head or '"status":"error"' in _content_head or 'traceback (most recent call last):' in _content_head or 'command not found' in _content_head or 'invalid syntax' in _content_head or 'fatal:' in _content_head or (('exit code: ' in _content_head or 'process exited with code' in _content_head) and not _exit_code_zero) or _content_head.startswith('exception:') or _content_head.startswith('failed to ')) %} + {%- set _weak_error = ('error:' in _content_head or 'err!' in _content_head) %} + {%- set _weak_suppressed = ('$ ' in _content_head or 'took ' in _content_head or content | length >= 600) %} + {%- if not _is_code_or_grep and (_strong_error or (_weak_error and not _weak_suppressed)) %} + {%- set ns2.consecutive_failures = ns2.consecutive_failures + 1 %} + {%- else %} + {%- set ns2.consecutive_failures = 0 %} + {%- endif %} + {%- if ns2.prev_role != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {%- set _is_json_payload = (_tool_format == 'json' and content | trim | length > 0 and (content | trim)[:1] in ('{', '[')) %} + {%- if not _is_json_payload and max_tool_response_chars > 0 and content | length > max_tool_response_chars %} + {%- set content = content[:max_tool_response_chars] + '\n[TRUNCATED - original length ' ~ (content | length | string) ~ ' chars]' %} + {%- endif %} + {{- '\n\n' + content }} + {%- if ns2.consecutive_failures >= 2 %} + {{- '\n\n⚠️ SYSTEM WARNING: ' ~ ns2.consecutive_failures ~ ' consecutive tool errors detected. Your previous approach is incorrect. You MUST use a fundamentally different approach or corrected arguments.' }} + {%- elif ns2.consecutive_failures == 1 %} + {{- '\n\n⚠️ SYSTEM WARNING: The previous tool call returned an error. Diagnose the failure and retry with completely corrected arguments.' }} + {%- endif %} + {{- '\n' }} + {%- if loop.last %} + {{- '<|im_end|>\n' }} + {%- else %} + {%- set _next_role = _msgs[loop.index0 + 1].role %} + {%- if _next_role != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- else %} + {{- '<|im_start|>user\n[' + message.role + ']: ' + content + '<|im_end|>\n' }} + {%- endif %} + {%- set ns2.prev_role = message.role %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if not ns_state.thinking %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/tests/targets/qwen3_6/test_frontend.cpp b/tests/targets/qwen3_6/test_frontend.cpp index 48edf21e8f..2915bdd68b 100644 --- a/tests/targets/qwen3_6/test_frontend.cpp +++ b/tests/targets/qwen3_6/test_frontend.cpp @@ -142,6 +142,14 @@ const std::string& reasoning_effort_template_source() { return source; } +const std::string& froggeric_v225_template_source() { + static const std::string source = read_template_fixture(NINFER_SOURCE_DIR + "/tests/fixtures/frontend/" + "reasoning_effort_froggeric_v225_" + "chat_template.jinja"); + return source; +} + const fi::CompiledChatTemplate& thinking_toggle_template() { static const fi::CompiledChatTemplate value = fi::CompiledChatTemplate::resolve(thinking_toggle_template_source()); @@ -154,6 +162,12 @@ const fi::CompiledChatTemplate& reasoning_effort_template() { return value; } +const fi::CompiledChatTemplate& froggeric_v225_template() { + static const fi::CompiledChatTemplate value = + fi::CompiledChatTemplate::resolve(froggeric_v225_template_source()); + return value; +} + nlohmann::json added(int id, std::string content, bool special = false) { return nlohmann::json{{"id", id}, {"content", std::move(content)}, @@ -904,6 +918,17 @@ int test_reasoning_effort_chat_template() { }), "reasoning effort and disabled thinking were accepted together"); + fi::ChatRenderOptions v225_off; + v225_off.enable_thinking = false; + failures += check(froggeric_v225_template() + .render({chat_message(ninfer::ChatRole::System, ""), + chat_message(ninfer::ChatRole::User, "hello")}, + v225_off) + .text == + "<|im_start|>user\nhello<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n", + "froggeric v22.5 template did not disable thinking on request"); + fi::ChatRenderOptions unsupported; unsupported.reasoning_effort = ninfer::ReasoningEffort::Low; failures += check(throws_invalid_argument([&] { @@ -942,6 +967,16 @@ int test_reasoning_effort_chat_template() { .text.ends_with("\n\n\n" "<|im_end|>\n"), "empty tool arguments did not follow the reasoning-effort template"); + + const ninfer::PromptCapabilities v225_capabilities = + froggeric_v225_template().capabilities(); + failures += check(v225_capabilities.enable_thinking && + v225_capabilities.reasoning_effort.low && + v225_capabilities.reasoning_effort.medium && + v225_capabilities.reasoning_effort.xhigh && + v225_capabilities.reasoning_effort.default_effort == + ninfer::ReasoningEffort::XHigh, + "froggeric v22.5 template did not advertise reasoning-effort capabilities"); return failures; } @@ -2203,6 +2238,29 @@ int test_media_preparation_cancellation() { } // namespace +int test_chat_template_file_override() { + static const std::filesystem::path kFixturePath = + NINFER_SOURCE_DIR "/tests/fixtures/frontend/reasoning_effort_froggeric_v225_chat_template.jinja"; + ninfer::targets::qwen3_6::FrontendOptions options; + options.max_context = std::numeric_limits::max(); + options.chat_template_path = kFixturePath; + const Frontend overridden = FrontendFactory::create_component(resources(), options); + const ninfer::PromptCapabilities capabilities = overridden.prompt_capabilities(); + int failures = check(capabilities.reasoning_effort.low && + capabilities.reasoning_effort.medium && + capabilities.reasoning_effort.xhigh && + capabilities.reasoning_effort.default_effort == + ninfer::ReasoningEffort::XHigh, + "chat-template override did not reach prompt capabilities"); + failures += check(throws_invalid_argument([&] { + ninfer::targets::qwen3_6::FrontendOptions missing = options; + missing.chat_template_path = "/ninfer-nonexistent-chat-template.jinja"; + (void)FrontendFactory::create_component(resources(), missing); + }), + "unreadable chat-template override was accepted"); + return failures; +} + int main() { const FrontendResources owned = resources(); const Frontend frontend = FrontendFactory::create_component(owned); @@ -2218,6 +2276,7 @@ int main() { failures += test_ordered_instruction_turns(); failures += test_assistant_continuation(); failures += test_reasoning_effort_chat_template(); + failures += test_chat_template_file_override(); failures += test_rewrite_checkpoint_trace(); failures += test_adjacent_tool_message_boundary(); failures += test_official_resource_guards(); diff --git a/tests/test_serve_options.cpp b/tests/test_serve_options.cpp index b2df413896..c50000569a 100644 --- a/tests/test_serve_options.cpp +++ b/tests/test_serve_options.cpp @@ -148,6 +148,8 @@ int main() { "--log-stats-interval-ms", "0", "--preserve-thinking", + "--chat-template", + "/tmp/ninfer-chat-template.jinja", "--media-cache-mib", "256", "--media-live-mib", @@ -162,6 +164,8 @@ int main() { failures += check(configured.enable_vision, "--vision did not enable Vision"); failures += check(configured.preserve_thinking, "--preserve-thinking did not reach serving options"); + failures += check(configured.chat_template_path == "/tmp/ninfer-chat-template.jinja", + "--chat-template did not reach serving options"); failures += check(configured.max_concurrency == 4, "--max-concurrency did not reach serving options"); failures += check(configured.max_context == 4096 && @@ -295,6 +299,9 @@ int main() { failures += check(serve_usage_text("ninfer-serve").find("--preserve-thinking") != std::string::npos, "serve help omits --preserve-thinking"); + failures += check(serve_usage_text("ninfer-serve").find("--chat-template") != + std::string::npos, + "serve help omits --chat-template"); failures += check(serve_usage_text("ninfer-serve").find("--default-thinking-budget") != std::string::npos, "serve help omits --default-thinking-budget");