diff --git a/src/serve/openai_chat_request.cpp b/src/serve/openai_chat_request.cpp index 2a7611c1e7..537638f8f9 100644 --- a/src/serve/openai_chat_request.cpp +++ b/src/serve/openai_chat_request.cpp @@ -753,6 +753,7 @@ void parse_parallel_tool_calls(const Json& body, const GenerationRequest& output } void parse_stop(const Json& body, GenerationRequest& output) { + output.ignore_eos = get_bool(body, "ignore_eos", false); if (!body.contains("stop") || body.at("stop").is_null()) { return; } output.stop_strings_apply_to_reasoning = true; const Json& stop = body.at("stop"); diff --git a/src/serve/request.h b/src/serve/request.h index cf87d3621f..fa7368bfd5 100644 --- a/src/serve/request.h +++ b/src/serve/request.h @@ -178,6 +178,10 @@ struct GenerationRequest { ToolChoice tool_choice; std::vector stop_strings; bool stop_strings_apply_to_reasoning = false; + // Benchmark/serving extension shared with vLLM, SGLang and llama.cpp: suppress the + // checkpoint's default stop tokens so generation runs to the requested token budget. + // Caller-supplied stop tokens and stop strings still apply. + bool ignore_eos = false; int max_tokens = 0; // resolved budget; zero means immediate output limit std::optional enable_thinking; // unset => use the server default std::optional thinking_budget; diff --git a/src/serve/translate.cpp b/src/serve/translate.cpp index c2effe323b..14279ce978 100644 --- a/src/serve/translate.cpp +++ b/src/serve/translate.cpp @@ -306,6 +306,7 @@ ninfer::RequestOptions to_request_options(const GenerationRequest& request, options.output.raw = false; options.output.preserve_special_tokens = request.uses_tools() || request.has_tool_history(); options.output.tool_name_max_length = static_cast(request.tool_name_max_length); + options.stop.include_model_defaults = !request.ignore_eos; options.stop.strings.reserve(request.stop_strings.size() * (request.stop_strings_apply_to_reasoning ? 2U : 1U)); for (const std::string& stop : request.stop_strings) {