From b003327c6ebb5dcf037d03ac75aa7ea7a7bee8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Sun, 6 Sep 2026 21:19:34 +0100 Subject: [PATCH 1/2] serve: honour ignore_eos on chat completions ninfer-serve accepts the ignore_eos field and ignores it, so a request generates the checkpoint's own number of tokens rather than max_tokens. With max_tokens 512 and ignore_eos true, a chat completion that stops naturally after 267 tokens should run to 512, which is what SGLang returns for the same request; ninfer returned 267 with finish_reason "stop". Any cross-engine throughput comparison then measures decode rates over windows of different lengths. StopPolicy already carries include_model_defaults (include/ninfer/types.h:234), and merge_stop_policy (src/targets/qwen3_6/impl/frontend/frontend.cpp:389) appends the tokenizer's default stop tokens only when it is set. The serve layer had no way to clear it. Add a bool ignore_eos to GenerationRequest, parse it with the existing get_bool helper in parse_stop, and map it in to_request_options as options.stop.include_model_defaults = !request.ignore_eos. Caller-supplied stop strings and stop token ids are unaffected: only the checkpoint's own defaults are suppressed, which is the documented meaning of the field in vLLM, SGLang and llama.cpp. The default is false, so a request that does not send the field behaves exactly as before, and only the OpenAI Chat Completions path is touched (/v1/messages and /v1/responses are not). Verified against the same server and prompt: ignore_eos=true gives completion_tokens=512 and finish_reason length; ignore_eos=false gives 267 and stop, unchanged. --- src/serve/openai_chat_request.cpp | 1 + src/serve/request.h | 4 ++++ src/serve/translate.cpp | 1 + 3 files changed, 6 insertions(+) diff --git a/src/serve/openai_chat_request.cpp b/src/serve/openai_chat_request.cpp index 2a7611c1e7..537638f8f9 100644 --- a/src/serve/openai_chat_request.cpp +++ b/src/serve/openai_chat_request.cpp @@ -753,6 +753,7 @@ void parse_parallel_tool_calls(const Json& body, const GenerationRequest& output } void parse_stop(const Json& body, GenerationRequest& output) { + output.ignore_eos = get_bool(body, "ignore_eos", false); if (!body.contains("stop") || body.at("stop").is_null()) { return; } output.stop_strings_apply_to_reasoning = true; const Json& stop = body.at("stop"); diff --git a/src/serve/request.h b/src/serve/request.h index cf87d3621f..fa7368bfd5 100644 --- a/src/serve/request.h +++ b/src/serve/request.h @@ -178,6 +178,10 @@ struct GenerationRequest { ToolChoice tool_choice; std::vector stop_strings; bool stop_strings_apply_to_reasoning = false; + // Benchmark/serving extension shared with vLLM, SGLang and llama.cpp: suppress the + // checkpoint's default stop tokens so generation runs to the requested token budget. + // Caller-supplied stop tokens and stop strings still apply. + bool ignore_eos = false; int max_tokens = 0; // resolved budget; zero means immediate output limit std::optional enable_thinking; // unset => use the server default std::optional thinking_budget; diff --git a/src/serve/translate.cpp b/src/serve/translate.cpp index c2effe323b..14279ce978 100644 --- a/src/serve/translate.cpp +++ b/src/serve/translate.cpp @@ -306,6 +306,7 @@ ninfer::RequestOptions to_request_options(const GenerationRequest& request, options.output.raw = false; options.output.preserve_special_tokens = request.uses_tools() || request.has_tool_history(); options.output.tool_name_max_length = static_cast(request.tool_name_max_length); + options.stop.include_model_defaults = !request.ignore_eos; options.stop.strings.reserve(request.stop_strings.size() * (request.stop_strings_apply_to_reasoning ? 2U : 1U)); for (const std::string& stop : request.stop_strings) { From 1b471efb22da1e0046e37ebfa5d456134929ee73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thireus=20=E2=98=A0?= Date: Mon, 21 Sep 2026 19:10:11 +0100 Subject: [PATCH 2/2] serve: document ignore_eos and pin it in the schema test AGENTS.md asks for the serving documentation and the schema tests to move with a protocol change, and the first commit moved neither. docs/serving.md now lists ignore_eos among the fields the Chat Completions endpoint supports, and says what true does and that caller stop strings and stop token ids still apply. tests/test_openai_schema.cpp pins the contract in test_stops_and_ranges: omitted and false keep the checkpoint's own stop tokens, true suppresses them, caller stop strings survive it, and a non-boolean value is rejected with param ignore_eos. --- docs/serving.md | 3 +++ tests/test_openai_schema.cpp | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/serving.md b/docs/serving.md index 5fb0bde843..4137089815 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -108,6 +108,9 @@ The endpoint supports: - `temperature`, `top_p`, presence/frequency penalties, and signed integer `seed`; - the compatible `top_k` (`0..20`) and `min_p` (`0..1`) sampler extensions; - up to four non-empty stop strings, applied to both reasoning and answer output; +- the boolean `ignore_eos` extension: `true` suppresses the checkpoint's own stop tokens so + generation runs to the requested token budget, while caller-supplied stop strings and stop token + ids still apply; omitted or `false` keeps them; - `n:1`, text-only `modalities`, and `response_format: {"type":"text"}`; - non-streaming responses and server-sent event streams; - `stream_options.include_usage`; diff --git a/tests/test_openai_schema.cpp b/tests/test_openai_schema.cpp index c4378233f4..395a93a8e2 100644 --- a/tests/test_openai_schema.cpp +++ b/tests/test_openai_schema.cpp @@ -580,6 +580,25 @@ int test_stops_and_ranges() { failures += check(api_error([&] { (void)parse(body); }).param == "stop", "empty stop string rejected"); + body = base_request(); + failures += check(options(parse(body).generation).stop.include_model_defaults, + "an omitted ignore_eos keeps the checkpoint's own stop tokens"); + body["ignore_eos"] = false; + failures += check(options(parse(body).generation).stop.include_model_defaults, + "ignore_eos false keeps the checkpoint's own stop tokens"); + body["ignore_eos"] = true; + failures += check(parse(body).generation.ignore_eos && + !options(parse(body).generation).stop.include_model_defaults, + "ignore_eos suppresses the checkpoint's own stop tokens"); + body["stop"] = Json::array({"A"}); + failures += check(options(parse(body).generation).stop.strings.size() == 2 && + !options(parse(body).generation).stop.include_model_defaults, + "ignore_eos leaves caller stop strings in place"); + body.erase("stop"); + body["ignore_eos"] = "true"; + failures += check(api_error([&] { (void)parse(body); }).param == "ignore_eos", + "a non-boolean ignore_eos is rejected"); + body = base_request(); body["top_k"] = 21; const GenerationRequest invalid_top_k = parse(body).generation;