From eb413c76ee6be473df18f37ce3f314f34eb01657 Mon Sep 17 00:00:00 2001 From: Aleksandr Iakimov Date: Thu, 20 Aug 2026 05:23:51 +0000 Subject: [PATCH] feat(serve): bound each image with a Vision-token budget preprocessor_config.json ships the model's capability ceiling: size.longest_edge there is large enough that a full-resolution screen capture is never resized, so a single image occupies thousands of prompt tokens. An agent client resends every screenshot it has read on every turn, so that number decides how many turns fit the context at all, and a serving endpoint wants a policy ceiling on top of the capability one, the way every hosted API applies one: fit-and-scale rather than reject. --image-token-budget N states that policy in the unit an operator reasons about. One Vision token is a 32x32 pixel square, which validate_pixel_pipeline already pins by requiring patch_size 16 and merge_size 2, so the budget lowers image_max_pixels and nothing else: no other consumer reads that value. Zero keeps the artifact's own number, so the served behavior is unchanged unless the operator asks for a ceiling, and videos are unaffected either way. The budget is per image and deliberately does not depend on how many images a request carries. Waterfilling a shared budget across items would move the prompt prefix as a conversation grows and invalidate prefix reuse on every turn. Measured on the 5090 NVFP4 build served with --vision --max-context 32768, as usage.prompt_tokens of a one-image chat request; the same request without an image reports 53: image default budget 1280 budget 256 2880x1800 5095 1315 295 1600x1200 1955 1285 289 Co-Authored-By: Claude Opus 5 --- docs/serving.md | 20 ++++++++++++++++ include/ninfer/types.h | 2 ++ src/serve/generation_service.cpp | 1 + src/serve/request_log.cpp | 1 + src/serve/serve_options.cpp | 7 +++++- src/serve/serve_options.h | 1 + .../export/ninfer/targets/qwen3_6/frontend.h | 2 ++ .../qwen3_6/impl/frontend/frontend.cpp | 23 +++++++++++++++++-- src/targets/qwen3_6_27b/impl/package.cpp | 1 + src/targets/qwen3_6_35b_a3b/impl/package.cpp | 1 + tests/test_serve_options.cpp | 7 ++++++ 11 files changed, 63 insertions(+), 3 deletions(-) diff --git a/docs/serving.md b/docs/serving.md index 3fe656cbf5..402aa3c0df 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -151,6 +151,25 @@ not invalidate a request reference, and live bytes are returned only when the fi released. A request-level preparation gate derived from the live limit prevents concurrent partial builds from deadlocking the memory account. +`preprocessor_config.json` ships the model's *capability* ceiling: `size.longest_edge` there is +large enough that a full-resolution screen capture is never resized, so a single image can occupy +thousands of prompt tokens. An agent client resends every screenshot it has read on every turn, +so that number decides how many turns fit the context at all. `--image-token-budget N` applies a +serving *policy* ceiling on top of the capability one, counted in Vision tokens, one Vision token +being a 32x32 pixel square: an image above the budget is scaled to fit rather than rejected, and +`0` keeps the artifact's own number. It is a per-image ceiling and deliberately does not depend on +how many images a request carries, because waterfilling a shared budget across items would move +the prompt prefix as a conversation grows and invalidate prefix reuse on every turn. Videos are +unaffected. + +On the Qwen3.8-27B NVFP4 artifact served with `--vision --max-context 32768`, a one-image chat +request reports these `usage.prompt_tokens`; the same request without an image reports 53: + +| image | default | `--image-token-budget 1280` | `--image-token-budget 256` | +| --- | ---: | ---: | ---: | +| 2880x1800 | 5,095 | 1,315 | 295 | +| 1600x1200 | 1,955 | 1,285 | 289 | + An expanded prompt beyond `--max-context` returns HTTP 400 `context_length_exceeded`, including the prepared token count and configured context ceiling. A media preprocessing resource rejection returns HTTP 400 `media_budget_exceeded`. HTTP 413 `request_too_large` is reserved for a raw request @@ -467,6 +486,7 @@ curl http://127.0.0.1:8080/v1/models \ | `--media-cache-mib N` | LRU-retained prepared BF16 media payloads; `0` disables retention | `1024` | | `--media-live-mib N` | all live prepared BF16 media payloads | `2048` | | `--media-preprocess-threads N` | bounded media preprocessing workers; `0` selects at most 16 from host concurrency | `0` | +| `--image-token-budget N` | per-image serving ceiling in Vision tokens; `0` keeps the artifact ceiling | `0` | | `--request-log-jsonl FILE` | append full-precision server/request records | disabled | | `--response-store-max-records N` | maximum locally retained Responses objects | `1024` | | `--response-store-max-mib N` | total local Response envelope/Item/context budget | `256` | diff --git a/include/ninfer/types.h b/include/ninfer/types.h index c074a3e04a..eef17062e5 100644 --- a/include/ninfer/types.h +++ b/include/ninfer/types.h @@ -87,6 +87,8 @@ struct EngineOptions { std::size_t media_live_bytes = kDefaultMediaLiveBytes; // Zero selects a bounded worker count from the detected host concurrency. std::uint32_t media_preprocess_threads = 0; + // Per-image serving ceiling in Vision tokens. Zero keeps the artifact's own ceiling. + std::uint32_t image_token_budget = 0; bool enable_vision = false; bool use_cuda_graph = true; LoadProgress load_progress; diff --git a/src/serve/generation_service.cpp b/src/serve/generation_service.cpp index b1c78d1c73..95aae24ada 100644 --- a/src/serve/generation_service.cpp +++ b/src/serve/generation_service.cpp @@ -241,6 +241,7 @@ GenerationService::GenerationService(ServeOptions options, LoadProgress load_pro engine_options.media_cache_bytes = options_.media_cache_bytes; engine_options.media_live_bytes = options_.media_live_bytes; engine_options.media_preprocess_threads = options_.media_preprocess_threads; + engine_options.image_token_budget = options_.image_token_budget; engine_options.load_progress = std::move(load_progress); engine_ = std::make_unique(std::move(engine_options)); prompt_capabilities_ = engine_->prompt_capabilities(); diff --git a/src/serve/request_log.cpp b/src/serve/request_log.cpp index b2dd984b69..4a744c5dcd 100644 --- a/src/serve/request_log.cpp +++ b/src/serve/request_log.cpp @@ -451,6 +451,7 @@ std::string format_server_start_json( {"media_cache_bytes", options.media_cache_bytes}, {"media_live_bytes", options.media_live_bytes}, {"media_preprocess_threads", options.media_preprocess_threads}, + {"image_token_budget", options.image_token_budget}, {"request_log_jsonl", options.request_log_jsonl}, {"default_output_tokens", options.default_max_tokens}, {"default_thinking", options.enable_thinking}, diff --git a/src/serve/serve_options.cpp b/src/serve/serve_options.cpp index c991e2cc85..7889bd514f 100644 --- a/src/serve/serve_options.cpp +++ b/src/serve/serve_options.cpp @@ -68,7 +68,7 @@ std::string serve_usage_text(const char* argv0) { "[--max-pending-requests N] [--pending-timeout-ms N] " "[--prefill-chunk N] [--log-stats-interval-ms N] [--device N] " "[--max-request-mib N] [--media-cache-mib N] [--media-live-mib N] " - "[--media-preprocess-threads N] " + "[--media-preprocess-threads N] [--image-token-budget N] " "[--request-log-jsonl FILE] " "[--response-store-max-records N] [--response-store-max-mib N] " "[--kv-dtype bf16|int8] [--spec mtp|dflash --draft-tokens N] " @@ -85,6 +85,8 @@ std::string serve_usage_text(const char* argv0) { " --media-cache-mib defaults to 1024; 0 disables retained media reuse\n" " --media-live-mib defaults to 2048 and bounds all live BF16 patch payloads\n" " --media-preprocess-threads defaults to 0 (auto, at most 16 workers)\n" + " --image-token-budget caps each image in Vision tokens (32x32 pixels each) " + "by scaling it to fit; 0 keeps the artifact ceiling\n" " --request-log-jsonl appends full-precision server/request records\n" " --model-id overrides the artifact identity.model_id reported by the server\n" " Responses state is process-local and bounded to 1024 records / 256 MiB by " @@ -188,6 +190,9 @@ ServeOptions parse_serve_options(int argc, char** argv) { throw std::invalid_argument("--media-preprocess-threads must be in [0,64]"); } options.media_preprocess_threads = static_cast(threads); + } else if (arg == "--image-token-budget") { + options.image_token_budget = static_cast( + parse_nonnegative_int(require_value("--image-token-budget"), "image-token-budget")); } else if (arg == "--request-log-jsonl") { options.request_log_jsonl = require_value("--request-log-jsonl"); if (options.request_log_jsonl.empty()) { diff --git a/src/serve/serve_options.h b/src/serve/serve_options.h index b6db8e4fd5..148deb12ec 100644 --- a/src/serve/serve_options.h +++ b/src/serve/serve_options.h @@ -37,6 +37,7 @@ struct ServeOptions { std::size_t media_cache_bytes = kDefaultMediaCacheBytes; std::size_t media_live_bytes = kDefaultMediaLiveBytes; std::uint32_t media_preprocess_threads = 0; + std::uint32_t image_token_budget = 0; std::size_t response_store_max_records = kDefaultResponseStoreRecords; std::size_t response_store_max_bytes = kDefaultResponseStoreBytes; int device = 0; diff --git a/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h b/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h index 3e66bdbcd2..3deb8f56df 100644 --- a/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h +++ b/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h @@ -20,6 +20,8 @@ struct FrontendOptions { std::size_t media_cache_bytes = kDefaultMediaCacheBytes; std::size_t media_live_bytes = kDefaultMediaLiveBytes; std::uint32_t media_preprocess_threads = 0; + // Per-image ceiling in Vision tokens. Zero keeps preprocessor_config.json's own value. + std::uint32_t image_token_budget = 0; }; struct FrontendResources; diff --git a/src/targets/qwen3_6/impl/frontend/frontend.cpp b/src/targets/qwen3_6/impl/frontend/frontend.cpp index 478f63c2c6..9ccfad55fb 100644 --- a/src/targets/qwen3_6/impl/frontend/frontend.cpp +++ b/src/targets/qwen3_6/impl/frontend/frontend.cpp @@ -42,6 +42,10 @@ constexpr double kVideoFps = 2.0; constexpr int kVideoMinFrames = 4; constexpr int kVideoMaxFrames = 768; +// validate_pixel_pipeline() pins patch_size to 16 and merge_size to 2, so one Vision token +// always covers a 32x32 pixel square. +constexpr std::uint64_t kPixelsPerVisionToken = (16ULL * 2ULL) * (16ULL * 2ULL); + constexpr std::array, 4> kVisionSpecialTokens = {{ {"<|vision_start|>", 248053}, {"<|vision_end|>", 248054}, @@ -141,7 +145,8 @@ void validate_pixel_pipeline(const Json& config, std::string_view resource) { } } -fi::ProcessorOptions processor_options(const FrontendResources& resources) { +fi::ProcessorOptions processor_options(const FrontendResources& resources, + std::uint32_t image_token_budget) { const Json image = parse_resource_json(resources.preprocessor_config_json, "preprocessor_config.json"); const Json video = parse_resource_json(resources.video_preprocessor_config_json, @@ -159,6 +164,19 @@ fi::ProcessorOptions processor_options(const FrontendResources& resources) { options.image_max_pixels = positive_u64(require_integer(image_size, "longest_edge", "preprocessor_config.json.size"), "image longest_edge"); + if (image_token_budget != 0) { + // preprocessor_config.json ships the model's *capability* ceiling: longest_edge there is + // large enough that a full-resolution screen capture is never resized, so a single image + // occupies thousands of prompt tokens. An agent conversation resends every screenshot it + // has read on every turn, so a serving endpoint wants a *policy* ceiling on top, the way + // every hosted API applies one: fit-and-scale rather than reject. The budget is per image + // and deliberately does not depend on how many images a request carries; waterfilling + // across items would move the prompt prefix as a conversation grows and invalidate prefix + // reuse on every turn. Nothing else in the pipeline reads image_max_pixels. + options.image_max_pixels = + std::min(options.image_max_pixels, + static_cast(image_token_budget) * kPixelsPerVisionToken); + } options.video_min_pixels = positive_u64( require_integer(video_size, "shortest_edge", "video_preprocessor_config.json.size"), "video shortest_edge"); @@ -600,7 +618,8 @@ class Frontend::Impl { fi::TokenizerResources{.tokenizer_json = resources.tokenizer_json, .tokenizer_config_json = resources.tokenizer_config_json, .generation_config_json = resources.generation_config_json})), - processor(processor_options(resources)), vision_enabled(options.vision_enabled) { + processor(processor_options(resources, options.image_token_budget)), + vision_enabled(options.vision_enabled) { if (options.max_context == 0) { throw std::invalid_argument("frontend max_context must be nonzero"); } diff --git a/src/targets/qwen3_6_27b/impl/package.cpp b/src/targets/qwen3_6_27b/impl/package.cpp index 67647994fe..7e4d01601b 100644 --- a/src/targets/qwen3_6_27b/impl/package.cpp +++ b/src/targets/qwen3_6_27b/impl/package.cpp @@ -124,6 +124,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO .media_cache_bytes = options.media_cache_bytes, .media_live_bytes = options.media_live_bytes, .media_preprocess_threads = options.media_preprocess_threads, + .image_token_budget = options.image_token_budget, }); } diff --git a/src/targets/qwen3_6_35b_a3b/impl/package.cpp b/src/targets/qwen3_6_35b_a3b/impl/package.cpp index 1bed0142e0..ac415030cf 100644 --- a/src/targets/qwen3_6_35b_a3b/impl/package.cpp +++ b/src/targets/qwen3_6_35b_a3b/impl/package.cpp @@ -96,6 +96,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO .media_cache_bytes = options.media_cache_bytes, .media_live_bytes = options.media_live_bytes, .media_preprocess_threads = options.media_preprocess_threads, + .image_token_budget = options.image_token_budget, }); } diff --git a/tests/test_serve_options.cpp b/tests/test_serve_options.cpp index 65231dff94..c149608733 100644 --- a/tests/test_serve_options.cpp +++ b/tests/test_serve_options.cpp @@ -41,6 +41,8 @@ int main() { defaults.media_live_bytes == ninfer::kDefaultMediaLiveBytes && defaults.media_preprocess_threads == 0, "media preparation resource defaults mismatch"); + failures += check(defaults.image_token_budget == 0, + "an image serving ceiling is unexpectedly applied by default"); failures += check(defaults.kv_capacity.mode == ninfer::KvCapacityMode::Explicit && defaults.kv_capacity.explicit_tokens == defaults.max_context, "default KV capacity does not follow max context"); @@ -72,6 +74,11 @@ int main() { } catch (const std::invalid_argument&) { empty_model_id_rejected = true; } failures += check(empty_model_id_rejected, "empty --model-id was accepted"); + const ServeOptions image_budget = + parse({"ninfer-serve", "model.ninfer", "--image-token-budget", "1280"}); + failures += check(image_budget.image_token_budget == 1280, + "--image-token-budget did not carry the per-image Vision-token ceiling"); + const ServeOptions dflash = parse({"ninfer-serve", "model.ninfer", "--spec", "dflash", "--draft-tokens", "15", "--lm-head-draft"}); failures += check(dflash.speculative.backend == ninfer::SpeculativeBackend::DFlash,