feat(serve): bound each image with a Vision-token budget

docs/serving.md

@@ -151,6 +151,25 @@ not invalidate a request reference, and live bytes are returned only when the fi
 released. A request-level preparation gate derived from the live limit prevents concurrent partial
 builds from deadlocking the memory account.
 
+`preprocessor_config.json` ships the model's *capability* ceiling: `size.longest_edge` there is
+large enough that a full-resolution screen capture is never resized, so a single image can occupy
+thousands of prompt tokens. An agent client resends every screenshot it has read on every turn,
+so that number decides how many turns fit the context at all. `--image-token-budget N` applies a
+serving *policy* ceiling on top of the capability one, counted in Vision tokens, one Vision token
+being a 32x32 pixel square: an image above the budget is scaled to fit rather than rejected, and
+`0` keeps the artifact's own number. It is a per-image ceiling and deliberately does not depend on
+how many images a request carries, because waterfilling a shared budget across items would move
+the prompt prefix as a conversation grows and invalidate prefix reuse on every turn. Videos are
+unaffected.
+
+On the Qwen3.8-27B NVFP4 artifact served with `--vision --max-context 32768`, a one-image chat
+request reports these `usage.prompt_tokens`; the same request without an image reports 53:
+
+| image | default | `--image-token-budget 1280` | `--image-token-budget 256` |
+| --- | ---: | ---: | ---: |
+| 2880x1800 | 5,095 | 1,315 | 295 |
+| 1600x1200 | 1,955 | 1,285 | 289 |
+
 An expanded prompt beyond `--max-context` returns HTTP 400 `context_length_exceeded`, including
 the prepared token count and configured context ceiling. A media preprocessing resource rejection
 returns HTTP 400 `media_budget_exceeded`. HTTP 413 `request_too_large` is reserved for a raw request
@@ -467,6 +486,7 @@ curl http://127.0.0.1:8080/v1/models \
 | `--media-cache-mib N` | LRU-retained prepared BF16 media payloads; `0` disables retention | `1024` |
 | `--media-live-mib N` | all live prepared BF16 media payloads | `2048` |
 | `--media-preprocess-threads N` | bounded media preprocessing workers; `0` selects at most 16 from host concurrency | `0` |
+| `--image-token-budget N` | per-image serving ceiling in Vision tokens; `0` keeps the artifact ceiling | `0` |
 | `--request-log-jsonl FILE` | append full-precision server/request records | disabled |
 | `--response-store-max-records N` | maximum locally retained Responses objects | `1024` |
 | `--response-store-max-mib N` | total local Response envelope/Item/context budget | `256` |

include/ninfer/types.h

@@ -87,6 +87,8 @@ struct EngineOptions {
     std::size_t media_live_bytes  = kDefaultMediaLiveBytes;
     // Zero selects a bounded worker count from the detected host concurrency.
     std::uint32_t media_preprocess_threads = 0;
+    // Per-image serving ceiling in Vision tokens. Zero keeps the artifact's own ceiling.
+    std::uint32_t image_token_budget       = 0;
     bool enable_vision                     = false;
     bool use_cuda_graph                    = true;
     LoadProgress load_progress;

src/serve/generation_service.cpp

@@ -241,6 +241,7 @@ GenerationService::GenerationService(ServeOptions options, LoadProgress load_pro
     engine_options.media_cache_bytes        = options_.media_cache_bytes;
     engine_options.media_live_bytes         = options_.media_live_bytes;
     engine_options.media_preprocess_threads = options_.media_preprocess_threads;
+    engine_options.image_token_budget       = options_.image_token_budget;
     engine_options.load_progress            = std::move(load_progress);
     engine_              = std::make_unique<ninfer::Engine>(std::move(engine_options));
     prompt_capabilities_ = engine_->prompt_capabilities();

src/serve/request_log.cpp

@@ -451,6 +451,7 @@ std::string format_server_start_json(
                               {"media_cache_bytes", options.media_cache_bytes},
                               {"media_live_bytes", options.media_live_bytes},
                               {"media_preprocess_threads", options.media_preprocess_threads},
+                              {"image_token_budget", options.image_token_budget},
                               {"request_log_jsonl", options.request_log_jsonl},
                               {"default_output_tokens", options.default_max_tokens},
                               {"default_thinking", options.enable_thinking},

src/serve/serve_options.cpp

@@ -68,7 +68,7 @@ std::string serve_usage_text(const char* argv0) {
            "[--max-pending-requests N] [--pending-timeout-ms N] "
            "[--prefill-chunk N] [--log-stats-interval-ms N] [--device N] "
            "[--max-request-mib N] [--media-cache-mib N] [--media-live-mib N] "
-           "[--media-preprocess-threads N] "
+           "[--media-preprocess-threads N] [--image-token-budget N] "
            "[--request-log-jsonl FILE] "
            "[--response-store-max-records N] [--response-store-max-mib N] "
            "[--kv-dtype bf16|int8] [--spec mtp|dflash --draft-tokens N] "
@@ -85,6 +85,8 @@ std::string serve_usage_text(const char* argv0) {
            "       --media-cache-mib defaults to 1024; 0 disables retained media reuse\n"
            "       --media-live-mib defaults to 2048 and bounds all live BF16 patch payloads\n"
            "       --media-preprocess-threads defaults to 0 (auto, at most 16 workers)\n"
+           "       --image-token-budget caps each image in Vision tokens (32x32 pixels each) "
+           "by scaling it to fit; 0 keeps the artifact ceiling\n"
            "       --request-log-jsonl appends full-precision server/request records\n"
            "       --model-id overrides the artifact identity.model_id reported by the server\n"
            "       Responses state is process-local and bounded to 1024 records / 256 MiB by "
@@ -188,6 +190,9 @@ ServeOptions parse_serve_options(int argc, char** argv) {
                 throw std::invalid_argument("--media-preprocess-threads must be in [0,64]");
             }
             options.media_preprocess_threads = static_cast<std::uint32_t>(threads);
+        } else if (arg == "--image-token-budget") {
+            options.image_token_budget = static_cast<std::uint32_t>(
+                parse_nonnegative_int(require_value("--image-token-budget"), "image-token-budget"));
         } else if (arg == "--request-log-jsonl") {
             options.request_log_jsonl = require_value("--request-log-jsonl");
             if (options.request_log_jsonl.empty()) {

src/serve/serve_options.h

@@ -37,6 +37,7 @@ struct ServeOptions {
     std::size_t media_cache_bytes          = kDefaultMediaCacheBytes;
     std::size_t media_live_bytes           = kDefaultMediaLiveBytes;
     std::uint32_t media_preprocess_threads = 0;
+    std::uint32_t image_token_budget       = 0;
     std::size_t response_store_max_records = kDefaultResponseStoreRecords;
     std::size_t response_store_max_bytes   = kDefaultResponseStoreBytes;
     int device                             = 0;

src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h

@@ -20,6 +20,8 @@ struct FrontendOptions {
     std::size_t media_cache_bytes          = kDefaultMediaCacheBytes;
     std::size_t media_live_bytes           = kDefaultMediaLiveBytes;
     std::uint32_t media_preprocess_threads = 0;
+    // Per-image ceiling in Vision tokens. Zero keeps preprocessor_config.json's own value.
+    std::uint32_t image_token_budget       = 0;
 };
 
 struct FrontendResources;

src/targets/qwen3_6/impl/frontend/frontend.cpp

@@ -42,6 +42,10 @@ constexpr double kVideoFps             = 2.0;
 constexpr int kVideoMinFrames          = 4;
 constexpr int kVideoMaxFrames          = 768;
 
+// validate_pixel_pipeline() pins patch_size to 16 and merge_size to 2, so one Vision token
+// always covers a 32x32 pixel square.
+constexpr std::uint64_t kPixelsPerVisionToken = (16ULL * 2ULL) * (16ULL * 2ULL);
+
 constexpr std::array<std::pair<std::string_view, TokenId>, 4> kVisionSpecialTokens = {{
     {"<|vision_start|>", 248053},
     {"<|vision_end|>", 248054},
@@ -141,7 +145,8 @@ void validate_pixel_pipeline(const Json& config, std::string_view resource) {
     }
 }
 
-fi::ProcessorOptions processor_options(const FrontendResources& resources) {
+fi::ProcessorOptions processor_options(const FrontendResources& resources,
+                                       std::uint32_t image_token_budget) {
     const Json image =
         parse_resource_json(resources.preprocessor_config_json, "preprocessor_config.json");
     const Json video = parse_resource_json(resources.video_preprocessor_config_json,
@@ -159,6 +164,19 @@ fi::ProcessorOptions processor_options(const FrontendResources& resources) {
     options.image_max_pixels =
         positive_u64(require_integer(image_size, "longest_edge", "preprocessor_config.json.size"),
                      "image longest_edge");
+    if (image_token_budget != 0) {
+        // preprocessor_config.json ships the model's *capability* ceiling: longest_edge there is
+        // large enough that a full-resolution screen capture is never resized, so a single image
+        // occupies thousands of prompt tokens. An agent conversation resends every screenshot it
+        // has read on every turn, so a serving endpoint wants a *policy* ceiling on top, the way
+        // every hosted API applies one: fit-and-scale rather than reject. The budget is per image
+        // and deliberately does not depend on how many images a request carries; waterfilling
+        // across items would move the prompt prefix as a conversation grows and invalidate prefix
+        // reuse on every turn. Nothing else in the pipeline reads image_max_pixels.
+        options.image_max_pixels =
+            std::min(options.image_max_pixels,
+                     static_cast<std::uint64_t>(image_token_budget) * kPixelsPerVisionToken);
+    }
     options.video_min_pixels = positive_u64(
         require_integer(video_size, "shortest_edge", "video_preprocessor_config.json.size"),
         "video shortest_edge");
@@ -600,7 +618,8 @@ class Frontend::Impl {
               fi::TokenizerResources{.tokenizer_json         = resources.tokenizer_json,
                                      .tokenizer_config_json  = resources.tokenizer_config_json,
                                      .generation_config_json = resources.generation_config_json})),
-          processor(processor_options(resources)), vision_enabled(options.vision_enabled) {
+          processor(processor_options(resources, options.image_token_budget)),
+          vision_enabled(options.vision_enabled) {
         if (options.max_context == 0) {
             throw std::invalid_argument("frontend max_context must be nonzero");
         }

src/targets/qwen3_6_27b/impl/package.cpp

@@ -124,6 +124,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO
                                       .media_cache_bytes        = options.media_cache_bytes,
                                       .media_live_bytes         = options.media_live_bytes,
                                       .media_preprocess_threads = options.media_preprocess_threads,
+                                      .image_token_budget       = options.image_token_budget,
                                   });
 }
 

src/targets/qwen3_6_35b_a3b/impl/package.cpp

@@ -96,6 +96,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO
                                       .media_cache_bytes        = options.media_cache_bytes,
                                       .media_live_bytes         = options.media_live_bytes,
                                       .media_preprocess_threads = options.media_preprocess_threads,
+                                      .image_token_budget       = options.image_token_budget,
                                   });
 }
 

tests/test_serve_options.cpp

@@ -41,6 +41,8 @@ int main() {
                           defaults.media_live_bytes == ninfer::kDefaultMediaLiveBytes &&
                           defaults.media_preprocess_threads == 0,
                       "media preparation resource defaults mismatch");
+    failures += check(defaults.image_token_budget == 0,
+                      "an image serving ceiling is unexpectedly applied by default");
     failures += check(defaults.kv_capacity.mode == ninfer::KvCapacityMode::Explicit &&
                           defaults.kv_capacity.explicit_tokens == defaults.max_context,
                       "default KV capacity does not follow max context");
@@ -72,6 +74,11 @@ int main() {
     } catch (const std::invalid_argument&) { empty_model_id_rejected = true; }
     failures += check(empty_model_id_rejected, "empty --model-id was accepted");
 
+    const ServeOptions image_budget =
+        parse({"ninfer-serve", "model.ninfer", "--image-token-budget", "1280"});
+    failures += check(image_budget.image_token_budget == 1280,
+                      "--image-token-budget did not carry the per-image Vision-token ceiling");
+
     const ServeOptions dflash = parse({"ninfer-serve", "model.ninfer", "--spec", "dflash",
                                        "--draft-tokens", "15", "--lm-head-draft"});
     failures += check(dflash.speculative.backend == ninfer::SpeculativeBackend::DFlash,