diff --git a/docs/serving.md b/docs/serving.md index 3fe656cbf5..402aa3c0df 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -151,6 +151,25 @@ not invalidate a request reference, and live bytes are returned only when the fi released. A request-level preparation gate derived from the live limit prevents concurrent partial builds from deadlocking the memory account. +`preprocessor_config.json` ships the model's *capability* ceiling: `size.longest_edge` there is +large enough that a full-resolution screen capture is never resized, so a single image can occupy +thousands of prompt tokens. An agent client resends every screenshot it has read on every turn, +so that number decides how many turns fit the context at all. `--image-token-budget N` applies a +serving *policy* ceiling on top of the capability one, counted in Vision tokens, one Vision token +being a 32x32 pixel square: an image above the budget is scaled to fit rather than rejected, and +`0` keeps the artifact's own number. It is a per-image ceiling and deliberately does not depend on +how many images a request carries, because waterfilling a shared budget across items would move +the prompt prefix as a conversation grows and invalidate prefix reuse on every turn. Videos are +unaffected. + +On the Qwen3.8-27B NVFP4 artifact served with `--vision --max-context 32768`, a one-image chat +request reports these `usage.prompt_tokens`; the same request without an image reports 53: + +| image | default | `--image-token-budget 1280` | `--image-token-budget 256` | +| --- | ---: | ---: | ---: | +| 2880x1800 | 5,095 | 1,315 | 295 | +| 1600x1200 | 1,955 | 1,285 | 289 | + An expanded prompt beyond `--max-context` returns HTTP 400 `context_length_exceeded`, including the prepared token count and configured context ceiling. A media preprocessing resource rejection returns HTTP 400 `media_budget_exceeded`. HTTP 413 `request_too_large` is reserved for a raw request @@ -467,6 +486,7 @@ curl http://127.0.0.1:8080/v1/models \ | `--media-cache-mib N` | LRU-retained prepared BF16 media payloads; `0` disables retention | `1024` | | `--media-live-mib N` | all live prepared BF16 media payloads | `2048` | | `--media-preprocess-threads N` | bounded media preprocessing workers; `0` selects at most 16 from host concurrency | `0` | +| `--image-token-budget N` | per-image serving ceiling in Vision tokens; `0` keeps the artifact ceiling | `0` | | `--request-log-jsonl FILE` | append full-precision server/request records | disabled | | `--response-store-max-records N` | maximum locally retained Responses objects | `1024` | | `--response-store-max-mib N` | total local Response envelope/Item/context budget | `256` | diff --git a/include/ninfer/types.h b/include/ninfer/types.h index c074a3e04a..eef17062e5 100644 --- a/include/ninfer/types.h +++ b/include/ninfer/types.h @@ -87,6 +87,8 @@ struct EngineOptions { std::size_t media_live_bytes = kDefaultMediaLiveBytes; // Zero selects a bounded worker count from the detected host concurrency. std::uint32_t media_preprocess_threads = 0; + // Per-image serving ceiling in Vision tokens. Zero keeps the artifact's own ceiling. + std::uint32_t image_token_budget = 0; bool enable_vision = false; bool use_cuda_graph = true; LoadProgress load_progress; diff --git a/src/serve/generation_service.cpp b/src/serve/generation_service.cpp index b1c78d1c73..95aae24ada 100644 --- a/src/serve/generation_service.cpp +++ b/src/serve/generation_service.cpp @@ -241,6 +241,7 @@ GenerationService::GenerationService(ServeOptions options, LoadProgress load_pro engine_options.media_cache_bytes = options_.media_cache_bytes; engine_options.media_live_bytes = options_.media_live_bytes; engine_options.media_preprocess_threads = options_.media_preprocess_threads; + engine_options.image_token_budget = options_.image_token_budget; engine_options.load_progress = std::move(load_progress); engine_ = std::make_unique(std::move(engine_options)); prompt_capabilities_ = engine_->prompt_capabilities(); diff --git a/src/serve/request_log.cpp b/src/serve/request_log.cpp index b2dd984b69..4a744c5dcd 100644 --- a/src/serve/request_log.cpp +++ b/src/serve/request_log.cpp @@ -451,6 +451,7 @@ std::string format_server_start_json( {"media_cache_bytes", options.media_cache_bytes}, {"media_live_bytes", options.media_live_bytes}, {"media_preprocess_threads", options.media_preprocess_threads}, + {"image_token_budget", options.image_token_budget}, {"request_log_jsonl", options.request_log_jsonl}, {"default_output_tokens", options.default_max_tokens}, {"default_thinking", options.enable_thinking}, diff --git a/src/serve/serve_options.cpp b/src/serve/serve_options.cpp index c991e2cc85..7889bd514f 100644 --- a/src/serve/serve_options.cpp +++ b/src/serve/serve_options.cpp @@ -68,7 +68,7 @@ std::string serve_usage_text(const char* argv0) { "[--max-pending-requests N] [--pending-timeout-ms N] " "[--prefill-chunk N] [--log-stats-interval-ms N] [--device N] " "[--max-request-mib N] [--media-cache-mib N] [--media-live-mib N] " - "[--media-preprocess-threads N] " + "[--media-preprocess-threads N] [--image-token-budget N] " "[--request-log-jsonl FILE] " "[--response-store-max-records N] [--response-store-max-mib N] " "[--kv-dtype bf16|int8] [--spec mtp|dflash --draft-tokens N] " @@ -85,6 +85,8 @@ std::string serve_usage_text(const char* argv0) { " --media-cache-mib defaults to 1024; 0 disables retained media reuse\n" " --media-live-mib defaults to 2048 and bounds all live BF16 patch payloads\n" " --media-preprocess-threads defaults to 0 (auto, at most 16 workers)\n" + " --image-token-budget caps each image in Vision tokens (32x32 pixels each) " + "by scaling it to fit; 0 keeps the artifact ceiling\n" " --request-log-jsonl appends full-precision server/request records\n" " --model-id overrides the artifact identity.model_id reported by the server\n" " Responses state is process-local and bounded to 1024 records / 256 MiB by " @@ -188,6 +190,9 @@ ServeOptions parse_serve_options(int argc, char** argv) { throw std::invalid_argument("--media-preprocess-threads must be in [0,64]"); } options.media_preprocess_threads = static_cast(threads); + } else if (arg == "--image-token-budget") { + options.image_token_budget = static_cast( + parse_nonnegative_int(require_value("--image-token-budget"), "image-token-budget")); } else if (arg == "--request-log-jsonl") { options.request_log_jsonl = require_value("--request-log-jsonl"); if (options.request_log_jsonl.empty()) { diff --git a/src/serve/serve_options.h b/src/serve/serve_options.h index b6db8e4fd5..148deb12ec 100644 --- a/src/serve/serve_options.h +++ b/src/serve/serve_options.h @@ -37,6 +37,7 @@ struct ServeOptions { std::size_t media_cache_bytes = kDefaultMediaCacheBytes; std::size_t media_live_bytes = kDefaultMediaLiveBytes; std::uint32_t media_preprocess_threads = 0; + std::uint32_t image_token_budget = 0; std::size_t response_store_max_records = kDefaultResponseStoreRecords; std::size_t response_store_max_bytes = kDefaultResponseStoreBytes; int device = 0; diff --git a/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h b/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h index 3e66bdbcd2..3deb8f56df 100644 --- a/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h +++ b/src/targets/qwen3_6/export/ninfer/targets/qwen3_6/frontend.h @@ -20,6 +20,8 @@ struct FrontendOptions { std::size_t media_cache_bytes = kDefaultMediaCacheBytes; std::size_t media_live_bytes = kDefaultMediaLiveBytes; std::uint32_t media_preprocess_threads = 0; + // Per-image ceiling in Vision tokens. Zero keeps preprocessor_config.json's own value. + std::uint32_t image_token_budget = 0; }; struct FrontendResources; diff --git a/src/targets/qwen3_6/impl/frontend/frontend.cpp b/src/targets/qwen3_6/impl/frontend/frontend.cpp index 478f63c2c6..9ccfad55fb 100644 --- a/src/targets/qwen3_6/impl/frontend/frontend.cpp +++ b/src/targets/qwen3_6/impl/frontend/frontend.cpp @@ -42,6 +42,10 @@ constexpr double kVideoFps = 2.0; constexpr int kVideoMinFrames = 4; constexpr int kVideoMaxFrames = 768; +// validate_pixel_pipeline() pins patch_size to 16 and merge_size to 2, so one Vision token +// always covers a 32x32 pixel square. +constexpr std::uint64_t kPixelsPerVisionToken = (16ULL * 2ULL) * (16ULL * 2ULL); + constexpr std::array, 4> kVisionSpecialTokens = {{ {"<|vision_start|>", 248053}, {"<|vision_end|>", 248054}, @@ -141,7 +145,8 @@ void validate_pixel_pipeline(const Json& config, std::string_view resource) { } } -fi::ProcessorOptions processor_options(const FrontendResources& resources) { +fi::ProcessorOptions processor_options(const FrontendResources& resources, + std::uint32_t image_token_budget) { const Json image = parse_resource_json(resources.preprocessor_config_json, "preprocessor_config.json"); const Json video = parse_resource_json(resources.video_preprocessor_config_json, @@ -159,6 +164,19 @@ fi::ProcessorOptions processor_options(const FrontendResources& resources) { options.image_max_pixels = positive_u64(require_integer(image_size, "longest_edge", "preprocessor_config.json.size"), "image longest_edge"); + if (image_token_budget != 0) { + // preprocessor_config.json ships the model's *capability* ceiling: longest_edge there is + // large enough that a full-resolution screen capture is never resized, so a single image + // occupies thousands of prompt tokens. An agent conversation resends every screenshot it + // has read on every turn, so a serving endpoint wants a *policy* ceiling on top, the way + // every hosted API applies one: fit-and-scale rather than reject. The budget is per image + // and deliberately does not depend on how many images a request carries; waterfilling + // across items would move the prompt prefix as a conversation grows and invalidate prefix + // reuse on every turn. Nothing else in the pipeline reads image_max_pixels. + options.image_max_pixels = + std::min(options.image_max_pixels, + static_cast(image_token_budget) * kPixelsPerVisionToken); + } options.video_min_pixels = positive_u64( require_integer(video_size, "shortest_edge", "video_preprocessor_config.json.size"), "video shortest_edge"); @@ -600,7 +618,8 @@ class Frontend::Impl { fi::TokenizerResources{.tokenizer_json = resources.tokenizer_json, .tokenizer_config_json = resources.tokenizer_config_json, .generation_config_json = resources.generation_config_json})), - processor(processor_options(resources)), vision_enabled(options.vision_enabled) { + processor(processor_options(resources, options.image_token_budget)), + vision_enabled(options.vision_enabled) { if (options.max_context == 0) { throw std::invalid_argument("frontend max_context must be nonzero"); } diff --git a/src/targets/qwen3_6_27b/impl/package.cpp b/src/targets/qwen3_6_27b/impl/package.cpp index 67647994fe..7e4d01601b 100644 --- a/src/targets/qwen3_6_27b/impl/package.cpp +++ b/src/targets/qwen3_6_27b/impl/package.cpp @@ -124,6 +124,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO .media_cache_bytes = options.media_cache_bytes, .media_live_bytes = options.media_live_bytes, .media_preprocess_threads = options.media_preprocess_threads, + .image_token_budget = options.image_token_budget, }); } diff --git a/src/targets/qwen3_6_35b_a3b/impl/package.cpp b/src/targets/qwen3_6_35b_a3b/impl/package.cpp index 1bed0142e0..ac415030cf 100644 --- a/src/targets/qwen3_6_35b_a3b/impl/package.cpp +++ b/src/targets/qwen3_6_35b_a3b/impl/package.cpp @@ -96,6 +96,7 @@ Package::Frontend Package::make_frontend(const LoadedModel& model, const EngineO .media_cache_bytes = options.media_cache_bytes, .media_live_bytes = options.media_live_bytes, .media_preprocess_threads = options.media_preprocess_threads, + .image_token_budget = options.image_token_budget, }); } diff --git a/tests/test_serve_options.cpp b/tests/test_serve_options.cpp index 65231dff94..c149608733 100644 --- a/tests/test_serve_options.cpp +++ b/tests/test_serve_options.cpp @@ -41,6 +41,8 @@ int main() { defaults.media_live_bytes == ninfer::kDefaultMediaLiveBytes && defaults.media_preprocess_threads == 0, "media preparation resource defaults mismatch"); + failures += check(defaults.image_token_budget == 0, + "an image serving ceiling is unexpectedly applied by default"); failures += check(defaults.kv_capacity.mode == ninfer::KvCapacityMode::Explicit && defaults.kv_capacity.explicit_tokens == defaults.max_context, "default KV capacity does not follow max context"); @@ -72,6 +74,11 @@ int main() { } catch (const std::invalid_argument&) { empty_model_id_rejected = true; } failures += check(empty_model_id_rejected, "empty --model-id was accepted"); + const ServeOptions image_budget = + parse({"ninfer-serve", "model.ninfer", "--image-token-budget", "1280"}); + failures += check(image_budget.image_token_budget == 1280, + "--image-token-budget did not carry the per-image Vision-token ceiling"); + const ServeOptions dflash = parse({"ninfer-serve", "model.ninfer", "--spec", "dflash", "--draft-tokens", "15", "--lm-head-draft"}); failures += check(dflash.speculative.backend == ninfer::SpeculativeBackend::DFlash,