From e02ecbf62aaaada45dea7cf51b588a697fe1b285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C6=B0=C6=A1ng=20Xu=C3=A2n=20B=C3=A1ch?= Date: Sun, 5 Jul 2026 12:46:46 +0700 Subject: [PATCH 1/3] tune prefill settings for trace round1 --- .../0705-0940/docker-compose-air_0705-0940.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml b/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml index 0163452..e0aeb02 100644 --- a/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml +++ b/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml @@ -15,15 +15,15 @@ services: - --tensor-parallel-size=1 - --enable-prefix-caching - --enable-chunked-prefill - - --max-num-batched-tokens=8192 - - --max-num-seqs=48 + - --max-num-batched-tokens=${MAX_NUM_BATCHED_TOKENS:-8192} + - --max-num-seqs=${MAX_NUM_SEQS:-48} - --max-cudagraph-capture-size=48 - --async-scheduling - --kv-cache-dtype=fp8_e4m3 - --quantization=fp8 - - --max-num-partial-prefills=2 - - --max-long-partial-prefills=1 - - --long-prefill-token-threshold=8192 + - --max-num-partial-prefills=${MAX_NUM_PARTIAL_PREFILLS:-2} + - --max-long-partial-prefills=${MAX_LONG_PARTIAL_PREFILLS:-2} + - --long-prefill-token-threshold=${LONG_PREFILL_TOKEN_THRESHOLD:-24576} ports: - "8000:8000" environment: From 90c3312c8e4582ac62ba463d5ca0fa047ab5dfef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C6=B0=C6=A1ng=20Xu=C3=A2n=20B=C3=A1ch?= Date: Sun, 5 Jul 2026 12:59:44 +0700 Subject: [PATCH 2/3] account for skipped waiting prefills --- .../patch_vllm_concurrent_prefill_v024.py | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py b/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py index c03995f..f5557e6 100644 --- a/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py +++ b/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py @@ -4,7 +4,7 @@ The upstream v0.24.0 image exposes max_num_partial_prefills and max_long_partial_prefills in SchedulerConfig, but arg_utils rejects non-default values at startup. This patch removes that guard and adds scheduler-side logic -to let short prefills share a step with one long prefill. +to chunk multiple active prefills fairly, including all-long tail waves. """ from __future__ import annotations @@ -221,24 +221,28 @@ def _build_partial_prefill_metadata(self) -> PartialPrefillMetadata: schedulable_prefills = active_prefills planned_long_prefills = active_long_prefills - for request in self.waiting: + waiting_queues = (self.skipped_waiting, self.waiting) + for request_queue in waiting_queues: + for request in request_queue: + if schedulable_prefills >= max_prefills: + break + if self._is_blocked_waiting_status(request.status): + continue + prefill_state = self._get_request_prefill_state( + request, request.num_computed_tokens + ) + if not prefill_state.is_prefill: + continue + if ( + prefill_state.is_long_prefill + and planned_long_prefills >= max_long_prefills + ): + continue + schedulable_prefills += 1 + if prefill_state.is_long_prefill: + planned_long_prefills += 1 if schedulable_prefills >= max_prefills: break - if self._is_blocked_waiting_status(request.status): - continue - prefill_state = self._get_request_prefill_state( - request, request.num_computed_tokens - ) - if not prefill_state.is_prefill: - continue - if ( - prefill_state.is_long_prefill - and planned_long_prefills >= max_long_prefills - ): - continue - schedulable_prefills += 1 - if prefill_state.is_long_prefill: - planned_long_prefills += 1 return PartialPrefillMetadata( schedulable_prefills=max(1, min(schedulable_prefills, max_prefills)), From 5d7f277e2c929badd8073652afc524183e44ca4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C6=B0=C6=A1ng=20Xu=C3=A2n=20B=C3=A1ch?= Date: Sun, 5 Jul 2026 13:08:58 +0700 Subject: [PATCH 3/3] favor decode latency in prefill chunks --- config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml b/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml index e0aeb02..049083b 100644 --- a/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml +++ b/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml @@ -15,9 +15,9 @@ services: - --tensor-parallel-size=1 - --enable-prefix-caching - --enable-chunked-prefill - - --max-num-batched-tokens=${MAX_NUM_BATCHED_TOKENS:-8192} + - --max-num-batched-tokens=${MAX_NUM_BATCHED_TOKENS:-4096} - --max-num-seqs=${MAX_NUM_SEQS:-48} - - --max-cudagraph-capture-size=48 + - --max-cudagraph-capture-size=${MAX_CUDAGRAPH_CAPTURE_SIZE:-48} - --async-scheduling - --kv-cache-dtype=fp8_e4m3 - --quantization=fp8