diff --git a/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml b/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml index 0163452..049083b 100644 --- a/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml +++ b/config_yml_aira/0705-0940/docker-compose-air_0705-0940.yml @@ -15,15 +15,15 @@ services: - --tensor-parallel-size=1 - --enable-prefix-caching - --enable-chunked-prefill - - --max-num-batched-tokens=8192 - - --max-num-seqs=48 - - --max-cudagraph-capture-size=48 + - --max-num-batched-tokens=${MAX_NUM_BATCHED_TOKENS:-4096} + - --max-num-seqs=${MAX_NUM_SEQS:-48} + - --max-cudagraph-capture-size=${MAX_CUDAGRAPH_CAPTURE_SIZE:-48} - --async-scheduling - --kv-cache-dtype=fp8_e4m3 - --quantization=fp8 - - --max-num-partial-prefills=2 - - --max-long-partial-prefills=1 - - --long-prefill-token-threshold=8192 + - --max-num-partial-prefills=${MAX_NUM_PARTIAL_PREFILLS:-2} + - --max-long-partial-prefills=${MAX_LONG_PARTIAL_PREFILLS:-2} + - --long-prefill-token-threshold=${LONG_PREFILL_TOKEN_THRESHOLD:-24576} ports: - "8000:8000" environment: diff --git a/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py b/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py index c03995f..f5557e6 100644 --- a/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py +++ b/config_yml_aira/0705-0940/patch_vllm_concurrent_prefill_v024.py @@ -4,7 +4,7 @@ The upstream v0.24.0 image exposes max_num_partial_prefills and max_long_partial_prefills in SchedulerConfig, but arg_utils rejects non-default values at startup. This patch removes that guard and adds scheduler-side logic -to let short prefills share a step with one long prefill. +to chunk multiple active prefills fairly, including all-long tail waves. """ from __future__ import annotations @@ -221,24 +221,28 @@ def _build_partial_prefill_metadata(self) -> PartialPrefillMetadata: schedulable_prefills = active_prefills planned_long_prefills = active_long_prefills - for request in self.waiting: + waiting_queues = (self.skipped_waiting, self.waiting) + for request_queue in waiting_queues: + for request in request_queue: + if schedulable_prefills >= max_prefills: + break + if self._is_blocked_waiting_status(request.status): + continue + prefill_state = self._get_request_prefill_state( + request, request.num_computed_tokens + ) + if not prefill_state.is_prefill: + continue + if ( + prefill_state.is_long_prefill + and planned_long_prefills >= max_long_prefills + ): + continue + schedulable_prefills += 1 + if prefill_state.is_long_prefill: + planned_long_prefills += 1 if schedulable_prefills >= max_prefills: break - if self._is_blocked_waiting_status(request.status): - continue - prefill_state = self._get_request_prefill_state( - request, request.num_computed_tokens - ) - if not prefill_state.is_prefill: - continue - if ( - prefill_state.is_long_prefill - and planned_long_prefills >= max_long_prefills - ): - continue - schedulable_prefills += 1 - if prefill_state.is_long_prefill: - planned_long_prefills += 1 return PartialPrefillMetadata( schedulable_prefills=max(1, min(schedulable_prefills, max_prefills)),