diff --git a/benchmarks/single_node/dsr1_fp8_mi300x.sh b/benchmarks/single_node/dsr1_fp8_mi300x.sh index a5f161960..3dbff11e4 100644 --- a/benchmarks/single_node/dsr1_fp8_mi300x.sh +++ b/benchmarks/single_node/dsr1_fp8_mi300x.sh @@ -48,10 +48,10 @@ set -x python3 -m sglang.launch_server \ --model-path=$MODEL --host=0.0.0.0 --port=$PORT --trust-remote-code \ --tensor-parallel-size=$TP \ ---mem-fraction-static=0.8 \ +--mem-fraction-static=0.85 \ --cuda-graph-max-bs=128 \ --chunked-prefill-size=131072 \ ---num-continuous-decode-steps=4 \ +--num-continuous-decode-steps=16 \ --max-prefill-tokens=131072 \ --kv-cache-dtype fp8_e4m3 \ --attention-backend aiter \ diff --git a/benchmarks/single_node/kimik2.5_int4_mi300x.sh b/benchmarks/single_node/kimik2.5_int4_mi300x.sh index a05baddeb..dc11bf27a 100755 --- a/benchmarks/single_node/kimik2.5_int4_mi300x.sh +++ b/benchmarks/single_node/kimik2.5_int4_mi300x.sh @@ -45,7 +45,7 @@ vllm serve $MODEL --port $PORT \ --block-size=64 \ --trust-remote-code \ --no-enable-prefix-caching \ ---max-num-seqs 256 \ +--max-num-seqs 128 \ --mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 16fea938d..1beb14f9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1,3 +1,15 @@ +- config-keys: + - kimik2.5-int4-mi300x-vllm + description: + - "Tune --max-num-seqs 256 → 128 (+34.2% avg output throughput gain)" + pr-link: TBD + +- config-keys: + - dsr1-fp8-mi300x-sglang + description: + - "Tune --mem-fraction-static 0.8 → 0.85, --num-continuous-decode-steps 4 → 16 (+4.2% avg output throughput gain)" + pr-link: TBD + - config-keys: - kimik2.5-fp4-gb200-dynamo-trt description: