From 837d5b35636d731fecca95c39dd0280a02013ba2 Mon Sep 17 00:00:00 2001 From: Guy Stone Date: Mon, 22 Jun 2026 23:01:46 +0000 Subject: [PATCH] [Perf] Skip detokenization in online beam search The online beam_search per-step SamplingParams requests 2*beam_width logprobs, and the engine detokenizes every one of those logprob token ids into strings on every decode step. Beam search ranks candidates by logprob and detokenizes only the final sequences itself (via tokenizer.decode), so the per-step detokenization is pure overhead. Set detokenize=False on the internal SamplingParams to skip it; final outputs are unchanged. Benchmark (vllm bench serve, Qwen3-1.7B, RTX PRO 6000, random 512-in/32-out, 20 prompts, openai-chat): mean E2EL bw20 1543->855 ms (1.80x), bw60 6612->2615 ms (2.53x); output throughput ~1.8-2.5x. nsys shows identical total GPU kernel time (~3.31s) with ~half the wall clock, and GPU SM utilization roughly doubles -- the GPU was being starved by CPU-side detokenization. Co-authored-by: Claude Signed-off-by: Guy Stone --- vllm/entrypoints/generate/beam_search/online.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/entrypoints/generate/beam_search/online.py b/vllm/entrypoints/generate/beam_search/online.py index 1daef9529bee..ac00689ca420 100644 --- a/vllm/entrypoints/generate/beam_search/online.py +++ b/vllm/entrypoints/generate/beam_search/online.py @@ -61,6 +61,11 @@ async def beam_search( logprobs=logprobs_num, max_tokens=1, temperature=temperature, + # Beam search ranks candidates by logprob and detokenizes the final + # sequences itself (see ``tokenizer.decode`` below), so the engine's + # per-step detokenization of all ``2 * beam_width`` logprob token ids + # is pure overhead -- O(beam_width) wasted decodes every step. Skip it. + detokenize=False, ) all_beams = [ BeamSearchSequence(