From e8443639e679c3f6191f1ddc851952d22bb604ff Mon Sep 17 00:00:00 2001 From: "zhangxu.709" Date: Wed, 15 Jul 2026 19:30:00 +0800 Subject: [PATCH 1/5] feat: auto-discover devices from visible-device mask, remove --devices/--device_id flags Remove the --devices, --device_id, and --draft_devices CLI flags along with their c_api/cc_api/pybind surfaces. Device selection now defaults to "auto": each process enumerates its visible devices via Platform::device_count(), honoring *_VISIBLE_DEVICES masks. Add a unified Platform::current_device() (NPU/CUDA/MLU/DCU/MUSA). Rework DistManager::setup_multi_node_workers for multi-process serving: the parsed devices vector is now the visible-card pool rather than "cards this process uses". Each process runs exactly one worker bound to devices[node_rank], so world_size == nnodes instead of devices.size()*nnodes. This keeps every process able to see all cards (so ATB can build the cross-card HCCL comm group) while deriving worker cardinality from nnodes/node_rank. Verified: 2-process TP=2 Qwen3-8B serves correctly, gsm8k mean_acc=0.9234. Update launch docs/examples to select devices via *_VISIBLE_DEVICES instead of --devices. Co-Authored-By: Claude --- docs/src/content/docs/en/cli_reference.md | 5 +- .../deepseek/deepseek_v4.md | 11 +--- .../autoregressive_models/glm/glm_5.md | 21 ++----- .../autoregressive_models/kimi/kimi2_5.md | 6 +- .../minmax/minmax_m2_7.md | 3 +- .../autoregressive_models/qwen/qwen3_5.md | 7 +-- .../cookbook/diffusion_models/flux/flux2.md | 6 +- .../src/content/docs/en/features/disagg_pd.md | 6 +- docs/src/content/docs/en/features/mtp.md | 4 +- .../docs/en/getting_started/launch_xllm.md | 27 ++------ .../docs/en/getting_started/multi_machine.md | 6 +- .../en/getting_started/offline_service.md | 4 +- .../content/docs/en/hardware/ascend_npu.md | 4 +- .../content/docs/en/hardware/cambricon_mlu.md | 5 +- docs/src/content/docs/en/hardware/dcu.md | 5 +- .../content/docs/en/hardware/metax_maca.md | 4 +- docs/src/content/docs/en/hardware/musa.md | 7 +-- .../content/docs/en/hardware/nvidia_gpu.md | 4 +- docs/src/content/docs/en/hardware/overview.md | 2 +- docs/src/content/docs/zh/cli_reference.md | 5 +- .../deepseek/deepseek_v4.md | 11 +--- .../autoregressive_models/glm/glm_5.md | 21 ++----- .../autoregressive_models/kimi/kimi2_5.md | 6 +- .../minmax/minmax_m2_7.md | 3 +- .../autoregressive_models/qwen/qwen3_5.md | 7 +-- .../cookbook/diffusion_models/flux/flux2.md | 6 +- .../src/content/docs/zh/features/disagg_pd.md | 6 +- docs/src/content/docs/zh/features/mtp.md | 4 +- .../docs/zh/getting_started/launch_xllm.md | 27 ++------ .../docs/zh/getting_started/multi_machine.md | 6 +- .../zh/getting_started/offline_service.md | 4 +- .../content/docs/zh/hardware/ascend_npu.md | 4 +- .../content/docs/zh/hardware/cambricon_mlu.md | 5 +- docs/src/content/docs/zh/hardware/dcu.md | 5 +- .../content/docs/zh/hardware/metax_maca.md | 4 +- docs/src/content/docs/zh/hardware/musa.md | 7 +-- .../content/docs/zh/hardware/nvidia_gpu.md | 4 +- examples/generate.py | 4 +- examples/generate_beam_search.py | 4 +- examples/generate_embedding.py | 4 +- examples/generate_vlm.py | 2 +- examples/sample.py | 4 +- tests/core/util/device_name_utils_test.cpp | 8 +-- xllm/c_api/default.h | 2 - .../test_press_rec_multimodal_completions.cpp | 8 +-- .../test_query_llm_chat_completions.cpp | 5 +- .../examples/test_query_llm_completions.cpp | 5 +- .../examples/test_query_rec_completions.cpp | 14 +---- .../test_query_rec_multimodal_completions.cpp | 5 +- xllm/c_api/internal/llm.cpp | 5 +- xllm/c_api/internal/rec.cpp | 5 +- xllm/c_api/llm.h | 7 +-- xllm/c_api/rec.h | 9 +-- xllm/c_api/test/utils.cpp | 4 -- xllm/c_api/test/utils.h | 1 - xllm/c_api/test/xllm_test.cpp | 20 +++--- xllm/c_api/types.h | 6 -- .../examples/multiple_llm_instances.cpp | 5 +- xllm/cc_api/examples/single_llm_instance.cpp | 3 +- xllm/cc_api/llm.cpp | 3 - xllm/cc_api/llm.h | 6 +- xllm/cc_api/types.h | 3 - xllm/core/common/global_flags.h | 6 -- xllm/core/common/options.cpp | 2 - xllm/core/common/options.h | 4 -- .../core/distributed_runtime/dist_manager.cpp | 62 ++++++++++--------- xllm/core/distributed_runtime/master.cpp | 7 +-- xllm/core/framework/config/model_config.cpp | 29 --------- xllm/core/framework/config/model_config.h | 6 -- .../framework/config/speculative_config.cpp | 16 ----- .../framework/config/speculative_config.h | 3 - xllm/core/platform/device_name_utils.cpp | 4 -- xllm/core/platform/device_name_utils.h | 2 - xllm/core/platform/platform.cpp | 14 +++++ xllm/core/platform/platform.h | 3 + xllm/launch_server.py | 21 ------- xllm/pybind/args.py | 2 - xllm/pybind/bind.cpp | 2 - xllm/pybind/embedding.py | 3 - xllm/pybind/llm.py | 4 -- xllm/pybind/vlm.py | 4 -- xllm/xllm.cpp | 11 +--- 82 files changed, 173 insertions(+), 446 deletions(-) diff --git a/docs/src/content/docs/en/cli_reference.md b/docs/src/content/docs/en/cli_reference.md index 66d8c75bdf..ad2ac5d08a 100644 --- a/docs/src/content/docs/en/cli_reference.md +++ b/docs/src/content/docs/en/cli_reference.md @@ -6,6 +6,8 @@ sidebar: xLLM uses gflags to manage service startup parameters. `--model ` is the only required flag. When `--config_json_file` is used, values in the JSON file override command-line flag values. The tables below are grouped by the Config classes in `/xllm/core/framework/config`, with one Config per section. The `ConfigJsonUtils` section contains the common JSON config-file flags. +> **Device selection**: xLLM no longer provides `--devices` / `--device_id` / `--draft_devices`. The devices to run on are determined by the visible-device mask environment variables (`ASCEND_RT_VISIBLE_DEVICES` for NPU, `CUDA_VISIBLE_DEVICES` for NVIDIA, `MLU_VISIBLE_DEVICES` for Cambricon, `HIP_VISIBLE_DEVICES` for DCU, `MUSA_VISIBLE_DEVICES` for Moore Threads). Each process uses all of its visible devices: a single-card process exposes one card, while single-process multi-card exposes several. The draft model always uses the same devices as the target model. + ## ConfigJsonUtils | Parameter | Type | Default | Description | @@ -37,8 +39,6 @@ xLLM uses gflags to manage service startup parameters. `--model ` is the o | `model` | `string` | `""` | Hugging Face model name or model path. | | `backend` | `string` | `""` | Backend model type: `llm` for text-only models, `vlm` for multimodal models, or `dit` for diffusion models. | | `task` | `string` | `"generate"` | Model task, for example `generate`, `embed`, or `mm_embed`. | -| `devices` | `string` | `""` | Deprecated; use `device_id` instead. Devices used by the current process, for example `npu:0` or `npu:0,npu:1`. | -| `device_id` | `int32` | `-1` | Device id to run the model on, for example `0`. | | `limit_image_per_prompt` | `int32` | `8` | Maximum number of images per prompt. Only applies to multimodal models. | | `max_encoder_cache_size` | `int64` | `0` | Maximum GPU/NPU memory size in MB for the encoder cache per worker. `0` disables the encoder cache. | | `reasoning_parser` | `string` | `""` | Reasoning parser, for example `auto`, `glm45`, `glm47`, `glm5`, `qwen3`, `qwen35`, or `deepseek-r1`. | @@ -181,7 +181,6 @@ xLLM uses gflags to manage service startup parameters. `--model ` is the o | Parameter | Type | Default | Description | |:----------|:-----|:--------|:------------| | `draft_model` | `string` | `""` | Draft model path. See [MTP](/en/features/mtp/) for MTP usage. | -| `draft_devices` | `string` | `""` | Devices used by the draft model, for example `npu:0` or `npu:0,npu:1`. If omitted, uses the target model devices when speculative decoding is enabled. | | `num_speculative_tokens` | `int32` | `0` | Number of speculative tokens generated per speculative decoding step. | | `speculative_algorithm` | `string` | `"MTP"` | Speculative decoding algorithm. Supported values: `MTP`, `Eagle3`, `Suffix`, `DFlash`. | | `speculative_suffix_cache_max_depth` | `int32` | `64` | Maximum suffix-tree depth for suffix speculative decoding. | diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md index f68b06b4af..62abadb2f2 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -140,11 +140,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup $XLLM_PATH -model-id ds \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -166,7 +165,6 @@ done # Variables required when MTP is enabled # --draft_model=$DRAFT_MODEL_PATH \ - # --draft_devices="npu:$DEVICE" \ # --num_speculative_tokens=1 \ # numactl -C xxxxx NUMA core binding (query with: npu-smi info -t topo) @@ -179,7 +177,6 @@ done #--enable_chunked_prefill Enable chunked prefill #--enable_graph Enable aclgraph #--draft_model MTP - MTP weights path -#--draft_devices MTP - MTP inference device (same as main model) #--num_speculative_tokens MTP - Number of speculative tokens ``` @@ -216,11 +213,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -246,11 +242,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md index fac8101a9d..9498b0b07d 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md @@ -132,10 +132,9 @@ do LOG_FILE="$LOG_DIR/node_$i.log" # Optional: bind CPU cores with numactl. Query NUMA affinity with: npu-smi info -t topo #nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ - nohup $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ --model $MODEL_PATH \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -150,7 +149,6 @@ do --communication_backend="hccl" \ --graph_decode_batch_size_limit=2 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=16 \ --dp_size=2 \ @@ -168,7 +166,6 @@ done # --enable_graph Enable aclgraph. It requires extra memory. # --acl_graph_decode_batch_size_limit Maximum decode batch size for graph capture. It must be <= 32 / (number of speculative tokens + 1). # --draft_model MTP draft-model weight path. -# --draft_devices MTP inference device, the same as the main model. # --num_speculative_tokens Number of speculative tokens predicted by MTP. ``` @@ -207,11 +204,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -226,7 +222,6 @@ do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -254,11 +249,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ @@ -273,7 +267,6 @@ do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -438,11 +431,10 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -455,7 +447,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ @@ -494,11 +485,10 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -511,7 +501,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md index 00c2156c32..c787adb619 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -153,10 +153,9 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -194,10 +193,9 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 89bb876f5d..05a3301ca5 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -176,11 +176,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md index 0166b86be0..6273c0ff9c 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -91,7 +91,6 @@ export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=14,15 export ASDOPS_LOG_TO_STDOUT=1 export ASDOPS_LOG_LEVEL=0 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -120,7 +119,7 @@ DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" MASTER_NODE_ADDR=":32764" START_PORT=18076 -START_DEVICE=0 +START_DEVICE=14 NNODES=2 export HCCL_IF_BASE_PORT=53433 @@ -130,9 +129,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ./xllm/build/xllm/core/server/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -151,7 +149,6 @@ do --max_concurrent_requests=8 \ --backend llm \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 3 \ >> $LOG_FILE 2>&1 & done diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md index 2d98560a70..4670381591 100644 --- a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md @@ -125,9 +125,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" - ./build/xllm/core/server/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -194,12 +193,11 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" - ./build/xllm/core/server/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ --max_memory_utilization=0.6 \ --backend="dit" \ --tp_size=2 \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --port $PORT \ diff --git a/docs/src/content/docs/en/features/disagg_pd.md b/docs/src/content/docs/en/features/disagg_pd.md index 2a0edad184..f1a6832592 100644 --- a/docs/src/content/docs/en/features/disagg_pd.md +++ b/docs/src/content/docs/en/features/disagg_pd.md @@ -43,9 +43,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. - Taking Qwen2-7B as an example - Start Prefill Instance ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=0 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8010 \ - --devices="npu:0" \ --master_node_addr="127.0.0.1:18888" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ @@ -59,9 +58,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. ``` - Start Decode Instance ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=1 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8020 \ - --devices="npu:1" \ --master_node_addr="127.0.0.1:18898" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ diff --git a/docs/src/content/docs/en/features/mtp.md b/docs/src/content/docs/en/features/mtp.md index f4d7ea5f17..93d0d9200d 100644 --- a/docs/src/content/docs/en/features/mtp.md +++ b/docs/src/content/docs/en/features/mtp.md @@ -103,14 +103,12 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup ./xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup ./xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --max_memory_utilization=0.90 \ --max_tokens_per_batch=10000 \ diff --git a/docs/src/content/docs/en/getting_started/launch_xllm.md b/docs/src/content/docs/en/getting_started/launch_xllm.md index 028a6398fb..b3c07edb78 100644 --- a/docs/src/content/docs/en/getting_started/launch_xllm.md +++ b/docs/src/content/docs/en/getting_started/launch_xllm.md @@ -15,7 +15,6 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/model/Qwen3-8B" # Model path @@ -32,9 +31,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -57,7 +55,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 @@ -75,9 +72,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -98,8 +94,6 @@ set -e rm -rf core.* -export MLU_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -114,9 +108,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MLU_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -133,8 +126,6 @@ set -e rm -rf core.* -export HIP_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -149,9 +140,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + HIP_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -168,7 +158,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" @@ -185,9 +174,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -208,8 +196,6 @@ set -e rm -rf core.* -export MUSA_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -224,9 +210,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MUSA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/getting_started/multi_machine.md b/docs/src/content/docs/en/getting_started/multi_machine.md index 199742acef..057bb884a4 100644 --- a/docs/src/content/docs/en/getting_started/multi_machine.md +++ b/docs/src/content/docs/en/getting_started/multi_machine.md @@ -37,11 +37,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - /path/to/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ @@ -89,11 +88,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - /path/to/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ diff --git a/docs/src/content/docs/en/getting_started/offline_service.md b/docs/src/content/docs/en/getting_started/offline_service.md index 5bc87cf513..703f3d162a 100644 --- a/docs/src/content/docs/en/getting_started/offline_service.md +++ b/docs/src/content/docs/en/getting_started/offline_service.md @@ -16,7 +16,9 @@ Use `BeamSearchParams` with `beam_width` greater than `1`, then call `llm.beam_s ```python from xllm import BeamSearchParams, LLM -llm = LLM(model="/path/models/Qwen2-7B-Instruct", devices="npu:0") +# The devices to run on are selected via ASCEND_RT_VISIBLE_DEVICES +# (or CUDA_VISIBLE_DEVICES / MLU_VISIBLE_DEVICES for other platforms). +llm = LLM(model="/path/models/Qwen2-7B-Instruct") params = BeamSearchParams( beam_width=2, top_logprobs=4, diff --git a/docs/src/content/docs/en/hardware/ascend_npu.md b/docs/src/content/docs/en/hardware/ascend_npu.md index fbdb261286..f1a12449cd 100644 --- a/docs/src/content/docs/en/hardware/ascend_npu.md +++ b/docs/src/content/docs/en/hardware/ascend_npu.md @@ -57,7 +57,6 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/model/Qwen3-8B" # Model path @@ -74,9 +73,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/cambricon_mlu.md b/docs/src/content/docs/en/hardware/cambricon_mlu.md index e1da637872..bfc3695c81 100644 --- a/docs/src/content/docs/en/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/en/hardware/cambricon_mlu.md @@ -33,8 +33,6 @@ set -e rm -rf core.* -export MLU_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -49,9 +47,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MLU_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/dcu.md b/docs/src/content/docs/en/hardware/dcu.md index 33ec7a6f6c..99d74a332e 100644 --- a/docs/src/content/docs/en/hardware/dcu.md +++ b/docs/src/content/docs/en/hardware/dcu.md @@ -45,8 +45,6 @@ set -e rm -rf core.* -export HIP_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -61,9 +59,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + HIP_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/metax_maca.md b/docs/src/content/docs/en/hardware/metax_maca.md index 073e60c804..acb769080f 100644 --- a/docs/src/content/docs/en/hardware/metax_maca.md +++ b/docs/src/content/docs/en/hardware/metax_maca.md @@ -47,7 +47,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" @@ -64,9 +63,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/musa.md b/docs/src/content/docs/en/hardware/musa.md index 5471398d08..c0d17432e5 100644 --- a/docs/src/content/docs/en/hardware/musa.md +++ b/docs/src/content/docs/en/hardware/musa.md @@ -5,7 +5,7 @@ sidebar: order: 7 --- -Use the MUSA device backend (`--devices=musa:`) when running xLLM on Mthreads MUSA hardware. +Use the MUSA device backend when running xLLM on Mthreads MUSA hardware; select the visible card(s) with `MUSA_VISIBLE_DEVICES`. ## Image and Container Startup @@ -44,8 +44,6 @@ set -e rm -rf core.* -export MUSA_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -60,9 +58,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MUSA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/nvidia_gpu.md b/docs/src/content/docs/en/hardware/nvidia_gpu.md index be797ce95d..b7d0d1f698 100644 --- a/docs/src/content/docs/en/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/en/hardware/nvidia_gpu.md @@ -41,7 +41,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 @@ -59,9 +58,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/overview.md b/docs/src/content/docs/en/hardware/overview.md index 417011e7c2..e95127b400 100644 --- a/docs/src/content/docs/en/hardware/overview.md +++ b/docs/src/content/docs/en/hardware/overview.md @@ -14,7 +14,7 @@ xLLM supports multiple accelerator backends for large-scale model inference. Thi - [Cambricon MLU](/en/hardware/cambricon_mlu/) - MLU backend setup and launch entry points. - [Hygon DCU](/en/hardware/dcu/) - Hygon DCU backend setup and launch entry points. - [MetaX MACA](/en/hardware/metax_maca/) - MetaX MACA backend setup and launch entry points. -- [Mthreads MUSA](/en/hardware/musa/) - Mthreads MUSA GPU image, build, and `--devices=musa` launch entry points. +- [Mthreads MUSA](/en/hardware/musa/) - Mthreads MUSA GPU image, build, and launch entry points (device selection via `MUSA_VISIBLE_DEVICES`). ## Common Workflow diff --git a/docs/src/content/docs/zh/cli_reference.md b/docs/src/content/docs/zh/cli_reference.md index 368bf908e5..e22b1b4ea6 100644 --- a/docs/src/content/docs/zh/cli_reference.md +++ b/docs/src/content/docs/zh/cli_reference.md @@ -6,6 +6,8 @@ sidebar: xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填参数。使用 `--config_json_file` 时，JSON 文件中的值会覆盖命令行 flag 值。下表按 `/xllm/core/framework/config` 下的 Config 类分组，一个 Config 对应一节；`ConfigJsonUtils` 一节包含配置文件相关的通用参数。 +> **设备选择**：xLLM 不再提供 `--devices` / `--device_id` / `--draft_devices` 参数。运行设备由可见设备掩码环境变量决定（NPU 用 `ASCEND_RT_VISIBLE_DEVICES`，NVIDIA 用 `CUDA_VISIBLE_DEVICES`，寒武纪用 `MLU_VISIBLE_DEVICES`，DCU 用 `HIP_VISIBLE_DEVICES`，摩尔线程用 `MUSA_VISIBLE_DEVICES`）。每个进程会使用其可见的全部设备：单卡进程只暴露一张卡，单进程多卡则暴露多张卡。draft 模型始终与 target 模型使用相同设备。 + ## ConfigJsonUtils | 参数名称 | 类型 | 默认值 | 参数含义 | @@ -37,8 +39,6 @@ xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填 | `model` | `string` | `""` | Hugging Face 模型名称或模型路径。 | | `backend` | `string` | `""` | 后端模型类型；`llm` 表示纯文本模型，`vlm` 表示多模态模型，`dit` 表示扩散模型。 | | `task` | `string` | `"generate"` | 模型任务类型，例如 `generate`、`embed`、`mm_embed`。 | -| `devices` | `string` | `""` | 已废弃，请改用 `device_id`。当前进程使用的设备，例如 `npu:0`、`npu:0,npu:1`。 | -| `device_id` | `int32` | `-1` | 运行模型的 device id，例如 `0`。 | | `limit_image_per_prompt` | `int32` | `8` | 每个 prompt 允许的最大图片数量，仅用于多模态模型。 | | `max_encoder_cache_size` | `int64` | `0` | 每个 worker 的 encoder cache 最大显存大小，单位 MB；`0` 表示禁用 encoder cache。 | | `reasoning_parser` | `string` | `""` | reasoning 交互解析器，例如 `auto`、`glm45`、`glm47`、`glm5`、`qwen3`、`qwen35`、`deepseek-r1`。 | @@ -181,7 +181,6 @@ xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填 | 参数名称 | 类型 | 默认值 | 参数含义 | |:---------|:-----|:-------|:---------| | `draft_model` | `string` | `""` | draft 模型路径；MTP 使用方式详见 [MTP](/zh/features/mtp/)。 | -| `draft_devices` | `string` | `""` | draft 模型使用的设备，例如 `npu:0`、`npu:0,npu:1`；未指定时，启用 speculative decoding 会使用 target 模型的设备。 | | `num_speculative_tokens` | `int32` | `0` | 每轮 speculative decoding 生成的 speculative token 数。 | | `speculative_algorithm` | `string` | `"MTP"` | Speculative decoding 算法，支持 `MTP`、`Eagle3`、`Suffix`、`DFlash`。 | | `speculative_suffix_cache_max_depth` | `int32` | `64` | Suffix speculative decoding 的后缀树最大深度。 | diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md index 0e2be15dec..a234f531b2 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -140,11 +140,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup $XLLM_PATH -model-id ds \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -166,7 +165,6 @@ done # 开启mtp时需要的变量 # --draft_model=$DRAFT_MODEL_PATH \ - # --draft_devices="npu:$DEVICE" \ # --num_speculative_tokens=1 \ # numactl -C xxxxx 亲和性绑核(NUMA亲和性查询命令： npu-smi info -t topo) @@ -179,7 +177,6 @@ done #--enable_chunked_prefill 开启chunked_prefill #--enable_graph 开启aclgraph #--draft_model mtp - mtp权重路径 -#--draft_devices mtp - mtp推理设备(与主模型同一) #--num_speculative_tokens mtp - 预测token数 ``` @@ -216,11 +213,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -246,11 +242,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md index 1037900c5c..181787a65f 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md @@ -133,10 +133,9 @@ do LOG_FILE="$LOG_DIR/node_$i.log" #可选：numactl绑核 (NUMA亲和性查询命令： npu-smi info -t topo) #nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ - nohup $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ --model $MODEL_PATH \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -151,7 +150,6 @@ do --communication_backend="hccl" \ --graph_decode_batch_size_limit=2 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=16 \ --dp_size=2 \ @@ -169,7 +167,6 @@ done # --enable_graph 开启aclgraph，需要额外显存 # --acl_graph_decode_batch_size_limit 抓图的最大bs，当前需<= 32 / (预测token数 + 1) # --draft_model mtp - mtp权重路径 -# --draft_devices mtp - mtp推理设备(与主模型同一) # --num_speculative_tokens mtp - 预测token数 ``` @@ -206,10 +203,9 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -224,7 +220,6 @@ for (( i=0; i<$LOCAL_NODES; i++ ))do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -250,10 +245,9 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ @@ -268,7 +262,6 @@ for (( i=0; i<$LOCAL_NODES; i++ ))do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -426,11 +419,10 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -443,7 +435,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ @@ -482,11 +473,10 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -499,7 +489,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md index 67c50d772b..d9d0c2fdf0 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -153,10 +153,9 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -194,10 +193,9 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 2fc02f1c05..3291cdcd0d 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -176,11 +176,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md index 39342f4493..c7d6b7c6f1 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -91,7 +91,6 @@ export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=14,15 export ASDOPS_LOG_TO_STDOUT=1 export ASDOPS_LOG_LEVEL=0 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -120,7 +119,7 @@ DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" MASTER_NODE_ADDR=":32764" START_PORT=18076 -START_DEVICE=0 +START_DEVICE=14 NNODES=2 export HCCL_IF_BASE_PORT=53433 @@ -130,9 +129,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ./xllm/build/xllm/core/server/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -151,7 +149,6 @@ do --max_concurrent_requests=8 \ --backend llm \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 3 \ >> $LOG_FILE 2>&1 & done diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md index 950945fda0..f6cc3d9331 100644 --- a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md @@ -126,9 +126,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" - ./build/xllm/core/server/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -195,12 +194,11 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" - ./build/xllm/core/server/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ --max_memory_utilization=0.6 \ --backend="dit" \ --tp_size=2 \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --port $PORT \ diff --git a/docs/src/content/docs/zh/features/disagg_pd.md b/docs/src/content/docs/zh/features/disagg_pd.md index 4dbdd6b310..db66c93d51 100644 --- a/docs/src/content/docs/zh/features/disagg_pd.md +++ b/docs/src/content/docs/zh/features/disagg_pd.md @@ -34,9 +34,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. 4. 以Qwen2-7B为例 - 启动Prefill实例 ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=0 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8010 \ - --devices="npu:0" \ --master_node_addr="127.0.0.1:18888" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ @@ -50,9 +49,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. ``` - 启动Decode实例 ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=1 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8020 \ - --devices="npu:1" \ --master_node_addr="127.0.0.1:18898" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ diff --git a/docs/src/content/docs/zh/features/mtp.md b/docs/src/content/docs/zh/features/mtp.md index eb06c21898..9ae032f355 100644 --- a/docs/src/content/docs/zh/features/mtp.md +++ b/docs/src/content/docs/zh/features/mtp.md @@ -101,14 +101,12 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - nohup ./xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup ./xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --max_memory_utilization=0.90 \ --max_tokens_per_batch=10000 \ diff --git a/docs/src/content/docs/zh/getting_started/launch_xllm.md b/docs/src/content/docs/zh/getting_started/launch_xllm.md index 2855124710..85f6aa9163 100644 --- a/docs/src/content/docs/zh/getting_started/launch_xllm.md +++ b/docs/src/content/docs/zh/getting_started/launch_xllm.md @@ -15,7 +15,6 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 @@ -32,9 +31,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -57,7 +55,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 @@ -75,9 +72,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -98,8 +94,6 @@ set -e rm -rf core.* -export MLU_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -114,9 +108,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MLU_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -133,8 +126,6 @@ set -e rm -rf core.* -export HIP_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -149,9 +140,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + HIP_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -168,7 +158,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" @@ -185,9 +174,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -208,8 +196,6 @@ set -e rm -rf core.* -export MUSA_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -224,9 +210,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MUSA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/getting_started/multi_machine.md b/docs/src/content/docs/zh/getting_started/multi_machine.md index 7eed9e114a..c6f77eeb6b 100644 --- a/docs/src/content/docs/zh/getting_started/multi_machine.md +++ b/docs/src/content/docs/zh/getting_started/multi_machine.md @@ -34,11 +34,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - /path/to/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ @@ -86,11 +85,10 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - /path/to/xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ diff --git a/docs/src/content/docs/zh/getting_started/offline_service.md b/docs/src/content/docs/zh/getting_started/offline_service.md index c9b9fbd9a2..3a9430dd28 100644 --- a/docs/src/content/docs/zh/getting_started/offline_service.md +++ b/docs/src/content/docs/zh/getting_started/offline_service.md @@ -16,7 +16,9 @@ LLM Beam Search 示例：[:simple-github: https://github.com/xLLM-AI/xllm/blob/m ```python from xllm import BeamSearchParams, LLM -llm = LLM(model="/path/models/Qwen2-7B-Instruct", devices="npu:0") +# 运行设备通过 ASCEND_RT_VISIBLE_DEVICES 选择 +# （其他平台使用 CUDA_VISIBLE_DEVICES / MLU_VISIBLE_DEVICES 等）。 +llm = LLM(model="/path/models/Qwen2-7B-Instruct") params = BeamSearchParams( beam_width=2, top_logprobs=4, diff --git a/docs/src/content/docs/zh/hardware/ascend_npu.md b/docs/src/content/docs/zh/hardware/ascend_npu.md index e87fecac9d..fa33455c28 100644 --- a/docs/src/content/docs/zh/hardware/ascend_npu.md +++ b/docs/src/content/docs/zh/hardware/ascend_npu.md @@ -57,7 +57,6 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 @@ -74,9 +73,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/hardware/cambricon_mlu.md b/docs/src/content/docs/zh/hardware/cambricon_mlu.md index a8e4e57726..690ecfb6af 100644 --- a/docs/src/content/docs/zh/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/zh/hardware/cambricon_mlu.md @@ -33,8 +33,6 @@ set -e rm -rf core.* -export MLU_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -49,9 +47,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MLU_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/dcu.md b/docs/src/content/docs/zh/hardware/dcu.md index 5397b7f9a9..bcaf1fe30f 100644 --- a/docs/src/content/docs/zh/hardware/dcu.md +++ b/docs/src/content/docs/zh/hardware/dcu.md @@ -45,8 +45,6 @@ set -e rm -rf core.* -export HIP_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -61,9 +59,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + HIP_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/metax_maca.md b/docs/src/content/docs/zh/hardware/metax_maca.md index 4f9890a8bb..013aa1d68e 100644 --- a/docs/src/content/docs/zh/hardware/metax_maca.md +++ b/docs/src/content/docs/zh/hardware/metax_maca.md @@ -47,7 +47,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" @@ -64,9 +63,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/musa.md b/docs/src/content/docs/zh/hardware/musa.md index 2a2fae5242..375254c95d 100644 --- a/docs/src/content/docs/zh/hardware/musa.md +++ b/docs/src/content/docs/zh/hardware/musa.md @@ -5,7 +5,7 @@ sidebar: order: 7 --- -在摩尔线程（Mthreads）MUSA GPU 上部署 xLLM 时使用 MUSA 设备后端（`--devices=musa:`）。 +在摩尔线程（Mthreads）MUSA GPU 上部署 xLLM 时使用 MUSA 设备后端；通过 `MUSA_VISIBLE_DEVICES` 选择可见的设备卡。 ## 镜像和容器启动命令 @@ -44,8 +44,6 @@ set -e rm -rf core.* -export MUSA_VISIBLE_DEVICES=0 - MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 @@ -60,9 +58,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + MUSA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/hardware/nvidia_gpu.md b/docs/src/content/docs/zh/hardware/nvidia_gpu.md index 120510fd72..0ad3898518 100644 --- a/docs/src/content/docs/zh/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/zh/hardware/nvidia_gpu.md @@ -39,7 +39,6 @@ set -e rm -rf core.* -export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 @@ -57,9 +56,8 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - xllm \ + CUDA_VISIBLE_DEVICES=$DEVICE xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/examples/generate.py b/examples/generate.py index 2a804e475d..55dfd3cf97 100644 --- a/examples/generate.py +++ b/examples/generate.py @@ -1,5 +1,5 @@ -# python examples/generate.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0' -# python generate.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/generate.py --model='/path/models/Qwen2-7B-Instruct' +# ASCEND_RT_VISIBLE_DEVICES=0,1 python generate.py --model='/path/models/Qwen2-7B-Instruct' from xllm import ArgumentParser, LLM, SamplingParams diff --git a/examples/generate_beam_search.py b/examples/generate_beam_search.py index a39a606ff3..78baa3801f 100644 --- a/examples/generate_beam_search.py +++ b/examples/generate_beam_search.py @@ -1,5 +1,5 @@ -# python examples/generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0' -# python generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' +# ASCEND_RT_VISIBLE_DEVICES=0,1 python generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' from xllm import ArgumentParser, BeamSearchParams, LLM diff --git a/examples/generate_embedding.py b/examples/generate_embedding.py index e0e3c55c95..7a6354fcb9 100644 --- a/examples/generate_embedding.py +++ b/examples/generate_embedding.py @@ -1,5 +1,5 @@ -# python examples/generate_embedding.py --model='/path/models/Qwen3-8B' --devices='npu:0' --runner pooling -# python generate_embedding.py --model='/path/models/Qwen3-8B' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/generate_embedding.py --model='/path/models/Qwen3-8B' --runner pooling +# ASCEND_RT_VISIBLE_DEVICES=0,1 python generate_embedding.py --model='/path/models/Qwen3-8B' from xllm import ArgumentParser, LLM, PoolingParams diff --git a/examples/generate_vlm.py b/examples/generate_vlm.py index 406d1591c6..05231be3f8 100755 --- a/examples/generate_vlm.py +++ b/examples/generate_vlm.py @@ -1,4 +1,4 @@ -# python generate_vlm.py --model /path/to/Qwen2.5-VL-7B-Instruct/ --max_seqs_per_batch 4 --devices='npu:0' --enable_shm +# ASCEND_RT_VISIBLE_DEVICES=0 python generate_vlm.py --model /path/to/Qwen2.5-VL-7B-Instruct/ --max_seqs_per_batch 4 --enable_shm from xllm import ArgumentParser, SamplingParams from xllm import LLM diff --git a/examples/sample.py b/examples/sample.py index 5c6cab869b..7613f67026 100644 --- a/examples/sample.py +++ b/examples/sample.py @@ -1,5 +1,5 @@ -# python examples/sample.py --model='/path/models/Qwen3-8B' --devices='npu:0' -# python examples/sample.py --model='/path/models/Qwen3-8B' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/sample.py --model='/path/models/Qwen3-8B' +# ASCEND_RT_VISIBLE_DEVICES=0,1 python examples/sample.py --model='/path/models/Qwen3-8B' from xllm import ArgumentParser, LLM, RequestParams diff --git a/tests/core/util/device_name_utils_test.cpp b/tests/core/util/device_name_utils_test.cpp index d0c4f7079c..093e71fe61 100644 --- a/tests/core/util/device_name_utils_test.cpp +++ b/tests/core/util/device_name_utils_test.cpp @@ -26,15 +26,9 @@ limitations under the License. namespace xllm { namespace { -TEST(DeviceNameUtilsTest, ToDeviceStringUsesCompiledDeviceType) { - const std::string device_string = DeviceNameUtils::to_device_string(3); - - EXPECT_EQ(device_string, Platform::type_str() + ":3"); -} - TEST(DeviceNameUtilsTest, ParseGeneratedDeviceString) { const std::vector devices = - DeviceNameUtils::parse_devices(DeviceNameUtils::to_device_string(2)); + DeviceNameUtils::parse_devices(Platform::type_str() + ":2"); ASSERT_EQ(devices.size(), 1); EXPECT_EQ(devices[0].type(), Platform::type_torch()); diff --git a/xllm/c_api/default.h b/xllm/c_api/default.h index 4ee08e327e..fa735fbe81 100644 --- a/xllm/c_api/default.h +++ b/xllm/c_api/default.h @@ -58,7 +58,6 @@ const XLLM_InitOptions XLLM_INIT_LLM_OPTIONS_DEFAULT = { .kv_cache_transfer_mode = "PUSH", .log_dir = "", .draft_model = "", - .draft_devices = "", .cpu_affinity = ""}; const XLLM_RequestParams XLLM_LLM_REQUEST_PARAMS_DEFAULT = { @@ -132,7 +131,6 @@ const XLLM_InitOptions XLLM_INIT_REC_OPTIONS_DEFAULT = { .kv_cache_transfer_mode = "PUSH", .log_dir = "", .draft_model = "", - .draft_devices = "", .cpu_affinity = ""}; const XLLM_RequestParams XLLM_REC_REQUEST_PARAMS_DEFAULT = { diff --git a/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp b/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp index 54a30e7693..5bc85309da 100644 --- a/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp +++ b/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp @@ -56,7 +56,6 @@ constexpr int32_t kMaxClientParallelism = 128; struct CliOptions { std::string model_path; - std::string devices = "cuda:0"; std::string master_node_addr = "127.0.0.1:18899"; int32_t prompt_size = 128; int32_t token_min_size = 1024; @@ -737,7 +736,6 @@ void PrintSummary(const CliOptions& options, std::cout << "=== stress_rec_multimodal_completions summary ===\n"; std::cout << "model_id=" << ResolveModelId(options.model_path) << "\n"; std::cout << "model_type=" << config.model_type << "\n"; - std::cout << "devices=" << options.devices << "\n"; std::cout << "master_node_addr=" << options.master_node_addr << "\n"; std::cout << "hidden_size=" << config.hidden_size << ", vocab_size=" << config.vocab_size @@ -802,10 +800,8 @@ int main(int argc, char** argv) { sizeof(init_options.master_node_addr), "%s", options.master_node_addr.c_str()); - const bool init_ok = xllm_rec_initialize(rec_handler, - options.model_path.c_str(), - options.devices.c_str(), - &init_options); + const bool init_ok = xllm_rec_initialize( + rec_handler, options.model_path.c_str(), &init_options); if (!init_ok) { xllm_rec_destroy(rec_handler); throw std::runtime_error("xllm_rec_initialize failed"); diff --git a/xllm/c_api/examples/test_query_llm_chat_completions.cpp b/xllm/c_api/examples/test_query_llm_chat_completions.cpp index 90ed928b85..aa7b193689 100644 --- a/xllm/c_api/examples/test_query_llm_chat_completions.cpp +++ b/xllm/c_api/examples/test_query_llm_chat_completions.cpp @@ -20,7 +20,6 @@ limitations under the License. #include "llm.h" -std::string devices = "cuda:4"; std::string model_name = "Qwen3-8B"; std::string model_path = "/export/home/models/Qwen3-8B"; @@ -34,8 +33,8 @@ XLLM_LLM_Handler* service_startup_hook() { snprintf( init_options.log_dir, sizeof(init_options.log_dir), "/export/xllm/log"); - bool ret = xllm_llm_initialize( - llm_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_llm_initialize(llm_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "LLM init failed" << std::endl; xllm_llm_destroy(llm_handler); diff --git a/xllm/c_api/examples/test_query_llm_completions.cpp b/xllm/c_api/examples/test_query_llm_completions.cpp index 70e56e9d30..5021238cef 100644 --- a/xllm/c_api/examples/test_query_llm_completions.cpp +++ b/xllm/c_api/examples/test_query_llm_completions.cpp @@ -20,7 +20,6 @@ limitations under the License. #include "llm.h" -std::string devices = "cuda:1"; std::string model_name = "Qwen3-8B"; std::string model_path = "/export/home/models/Qwen3-8B"; @@ -35,8 +34,8 @@ XLLM_LLM_Handler* service_startup_hook() { snprintf( init_options.log_dir, sizeof(init_options.log_dir), "/export/xllm/log"); - bool ret = xllm_llm_initialize( - llm_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_llm_initialize(llm_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "LLM init failed" << std::endl; xllm_llm_destroy(llm_handler); diff --git a/xllm/c_api/examples/test_query_rec_completions.cpp b/xllm/c_api/examples/test_query_rec_completions.cpp index a1926f9d12..78d7f67a8f 100644 --- a/xllm/c_api/examples/test_query_rec_completions.cpp +++ b/xllm/c_api/examples/test_query_rec_completions.cpp @@ -21,13 +21,6 @@ limitations under the License. #include "rec.h" -#if defined(USE_NPU) -std::string devices = "npu:14"; -#elif defined(USE_CUDA) -std::string devices = "cuda:0"; -#else -std::string devices = "npu:14"; -#endif std::string model_name = "Qwen3-0.6B"; std::string model_path = "/export/home/models/Qwen3-0.6B"; @@ -54,8 +47,8 @@ XLLM_REC_Handler* service_startup_hook() { init_options.rec_worker_max_concurrency = 1; #endif - bool ret = xllm_rec_initialize( - rec_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_rec_initialize(rec_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "REC init failed" << std::endl; xllm_rec_destroy(rec_handler); @@ -74,11 +67,10 @@ void service_stop_hook(XLLM_REC_Handler* rec_handler) { int main(int argc, char** argv) { if (argc > 1) { - devices = argv[1]; + model_path = argv[1]; } std::cout << "Using model path: " << model_path << std::endl; - std::cout << "Using devices: " << devices << std::endl; XLLM_REC_Handler* rec_handler = service_startup_hook(); if (nullptr == rec_handler) { diff --git a/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp b/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp index 56d2bd323e..59127f9401 100644 --- a/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp +++ b/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp @@ -27,7 +27,6 @@ limitations under the License. #include "rec.h" -std::string devices = "cuda:0"; std::string model_name = "homepage_qwen_06b_6_raw"; std::string model_path = "/export/home/models/homepage_qwen_06b_6_raw"; @@ -185,8 +184,8 @@ XLLM_REC_Handler* service_startup_hook() { sizeof(init_options.log_dir), "/export/home/huheng7/log"); - bool ret = xllm_rec_initialize( - rec_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_rec_initialize(rec_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "REC init failed" << std::endl; xllm_rec_destroy(rec_handler); diff --git a/xllm/c_api/internal/llm.cpp b/xllm/c_api/internal/llm.cpp index 3c2ffd18bb..5683a52be8 100644 --- a/xllm/c_api/internal/llm.cpp +++ b/xllm/c_api/internal/llm.cpp @@ -59,9 +59,8 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( XLLM_CAPI_EXPORT bool xllm_llm_initialize( XLLM_LLM_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options) { - if (!handler || !model_path || !devices) return false; + if (!handler || !model_path) return false; try { XLLM_InitOptions xllm_init_options; @@ -81,9 +80,7 @@ XLLM_CAPI_EXPORT bool xllm_llm_initialize( xllm::Options options; options.model_path(model_path) .task_type(xllm_init_options.task) - .devices(devices) .draft_model_path(xllm_init_options.draft_model) - .draft_devices(xllm_init_options.draft_devices) .backend("llm") .block_size(xllm_init_options.block_size) .max_cache_size(xllm_init_options.max_cache_size) diff --git a/xllm/c_api/internal/rec.cpp b/xllm/c_api/internal/rec.cpp index bc0348c941..6048a63b17 100644 --- a/xllm/c_api/internal/rec.cpp +++ b/xllm/c_api/internal/rec.cpp @@ -66,9 +66,8 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( XLLM_CAPI_EXPORT bool xllm_rec_initialize( XLLM_REC_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options) { - if (!handler || !model_path || !devices) return false; + if (!handler || !model_path) return false; try { xllm::CpuAffinity::get_instance().set_cpu_affinity( @@ -91,9 +90,7 @@ XLLM_CAPI_EXPORT bool xllm_rec_initialize( xllm::Options options; options.model_path(model_path) .task_type(xllm_init_options.task) - .devices(devices) .draft_model_path(xllm_init_options.draft_model) - .draft_devices(xllm_init_options.draft_devices) .backend("rec") .block_size(xllm_init_options.block_size) .max_cache_size(xllm_init_options.max_cache_size) diff --git a/xllm/c_api/llm.h b/xllm/c_api/llm.h index 5460ae0da5..72d321c14c 100644 --- a/xllm/c_api/llm.h +++ b/xllm/c_api/llm.h @@ -82,6 +82,8 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( * Loads model weights from the specified path, configures target devices, * initializes compute contexts, and prepares the inference runtime. * Must be called exactly once per handler before using completion/chat APIs. + * The target devices are auto-detected from the visible device mask + * (e.g. ASCEND_RT_VISIBLE_DEVICES / CUDA_VISIBLE_DEVICES). * * If init_options is NULL, this function automatically uses the default values * from XLLM_INIT_LLM_OPTIONS_DEFAULT (via xllm_llm_init_options_default()). @@ -89,9 +91,6 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( * @param handler Valid LLM instance handle (must not be NULL) * @param model_path Null-terminated string of the model directory/file path * (supports .bin/.pth/.safetensors formats) - * @param devices Null-terminated string specifying target devices (format: - * "npu:0,1" (specific NPUs), "cuda:0" (single GPU), "auto" - * (automatic selection)) * @param init_options Advanced initialization options (NULL = use defaults) * * @return true if initialization succeeds; false on failure (see failure causes @@ -100,7 +99,6 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( * @failure_causes * - Invalid handler (NULL or already destroyed) * - Invalid model_path (non-existent, corrupted, or unsupported format) - * - Invalid devices string (malformed format or unavailable devices) * - Model load error (mismatched model architecture or weight corruption) * - Device initialization failure (out of memory, driver error) * @@ -109,7 +107,6 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( */ XLLM_CAPI_EXPORT bool xllm_llm_initialize(XLLM_LLM_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options); /** diff --git a/xllm/c_api/rec.h b/xllm/c_api/rec.h index ef795006f6..2d8fa73657 100644 --- a/xllm/c_api/rec.h +++ b/xllm/c_api/rec.h @@ -75,13 +75,12 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( * @brief Initialize the Generative Recommendation (REC) model and runtime * environment Loads generative recommendation model weights from the specified * path, configures target devices, initializes compute contexts, and prepares - * the recommendation inference runtime + * the recommendation inference runtime. The target devices are auto-detected + * from the visible device mask (e.g. ASCEND_RT_VISIBLE_DEVICES / + * CUDA_VISIBLE_DEVICES). * @param handler Valid REC inference instance handle (must not be NULL) * @param model_path Null-terminated string of the REC model directory/file path * (supports .bin/.pth/.safetensors formats with ranking head) - * @param devices Null-terminated string specifying target devices (format: - * "npu:0,1" (specific NPUs), "cuda:0" (single GPU), "auto" - * (automatic selection)) * @param init_options Advanced initialization options (NULL = use REC defaults) * @return true if initialization succeeds; false on failure (see failure causes * below) @@ -89,7 +88,6 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( * - Invalid handler (NULL or already destroyed) * - Invalid model_path (non-existent, corrupted, or missing ranking head * weights) - * - Invalid devices string (malformed format or unavailable devices) * - Model load error (mismatched REC model architecture or embedding table * corruption) * - Device initialization failure (out of memory, driver error, insufficient @@ -99,7 +97,6 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( */ XLLM_CAPI_EXPORT bool xllm_rec_initialize(XLLM_REC_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options); /** diff --git a/xllm/c_api/test/utils.cpp b/xllm/c_api/test/utils.cpp index e0724df61b..a8e58f9536 100644 --- a/xllm/c_api/test/utils.cpp +++ b/xllm/c_api/test/utils.cpp @@ -23,7 +23,6 @@ limitations under the License. // --- Server + XLLM_InitOptions gflags (defaults aligned with REC defaults) --- DEFINE_string(model_path, "", "Path to REC model weights"); -DEFINE_string(devices, "auto", "Devices string, e.g. npu:0 or auto"); DEFINE_int32(port, 8000, "brpc TCP port"); DEFINE_string(listen_addr, "", @@ -74,7 +73,6 @@ DEFINE_string(kv_cache_transfer_mode, "PUSH", ""); // Not named "log_dir": glog already registers FLAGS_log_dir. DEFINE_string(xllm_init_log_dir, "", ""); DEFINE_string(draft_model, "", ""); -DEFINE_string(draft_devices, "", ""); namespace xllm_capi_test { @@ -192,8 +190,6 @@ void ApplyGflagsToXllmInitOptions(XLLM_InitOptions* o) { o->log_dir, FLAGS_xllm_init_log_dir, XLLM_META_STRING_FIELD_MAX_LEN); CopyToFixed( o->draft_model, FLAGS_draft_model, XLLM_META_STRING_FIELD_MAX_LEN); - CopyToFixed( - o->draft_devices, FLAGS_draft_devices, XLLM_META_STRING_FIELD_MAX_LEN); } void PbToXllmDims(const c_api_test::XLLM_Dims& pb, XLLM_Dims* out) { diff --git a/xllm/c_api/test/utils.h b/xllm/c_api/test/utils.h index a875e733f9..5246291c76 100644 --- a/xllm/c_api/test/utils.h +++ b/xllm/c_api/test/utils.h @@ -26,7 +26,6 @@ limitations under the License. #include "xllm_test.pb.h" DECLARE_string(model_path); -DECLARE_string(devices); DECLARE_int32(port); DECLARE_string(listen_addr); DECLARE_int32(idle_timeout_s); diff --git a/xllm/c_api/test/xllm_test.cpp b/xllm/c_api/test/xllm_test.cpp index b0d0a608d9..8d2873b697 100644 --- a/xllm/c_api/test/xllm_test.cpp +++ b/xllm/c_api/test/xllm_test.cpp @@ -259,12 +259,10 @@ int main(int argc, char* argv[]) { XLLM_InitOptions init{}; xllm_rec_init_options_default(&init); xllm_capi_test::ApplyGflagsToXllmInitOptions(&init); - if (!xllm_rec_initialize(rec_holder.get(), - FLAGS_model_path.c_str(), - FLAGS_devices.c_str(), - &init)) { - LOG(ERROR) << "xllm_rec_initialize failed model_path=" << FLAGS_model_path - << " devices=" << FLAGS_devices; + if (!xllm_rec_initialize( + rec_holder.get(), FLAGS_model_path.c_str(), &init)) { + LOG(ERROR) << "xllm_rec_initialize failed model_path=" + << FLAGS_model_path; return -4; } rec_ptr = rec_holder.get(); @@ -277,12 +275,10 @@ int main(int argc, char* argv[]) { XLLM_InitOptions init{}; xllm_llm_init_options_default(&init); xllm_capi_test::ApplyGflagsToXllmInitOptions(&init); - if (!xllm_llm_initialize(llm_holder.get(), - FLAGS_model_path.c_str(), - FLAGS_devices.c_str(), - &init)) { - LOG(ERROR) << "xllm_llm_initialize failed model_path=" << FLAGS_model_path - << " devices=" << FLAGS_devices; + if (!xllm_llm_initialize( + llm_holder.get(), FLAGS_model_path.c_str(), &init)) { + LOG(ERROR) << "xllm_llm_initialize failed model_path=" + << FLAGS_model_path; return -6; } llm_ptr = llm_holder.get(); diff --git a/xllm/c_api/types.h b/xllm/c_api/types.h index d4a9c2ce01..ad2e4e7a2e 100644 --- a/xllm/c_api/types.h +++ b/xllm/c_api/types.h @@ -178,12 +178,6 @@ typedef struct XLLM_CAPI_EXPORT XLLM_InitOptions { /** Draft hf model path (empty string = no draft model) */ char draft_model[XLLM_META_STRING_FIELD_MAX_LEN]; - /** - * Devices to run the draft model on (e.g. npu:0, npu:0,npu:1). - * Empty string = use the same devices as main model - */ - char draft_devices[XLLM_META_STRING_FIELD_MAX_LEN]; - // CPU affinity settings, format be like: // 1,2,3 // meaing ith thread attach to logical processor 1,2,3 diff --git a/xllm/cc_api/examples/multiple_llm_instances.cpp b/xllm/cc_api/examples/multiple_llm_instances.cpp index eb460e8fc4..49814cb563 100644 --- a/xllm/cc_api/examples/multiple_llm_instances.cpp +++ b/xllm/cc_api/examples/multiple_llm_instances.cpp @@ -26,7 +26,6 @@ limitations under the License. * multiple models needs to be supported, then you can follow this example. */ -std::string devices = "npu:0"; std::string model_path = "/export/home/models/Qwen3-4B"; int main(int argc, char** argv) { @@ -35,7 +34,7 @@ int main(int argc, char** argv) { std::shared_ptr llm_instance_01 = std::make_shared(); xllm::XLLM_InitLLMOptions options_01; options_01.max_memory_utilization = 0.45; - bool ret = llm_instance_01->Initialize(model_path, devices, options_01); + bool ret = llm_instance_01->Initialize(model_path, options_01); if (!ret) { std::cout << "LLM instance init failed." << std::endl; return -1; @@ -60,7 +59,7 @@ int main(int argc, char** argv) { options_02.server_idx = 1; options_02.max_memory_utilization = 0.9; - ret = llm_instance_02->Initialize(model_path, devices, options_02); + ret = llm_instance_02->Initialize(model_path, options_02); if (!ret) { std::cout << "LLM instance init failed." << std::endl; return -1; diff --git a/xllm/cc_api/examples/single_llm_instance.cpp b/xllm/cc_api/examples/single_llm_instance.cpp index 8490d4fdd3..b3bbf45013 100644 --- a/xllm/cc_api/examples/single_llm_instance.cpp +++ b/xllm/cc_api/examples/single_llm_instance.cpp @@ -25,14 +25,13 @@ limitations under the License. * inference engine. */ -std::string devices = "npu:0"; std::string model_path = "/export/home/models/Qwen3-4B"; int main(int argc, char** argv) { xllm::LLM llm_instance; xllm::XLLM_InitLLMOptions options; options.cp_size = 1; - bool ret = llm_instance.Initialize(model_path, devices, options); + bool ret = llm_instance.Initialize(model_path, options); if (!ret) { std::cout << "LLM init failed." << std::endl; return -1; diff --git a/xllm/cc_api/llm.cpp b/xllm/cc_api/llm.cpp index e6c45dc8f6..37d59a3b6d 100644 --- a/xllm/cc_api/llm.cpp +++ b/xllm/cc_api/llm.cpp @@ -59,7 +59,6 @@ LLM::~LLM() { } bool LLM::Initialize(const std::string& model_path, - const std::string& devices, const XLLM_InitLLMOptions& init_options) { if (!init_options.log_dir.empty()) { InitGlog(init_options.log_dir); @@ -74,9 +73,7 @@ bool LLM::Initialize(const std::string& model_path, Options options; options.model_path(model_path) .task_type(init_options.task) - .devices(devices) .draft_model_path(init_options.draft_model) - .draft_devices(init_options.draft_devices) .backend("llm") .block_size(init_options.block_size) .max_cache_size(init_options.max_cache_size) diff --git a/xllm/cc_api/llm.h b/xllm/cc_api/llm.h index 541ab3c110..9e6ab32646 100644 --- a/xllm/cc_api/llm.h +++ b/xllm/cc_api/llm.h @@ -44,16 +44,14 @@ class XLLM_CAPI_EXPORT LLM { * @brief Initialize the model: Load model files and configure runtime * environment * @param model_path Path to model files - * @param devices Device configuration (format: "npu:1" for specific NPU, - * "auto" for auto-selection) * @param init_options Advanced initialization options, Provided default * configuration * @return bool true if initialization succeeds; false if fails * @note Must be called before Completions/ChatCompletions, and only needs to - * be called once + * be called once. The devices to run on are auto-detected from the visible + * device mask (e.g. ASCEND_RT_VISIBLE_DEVICES / CUDA_VISIBLE_DEVICES). */ bool Initialize(const std::string& model_path, - const std::string& devices, const XLLM_InitLLMOptions& init_options); /** diff --git a/xllm/cc_api/types.h b/xllm/cc_api/types.h index f690c82a38..d8ea2b9145 100644 --- a/xllm/cc_api/types.h +++ b/xllm/cc_api/types.h @@ -137,9 +137,6 @@ struct XLLM_CAPI_EXPORT XLLM_InitLLMOptions { // draft hf model path to the model file std::optional draft_model = std::nullopt; - // Devices to run the draft model on, e.g. npu:0, npu:0,npu:1 - std::optional draft_devices = std::nullopt; - // Index ID for internal server ID, which must be set different values // if the model supports multiple version or there are multiple models. int64_t server_idx = 0; diff --git a/xllm/core/common/global_flags.h b/xllm/core/common/global_flags.h index ecc6a13d03..a9d65171fd 100755 --- a/xllm/core/common/global_flags.h +++ b/xllm/core/common/global_flags.h @@ -43,10 +43,6 @@ DECLARE_string(python_model_path); DECLARE_string(task); -DECLARE_string(devices); - -DECLARE_int32(device_id); - DECLARE_int32(limit_image_per_prompt); DECLARE_string(mm_download_headers); @@ -76,8 +72,6 @@ DECLARE_int32(max_tokens_per_chunk_for_prefill); // --- speculative inference config --- DECLARE_string(draft_model); -DECLARE_string(draft_devices); - DECLARE_int32(num_speculative_tokens); DECLARE_string(speculative_algorithm); diff --git a/xllm/core/common/options.cpp b/xllm/core/common/options.cpp index 7bbc65a030..bfe237b1f3 100644 --- a/xllm/core/common/options.cpp +++ b/xllm/core/common/options.cpp @@ -20,9 +20,7 @@ std::string Options::to_string() const { std::stringstream ss; ss << "Options: ["; ss << "model_path: " << model_path() - << ", devices: " << devices().value_or("null") << ", draft_model_path: " << draft_model_path().value_or("null") - << ", draft_devices: " << draft_devices().value_or("null") << ", backend: " << backend() << ", limit_image_per_prompt: " << limit_image_per_prompt() << ", block_size: " << block_size() diff --git a/xllm/core/common/options.h b/xllm/core/common/options.h index 63deb6116a..e1c56661fc 100644 --- a/xllm/core/common/options.h +++ b/xllm/core/common/options.h @@ -39,12 +39,8 @@ class Options { PROPERTY(std::string, model_id); - PROPERTY(std::optional, devices); - PROPERTY(std::optional, draft_model_path); - PROPERTY(std::optional, draft_devices); - // model backend PROPERTY(std::string, backend); diff --git a/xllm/core/distributed_runtime/dist_manager.cpp b/xllm/core/distributed_runtime/dist_manager.cpp index eb66ba45d5..37d4e162bb 100644 --- a/xllm/core/distributed_runtime/dist_manager.cpp +++ b/xllm/core/distributed_runtime/dist_manager.cpp @@ -167,16 +167,24 @@ void DistManager::setup_multi_node_workers( // class. In this class, we create processes, initialize NCCL ProcessGroup, // set up GRPC servers, and so on. - std::vector> dones(devices.size()); - for (size_t i = 0; i < devices.size(); ++i) { - dones[i].store(false, std::memory_order_relaxed); - } + std::vector> dones(1); + dones[0].store(false, std::memory_order_relaxed); CHECK_GE(options.nnodes(), 1) << "At least one node is required"; CHECK_GE(options.node_rank(), 0) << "Node rank must >= 0."; - const int32_t each_node_ranks = static_cast(devices.size()); - const int32_t world_size = each_node_ranks * options.nnodes(); - const int32_t base_rank = options.node_rank() * each_node_ranks; + // Multi-process serving: each process runs exactly one worker. `devices` is + // the visible-device pool (all cards each process can see, required so ATB + // can build the cross-card HCCL comm group); this process binds the single + // card selected by its node_rank. World size is therefore the node count. + const int32_t each_node_ranks = 1; + const int32_t world_size = options.nnodes(); + const int32_t base_rank = options.node_rank(); + const int32_t local_device_index = options.node_rank(); + CHECK_LT(local_device_index, static_cast(devices.size())) + << "node_rank " << local_device_index << " exceeds the number of visible " + << "devices " << devices.size() + << ". Ensure *_VISIBLE_DEVICES exposes all cards used across processes."; + const torch::Device local_device = devices[local_device_index]; const int32_t dp_size = options.dp_size(); const int32_t cp_size = options.cp_size(); const int32_t ep_size = options.ep_size(); @@ -238,26 +246,22 @@ void DistManager::setup_multi_node_workers( } else { LOG(FATAL) << "Unsupported " << model_backend << " in multi-node."; } - // create local workers - for (int32_t i = 0; i < each_node_ranks; ++i) { - // worldsize = 8 - // Node1: 0, 1, 2, 3 - // Node2: 0+4, 1+4, 2+4, 3+4 - const int32_t rank = i + base_rank; + // create the single local worker for this process + { + const int32_t i = 0; + const int32_t rank = base_rank; worker_server_options.server_idx(rank); - // we use spawn process worker to launch a xllm instance - // when start a offline inference task with multi-gpu/npu/mpu/... #if defined(USE_CUDA) || defined(USE_MLU) || defined(USE_DCU) - bool use_spawn_worker = (options.enable_offline_inference() && i > 0) || - force_spawn_for_numa_isolation[i]; - if (force_spawn_for_numa_isolation[i]) { - LOG(INFO) << "Force spawn worker for local rank " << i << " (device " - << devices[i].index() << ", NUMA " << device_numa_nodes[i] + bool use_spawn_worker = force_spawn_for_numa_isolation[local_device_index]; + if (use_spawn_worker) { + LOG(INFO) << "Force spawn worker for node_rank " << local_device_index + << " (device " << local_device.index() << ", NUMA " + << device_numa_nodes[local_device_index] << ") to keep each process within a single NUMA region"; } #else - bool use_spawn_worker = options.enable_offline_inference() && i > 0; + bool use_spawn_worker = false; #endif ParallelArgs parallel_args( rank, world_size, dp_size, cp_size, nullptr, ep_size); @@ -265,9 +269,9 @@ void DistManager::setup_multi_node_workers( servers_.emplace_back(std::make_unique(i, master_node_addr, // done, - dones[i], + dones[0], parallel_args, - devices[i], + local_device, worker_server_options, worker_type, use_spawn_worker)); @@ -297,6 +301,9 @@ void DistManager::setup_multi_node_workers( // check if all workers connected // and then create worker clients + CHECK_LE(static_cast(world_size), devices.size()) + << "world_size " << world_size << " exceeds the visible-device pool " + << devices.size() << "; each global rank maps to one visible card."; for (size_t r = 0; r < world_size; ++r) { if (worker_addrs_map.find(r) == worker_addrs_map.end()) { LOG(FATAL) << "Not all worker connect to engine server. Miss rank is " @@ -306,11 +313,10 @@ void DistManager::setup_multi_node_workers( /* TODO(CP): support smem + CP */ auto channel = create_channel(worker_addrs_map[r], r, dp_local_tp_size, options); - worker_clients_.emplace_back( - std::make_unique(r, - worker_addrs_map[r], - devices[r % each_node_ranks], - std::move(channel))); + // Global rank r runs on the process whose node_rank == r, which binds the + // card devices[r] from the shared visible-device pool. + worker_clients_.emplace_back(std::make_unique( + r, worker_addrs_map[r], devices[r], std::move(channel))); } // Register health check for each worker and start background health check diff --git a/xllm/core/distributed_runtime/master.cpp b/xllm/core/distributed_runtime/master.cpp index 0012e6b15e..e9e754bcd9 100644 --- a/xllm/core/distributed_runtime/master.cpp +++ b/xllm/core/distributed_runtime/master.cpp @@ -199,8 +199,7 @@ Master::Master(const Options& options, EngineType type) master_status_(options.master_status()) { const auto model_path = std::filesystem::path(options_.model_path()).lexically_normal(); - const auto devices = - DeviceNameUtils::parse_devices(options_.devices().value_or("auto")); + const auto devices = DeviceNameUtils::parse_devices("auto"); const int32_t global_world_size = options_.nnodes() * static_cast(devices.size()); std::string cp_model_type; @@ -341,8 +340,8 @@ Master::Master(const Options& options, EngineType type) const bool use_suffix_spec = options_.speculative_algorithm() == "Suffix"; CHECK(use_suffix_spec || !draft_model_path.empty()) << "draft model path is required unless --speculative_algorithm=Suffix"; - const auto draft_devices = DeviceNameUtils::parse_devices( - options_.draft_devices().value_or("auto")); + // Draft model shares the same devices as the target model. + const auto& draft_devices = devices; LOG(INFO) << "Using draft devices: " << DeviceNameUtils::to_string(draft_devices); runtime::Options spec_options; diff --git a/xllm/core/framework/config/model_config.cpp b/xllm/core/framework/config/model_config.cpp index 66bd75a3e3..c4719a75c0 100644 --- a/xllm/core/framework/config/model_config.cpp +++ b/xllm/core/framework/config/model_config.cpp @@ -19,7 +19,6 @@ limitations under the License. #include "core/common/global_flags.h" #include "core/framework/config/config_utils.h" -#include "core/platform/device_name_utils.h" DEFINE_string(model_id, "", "hf model name."); @@ -48,13 +47,6 @@ DEFINE_string(task, "generate", "The task to use the model for(e.g. generate, embed, mm_embed)."); -DEFINE_string(devices, - "", - "Deprecated. Use --device_id instead. Devices to run the model " - "on, e.g. npu:0, npu:0,npu:1."); - -DEFINE_int32(device_id, -1, "Device id to run the model on, e.g. 0."); - DEFINE_int32(limit_image_per_prompt, 8, "Maximum number of image per prompt. Only applicable for " @@ -123,19 +115,6 @@ void ModelConfig::from_flags() { XLLM_CONFIG_ASSIGN_FROM_FLAG(python_model_path); XLLM_CONFIG_ASSIGN_FROM_FLAG(backend); XLLM_CONFIG_ASSIGN_FROM_FLAG(task); - XLLM_CONFIG_ASSIGN_FROM_FLAG(device_id); - const bool devices_specified = config::is_flag_specified("devices"); - const bool device_id_specified = config::is_flag_specified("device_id"); - if (devices_specified) { - LOG(WARNING) << "--devices is deprecated and will be removed in a future " - "release. Use --device_id instead."; - } - if (devices_specified && !device_id_specified) { - XLLM_CONFIG_ASSIGN_FROM_FLAG(devices); - } else { - CHECK(device_id() >= 0) << "--device_id must be >= 0."; - devices(DeviceNameUtils::to_device_string(device_id())); - } XLLM_CONFIG_ASSIGN_FROM_FLAG(limit_image_per_prompt); XLLM_CONFIG_ASSIGN_FROM_FLAG(max_encoder_cache_size); XLLM_CONFIG_ASSIGN_FROM_FLAG(reasoning_parser); @@ -176,9 +155,6 @@ void ModelConfig::from_json(const JsonReader& json) { XLLM_CONFIG_ASSIGN_FROM_JSON(python_model_path); XLLM_CONFIG_ASSIGN_FROM_JSON(backend); XLLM_CONFIG_ASSIGN_FROM_JSON(task); - // don't read rank-related config - // XLLM_CONFIG_ASSIGN_FROM_JSON(device_id); - // XLLM_CONFIG_ASSIGN_FROM_JSON(devices); XLLM_CONFIG_ASSIGN_FROM_JSON(limit_image_per_prompt); XLLM_CONFIG_ASSIGN_FROM_JSON(max_encoder_cache_size); XLLM_CONFIG_ASSIGN_FROM_JSON(reasoning_parser); @@ -203,11 +179,6 @@ void ModelConfig::append_config_json( config_json, default_config, python_model_path); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, backend); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, task); - // don't dump rank-related config - // APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, - // device_id); - // APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, - // devices); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, limit_image_per_prompt); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( diff --git a/xllm/core/framework/config/model_config.h b/xllm/core/framework/config/model_config.h index f2a4f29b42..8566de744c 100644 --- a/xllm/core/framework/config/model_config.h +++ b/xllm/core/framework/config/model_config.h @@ -50,8 +50,6 @@ class ModelConfig final { "model_impl", "backend", "task", - "device_id", - "devices", "python_model_path", "limit_image_per_prompt", "max_encoder_cache_size", @@ -78,10 +76,6 @@ class ModelConfig final { PROPERTY(std::string, task) = "generate"; - PROPERTY(int32_t, device_id) = -1; - - PROPERTY(std::string, devices) = ""; - PROPERTY(int32_t, limit_image_per_prompt) = 8; PROPERTY(int64_t, max_encoder_cache_size) = 0; diff --git a/xllm/core/framework/config/speculative_config.cpp b/xllm/core/framework/config/speculative_config.cpp index d74b498e0b..beb20e93ce 100644 --- a/xllm/core/framework/config/speculative_config.cpp +++ b/xllm/core/framework/config/speculative_config.cpp @@ -22,12 +22,6 @@ limitations under the License. DEFINE_string(draft_model, "", "draft hf model path to the model file."); -DEFINE_string(draft_devices, - "", - "Devices to run the draft model on, e.g. npu:0, npu:0,npu:1. " - "If omitted, uses the target model devices when speculative " - "decoding is enabled."); - DEFINE_int32(num_speculative_tokens, 0, "Number of speculative tokens."); DEFINE_string(speculative_algorithm, @@ -80,11 +74,6 @@ namespace xllm { void SpeculativeConfig::from_flags() { XLLM_CONFIG_ASSIGN_FROM_FLAG(draft_model); - if (config::is_flag_specified("draft_devices")) { - LOG(WARNING) << "--draft_devices is deprecated and will be removed in a " - "future release. Because it's same as --devices."; - } - XLLM_CONFIG_ASSIGN_FROM_FLAG(draft_devices); XLLM_CONFIG_ASSIGN_FROM_FLAG(num_speculative_tokens); XLLM_CONFIG_ASSIGN_FROM_FLAG(speculative_algorithm); XLLM_CONFIG_ASSIGN_FROM_FLAG(speculative_suffix_cache_max_depth); @@ -100,8 +89,6 @@ void SpeculativeConfig::from_flags() { void SpeculativeConfig::from_json(const JsonReader& json) { XLLM_CONFIG_ASSIGN_FROM_JSON(draft_model); - // don't read rank-related config - // XLLM_CONFIG_ASSIGN_FROM_JSON(draft_devices); XLLM_CONFIG_ASSIGN_FROM_JSON(num_speculative_tokens); XLLM_CONFIG_ASSIGN_FROM_JSON(speculative_algorithm); XLLM_CONFIG_ASSIGN_FROM_JSON(speculative_suffix_cache_max_depth); @@ -120,9 +107,6 @@ void SpeculativeConfig::append_config_json( const SpeculativeConfig default_config; APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, draft_model); - // don't dump rank-related config - // APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( - // config_json, default_config, draft_devices); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, num_speculative_tokens); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( diff --git a/xllm/core/framework/config/speculative_config.h b/xllm/core/framework/config/speculative_config.h index 54c98b9db9..b259236b52 100644 --- a/xllm/core/framework/config/speculative_config.h +++ b/xllm/core/framework/config/speculative_config.h @@ -55,7 +55,6 @@ class SpeculativeConfig final { static const OptionCategory kOptionCategory = { "SPECULATIVE OPTIONS", {"draft_model", - "draft_devices", "num_speculative_tokens", "speculative_algorithm", "speculative_suffix_cache_max_depth", @@ -72,8 +71,6 @@ class SpeculativeConfig final { PROPERTY(std::string, draft_model); - PROPERTY(std::string, draft_devices) = ""; - PROPERTY(int32_t, num_speculative_tokens) = 0; PROPERTY(std::string, speculative_algorithm) = "MTP"; diff --git a/xllm/core/platform/device_name_utils.cpp b/xllm/core/platform/device_name_utils.cpp index 6efdb523dc..6b06324875 100644 --- a/xllm/core/platform/device_name_utils.cpp +++ b/xllm/core/platform/device_name_utils.cpp @@ -29,10 +29,6 @@ limitations under the License. namespace xllm { -std::string DeviceNameUtils::to_device_string(int32_t device_id) { - return Platform::type_str() + ":" + std::to_string(device_id); -} - std::vector DeviceNameUtils::parse_devices( const std::string& device_str) { std::vector devices; diff --git a/xllm/core/platform/device_name_utils.h b/xllm/core/platform/device_name_utils.h index ca4c0aa872..3adc3849ab 100644 --- a/xllm/core/platform/device_name_utils.h +++ b/xllm/core/platform/device_name_utils.h @@ -27,8 +27,6 @@ namespace xllm { class DeviceNameUtils { public: - static std::string to_device_string(int32_t device_id); - static std::vector parse_devices( const std::string& device_str); diff --git a/xllm/core/platform/platform.cpp b/xllm/core/platform/platform.cpp index 6503a60ebc..a12d3910e4 100644 --- a/xllm/core/platform/platform.cpp +++ b/xllm/core/platform/platform.cpp @@ -84,6 +84,20 @@ int32_t Platform::device_count() { #endif } +int32_t Platform::current_device() { +#if defined(USE_NPU) + return static_cast(c10_npu::current_device()); +#elif defined(USE_MLU) + return static_cast(torch_mlu::current_device()); +#elif defined(USE_CUDA) || defined(USE_ILU) + return static_cast(c10::cuda::current_device()); +#elif defined(USE_MUSA) + return static_cast(c10::musa::current_device()); +#elif defined(USE_DCU) + return static_cast(c10::hip::current_device()); +#endif +} + bool Platform::is_enable_pdl() { return enable_pdl_; } int32_t Platform::sm_count() { return sm_count_; } diff --git a/xllm/core/platform/platform.h b/xllm/core/platform/platform.h index c2a29ea7e3..67c2ed77c6 100644 --- a/xllm/core/platform/platform.h +++ b/xllm/core/platform/platform.h @@ -81,6 +81,9 @@ class Platform final { } static int32_t device_count(); + // Returns the logical index of the device bound to the current thread. + // Valid only after the device has been set (e.g. via Device::set_device). + static int32_t current_device(); static int32_t sm_count(); static bool is_enable_pdl(); static bool is_support_sm90a(); diff --git a/xllm/launch_server.py b/xllm/launch_server.py index 142b47beb7..c4c5c0c888 100644 --- a/xllm/launch_server.py +++ b/xllm/launch_server.py @@ -100,14 +100,6 @@ def _build_parser() -> argparse.ArgumentParser: default=None, help="Launch only this rank. If omitted, local ranks 0..nnodes-1 are launched.", ) - parser.add_argument( - "--start-device-id", - "--start_device_id", - dest="start_device_id", - type=int, - default=0, - help="Base logical device id. Local multi-rank launch uses id + rank.", - ) parser.add_argument( "--log-dir", "--log_dir", @@ -195,8 +187,6 @@ def _validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> parser.error("--nnodes must be greater than 0") if args.start_port < 1 or args.start_port > 65535: parser.error("--port/--start-port must be in range [1, 65535]") - if args.start_device_id < 0: - parser.error("--start-device-id must be greater than or equal to 0") if args.node_rank is not None and ( args.node_rank < 0 or args.node_rank >= args.nnodes ): @@ -224,15 +214,6 @@ def _resolve_port( return args.start_port -def _resolve_device_id( - args: argparse.Namespace, - rank: int, - launches_all_local_ranks: bool, -) -> int: - rank_offset = rank if launches_all_local_ranks else 0 - return args.start_device_id + rank_offset - - def _build_command( binary_path: str, args: argparse.Namespace, @@ -241,7 +222,6 @@ def _build_command( launches_all_local_ranks: bool, ) -> list[str]: port = _resolve_port(args, rank, launches_all_local_ranks) - device_id = _resolve_device_id(args, rank, launches_all_local_ranks) command = [binary_path] if args.config_json_file is not None: @@ -249,7 +229,6 @@ def _build_command( command.append(f"--port={port}") command.append(f"--nnodes={args.nnodes}") command.append(f"--node_rank={rank}") - command.append(f"--device_id={device_id}") command.extend(extra_args) return command diff --git a/xllm/pybind/args.py b/xllm/pybind/args.py index 7e4767c6a7..6dd34a4936 100644 --- a/xllm/pybind/args.py +++ b/xllm/pybind/args.py @@ -34,9 +34,7 @@ def __init__(self) -> None: self.parser.add_argument('--model', type=str, help='"Name or path of the huggingface model to use."') self.parser.add_argument('--task', type=str, default="generate", help='The task to use the model for. generate/embed.') self.parser.add_argument('--runner', type=str, choices=['pooling'], default=None, help='Optional runner mode for LLM. Currently supports: pooling.') - self.parser.add_argument('--devices', type=str, default='npu:0', help='Devices to run the model on, e.g. npu:0,npu:1.') self.parser.add_argument('--draft_model', type=str, default='', help='draft hf model path to the model file.') - self.parser.add_argument('--draft_devices', type=str, default='npu:0', help='Devices to run the draft model on, e.g. npu:0,npu:1.') self.parser.add_argument('--limit_image_per_prompt', type=int, default=8, help='Maximum number of images per prompt.') self.parser.add_argument('--block_size', type=int, default=128, help='Number of slots per kv cache block. Default is 128.') self.parser.add_argument('--max_cache_size', type=int, default=0, help='Max gpu memory size for kv cache. Default is 0, which means cache size is calculated by available memory.') diff --git a/xllm/pybind/bind.cpp b/xllm/pybind/bind.cpp index 412535cd24..aab3abf560 100644 --- a/xllm/pybind/bind.cpp +++ b/xllm/pybind/bind.cpp @@ -40,9 +40,7 @@ PYBIND11_MODULE(xllm_export, m) { py::class_(m, "Options") .def(py::init()) .def_readwrite("model_path", &Options::model_path_) - .def_readwrite("devices", &Options::devices_) .def_readwrite("draft_model_path", &Options::draft_model_path_) - .def_readwrite("draft_devices", &Options::draft_devices_) .def_readwrite("backend", &Options::backend_) .def_readwrite("limit_image_per_prompt", &Options::limit_image_per_prompt_) diff --git a/xllm/pybind/embedding.py b/xllm/pybind/embedding.py index 024f7f88b3..3593b7d41d 100644 --- a/xllm/pybind/embedding.py +++ b/xllm/pybind/embedding.py @@ -28,7 +28,6 @@ class Embedding: def __init__( self, model: str, - devices: str = 'npu:0', limit_image_per_prompt: int = 8, block_size: int = 128, max_cache_size: int = 0, @@ -76,9 +75,7 @@ def __init__( options = Options() options.model_path = model options.task_type = "embed" - options.devices = devices options.draft_model_path = None - options.draft_devices = None options.backend = "llm" options.limit_image_per_prompt = limit_image_per_prompt options.block_size = block_size diff --git a/xllm/pybind/llm.py b/xllm/pybind/llm.py index c15fdf51f7..969fb7f3b1 100644 --- a/xllm/pybind/llm.py +++ b/xllm/pybind/llm.py @@ -93,9 +93,7 @@ def __init__( model: str, task: str = "generate", runner: Optional[str] = None, - devices: str = 'npu:0', draft_model: Optional[str] = '', - draft_devices: Optional[str] = 'npu:0', limit_image_per_prompt: int = 8, block_size: int = 128, max_cache_size: int = 0, @@ -165,9 +163,7 @@ def __init__( options = Options() options.model_path = model options.task_type = task - options.devices = devices options.draft_model_path = draft_model - options.draft_devices = draft_devices options.backend = backend options.limit_image_per_prompt = limit_image_per_prompt options.block_size = block_size diff --git a/xllm/pybind/vlm.py b/xllm/pybind/vlm.py index 2f52ea5560..84c85da03b 100644 --- a/xllm/pybind/vlm.py +++ b/xllm/pybind/vlm.py @@ -50,9 +50,7 @@ def __init__( self, model: str, task: str = "generate", - devices: str = 'npu:0', draft_model: Optional[str] = '', - draft_devices: Optional[str] = 'npu:0', limit_image_per_prompt: int = 8, block_size: int = 128, max_cache_size: int = 0, @@ -107,9 +105,7 @@ def __init__( options = Options() options.model_path = model options.task_type = task - options.devices = devices options.draft_model_path = draft_model - options.draft_devices = draft_devices options.backend ="vlm" options.limit_image_per_prompt = limit_image_per_prompt options.block_size = block_size diff --git a/xllm/xllm.cpp b/xllm/xllm.cpp index e39787d061..d0ebb126fe 100644 --- a/xllm/xllm.cpp +++ b/xllm/xllm.cpp @@ -119,7 +119,6 @@ Options create_options(const std::string& instance_name, bool is_local) { options.model_path(model_config.model()) .model_id(model_config.model_id()) .task_type(model_config.task()) - .devices(model_config.devices()) .draft_model_path(speculative_config.draft_model()) .backend(model_config.backend()) .limit_image_per_prompt(model_config.limit_image_per_prompt()) @@ -224,13 +223,6 @@ Options create_options(const std::string& instance_name, bool is_local) { static_cast(rec_config.rec_worker_max_concurrency())) .is_local(is_local); - if (speculative_config.num_speculative_tokens() > 0) { - const std::string draft_devices = speculative_config.draft_devices().empty() - ? model_config.devices() - : speculative_config.draft_devices(); - options.draft_devices(draft_devices); - } - return options; } @@ -427,8 +419,7 @@ int run() { // init XTensor allocator and PhyPagePool for xtensor mode if (kv_cache_config.enable_xtensor()) { // Parse devices - const auto devices = - DeviceNameUtils::parse_devices(options.devices().value_or("auto")); + const auto devices = DeviceNameUtils::parse_devices("auto"); // Initialize XTensorAllocator with first device auto& allocator = XTensorAllocator::get_instance(); From 0c9a48914f5a491038e72167dc8771be12bc6439 Mon Sep 17 00:00:00 2001 From: "zhangxu.709" Date: Fri, 17 Jul 2026 12:29:54 +0800 Subject: [PATCH 2/5] fix: derive CP-config global_world_size from nnodes, not devices.size() The CP-config validation in Master computed global_world_size as nnodes * devices.size(), which double-counts now that the devices vector is the visible-card pool rather than the per-process worker list. Multi-process serving runs one worker per process, so the global world size is the node count. Aligns this path with DistManager::setup_multi_node_workers. Co-Authored-By: Claude --- xllm/core/distributed_runtime/master.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xllm/core/distributed_runtime/master.cpp b/xllm/core/distributed_runtime/master.cpp index e9e754bcd9..62ad1447cb 100644 --- a/xllm/core/distributed_runtime/master.cpp +++ b/xllm/core/distributed_runtime/master.cpp @@ -200,8 +200,9 @@ Master::Master(const Options& options, EngineType type) const auto model_path = std::filesystem::path(options_.model_path()).lexically_normal(); const auto devices = DeviceNameUtils::parse_devices("auto"); - const int32_t global_world_size = - options_.nnodes() * static_cast(devices.size()); + // Multi-process serving runs one worker per process, so the global world + // size is the node count. `devices` is only the visible-card pool here. + const int32_t global_world_size = options_.nnodes(); std::string cp_model_type; if (options_.cp_size() > 1 && Platform::uses_model_cp_partition()) { cp_model_type = util::get_model_type(model_path, options_.backend()); From acf31cd638a35cb38e76159d3fcc13685d33b7e4 Mon Sep 17 00:00:00 2001 From: "zhangxu.709" Date: Fri, 17 Jul 2026 13:47:00 +0800 Subject: [PATCH 3/5] docs: launch without a per-process device mask so all visible cards are used Multi-process serving requires every process to see all cards (so ATB can build the cross-card HCCL comm group); each process then binds the card selected by its --node_rank. Masking each process to a single card (_VISIBLE_DEVICES=$DEVICE) makes the visible-device pool size 1 and trips the node_rank bounds check for rank>=1. Revert the launch snippets to invoke the binary without a per-process mask (no mask means all cards are visible), distinguishing processes only by --node_rank. Preserve the numactl CPU-affinity $DEVICE usage and fix stray backslash-continuation lines. Co-Authored-By: Claude --- .../autoregressive_models/deepseek/deepseek_v4.md | 6 +++--- .../en/cookbook/autoregressive_models/glm/glm_5.md | 10 +++++----- .../cookbook/autoregressive_models/kimi/kimi2_5.md | 6 ++++-- .../autoregressive_models/minmax/minmax_m2_7.md | 2 +- .../cookbook/autoregressive_models/qwen/qwen3_5.md | 2 +- .../docs/en/cookbook/diffusion_models/flux/flux2.md | 4 ++-- docs/src/content/docs/en/features/mtp.md | 2 +- .../content/docs/en/getting_started/launch_xllm.md | 12 ++++++------ .../content/docs/en/getting_started/multi_machine.md | 4 ++-- docs/src/content/docs/en/hardware/ascend_npu.md | 2 +- docs/src/content/docs/en/hardware/cambricon_mlu.md | 2 +- docs/src/content/docs/en/hardware/dcu.md | 2 +- docs/src/content/docs/en/hardware/metax_maca.md | 2 +- docs/src/content/docs/en/hardware/musa.md | 2 +- docs/src/content/docs/en/hardware/nvidia_gpu.md | 2 +- .../autoregressive_models/deepseek/deepseek_v4.md | 6 +++--- .../zh/cookbook/autoregressive_models/glm/glm_5.md | 12 +++++++----- .../cookbook/autoregressive_models/kimi/kimi2_5.md | 6 ++++-- .../autoregressive_models/minmax/minmax_m2_7.md | 2 +- .../cookbook/autoregressive_models/qwen/qwen3_5.md | 2 +- .../docs/zh/cookbook/diffusion_models/flux/flux2.md | 4 ++-- docs/src/content/docs/zh/features/mtp.md | 2 +- .../content/docs/zh/getting_started/launch_xllm.md | 12 ++++++------ .../content/docs/zh/getting_started/multi_machine.md | 4 ++-- docs/src/content/docs/zh/hardware/ascend_npu.md | 2 +- docs/src/content/docs/zh/hardware/cambricon_mlu.md | 2 +- docs/src/content/docs/zh/hardware/dcu.md | 2 +- docs/src/content/docs/zh/hardware/metax_maca.md | 2 +- docs/src/content/docs/zh/hardware/musa.md | 2 +- docs/src/content/docs/zh/hardware/nvidia_gpu.md | 2 +- 30 files changed, 64 insertions(+), 58 deletions(-) diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md index 62abadb2f2..d430e8a220 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -140,7 +140,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH -model-id ds \ + nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -213,7 +213,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ + nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -242,7 +242,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ + nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md index 9498b0b07d..9a57acda10 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md @@ -132,7 +132,7 @@ do LOG_FILE="$LOG_DIR/node_$i.log" # Optional: bind CPU cores with numactl. Query NUMA affinity with: npu-smi info -t topo #nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ + nohup $XLLM_PATH \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -204,7 +204,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -249,7 +249,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -431,7 +431,7 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ @@ -485,7 +485,7 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md index c787adb619..c6f4eeea4e 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -153,7 +153,8 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -193,7 +194,8 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 05a3301ca5..5b7bdabf69 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -176,7 +176,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md index 6273c0ff9c..2845bf54b0 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -129,7 +129,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./xllm/build/xllm/core/server/xllm \ + ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md index 4670381591..740421f3cf 100644 --- a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md @@ -125,7 +125,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ + ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -193,7 +193,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ + ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ --max_memory_utilization=0.6 \ --backend="dit" \ diff --git a/docs/src/content/docs/en/features/mtp.md b/docs/src/content/docs/en/features/mtp.md index 93d0d9200d..2f11874b3e 100644 --- a/docs/src/content/docs/en/features/mtp.md +++ b/docs/src/content/docs/en/features/mtp.md @@ -103,7 +103,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup ./xllm \ + nohup ./xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/getting_started/launch_xllm.md b/docs/src/content/docs/en/getting_started/launch_xllm.md index b3c07edb78..5eb76b006f 100644 --- a/docs/src/content/docs/en/getting_started/launch_xllm.md +++ b/docs/src/content/docs/en/getting_started/launch_xllm.md @@ -31,7 +31,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -72,7 +72,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -108,7 +108,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MLU_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ @@ -140,7 +140,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - HIP_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ @@ -174,7 +174,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ @@ -210,7 +210,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MUSA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/getting_started/multi_machine.md b/docs/src/content/docs/en/getting_started/multi_machine.md index 057bb884a4..69b57627e1 100644 --- a/docs/src/content/docs/en/getting_started/multi_machine.md +++ b/docs/src/content/docs/en/getting_started/multi_machine.md @@ -37,7 +37,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ + /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -88,7 +88,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ + /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/en/hardware/ascend_npu.md b/docs/src/content/docs/en/hardware/ascend_npu.md index f1a12449cd..302f8f0c9f 100644 --- a/docs/src/content/docs/en/hardware/ascend_npu.md +++ b/docs/src/content/docs/en/hardware/ascend_npu.md @@ -73,7 +73,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/cambricon_mlu.md b/docs/src/content/docs/en/hardware/cambricon_mlu.md index bfc3695c81..538bef5c8d 100644 --- a/docs/src/content/docs/en/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/en/hardware/cambricon_mlu.md @@ -47,7 +47,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MLU_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/dcu.md b/docs/src/content/docs/en/hardware/dcu.md index 99d74a332e..3df9cbe58f 100644 --- a/docs/src/content/docs/en/hardware/dcu.md +++ b/docs/src/content/docs/en/hardware/dcu.md @@ -59,7 +59,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - HIP_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/metax_maca.md b/docs/src/content/docs/en/hardware/metax_maca.md index acb769080f..f1cff689be 100644 --- a/docs/src/content/docs/en/hardware/metax_maca.md +++ b/docs/src/content/docs/en/hardware/metax_maca.md @@ -63,7 +63,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/musa.md b/docs/src/content/docs/en/hardware/musa.md index c0d17432e5..60c058a561 100644 --- a/docs/src/content/docs/en/hardware/musa.md +++ b/docs/src/content/docs/en/hardware/musa.md @@ -58,7 +58,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MUSA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/nvidia_gpu.md b/docs/src/content/docs/en/hardware/nvidia_gpu.md index b7d0d1f698..832752a0fb 100644 --- a/docs/src/content/docs/en/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/en/hardware/nvidia_gpu.md @@ -58,7 +58,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md index a234f531b2..b7c3df5d35 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -140,7 +140,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH -model-id ds \ + nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -213,7 +213,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ + nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -242,7 +242,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ + nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md index 181787a65f..12b7b0b9a0 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md @@ -133,7 +133,7 @@ do LOG_FILE="$LOG_DIR/node_$i.log" #可选：numactl绑核 (NUMA亲和性查询命令： npu-smi info -t topo) #nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup $XLLM_PATH \ + nohup $XLLM_PATH \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -203,7 +203,8 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -245,7 +246,8 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -419,7 +421,7 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ @@ -473,7 +475,7 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md index d9d0c2fdf0..ca689cc45e 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -153,7 +153,8 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -193,7 +194,8 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 3291cdcd0d..1161a5e2d4 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -176,7 +176,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ + nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md index c7d6b7c6f1..e2de4fd4fc 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -129,7 +129,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./xllm/build/xllm/core/server/xllm \ + ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md index f6cc3d9331..457745b8f4 100644 --- a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md @@ -126,7 +126,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ + ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -194,7 +194,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE ./build/xllm/core/server/xllm \ + ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ --max_memory_utilization=0.6 \ --backend="dit" \ diff --git a/docs/src/content/docs/zh/features/mtp.md b/docs/src/content/docs/zh/features/mtp.md index 9ae032f355..7e99645f3c 100644 --- a/docs/src/content/docs/zh/features/mtp.md +++ b/docs/src/content/docs/zh/features/mtp.md @@ -101,7 +101,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE nohup ./xllm \ + nohup ./xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/getting_started/launch_xllm.md b/docs/src/content/docs/zh/getting_started/launch_xllm.md index 85f6aa9163..8149f3d8df 100644 --- a/docs/src/content/docs/zh/getting_started/launch_xllm.md +++ b/docs/src/content/docs/zh/getting_started/launch_xllm.md @@ -31,7 +31,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -72,7 +72,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -108,7 +108,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MLU_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ @@ -140,7 +140,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - HIP_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ @@ -174,7 +174,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ @@ -210,7 +210,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MUSA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/getting_started/multi_machine.md b/docs/src/content/docs/zh/getting_started/multi_machine.md index c6f77eeb6b..fbff2308ef 100644 --- a/docs/src/content/docs/zh/getting_started/multi_machine.md +++ b/docs/src/content/docs/zh/getting_started/multi_machine.md @@ -34,7 +34,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ + /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ @@ -85,7 +85,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE /path/to/xllm \ + /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ diff --git a/docs/src/content/docs/zh/hardware/ascend_npu.md b/docs/src/content/docs/zh/hardware/ascend_npu.md index fa33455c28..1cad1429db 100644 --- a/docs/src/content/docs/zh/hardware/ascend_npu.md +++ b/docs/src/content/docs/zh/hardware/ascend_npu.md @@ -73,7 +73,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - ASCEND_RT_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/cambricon_mlu.md b/docs/src/content/docs/zh/hardware/cambricon_mlu.md index 690ecfb6af..899ec1a222 100644 --- a/docs/src/content/docs/zh/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/zh/hardware/cambricon_mlu.md @@ -47,7 +47,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MLU_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/hardware/dcu.md b/docs/src/content/docs/zh/hardware/dcu.md index bcaf1fe30f..19b668004f 100644 --- a/docs/src/content/docs/zh/hardware/dcu.md +++ b/docs/src/content/docs/zh/hardware/dcu.md @@ -59,7 +59,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - HIP_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/hardware/metax_maca.md b/docs/src/content/docs/zh/hardware/metax_maca.md index 013aa1d68e..3a4a88c264 100644 --- a/docs/src/content/docs/zh/hardware/metax_maca.md +++ b/docs/src/content/docs/zh/hardware/metax_maca.md @@ -63,7 +63,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/hardware/musa.md b/docs/src/content/docs/zh/hardware/musa.md index 375254c95d..c9e684a19d 100644 --- a/docs/src/content/docs/zh/hardware/musa.md +++ b/docs/src/content/docs/zh/hardware/musa.md @@ -58,7 +58,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - MUSA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/nvidia_gpu.md b/docs/src/content/docs/zh/hardware/nvidia_gpu.md index 0ad3898518..7207f5e41e 100644 --- a/docs/src/content/docs/zh/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/zh/hardware/nvidia_gpu.md @@ -56,7 +56,7 @@ do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" - CUDA_VISIBLE_DEVICES=$DEVICE xllm \ + xllm \ --model $MODEL_PATH \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ From ce99471574d0327fabc9e000f1531571bb4d8e57 Mon Sep 17 00:00:00 2001 From: "zhangxu.709" Date: Fri, 17 Jul 2026 14:27:03 +0800 Subject: [PATCH 4/5] docs: restore global VISIBLE_DEVICES export in launch scripts The earlier flag-removal pass dropped the global export _VISIBLE_DEVICES line and left dead START_DEVICE/DEVICE shell vars. Restore the global export (the intended way to pick cards now that --devices is gone; a process sees the masked cards and binds by --node_rank) and remove the dead vars. Keep DEVICE in glm_5/kimi2_5 where numactl still uses it for CPU affinity. Multi-card cookbook examples that never had a global export stay bare (no mask means all cards visible). Co-Authored-By: Claude --- .../deepseek/deepseek_v4.md | 8 ++----- .../minmax/minmax_m2_7.md | 2 -- .../autoregressive_models/qwen/qwen3_5.md | 3 +-- .../cookbook/diffusion_models/flux/flux2.md | 4 ---- docs/src/content/docs/en/features/mtp.md | 2 -- .../docs/en/getting_started/launch_xllm.md | 21 ++++++++----------- .../docs/en/getting_started/multi_machine.md | 4 ---- .../content/docs/en/hardware/ascend_npu.md | 3 +-- .../content/docs/en/hardware/cambricon_mlu.md | 4 ++-- docs/src/content/docs/en/hardware/dcu.md | 4 ++-- .../content/docs/en/hardware/metax_maca.md | 3 +-- docs/src/content/docs/en/hardware/musa.md | 4 ++-- .../content/docs/en/hardware/nvidia_gpu.md | 3 +-- .../deepseek/deepseek_v4.md | 8 ++----- .../minmax/minmax_m2_7.md | 2 -- .../autoregressive_models/qwen/qwen3_5.md | 3 +-- .../cookbook/diffusion_models/flux/flux2.md | 4 ---- docs/src/content/docs/zh/features/mtp.md | 2 -- .../docs/zh/getting_started/launch_xllm.md | 21 ++++++++----------- .../docs/zh/getting_started/multi_machine.md | 4 ---- .../content/docs/zh/hardware/ascend_npu.md | 3 +-- .../content/docs/zh/hardware/cambricon_mlu.md | 4 ++-- docs/src/content/docs/zh/hardware/dcu.md | 4 ++-- .../content/docs/zh/hardware/metax_maca.md | 3 +-- docs/src/content/docs/zh/hardware/musa.md | 4 ++-- .../content/docs/zh/hardware/nvidia_gpu.md | 3 +-- 26 files changed, 42 insertions(+), 88 deletions(-) diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md index d430e8a220..509e2685d5 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -131,14 +131,12 @@ MASTER_NODE_ADDR="11.87.49.110:10015" LOCAL_HOST="11.87.49.110" # Service Port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=8 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ @@ -203,7 +201,6 @@ export ATB_MATMUL_SHUFFLE_K_ENABLE=0 MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.110" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -212,7 +209,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ @@ -232,7 +229,6 @@ done MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.111" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -241,7 +237,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 5b7bdabf69..01e73b7118 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -167,14 +167,12 @@ MASTER_NODE_ADDR="10.143.3.204:10015" LOCAL_HOST="10.143.3.204" # Service port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md index 2845bf54b0..92d39b3895 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -119,15 +119,14 @@ DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" MASTER_NODE_ADDR=":32764" START_PORT=18076 -START_DEVICE=14 NNODES=2 +export ASCEND_RT_VISIBLE_DEVICES=14,15 export HCCL_IF_BASE_PORT=53433 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md index 740421f3cf..a8e6cf1562 100644 --- a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md @@ -114,7 +114,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/flux2/text_encoder/" # text_encoder path MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be consistent globally) START_PORT=18001 # Service starting port -START_DEVICE=2 # Starting logical device ID LOG_DIR="log" # Log directory NNODES=2 # Number of nodes (this script starts 1 process) @@ -123,7 +122,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ @@ -184,14 +182,12 @@ export HCCL_IF_BASE_PORT=43432 # HCCL communication base port ```bash MASTER_NODE_ADDR="127.0.0.1:8999" # Master node address (must be consistent globally) START_PORT=18018 # Service starting port -START_DEVICE=4 # Starting NPU logical device ID LOG_DIR="log" # Log directory NNODES=2 # Number of nodes (this script starts 2 processes) for (( i=0; i<2; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ diff --git a/docs/src/content/docs/en/features/mtp.md b/docs/src/content/docs/en/features/mtp.md index 2f11874b3e..91a458c0da 100644 --- a/docs/src/content/docs/en/features/mtp.md +++ b/docs/src/content/docs/en/features/mtp.md @@ -94,14 +94,12 @@ MODEL_PATH="/models/DeepSeek-V3" DRAFT_MODEL_PATH="/models/DeepSeek-V3-mtp" MASTER_NODE_ADDR="127.0.0.1:42123" START_PORT=13222 -START_DEVICE=0 LOG_DIR="log" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup ./xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/getting_started/launch_xllm.md b/docs/src/content/docs/en/getting_started/launch_xllm.md index 5eb76b006f..0c35d036d9 100644 --- a/docs/src/content/docs/en/getting_started/launch_xllm.md +++ b/docs/src/content/docs/en/getting_started/launch_xllm.md @@ -15,12 +15,12 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/model/Qwen3-8B" # Model path MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be globally consistent) START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting logical device number LOG_DIR="log" # Log directory NNODES=1 # Number of nodes (current script launches 1 process) @@ -29,7 +29,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -55,13 +54,13 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -70,7 +69,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -94,10 +92,11 @@ set -e rm -rf core.* +export MLU_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -106,7 +105,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -126,10 +124,11 @@ set -e rm -rf core.* +export HIP_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -138,7 +137,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -158,12 +156,12 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -172,7 +170,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -196,10 +193,11 @@ set -e rm -rf core.* +export MUSA_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -208,7 +206,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/getting_started/multi_machine.md b/docs/src/content/docs/en/getting_started/multi_machine.md index 69b57627e1..4b4c43c032 100644 --- a/docs/src/content/docs/en/getting_started/multi_machine.md +++ b/docs/src/content/docs/en/getting_started/multi_machine.md @@ -25,7 +25,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # Model path MASTER_NODE_ADDR="123.123.123.123:9748" # Master node address (must be globally consistent) LOCAL_HOST=123.123.123.123 # Local IP for service launch START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting NPU logical device number LOCAL_NODES=16 # Number of local processes (this script launches 16 processes) LOG_DIR="log" # Log directory NNODES=32 # Total number of GPUs/NPUs (32 in this 2-machine example) @@ -35,7 +34,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ @@ -76,7 +74,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # Model path MASTER_NODE_ADDR="123.123.123.123:9748" # Master node address (must be globally consistent) LOCAL_HOST=456.456.456.456 # Local IP for service launch START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting NPU logical device number LOCAL_NODES=16 # Number of local processes (this script launches 16 processes) LOG_DIR="log" # Log directory NNODES=32 # Total number of GPUs/NPUs (32 in this 2-machine example) @@ -86,7 +83,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/hardware/ascend_npu.md b/docs/src/content/docs/en/hardware/ascend_npu.md index 302f8f0c9f..1315370b6c 100644 --- a/docs/src/content/docs/en/hardware/ascend_npu.md +++ b/docs/src/content/docs/en/hardware/ascend_npu.md @@ -57,12 +57,12 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/model/Qwen3-8B" # Model path MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be globally consistent) START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting logical device number LOG_DIR="log" # Log directory NNODES=1 # Number of nodes (current script launches 1 process) @@ -71,7 +71,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/hardware/cambricon_mlu.md b/docs/src/content/docs/en/hardware/cambricon_mlu.md index 538bef5c8d..98b1f7461e 100644 --- a/docs/src/content/docs/en/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/en/hardware/cambricon_mlu.md @@ -33,10 +33,11 @@ set -e rm -rf core.* +export MLU_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -45,7 +46,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/hardware/dcu.md b/docs/src/content/docs/en/hardware/dcu.md index 3df9cbe58f..afcceb8df7 100644 --- a/docs/src/content/docs/en/hardware/dcu.md +++ b/docs/src/content/docs/en/hardware/dcu.md @@ -45,10 +45,11 @@ set -e rm -rf core.* +export HIP_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -57,7 +58,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/hardware/metax_maca.md b/docs/src/content/docs/en/hardware/metax_maca.md index f1cff689be..e661315b0a 100644 --- a/docs/src/content/docs/en/hardware/metax_maca.md +++ b/docs/src/content/docs/en/hardware/metax_maca.md @@ -47,12 +47,12 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -61,7 +61,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/hardware/musa.md b/docs/src/content/docs/en/hardware/musa.md index 60c058a561..3625ba5764 100644 --- a/docs/src/content/docs/en/hardware/musa.md +++ b/docs/src/content/docs/en/hardware/musa.md @@ -44,10 +44,11 @@ set -e rm -rf core.* +export MUSA_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -56,7 +57,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/en/hardware/nvidia_gpu.md b/docs/src/content/docs/en/hardware/nvidia_gpu.md index 832752a0fb..01fe120e06 100644 --- a/docs/src/content/docs/en/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/en/hardware/nvidia_gpu.md @@ -41,13 +41,13 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -56,7 +56,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md index b7c3df5d35..d86e557a06 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -131,14 +131,12 @@ MASTER_NODE_ADDR="11.87.49.110:10015" LOCAL_HOST="11.87.49.110" # Service Port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=8 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ @@ -203,7 +201,6 @@ export ATB_MATMUL_SHUFFLE_K_ENABLE=0 MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.110" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -212,7 +209,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ @@ -232,7 +229,6 @@ done MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.111" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -241,7 +237,7 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 1161a5e2d4..d2e290badf 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -167,14 +167,12 @@ MASTER_NODE_ADDR="10.143.3.204:10015" LOCAL_HOST="10.143.3.204" # Service Port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md index e2de4fd4fc..9130936415 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -119,15 +119,14 @@ DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" MASTER_NODE_ADDR=":32764" START_PORT=18076 -START_DEVICE=14 NNODES=2 +export ASCEND_RT_VISIBLE_DEVICES=14,15 export HCCL_IF_BASE_PORT=53433 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md index 457745b8f4..19d9588f44 100644 --- a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md @@ -115,7 +115,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/flux2/text_encoder/" # text_encoder路径 MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址（需全局一致） START_PORT=18001 # 服务起始端口 -START_DEVICE=2 # 起始逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=2 # 节点数（当前脚本启动 1 个进程） @@ -124,7 +123,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ @@ -185,14 +183,12 @@ export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 ```bash MASTER_NODE_ADDR="127.0.0.1:8999" # Master 节点地址（需全局一致） START_PORT=18018 # 服务起始端口 -START_DEVICE=4 # 起始 NPU 逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=2 # 节点数（当前脚本启动 2 个进程） for (( i=0; i<2; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ diff --git a/docs/src/content/docs/zh/features/mtp.md b/docs/src/content/docs/zh/features/mtp.md index 7e99645f3c..6df813375e 100644 --- a/docs/src/content/docs/zh/features/mtp.md +++ b/docs/src/content/docs/zh/features/mtp.md @@ -92,14 +92,12 @@ MODEL_PATH="/models/DeepSeek-V3" DRAFT_MODEL_PATH="/models/DeepSeek-V3-mtp" MASTER_NODE_ADDR="127.0.0.1:42123" START_PORT=13222 -START_DEVICE=0 LOG_DIR="log" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup ./xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/getting_started/launch_xllm.md b/docs/src/content/docs/zh/getting_started/launch_xllm.md index 8149f3d8df..d46e881992 100644 --- a/docs/src/content/docs/zh/getting_started/launch_xllm.md +++ b/docs/src/content/docs/zh/getting_started/launch_xllm.md @@ -15,12 +15,12 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址（需全局一致） START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=1 # 节点数（当前脚本启动 1 个进程） @@ -29,7 +29,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -55,13 +54,13 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -70,7 +69,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -94,10 +92,11 @@ set -e rm -rf core.* +export MLU_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -106,7 +105,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -126,10 +124,11 @@ set -e rm -rf core.* +export HIP_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -138,7 +137,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -158,12 +156,12 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -172,7 +170,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ @@ -196,10 +193,11 @@ set -e rm -rf core.* +export MUSA_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -208,7 +206,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/getting_started/multi_machine.md b/docs/src/content/docs/zh/getting_started/multi_machine.md index fbff2308ef..0683e95ac3 100644 --- a/docs/src/content/docs/zh/getting_started/multi_machine.md +++ b/docs/src/content/docs/zh/getting_started/multi_machine.md @@ -22,7 +22,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # 模型路径 MASTER_NODE_ADDR="123.123.123.123:9748" # Master 节点地址（需全局一致） LOCAL_HOST=123.123.123.123 # 本机服务启动IP START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始 NPU 逻辑设备号 LOG_DIR="log" # 日志目录 LOCAL_NODES=16 # 单机节点数（当前脚本启动 16 个进程） NNODES=32 # 总卡数（该示例为2机32卡） @@ -32,7 +31,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ @@ -73,7 +71,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # 模型路径 MASTER_NODE_ADDR="123.123.123.123:9748" # Master 节点地址（需全局一致） LOCAL_HOST=456.456.456.456 # 本机服务启动IP START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始 NPU 逻辑设备号 LOG_DIR="log" # 日志目录 LOCAL_NODES=16 # 单机节点数（当前脚本启动 16 个进程） NNODES=32 # 总卡数（该示例为2机32卡） @@ -83,7 +80,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/hardware/ascend_npu.md b/docs/src/content/docs/zh/hardware/ascend_npu.md index 1cad1429db..df9b744100 100644 --- a/docs/src/content/docs/zh/hardware/ascend_npu.md +++ b/docs/src/content/docs/zh/hardware/ascend_npu.md @@ -57,12 +57,12 @@ rm -rf core.* source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址（需全局一致） START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=1 # 节点数（当前脚本启动 1 个进程） @@ -71,7 +71,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/hardware/cambricon_mlu.md b/docs/src/content/docs/zh/hardware/cambricon_mlu.md index 899ec1a222..312377b0d9 100644 --- a/docs/src/content/docs/zh/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/zh/hardware/cambricon_mlu.md @@ -33,10 +33,11 @@ set -e rm -rf core.* +export MLU_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -45,7 +46,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/hardware/dcu.md b/docs/src/content/docs/zh/hardware/dcu.md index 19b668004f..8720e9ecb1 100644 --- a/docs/src/content/docs/zh/hardware/dcu.md +++ b/docs/src/content/docs/zh/hardware/dcu.md @@ -45,10 +45,11 @@ set -e rm -rf core.* +export HIP_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -57,7 +58,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/hardware/metax_maca.md b/docs/src/content/docs/zh/hardware/metax_maca.md index 3a4a88c264..c9f53d53a0 100644 --- a/docs/src/content/docs/zh/hardware/metax_maca.md +++ b/docs/src/content/docs/zh/hardware/metax_maca.md @@ -47,12 +47,12 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/data/aot/ MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -61,7 +61,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/hardware/musa.md b/docs/src/content/docs/zh/hardware/musa.md index c9e684a19d..0b6b2a606c 100644 --- a/docs/src/content/docs/zh/hardware/musa.md +++ b/docs/src/content/docs/zh/hardware/musa.md @@ -44,10 +44,11 @@ set -e rm -rf core.* +export MUSA_VISIBLE_DEVICES=0 + MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -56,7 +57,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ diff --git a/docs/src/content/docs/zh/hardware/nvidia_gpu.md b/docs/src/content/docs/zh/hardware/nvidia_gpu.md index 7207f5e41e..52ea0b7bd8 100644 --- a/docs/src/content/docs/zh/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/zh/hardware/nvidia_gpu.md @@ -39,13 +39,13 @@ set -e rm -rf core.* +export CUDA_VISIBLE_DEVICES=0 # for debug # export CUDA_LAUNCH_BLOCKING=1 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -54,7 +54,6 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ From 824c60e0fa7a305d28fc33c7a8794b2359f35c0a Mon Sep 17 00:00:00 2001 From: "zhangxu.709" Date: Fri, 17 Jul 2026 16:03:02 +0800 Subject: [PATCH 5/5] fix: keep per-process devices single-element to match legacy semantics Select the card this process owns from the visible pool by node_rank, so options.devices() holds exactly one device (the card this process uses) instead of the whole visible pool. This mirrors the historical single-element devices semantics (previously set via --devices="npu:$DEVICE"): DeviceNameUtils:: to_string(devices) prints "npu:N" rather than "npu:0,npu:1", and DeviceMonitor registers only this process's card. dist_manager reverts to deriving each_node_ranks/world_size from devices.size()==1, so world_size==nnodes. Verified: 2-process TP=2 Qwen3-8B, node_0 devices=npu:0, node_1 devices=npu:1, world_size=2, inference correct. Co-Authored-By: Claude --- .../core/distributed_runtime/dist_manager.cpp | 62 +++++++++---------- xllm/core/distributed_runtime/master.cpp | 15 ++++- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/xllm/core/distributed_runtime/dist_manager.cpp b/xllm/core/distributed_runtime/dist_manager.cpp index 37d4e162bb..eb66ba45d5 100644 --- a/xllm/core/distributed_runtime/dist_manager.cpp +++ b/xllm/core/distributed_runtime/dist_manager.cpp @@ -167,24 +167,16 @@ void DistManager::setup_multi_node_workers( // class. In this class, we create processes, initialize NCCL ProcessGroup, // set up GRPC servers, and so on. - std::vector> dones(1); - dones[0].store(false, std::memory_order_relaxed); + std::vector> dones(devices.size()); + for (size_t i = 0; i < devices.size(); ++i) { + dones[i].store(false, std::memory_order_relaxed); + } CHECK_GE(options.nnodes(), 1) << "At least one node is required"; CHECK_GE(options.node_rank(), 0) << "Node rank must >= 0."; - // Multi-process serving: each process runs exactly one worker. `devices` is - // the visible-device pool (all cards each process can see, required so ATB - // can build the cross-card HCCL comm group); this process binds the single - // card selected by its node_rank. World size is therefore the node count. - const int32_t each_node_ranks = 1; - const int32_t world_size = options.nnodes(); - const int32_t base_rank = options.node_rank(); - const int32_t local_device_index = options.node_rank(); - CHECK_LT(local_device_index, static_cast(devices.size())) - << "node_rank " << local_device_index << " exceeds the number of visible " - << "devices " << devices.size() - << ". Ensure *_VISIBLE_DEVICES exposes all cards used across processes."; - const torch::Device local_device = devices[local_device_index]; + const int32_t each_node_ranks = static_cast(devices.size()); + const int32_t world_size = each_node_ranks * options.nnodes(); + const int32_t base_rank = options.node_rank() * each_node_ranks; const int32_t dp_size = options.dp_size(); const int32_t cp_size = options.cp_size(); const int32_t ep_size = options.ep_size(); @@ -246,22 +238,26 @@ void DistManager::setup_multi_node_workers( } else { LOG(FATAL) << "Unsupported " << model_backend << " in multi-node."; } - // create the single local worker for this process - { - const int32_t i = 0; - const int32_t rank = base_rank; + // create local workers + for (int32_t i = 0; i < each_node_ranks; ++i) { + // worldsize = 8 + // Node1: 0, 1, 2, 3 + // Node2: 0+4, 1+4, 2+4, 3+4 + const int32_t rank = i + base_rank; worker_server_options.server_idx(rank); + // we use spawn process worker to launch a xllm instance + // when start a offline inference task with multi-gpu/npu/mpu/... #if defined(USE_CUDA) || defined(USE_MLU) || defined(USE_DCU) - bool use_spawn_worker = force_spawn_for_numa_isolation[local_device_index]; - if (use_spawn_worker) { - LOG(INFO) << "Force spawn worker for node_rank " << local_device_index - << " (device " << local_device.index() << ", NUMA " - << device_numa_nodes[local_device_index] + bool use_spawn_worker = (options.enable_offline_inference() && i > 0) || + force_spawn_for_numa_isolation[i]; + if (force_spawn_for_numa_isolation[i]) { + LOG(INFO) << "Force spawn worker for local rank " << i << " (device " + << devices[i].index() << ", NUMA " << device_numa_nodes[i] << ") to keep each process within a single NUMA region"; } #else - bool use_spawn_worker = false; + bool use_spawn_worker = options.enable_offline_inference() && i > 0; #endif ParallelArgs parallel_args( rank, world_size, dp_size, cp_size, nullptr, ep_size); @@ -269,9 +265,9 @@ void DistManager::setup_multi_node_workers( servers_.emplace_back(std::make_unique(i, master_node_addr, // done, - dones[0], + dones[i], parallel_args, - local_device, + devices[i], worker_server_options, worker_type, use_spawn_worker)); @@ -301,9 +297,6 @@ void DistManager::setup_multi_node_workers( // check if all workers connected // and then create worker clients - CHECK_LE(static_cast(world_size), devices.size()) - << "world_size " << world_size << " exceeds the visible-device pool " - << devices.size() << "; each global rank maps to one visible card."; for (size_t r = 0; r < world_size; ++r) { if (worker_addrs_map.find(r) == worker_addrs_map.end()) { LOG(FATAL) << "Not all worker connect to engine server. Miss rank is " @@ -313,10 +306,11 @@ void DistManager::setup_multi_node_workers( /* TODO(CP): support smem + CP */ auto channel = create_channel(worker_addrs_map[r], r, dp_local_tp_size, options); - // Global rank r runs on the process whose node_rank == r, which binds the - // card devices[r] from the shared visible-device pool. - worker_clients_.emplace_back(std::make_unique( - r, worker_addrs_map[r], devices[r], std::move(channel))); + worker_clients_.emplace_back( + std::make_unique(r, + worker_addrs_map[r], + devices[r % each_node_ranks], + std::move(channel))); } // Register health check for each worker and start background health check diff --git a/xllm/core/distributed_runtime/master.cpp b/xllm/core/distributed_runtime/master.cpp index 62ad1447cb..b1743ddde2 100644 --- a/xllm/core/distributed_runtime/master.cpp +++ b/xllm/core/distributed_runtime/master.cpp @@ -199,9 +199,18 @@ Master::Master(const Options& options, EngineType type) master_status_(options.master_status()) { const auto model_path = std::filesystem::path(options_.model_path()).lexically_normal(); - const auto devices = DeviceNameUtils::parse_devices("auto"); - // Multi-process serving runs one worker per process, so the global world - // size is the node count. `devices` is only the visible-card pool here. + // Multi-process serving runs one worker per process. Enumerate the visible + // cards (honoring *_VISIBLE_DEVICES) and select the single card this process + // owns by its node_rank, so `devices` holds exactly the card this process + // uses -- mirroring the historical single-element devices semantics. + const auto visible_devices = DeviceNameUtils::parse_devices("auto"); + CHECK_LT(options_.node_rank(), static_cast(visible_devices.size())) + << "node_rank " << options_.node_rank() + << " exceeds the number of visible devices " << visible_devices.size() + << ". Ensure *_VISIBLE_DEVICES exposes all cards used across processes."; + const std::vector devices = { + visible_devices[options_.node_rank()]}; + // World size is the node count (one worker per process). const int32_t global_world_size = options_.nnodes(); std::string cp_model_type; if (options_.cp_size() > 1 && Platform::uses_model_cp_partition()) {