diff --git a/docs/src/content/docs/en/cli_reference.md b/docs/src/content/docs/en/cli_reference.md index 66d8c75bdf..ad2ac5d08a 100644 --- a/docs/src/content/docs/en/cli_reference.md +++ b/docs/src/content/docs/en/cli_reference.md @@ -6,6 +6,8 @@ sidebar: xLLM uses gflags to manage service startup parameters. `--model ` is the only required flag. When `--config_json_file` is used, values in the JSON file override command-line flag values. The tables below are grouped by the Config classes in `/xllm/core/framework/config`, with one Config per section. The `ConfigJsonUtils` section contains the common JSON config-file flags. +> **Device selection**: xLLM no longer provides `--devices` / `--device_id` / `--draft_devices`. The devices to run on are determined by the visible-device mask environment variables (`ASCEND_RT_VISIBLE_DEVICES` for NPU, `CUDA_VISIBLE_DEVICES` for NVIDIA, `MLU_VISIBLE_DEVICES` for Cambricon, `HIP_VISIBLE_DEVICES` for DCU, `MUSA_VISIBLE_DEVICES` for Moore Threads). Each process uses all of its visible devices: a single-card process exposes one card, while single-process multi-card exposes several. The draft model always uses the same devices as the target model. + ## ConfigJsonUtils | Parameter | Type | Default | Description | @@ -37,8 +39,6 @@ xLLM uses gflags to manage service startup parameters. `--model ` is the o | `model` | `string` | `""` | Hugging Face model name or model path. | | `backend` | `string` | `""` | Backend model type: `llm` for text-only models, `vlm` for multimodal models, or `dit` for diffusion models. | | `task` | `string` | `"generate"` | Model task, for example `generate`, `embed`, or `mm_embed`. | -| `devices` | `string` | `""` | Deprecated; use `device_id` instead. Devices used by the current process, for example `npu:0` or `npu:0,npu:1`. | -| `device_id` | `int32` | `-1` | Device id to run the model on, for example `0`. | | `limit_image_per_prompt` | `int32` | `8` | Maximum number of images per prompt. Only applies to multimodal models. | | `max_encoder_cache_size` | `int64` | `0` | Maximum GPU/NPU memory size in MB for the encoder cache per worker. `0` disables the encoder cache. | | `reasoning_parser` | `string` | `""` | Reasoning parser, for example `auto`, `glm45`, `glm47`, `glm5`, `qwen3`, `qwen35`, or `deepseek-r1`. | @@ -181,7 +181,6 @@ xLLM uses gflags to manage service startup parameters. `--model ` is the o | Parameter | Type | Default | Description | |:----------|:-----|:--------|:------------| | `draft_model` | `string` | `""` | Draft model path. See [MTP](/en/features/mtp/) for MTP usage. | -| `draft_devices` | `string` | `""` | Devices used by the draft model, for example `npu:0` or `npu:0,npu:1`. If omitted, uses the target model devices when speculative decoding is enabled. | | `num_speculative_tokens` | `int32` | `0` | Number of speculative tokens generated per speculative decoding step. | | `speculative_algorithm` | `string` | `"MTP"` | Speculative decoding algorithm. Supported values: `MTP`, `Eagle3`, `Suffix`, `DFlash`. | | `speculative_suffix_cache_max_depth` | `int32` | `64` | Maximum suffix-tree depth for suffix speculative decoding. | diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md index f68b06b4af..509e2685d5 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -131,20 +131,17 @@ MASTER_NODE_ADDR="11.87.49.110:10015" LOCAL_HOST="11.87.49.110" # Service Port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=8 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -166,7 +163,6 @@ done # Variables required when MTP is enabled # --draft_model=$DRAFT_MODEL_PATH \ - # --draft_devices="npu:$DEVICE" \ # --num_speculative_tokens=1 \ # numactl -C xxxxx NUMA core binding (query with: npu-smi info -t topo) @@ -179,7 +175,6 @@ done #--enable_chunked_prefill Enable chunked prefill #--enable_graph Enable aclgraph #--draft_model MTP - MTP weights path -#--draft_devices MTP - MTP inference device (same as main model) #--num_speculative_tokens MTP - Number of speculative tokens ``` @@ -206,7 +201,6 @@ export ATB_MATMUL_SHUFFLE_K_ENABLE=0 MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.110" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -215,12 +209,11 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -236,7 +229,6 @@ done MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.111" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -245,12 +237,11 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md index fac8101a9d..9a57acda10 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/glm/glm_5.md @@ -135,7 +135,6 @@ do nohup $XLLM_PATH \ --model $MODEL_PATH \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -150,7 +149,6 @@ do --communication_backend="hccl" \ --graph_decode_batch_size_limit=2 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=16 \ --dp_size=2 \ @@ -168,7 +166,6 @@ done # --enable_graph Enable aclgraph. It requires extra memory. # --acl_graph_decode_batch_size_limit Maximum decode batch size for graph capture. It must be <= 32 / (number of speculative tokens + 1). # --draft_model MTP draft-model weight path. -# --draft_devices MTP inference device, the same as the main model. # --num_speculative_tokens Number of speculative tokens predicted by MTP. ``` @@ -211,7 +208,6 @@ do --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -226,7 +222,6 @@ do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -258,7 +253,6 @@ do --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ @@ -273,7 +267,6 @@ do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -442,7 +435,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -455,7 +447,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ @@ -498,7 +489,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -511,7 +501,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md index 00c2156c32..c6f4eeea4e 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -153,10 +153,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -194,10 +194,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 89bb876f5d..01e73b7118 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -167,20 +167,17 @@ MASTER_NODE_ADDR="10.143.3.204:10015" LOCAL_HOST="10.143.3.204" # Service port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ diff --git a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md index 0166b86be0..92d39b3895 100644 --- a/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/en/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -91,7 +91,6 @@ export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=14,15 export ASDOPS_LOG_TO_STDOUT=1 export ASDOPS_LOG_LEVEL=0 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -120,19 +119,17 @@ DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" MASTER_NODE_ADDR=":32764" START_PORT=18076 -START_DEVICE=0 NNODES=2 +export ASCEND_RT_VISIBLE_DEVICES=14,15 export HCCL_IF_BASE_PORT=53433 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -151,7 +148,6 @@ do --max_concurrent_requests=8 \ --backend llm \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 3 \ >> $LOG_FILE 2>&1 & done diff --git a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md index 2d98560a70..a8e6cf1562 100644 --- a/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/en/cookbook/diffusion_models/flux/flux2.md @@ -114,7 +114,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/flux2/text_encoder/" # text_encoder path MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be consistent globally) START_PORT=18001 # Service starting port -START_DEVICE=2 # Starting logical device ID LOG_DIR="log" # Log directory NNODES=2 # Number of nodes (this script starts 1 process) @@ -123,11 +122,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -185,21 +182,18 @@ export HCCL_IF_BASE_PORT=43432 # HCCL communication base port ```bash MASTER_NODE_ADDR="127.0.0.1:8999" # Master node address (must be consistent globally) START_PORT=18018 # Service starting port -START_DEVICE=4 # Starting NPU logical device ID LOG_DIR="log" # Log directory NNODES=2 # Number of nodes (this script starts 2 processes) for (( i=0; i<2; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ --max_memory_utilization=0.6 \ --backend="dit" \ --tp_size=2 \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --port $PORT \ diff --git a/docs/src/content/docs/en/features/disagg_pd.md b/docs/src/content/docs/en/features/disagg_pd.md index 2a0edad184..f1a6832592 100644 --- a/docs/src/content/docs/en/features/disagg_pd.md +++ b/docs/src/content/docs/en/features/disagg_pd.md @@ -43,9 +43,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. - Taking Qwen2-7B as an example - Start Prefill Instance ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=0 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8010 \ - --devices="npu:0" \ --master_node_addr="127.0.0.1:18888" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ @@ -59,9 +58,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. ``` - Start Decode Instance ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=1 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8020 \ - --devices="npu:1" \ --master_node_addr="127.0.0.1:18898" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ diff --git a/docs/src/content/docs/en/features/mtp.md b/docs/src/content/docs/en/features/mtp.md index f4d7ea5f17..91a458c0da 100644 --- a/docs/src/content/docs/en/features/mtp.md +++ b/docs/src/content/docs/en/features/mtp.md @@ -94,23 +94,19 @@ MODEL_PATH="/models/DeepSeek-V3" DRAFT_MODEL_PATH="/models/DeepSeek-V3-mtp" MASTER_NODE_ADDR="127.0.0.1:42123" START_PORT=13222 -START_DEVICE=0 LOG_DIR="log" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup ./xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --max_memory_utilization=0.90 \ --max_tokens_per_batch=10000 \ diff --git a/docs/src/content/docs/en/getting_started/launch_xllm.md b/docs/src/content/docs/en/getting_started/launch_xllm.md index 028a6398fb..0c35d036d9 100644 --- a/docs/src/content/docs/en/getting_started/launch_xllm.md +++ b/docs/src/content/docs/en/getting_started/launch_xllm.md @@ -21,7 +21,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/model/Qwen3-8B" # Model path MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be globally consistent) START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting logical device number LOG_DIR="log" # Log directory NNODES=1 # Number of nodes (current script launches 1 process) @@ -30,11 +29,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -64,7 +61,6 @@ export CUDA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -73,11 +69,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -103,7 +97,6 @@ export MLU_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -112,11 +105,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -138,7 +129,6 @@ export HIP_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -147,11 +137,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -174,7 +162,6 @@ export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/da MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -183,11 +170,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -213,7 +198,6 @@ export MUSA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -222,11 +206,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/getting_started/multi_machine.md b/docs/src/content/docs/en/getting_started/multi_machine.md index 199742acef..4b4c43c032 100644 --- a/docs/src/content/docs/en/getting_started/multi_machine.md +++ b/docs/src/content/docs/en/getting_started/multi_machine.md @@ -25,7 +25,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # Model path MASTER_NODE_ADDR="123.123.123.123:9748" # Master node address (must be globally consistent) LOCAL_HOST=123.123.123.123 # Local IP for service launch START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting NPU logical device number LOCAL_NODES=16 # Number of local processes (this script launches 16 processes) LOG_DIR="log" # Log directory NNODES=32 # Total number of GPUs/NPUs (32 in this 2-machine example) @@ -35,13 +34,11 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ @@ -77,7 +74,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # Model path MASTER_NODE_ADDR="123.123.123.123:9748" # Master node address (must be globally consistent) LOCAL_HOST=456.456.456.456 # Local IP for service launch START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting NPU logical device number LOCAL_NODES=16 # Number of local processes (this script launches 16 processes) LOG_DIR="log" # Log directory NNODES=32 # Total number of GPUs/NPUs (32 in this 2-machine example) @@ -87,13 +83,11 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ diff --git a/docs/src/content/docs/en/getting_started/offline_service.md b/docs/src/content/docs/en/getting_started/offline_service.md index 5bc87cf513..703f3d162a 100644 --- a/docs/src/content/docs/en/getting_started/offline_service.md +++ b/docs/src/content/docs/en/getting_started/offline_service.md @@ -16,7 +16,9 @@ Use `BeamSearchParams` with `beam_width` greater than `1`, then call `llm.beam_s ```python from xllm import BeamSearchParams, LLM -llm = LLM(model="/path/models/Qwen2-7B-Instruct", devices="npu:0") +# The devices to run on are selected via ASCEND_RT_VISIBLE_DEVICES +# (or CUDA_VISIBLE_DEVICES / MLU_VISIBLE_DEVICES for other platforms). +llm = LLM(model="/path/models/Qwen2-7B-Instruct") params = BeamSearchParams( beam_width=2, top_logprobs=4, diff --git a/docs/src/content/docs/en/hardware/ascend_npu.md b/docs/src/content/docs/en/hardware/ascend_npu.md index fbdb261286..1315370b6c 100644 --- a/docs/src/content/docs/en/hardware/ascend_npu.md +++ b/docs/src/content/docs/en/hardware/ascend_npu.md @@ -63,7 +63,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL communication base port MODEL_PATH="/path/to/model/Qwen3-8B" # Model path MASTER_NODE_ADDR="127.0.0.1:9748" # Master node address (must be globally consistent) START_PORT=18000 # Service starting port -START_DEVICE=0 # Starting logical device number LOG_DIR="log" # Log directory NNODES=1 # Number of nodes (current script launches 1 process) @@ -72,11 +71,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/cambricon_mlu.md b/docs/src/content/docs/en/hardware/cambricon_mlu.md index e1da637872..98b1f7461e 100644 --- a/docs/src/content/docs/en/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/en/hardware/cambricon_mlu.md @@ -38,7 +38,6 @@ export MLU_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -47,11 +46,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/dcu.md b/docs/src/content/docs/en/hardware/dcu.md index 33ec7a6f6c..afcceb8df7 100644 --- a/docs/src/content/docs/en/hardware/dcu.md +++ b/docs/src/content/docs/en/hardware/dcu.md @@ -50,7 +50,6 @@ export HIP_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -59,11 +58,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/metax_maca.md b/docs/src/content/docs/en/hardware/metax_maca.md index 073e60c804..e661315b0a 100644 --- a/docs/src/content/docs/en/hardware/metax_maca.md +++ b/docs/src/content/docs/en/hardware/metax_maca.md @@ -53,7 +53,6 @@ export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/da MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -62,11 +61,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/en/hardware/musa.md b/docs/src/content/docs/en/hardware/musa.md index 5471398d08..3625ba5764 100644 --- a/docs/src/content/docs/en/hardware/musa.md +++ b/docs/src/content/docs/en/hardware/musa.md @@ -5,7 +5,7 @@ sidebar: order: 7 --- -Use the MUSA device backend (`--devices=musa:`) when running xLLM on Mthreads MUSA hardware. +Use the MUSA device backend when running xLLM on Mthreads MUSA hardware; select the visible card(s) with `MUSA_VISIBLE_DEVICES`. ## Image and Container Startup @@ -49,7 +49,6 @@ export MUSA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -58,11 +57,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/nvidia_gpu.md b/docs/src/content/docs/en/hardware/nvidia_gpu.md index be797ce95d..01fe120e06 100644 --- a/docs/src/content/docs/en/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/en/hardware/nvidia_gpu.md @@ -48,7 +48,6 @@ export CUDA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -57,11 +56,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/en/hardware/overview.md b/docs/src/content/docs/en/hardware/overview.md index 417011e7c2..e95127b400 100644 --- a/docs/src/content/docs/en/hardware/overview.md +++ b/docs/src/content/docs/en/hardware/overview.md @@ -14,7 +14,7 @@ xLLM supports multiple accelerator backends for large-scale model inference. Thi - [Cambricon MLU](/en/hardware/cambricon_mlu/) - MLU backend setup and launch entry points. - [Hygon DCU](/en/hardware/dcu/) - Hygon DCU backend setup and launch entry points. - [MetaX MACA](/en/hardware/metax_maca/) - MetaX MACA backend setup and launch entry points. -- [Mthreads MUSA](/en/hardware/musa/) - Mthreads MUSA GPU image, build, and `--devices=musa` launch entry points. +- [Mthreads MUSA](/en/hardware/musa/) - Mthreads MUSA GPU image, build, and launch entry points (device selection via `MUSA_VISIBLE_DEVICES`). ## Common Workflow diff --git a/docs/src/content/docs/zh/cli_reference.md b/docs/src/content/docs/zh/cli_reference.md index 368bf908e5..e22b1b4ea6 100644 --- a/docs/src/content/docs/zh/cli_reference.md +++ b/docs/src/content/docs/zh/cli_reference.md @@ -6,6 +6,8 @@ sidebar: xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填参数。使用 `--config_json_file` 时,JSON 文件中的值会覆盖命令行 flag 值。下表按 `/xllm/core/framework/config` 下的 Config 类分组,一个 Config 对应一节;`ConfigJsonUtils` 一节包含配置文件相关的通用参数。 +> **设备选择**:xLLM 不再提供 `--devices` / `--device_id` / `--draft_devices` 参数。运行设备由可见设备掩码环境变量决定(NPU 用 `ASCEND_RT_VISIBLE_DEVICES`,NVIDIA 用 `CUDA_VISIBLE_DEVICES`,寒武纪用 `MLU_VISIBLE_DEVICES`,DCU 用 `HIP_VISIBLE_DEVICES`,摩尔线程用 `MUSA_VISIBLE_DEVICES`)。每个进程会使用其可见的全部设备:单卡进程只暴露一张卡,单进程多卡则暴露多张卡。draft 模型始终与 target 模型使用相同设备。 + ## ConfigJsonUtils | 参数名称 | 类型 | 默认值 | 参数含义 | @@ -37,8 +39,6 @@ xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填 | `model` | `string` | `""` | Hugging Face 模型名称或模型路径。 | | `backend` | `string` | `""` | 后端模型类型;`llm` 表示纯文本模型,`vlm` 表示多模态模型,`dit` 表示扩散模型。 | | `task` | `string` | `"generate"` | 模型任务类型,例如 `generate`、`embed`、`mm_embed`。 | -| `devices` | `string` | `""` | 已废弃,请改用 `device_id`。当前进程使用的设备,例如 `npu:0`、`npu:0,npu:1`。 | -| `device_id` | `int32` | `-1` | 运行模型的 device id,例如 `0`。 | | `limit_image_per_prompt` | `int32` | `8` | 每个 prompt 允许的最大图片数量,仅用于多模态模型。 | | `max_encoder_cache_size` | `int64` | `0` | 每个 worker 的 encoder cache 最大显存大小,单位 MB;`0` 表示禁用 encoder cache。 | | `reasoning_parser` | `string` | `""` | reasoning 交互解析器,例如 `auto`、`glm45`、`glm47`、`glm5`、`qwen3`、`qwen35`、`deepseek-r1`。 | @@ -181,7 +181,6 @@ xLLM 使用 gflags 管理服务启动参数。`--model ` 是唯一必填 | 参数名称 | 类型 | 默认值 | 参数含义 | |:---------|:-----|:-------|:---------| | `draft_model` | `string` | `""` | draft 模型路径;MTP 使用方式详见 [MTP](/zh/features/mtp/)。 | -| `draft_devices` | `string` | `""` | draft 模型使用的设备,例如 `npu:0`、`npu:0,npu:1`;未指定时,启用 speculative decoding 会使用 target 模型的设备。 | | `num_speculative_tokens` | `int32` | `0` | 每轮 speculative decoding 生成的 speculative token 数。 | | `speculative_algorithm` | `string` | `"MTP"` | Speculative decoding 算法,支持 `MTP`、`Eagle3`、`Suffix`、`DFlash`。 | | `speculative_suffix_cache_max_depth` | `int32` | `64` | Suffix speculative decoding 的后缀树最大深度。 | diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md index 0e2be15dec..d86e557a06 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/deepseek/deepseek_v4.md @@ -131,20 +131,17 @@ MASTER_NODE_ADDR="11.87.49.110:10015" LOCAL_HOST="11.87.49.110" # Service Port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=8 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH -model-id ds \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -166,7 +163,6 @@ done # 开启mtp时需要的变量 # --draft_model=$DRAFT_MODEL_PATH \ - # --draft_devices="npu:$DEVICE" \ # --num_speculative_tokens=1 \ # numactl -C xxxxx 亲和性绑核(NUMA亲和性查询命令: npu-smi info -t topo) @@ -179,7 +175,6 @@ done #--enable_chunked_prefill 开启chunked_prefill #--enable_graph 开启aclgraph #--draft_model mtp - mtp权重路径 -#--draft_devices mtp - mtp推理设备(与主模型同一) #--num_speculative_tokens mtp - 预测token数 ``` @@ -206,7 +201,6 @@ export ATB_MATMUL_SHUFFLE_K_ENABLE=0 MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.110" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -215,12 +209,11 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -236,7 +229,6 @@ done MASTER_NODE_ADDR="11.87.49.110:19990" LOCAL_HOST="11.87.49.111" START_PORT=15890 -START_DEVICE=0 LOG_DIR="logs" NNODES=32 LOCAL_NODES=16 @@ -245,12 +237,11 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ )); do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" + LOG_FILE="$LOG_DIR/node_$i.log" nohup $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md index 1037900c5c..12b7b0b9a0 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/glm/glm_5.md @@ -136,7 +136,6 @@ do nohup $XLLM_PATH \ --model $MODEL_PATH \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -151,7 +150,6 @@ do --communication_backend="hccl" \ --graph_decode_batch_size_limit=2 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=16 \ --dp_size=2 \ @@ -169,7 +167,6 @@ done # --enable_graph 开启aclgraph,需要额外显存 # --acl_graph_decode_batch_size_limit 抓图的最大bs,当前需<= 32 / (预测token数 + 1) # --draft_model mtp - mtp权重路径 -# --draft_devices mtp - mtp推理设备(与主模型同一) # --num_speculative_tokens mtp - 预测token数 ``` @@ -206,10 +203,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -224,7 +221,6 @@ for (( i=0; i<$LOCAL_NODES; i++ ))do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -250,10 +246,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ @@ -268,7 +264,6 @@ for (( i=0; i<$LOCAL_NODES; i++ ))do --enable_graph=true \ --acl_graph_decode_batch_size_limit=4 \ --draft_model=$DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens=3 \ --ep_size=32 \ --dp_size=4 \ @@ -430,7 +425,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -443,7 +437,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ @@ -486,7 +479,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --model $MODEL_PATH --model_id glmmoe \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -499,7 +491,6 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ../xllm-service/build/xllm_service/xllm_m --enable_chunked_prefill=false \ --enable_graph=true \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --tool_call_parser=auto \ --enable_disagg_pd=true \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md index 67c50d772b..ca689cc45e 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/kimi/kimi2_5.md @@ -153,10 +153,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ @@ -194,10 +194,10 @@ unset HCCL_OP_EXPANSION_MODE for (( i=0; i<$LOCAL_NODES; i++ ))do PORT=$((START_PORT + i)) DEVICE=$((START_DEVICE + i)); LOG_FILE="$LOG_DIR/node_$i.log" - nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ + nohup numactl -C $((DEVICE*40))-$((DEVICE*40+39)) $XLLM_PATH \ + --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$((i + LOCAL_NODES)) \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md index 2fc02f1c05..d2e290badf 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/minmax/minmax_m2_7.md @@ -167,20 +167,17 @@ MASTER_NODE_ADDR="10.143.3.204:10015" LOCAL_HOST="10.143.3.204" # Service Port START_PORT=18994 -START_DEVICE=0 LOG_DIR="logs" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup numactl -C $((i*40))-$((i*40+39)) $XLLM_PATH \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --node_rank=$i \ diff --git a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md index 39342f4493..9130936415 100644 --- a/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md +++ b/docs/src/content/docs/zh/cookbook/autoregressive_models/qwen/qwen3_5.md @@ -91,7 +91,6 @@ export LD_LIBRARY_PATH=/usr/local/libtorch_npu/lib:$LD_LIBRARY_PATH source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export ASCEND_RT_VISIBLE_DEVICES=14,15 export ASDOPS_LOG_TO_STDOUT=1 export ASDOPS_LOG_LEVEL=0 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -120,19 +119,17 @@ DRAFT_MODEL_PATH="/path/to/Qwen3.5-27B-mtp" MASTER_NODE_ADDR=":32764" START_PORT=18076 -START_DEVICE=0 NNODES=2 +export ASCEND_RT_VISIBLE_DEVICES=14,15 export HCCL_IF_BASE_PORT=53433 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" ./xllm/build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -151,7 +148,6 @@ do --max_concurrent_requests=8 \ --backend llm \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 3 \ >> $LOG_FILE 2>&1 & done diff --git a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md index 950945fda0..19d9588f44 100644 --- a/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md +++ b/docs/src/content/docs/zh/cookbook/diffusion_models/flux/flux2.md @@ -115,7 +115,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/flux2/text_encoder/" # text_encoder路径 MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址(需全局一致) START_PORT=18001 # 服务起始端口 -START_DEVICE=2 # 起始逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=2 # 节点数(当前脚本启动 1 个进程) @@ -124,11 +123,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/mistral_node_$i.log" ./build/xllm/core/server/xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -186,21 +183,18 @@ export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 ```bash MASTER_NODE_ADDR="127.0.0.1:8999" # Master 节点地址(需全局一致) START_PORT=18018 # 服务起始端口 -START_DEVICE=4 # 起始 NPU 逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=2 # 节点数(当前脚本启动 2 个进程) for (( i=0; i<2; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/dit_node_$i.log" ./build/xllm/core/server/xllm \ --model="/path/to/flux2/" \ --max_memory_utilization=0.6 \ --backend="dit" \ --tp_size=2 \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --port $PORT \ diff --git a/docs/src/content/docs/zh/features/disagg_pd.md b/docs/src/content/docs/zh/features/disagg_pd.md index 4dbdd6b310..db66c93d51 100644 --- a/docs/src/content/docs/zh/features/disagg_pd.md +++ b/docs/src/content/docs/zh/features/disagg_pd.md @@ -34,9 +34,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. 4. 以Qwen2-7B为例 - 启动Prefill实例 ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=0 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8010 \ - --devices="npu:0" \ --master_node_addr="127.0.0.1:18888" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ @@ -50,9 +49,8 @@ ENABLE_DECODE_RESPONSE_TO_SERVICE=true ./xllm_master_serving --etcd_addr="127.0. ``` - 启动Decode实例 ```bash - /path/to/xllm --model=Qwen2-7B-Instruct \ + ASCEND_RT_VISIBLE_DEVICES=1 /path/to/xllm --model=Qwen2-7B-Instruct \ --port=8020 \ - --devices="npu:1" \ --master_node_addr="127.0.0.1:18898" \ --enable_prefix_cache=false \ --enable_chunked_prefill=false \ diff --git a/docs/src/content/docs/zh/features/mtp.md b/docs/src/content/docs/zh/features/mtp.md index eb06c21898..6df813375e 100644 --- a/docs/src/content/docs/zh/features/mtp.md +++ b/docs/src/content/docs/zh/features/mtp.md @@ -92,23 +92,19 @@ MODEL_PATH="/models/DeepSeek-V3" DRAFT_MODEL_PATH="/models/DeepSeek-V3-mtp" MASTER_NODE_ADDR="127.0.0.1:42123" START_PORT=13222 -START_DEVICE=0 LOG_DIR="log" NNODES=16 for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" nohup ./xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --draft_model $DRAFT_MODEL_PATH \ - --draft_devices="npu:$DEVICE" \ --num_speculative_tokens 1 \ --max_memory_utilization=0.90 \ --max_tokens_per_batch=10000 \ diff --git a/docs/src/content/docs/zh/getting_started/launch_xllm.md b/docs/src/content/docs/zh/getting_started/launch_xllm.md index 2855124710..d46e881992 100644 --- a/docs/src/content/docs/zh/getting_started/launch_xllm.md +++ b/docs/src/content/docs/zh/getting_started/launch_xllm.md @@ -21,7 +21,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址(需全局一致) START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=1 # 节点数(当前脚本启动 1 个进程) @@ -30,11 +29,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -64,7 +61,6 @@ export CUDA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -73,11 +69,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ @@ -103,7 +97,6 @@ export MLU_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -112,11 +105,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -138,7 +129,6 @@ export HIP_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -147,11 +137,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -174,7 +162,6 @@ export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/da MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -183,11 +170,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ @@ -213,7 +198,6 @@ export MUSA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -222,11 +206,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/getting_started/multi_machine.md b/docs/src/content/docs/zh/getting_started/multi_machine.md index 7eed9e114a..0683e95ac3 100644 --- a/docs/src/content/docs/zh/getting_started/multi_machine.md +++ b/docs/src/content/docs/zh/getting_started/multi_machine.md @@ -22,7 +22,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # 模型路径 MASTER_NODE_ADDR="123.123.123.123:9748" # Master 节点地址(需全局一致) LOCAL_HOST=123.123.123.123 # 本机服务启动IP START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始 NPU 逻辑设备号 LOG_DIR="log" # 日志目录 LOCAL_NODES=16 # 单机节点数(当前脚本启动 16 个进程) NNODES=32 # 总卡数(该示例为2机32卡) @@ -32,13 +31,11 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ @@ -74,7 +71,6 @@ MODEL_PATH="/path/to/your/DeepSeek-R1" # 模型路径 MASTER_NODE_ADDR="123.123.123.123:9748" # Master 节点地址(需全局一致) LOCAL_HOST=456.456.456.456 # 本机服务启动IP START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始 NPU 逻辑设备号 LOG_DIR="log" # 日志目录 LOCAL_NODES=16 # 单机节点数(当前脚本启动 16 个进程) NNODES=32 # 总卡数(该示例为2机32卡) @@ -84,13 +80,11 @@ mkdir -p $LOG_DIR for (( i=0; i<$LOCAL_NODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" /path/to/xllm \ --model $MODEL_PATH \ --host $LOCAL_HOST \ --port $PORT \ - --devices="npu:$DEVICE" \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ --max_memory_utilization=0.86 \ diff --git a/docs/src/content/docs/zh/getting_started/offline_service.md b/docs/src/content/docs/zh/getting_started/offline_service.md index c9b9fbd9a2..3a9430dd28 100644 --- a/docs/src/content/docs/zh/getting_started/offline_service.md +++ b/docs/src/content/docs/zh/getting_started/offline_service.md @@ -16,7 +16,9 @@ LLM Beam Search 示例:[:simple-github: https://github.com/xLLM-AI/xllm/blob/m ```python from xllm import BeamSearchParams, LLM -llm = LLM(model="/path/models/Qwen2-7B-Instruct", devices="npu:0") +# 运行设备通过 ASCEND_RT_VISIBLE_DEVICES 选择 +# (其他平台使用 CUDA_VISIBLE_DEVICES / MLU_VISIBLE_DEVICES 等)。 +llm = LLM(model="/path/models/Qwen2-7B-Instruct") params = BeamSearchParams( beam_width=2, top_logprobs=4, diff --git a/docs/src/content/docs/zh/hardware/ascend_npu.md b/docs/src/content/docs/zh/hardware/ascend_npu.md index e87fecac9d..df9b744100 100644 --- a/docs/src/content/docs/zh/hardware/ascend_npu.md +++ b/docs/src/content/docs/zh/hardware/ascend_npu.md @@ -63,7 +63,6 @@ export HCCL_IF_BASE_PORT=43432 # HCCL 通信基础端口 MODEL_PATH="/path/to/model/Qwen3-8B" # 模型路径 MASTER_NODE_ADDR="127.0.0.1:9748" # Master 节点地址(需全局一致) START_PORT=18000 # 服务起始端口 -START_DEVICE=0 # 起始逻辑设备号 LOG_DIR="log" # 日志目录 NNODES=1 # 节点数(当前脚本启动 1 个进程) @@ -72,11 +71,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="npu:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/hardware/cambricon_mlu.md b/docs/src/content/docs/zh/hardware/cambricon_mlu.md index a8e4e57726..312377b0d9 100644 --- a/docs/src/content/docs/zh/hardware/cambricon_mlu.md +++ b/docs/src/content/docs/zh/hardware/cambricon_mlu.md @@ -38,7 +38,6 @@ export MLU_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -47,11 +46,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="mlu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/dcu.md b/docs/src/content/docs/zh/hardware/dcu.md index 5397b7f9a9..8720e9ecb1 100644 --- a/docs/src/content/docs/zh/hardware/dcu.md +++ b/docs/src/content/docs/zh/hardware/dcu.md @@ -50,7 +50,6 @@ export HIP_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -59,11 +58,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="dcu:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/metax_maca.md b/docs/src/content/docs/zh/hardware/metax_maca.md index 4f9890a8bb..c9f53d53a0 100644 --- a/docs/src/content/docs/zh/hardware/metax_maca.md +++ b/docs/src/content/docs/zh/hardware/metax_maca.md @@ -53,7 +53,6 @@ export FLASHINFER_OPS_PATH=/opt/conda/lib/python3.10/site-packages/flashinfer/da MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -62,11 +61,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --nnodes=$NNODES \ --master_node_addr=$MASTER_NODE_ADDR \ diff --git a/docs/src/content/docs/zh/hardware/musa.md b/docs/src/content/docs/zh/hardware/musa.md index 2a2fae5242..0b6b2a606c 100644 --- a/docs/src/content/docs/zh/hardware/musa.md +++ b/docs/src/content/docs/zh/hardware/musa.md @@ -5,7 +5,7 @@ sidebar: order: 7 --- -在摩尔线程(Mthreads)MUSA GPU 上部署 xLLM 时使用 MUSA 设备后端(`--devices=musa:`)。 +在摩尔线程(Mthreads)MUSA GPU 上部署 xLLM 时使用 MUSA 设备后端;通过 `MUSA_VISIBLE_DEVICES` 选择可见的设备卡。 ## 镜像和容器启动命令 @@ -49,7 +49,6 @@ export MUSA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3.5-27B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -58,11 +57,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="musa:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/docs/src/content/docs/zh/hardware/nvidia_gpu.md b/docs/src/content/docs/zh/hardware/nvidia_gpu.md index 120510fd72..52ea0b7bd8 100644 --- a/docs/src/content/docs/zh/hardware/nvidia_gpu.md +++ b/docs/src/content/docs/zh/hardware/nvidia_gpu.md @@ -46,7 +46,6 @@ export CUDA_VISIBLE_DEVICES=0 MODEL_PATH="/path/to/model/Qwen3-8B" MASTER_NODE_ADDR="127.0.0.1:9748" START_PORT=18000 -START_DEVICE=0 LOG_DIR="log" NNODES=1 @@ -55,11 +54,9 @@ mkdir -p $LOG_DIR for (( i=0; i<$NNODES; i++ )) do PORT=$((START_PORT + i)) - DEVICE=$((START_DEVICE + i)) LOG_FILE="$LOG_DIR/node_$i.log" xllm \ --model $MODEL_PATH \ - --devices="cuda:$DEVICE" \ --port $PORT \ --master_node_addr=$MASTER_NODE_ADDR \ --nnodes=$NNODES \ diff --git a/examples/generate.py b/examples/generate.py index 2a804e475d..55dfd3cf97 100644 --- a/examples/generate.py +++ b/examples/generate.py @@ -1,5 +1,5 @@ -# python examples/generate.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0' -# python generate.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/generate.py --model='/path/models/Qwen2-7B-Instruct' +# ASCEND_RT_VISIBLE_DEVICES=0,1 python generate.py --model='/path/models/Qwen2-7B-Instruct' from xllm import ArgumentParser, LLM, SamplingParams diff --git a/examples/generate_beam_search.py b/examples/generate_beam_search.py index a39a606ff3..78baa3801f 100644 --- a/examples/generate_beam_search.py +++ b/examples/generate_beam_search.py @@ -1,5 +1,5 @@ -# python examples/generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0' -# python generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' +# ASCEND_RT_VISIBLE_DEVICES=0,1 python generate_beam_search.py --model='/path/models/Qwen2-7B-Instruct' from xllm import ArgumentParser, BeamSearchParams, LLM diff --git a/examples/generate_embedding.py b/examples/generate_embedding.py index e0e3c55c95..7a6354fcb9 100644 --- a/examples/generate_embedding.py +++ b/examples/generate_embedding.py @@ -1,5 +1,5 @@ -# python examples/generate_embedding.py --model='/path/models/Qwen3-8B' --devices='npu:0' --runner pooling -# python generate_embedding.py --model='/path/models/Qwen3-8B' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/generate_embedding.py --model='/path/models/Qwen3-8B' --runner pooling +# ASCEND_RT_VISIBLE_DEVICES=0,1 python generate_embedding.py --model='/path/models/Qwen3-8B' from xllm import ArgumentParser, LLM, PoolingParams diff --git a/examples/generate_vlm.py b/examples/generate_vlm.py index 406d1591c6..05231be3f8 100755 --- a/examples/generate_vlm.py +++ b/examples/generate_vlm.py @@ -1,4 +1,4 @@ -# python generate_vlm.py --model /path/to/Qwen2.5-VL-7B-Instruct/ --max_seqs_per_batch 4 --devices='npu:0' --enable_shm +# ASCEND_RT_VISIBLE_DEVICES=0 python generate_vlm.py --model /path/to/Qwen2.5-VL-7B-Instruct/ --max_seqs_per_batch 4 --enable_shm from xllm import ArgumentParser, SamplingParams from xllm import LLM diff --git a/examples/sample.py b/examples/sample.py index 5c6cab869b..7613f67026 100644 --- a/examples/sample.py +++ b/examples/sample.py @@ -1,5 +1,5 @@ -# python examples/sample.py --model='/path/models/Qwen3-8B' --devices='npu:0' -# python examples/sample.py --model='/path/models/Qwen3-8B' --devices='npu:0,npu:1' +# ASCEND_RT_VISIBLE_DEVICES=0 python examples/sample.py --model='/path/models/Qwen3-8B' +# ASCEND_RT_VISIBLE_DEVICES=0,1 python examples/sample.py --model='/path/models/Qwen3-8B' from xllm import ArgumentParser, LLM, RequestParams diff --git a/tests/core/util/device_name_utils_test.cpp b/tests/core/util/device_name_utils_test.cpp index d0c4f7079c..093e71fe61 100644 --- a/tests/core/util/device_name_utils_test.cpp +++ b/tests/core/util/device_name_utils_test.cpp @@ -26,15 +26,9 @@ limitations under the License. namespace xllm { namespace { -TEST(DeviceNameUtilsTest, ToDeviceStringUsesCompiledDeviceType) { - const std::string device_string = DeviceNameUtils::to_device_string(3); - - EXPECT_EQ(device_string, Platform::type_str() + ":3"); -} - TEST(DeviceNameUtilsTest, ParseGeneratedDeviceString) { const std::vector devices = - DeviceNameUtils::parse_devices(DeviceNameUtils::to_device_string(2)); + DeviceNameUtils::parse_devices(Platform::type_str() + ":2"); ASSERT_EQ(devices.size(), 1); EXPECT_EQ(devices[0].type(), Platform::type_torch()); diff --git a/xllm/c_api/default.h b/xllm/c_api/default.h index 4ee08e327e..fa735fbe81 100644 --- a/xllm/c_api/default.h +++ b/xllm/c_api/default.h @@ -58,7 +58,6 @@ const XLLM_InitOptions XLLM_INIT_LLM_OPTIONS_DEFAULT = { .kv_cache_transfer_mode = "PUSH", .log_dir = "", .draft_model = "", - .draft_devices = "", .cpu_affinity = ""}; const XLLM_RequestParams XLLM_LLM_REQUEST_PARAMS_DEFAULT = { @@ -132,7 +131,6 @@ const XLLM_InitOptions XLLM_INIT_REC_OPTIONS_DEFAULT = { .kv_cache_transfer_mode = "PUSH", .log_dir = "", .draft_model = "", - .draft_devices = "", .cpu_affinity = ""}; const XLLM_RequestParams XLLM_REC_REQUEST_PARAMS_DEFAULT = { diff --git a/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp b/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp index 54a30e7693..5bc85309da 100644 --- a/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp +++ b/xllm/c_api/examples/test_press_rec_multimodal_completions.cpp @@ -56,7 +56,6 @@ constexpr int32_t kMaxClientParallelism = 128; struct CliOptions { std::string model_path; - std::string devices = "cuda:0"; std::string master_node_addr = "127.0.0.1:18899"; int32_t prompt_size = 128; int32_t token_min_size = 1024; @@ -737,7 +736,6 @@ void PrintSummary(const CliOptions& options, std::cout << "=== stress_rec_multimodal_completions summary ===\n"; std::cout << "model_id=" << ResolveModelId(options.model_path) << "\n"; std::cout << "model_type=" << config.model_type << "\n"; - std::cout << "devices=" << options.devices << "\n"; std::cout << "master_node_addr=" << options.master_node_addr << "\n"; std::cout << "hidden_size=" << config.hidden_size << ", vocab_size=" << config.vocab_size @@ -802,10 +800,8 @@ int main(int argc, char** argv) { sizeof(init_options.master_node_addr), "%s", options.master_node_addr.c_str()); - const bool init_ok = xllm_rec_initialize(rec_handler, - options.model_path.c_str(), - options.devices.c_str(), - &init_options); + const bool init_ok = xllm_rec_initialize( + rec_handler, options.model_path.c_str(), &init_options); if (!init_ok) { xllm_rec_destroy(rec_handler); throw std::runtime_error("xllm_rec_initialize failed"); diff --git a/xllm/c_api/examples/test_query_llm_chat_completions.cpp b/xllm/c_api/examples/test_query_llm_chat_completions.cpp index 90ed928b85..aa7b193689 100644 --- a/xllm/c_api/examples/test_query_llm_chat_completions.cpp +++ b/xllm/c_api/examples/test_query_llm_chat_completions.cpp @@ -20,7 +20,6 @@ limitations under the License. #include "llm.h" -std::string devices = "cuda:4"; std::string model_name = "Qwen3-8B"; std::string model_path = "/export/home/models/Qwen3-8B"; @@ -34,8 +33,8 @@ XLLM_LLM_Handler* service_startup_hook() { snprintf( init_options.log_dir, sizeof(init_options.log_dir), "/export/xllm/log"); - bool ret = xllm_llm_initialize( - llm_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_llm_initialize(llm_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "LLM init failed" << std::endl; xllm_llm_destroy(llm_handler); diff --git a/xllm/c_api/examples/test_query_llm_completions.cpp b/xllm/c_api/examples/test_query_llm_completions.cpp index 70e56e9d30..5021238cef 100644 --- a/xllm/c_api/examples/test_query_llm_completions.cpp +++ b/xllm/c_api/examples/test_query_llm_completions.cpp @@ -20,7 +20,6 @@ limitations under the License. #include "llm.h" -std::string devices = "cuda:1"; std::string model_name = "Qwen3-8B"; std::string model_path = "/export/home/models/Qwen3-8B"; @@ -35,8 +34,8 @@ XLLM_LLM_Handler* service_startup_hook() { snprintf( init_options.log_dir, sizeof(init_options.log_dir), "/export/xllm/log"); - bool ret = xllm_llm_initialize( - llm_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_llm_initialize(llm_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "LLM init failed" << std::endl; xllm_llm_destroy(llm_handler); diff --git a/xllm/c_api/examples/test_query_rec_completions.cpp b/xllm/c_api/examples/test_query_rec_completions.cpp index a1926f9d12..78d7f67a8f 100644 --- a/xllm/c_api/examples/test_query_rec_completions.cpp +++ b/xllm/c_api/examples/test_query_rec_completions.cpp @@ -21,13 +21,6 @@ limitations under the License. #include "rec.h" -#if defined(USE_NPU) -std::string devices = "npu:14"; -#elif defined(USE_CUDA) -std::string devices = "cuda:0"; -#else -std::string devices = "npu:14"; -#endif std::string model_name = "Qwen3-0.6B"; std::string model_path = "/export/home/models/Qwen3-0.6B"; @@ -54,8 +47,8 @@ XLLM_REC_Handler* service_startup_hook() { init_options.rec_worker_max_concurrency = 1; #endif - bool ret = xllm_rec_initialize( - rec_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_rec_initialize(rec_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "REC init failed" << std::endl; xllm_rec_destroy(rec_handler); @@ -74,11 +67,10 @@ void service_stop_hook(XLLM_REC_Handler* rec_handler) { int main(int argc, char** argv) { if (argc > 1) { - devices = argv[1]; + model_path = argv[1]; } std::cout << "Using model path: " << model_path << std::endl; - std::cout << "Using devices: " << devices << std::endl; XLLM_REC_Handler* rec_handler = service_startup_hook(); if (nullptr == rec_handler) { diff --git a/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp b/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp index 56d2bd323e..59127f9401 100644 --- a/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp +++ b/xllm/c_api/examples/test_query_rec_multimodal_completions.cpp @@ -27,7 +27,6 @@ limitations under the License. #include "rec.h" -std::string devices = "cuda:0"; std::string model_name = "homepage_qwen_06b_6_raw"; std::string model_path = "/export/home/models/homepage_qwen_06b_6_raw"; @@ -185,8 +184,8 @@ XLLM_REC_Handler* service_startup_hook() { sizeof(init_options.log_dir), "/export/home/huheng7/log"); - bool ret = xllm_rec_initialize( - rec_handler, model_path.c_str(), devices.c_str(), &init_options); + bool ret = + xllm_rec_initialize(rec_handler, model_path.c_str(), &init_options); if (!ret) { std::cout << "REC init failed" << std::endl; xllm_rec_destroy(rec_handler); diff --git a/xllm/c_api/internal/llm.cpp b/xllm/c_api/internal/llm.cpp index 3c2ffd18bb..5683a52be8 100644 --- a/xllm/c_api/internal/llm.cpp +++ b/xllm/c_api/internal/llm.cpp @@ -59,9 +59,8 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( XLLM_CAPI_EXPORT bool xllm_llm_initialize( XLLM_LLM_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options) { - if (!handler || !model_path || !devices) return false; + if (!handler || !model_path) return false; try { XLLM_InitOptions xllm_init_options; @@ -81,9 +80,7 @@ XLLM_CAPI_EXPORT bool xllm_llm_initialize( xllm::Options options; options.model_path(model_path) .task_type(xllm_init_options.task) - .devices(devices) .draft_model_path(xllm_init_options.draft_model) - .draft_devices(xllm_init_options.draft_devices) .backend("llm") .block_size(xllm_init_options.block_size) .max_cache_size(xllm_init_options.max_cache_size) diff --git a/xllm/c_api/internal/rec.cpp b/xllm/c_api/internal/rec.cpp index bc0348c941..6048a63b17 100644 --- a/xllm/c_api/internal/rec.cpp +++ b/xllm/c_api/internal/rec.cpp @@ -66,9 +66,8 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( XLLM_CAPI_EXPORT bool xllm_rec_initialize( XLLM_REC_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options) { - if (!handler || !model_path || !devices) return false; + if (!handler || !model_path) return false; try { xllm::CpuAffinity::get_instance().set_cpu_affinity( @@ -91,9 +90,7 @@ XLLM_CAPI_EXPORT bool xllm_rec_initialize( xllm::Options options; options.model_path(model_path) .task_type(xllm_init_options.task) - .devices(devices) .draft_model_path(xllm_init_options.draft_model) - .draft_devices(xllm_init_options.draft_devices) .backend("rec") .block_size(xllm_init_options.block_size) .max_cache_size(xllm_init_options.max_cache_size) diff --git a/xllm/c_api/llm.h b/xllm/c_api/llm.h index 5460ae0da5..72d321c14c 100644 --- a/xllm/c_api/llm.h +++ b/xllm/c_api/llm.h @@ -82,6 +82,8 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( * Loads model weights from the specified path, configures target devices, * initializes compute contexts, and prepares the inference runtime. * Must be called exactly once per handler before using completion/chat APIs. + * The target devices are auto-detected from the visible device mask + * (e.g. ASCEND_RT_VISIBLE_DEVICES / CUDA_VISIBLE_DEVICES). * * If init_options is NULL, this function automatically uses the default values * from XLLM_INIT_LLM_OPTIONS_DEFAULT (via xllm_llm_init_options_default()). @@ -89,9 +91,6 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( * @param handler Valid LLM instance handle (must not be NULL) * @param model_path Null-terminated string of the model directory/file path * (supports .bin/.pth/.safetensors formats) - * @param devices Null-terminated string specifying target devices (format: - * "npu:0,1" (specific NPUs), "cuda:0" (single GPU), "auto" - * (automatic selection)) * @param init_options Advanced initialization options (NULL = use defaults) * * @return true if initialization succeeds; false on failure (see failure causes @@ -100,7 +99,6 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( * @failure_causes * - Invalid handler (NULL or already destroyed) * - Invalid model_path (non-existent, corrupted, or unsupported format) - * - Invalid devices string (malformed format or unavailable devices) * - Model load error (mismatched model architecture or weight corruption) * - Device initialization failure (out of memory, driver error) * @@ -109,7 +107,6 @@ XLLM_CAPI_EXPORT void xllm_llm_init_options_default( */ XLLM_CAPI_EXPORT bool xllm_llm_initialize(XLLM_LLM_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options); /** diff --git a/xllm/c_api/rec.h b/xllm/c_api/rec.h index ef795006f6..2d8fa73657 100644 --- a/xllm/c_api/rec.h +++ b/xllm/c_api/rec.h @@ -75,13 +75,12 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( * @brief Initialize the Generative Recommendation (REC) model and runtime * environment Loads generative recommendation model weights from the specified * path, configures target devices, initializes compute contexts, and prepares - * the recommendation inference runtime + * the recommendation inference runtime. The target devices are auto-detected + * from the visible device mask (e.g. ASCEND_RT_VISIBLE_DEVICES / + * CUDA_VISIBLE_DEVICES). * @param handler Valid REC inference instance handle (must not be NULL) * @param model_path Null-terminated string of the REC model directory/file path * (supports .bin/.pth/.safetensors formats with ranking head) - * @param devices Null-terminated string specifying target devices (format: - * "npu:0,1" (specific NPUs), "cuda:0" (single GPU), "auto" - * (automatic selection)) * @param init_options Advanced initialization options (NULL = use REC defaults) * @return true if initialization succeeds; false on failure (see failure causes * below) @@ -89,7 +88,6 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( * - Invalid handler (NULL or already destroyed) * - Invalid model_path (non-existent, corrupted, or missing ranking head * weights) - * - Invalid devices string (malformed format or unavailable devices) * - Model load error (mismatched REC model architecture or embedding table * corruption) * - Device initialization failure (out of memory, driver error, insufficient @@ -99,7 +97,6 @@ XLLM_CAPI_EXPORT void xllm_rec_init_options_default( */ XLLM_CAPI_EXPORT bool xllm_rec_initialize(XLLM_REC_Handler* handler, const char* model_path, - const char* devices, const XLLM_InitOptions* init_options); /** diff --git a/xllm/c_api/test/utils.cpp b/xllm/c_api/test/utils.cpp index e0724df61b..a8e58f9536 100644 --- a/xllm/c_api/test/utils.cpp +++ b/xllm/c_api/test/utils.cpp @@ -23,7 +23,6 @@ limitations under the License. // --- Server + XLLM_InitOptions gflags (defaults aligned with REC defaults) --- DEFINE_string(model_path, "", "Path to REC model weights"); -DEFINE_string(devices, "auto", "Devices string, e.g. npu:0 or auto"); DEFINE_int32(port, 8000, "brpc TCP port"); DEFINE_string(listen_addr, "", @@ -74,7 +73,6 @@ DEFINE_string(kv_cache_transfer_mode, "PUSH", ""); // Not named "log_dir": glog already registers FLAGS_log_dir. DEFINE_string(xllm_init_log_dir, "", ""); DEFINE_string(draft_model, "", ""); -DEFINE_string(draft_devices, "", ""); namespace xllm_capi_test { @@ -192,8 +190,6 @@ void ApplyGflagsToXllmInitOptions(XLLM_InitOptions* o) { o->log_dir, FLAGS_xllm_init_log_dir, XLLM_META_STRING_FIELD_MAX_LEN); CopyToFixed( o->draft_model, FLAGS_draft_model, XLLM_META_STRING_FIELD_MAX_LEN); - CopyToFixed( - o->draft_devices, FLAGS_draft_devices, XLLM_META_STRING_FIELD_MAX_LEN); } void PbToXllmDims(const c_api_test::XLLM_Dims& pb, XLLM_Dims* out) { diff --git a/xllm/c_api/test/utils.h b/xllm/c_api/test/utils.h index a875e733f9..5246291c76 100644 --- a/xllm/c_api/test/utils.h +++ b/xllm/c_api/test/utils.h @@ -26,7 +26,6 @@ limitations under the License. #include "xllm_test.pb.h" DECLARE_string(model_path); -DECLARE_string(devices); DECLARE_int32(port); DECLARE_string(listen_addr); DECLARE_int32(idle_timeout_s); diff --git a/xllm/c_api/test/xllm_test.cpp b/xllm/c_api/test/xllm_test.cpp index b0d0a608d9..8d2873b697 100644 --- a/xllm/c_api/test/xllm_test.cpp +++ b/xllm/c_api/test/xllm_test.cpp @@ -259,12 +259,10 @@ int main(int argc, char* argv[]) { XLLM_InitOptions init{}; xllm_rec_init_options_default(&init); xllm_capi_test::ApplyGflagsToXllmInitOptions(&init); - if (!xllm_rec_initialize(rec_holder.get(), - FLAGS_model_path.c_str(), - FLAGS_devices.c_str(), - &init)) { - LOG(ERROR) << "xllm_rec_initialize failed model_path=" << FLAGS_model_path - << " devices=" << FLAGS_devices; + if (!xllm_rec_initialize( + rec_holder.get(), FLAGS_model_path.c_str(), &init)) { + LOG(ERROR) << "xllm_rec_initialize failed model_path=" + << FLAGS_model_path; return -4; } rec_ptr = rec_holder.get(); @@ -277,12 +275,10 @@ int main(int argc, char* argv[]) { XLLM_InitOptions init{}; xllm_llm_init_options_default(&init); xllm_capi_test::ApplyGflagsToXllmInitOptions(&init); - if (!xllm_llm_initialize(llm_holder.get(), - FLAGS_model_path.c_str(), - FLAGS_devices.c_str(), - &init)) { - LOG(ERROR) << "xllm_llm_initialize failed model_path=" << FLAGS_model_path - << " devices=" << FLAGS_devices; + if (!xllm_llm_initialize( + llm_holder.get(), FLAGS_model_path.c_str(), &init)) { + LOG(ERROR) << "xllm_llm_initialize failed model_path=" + << FLAGS_model_path; return -6; } llm_ptr = llm_holder.get(); diff --git a/xllm/c_api/types.h b/xllm/c_api/types.h index d4a9c2ce01..ad2e4e7a2e 100644 --- a/xllm/c_api/types.h +++ b/xllm/c_api/types.h @@ -178,12 +178,6 @@ typedef struct XLLM_CAPI_EXPORT XLLM_InitOptions { /** Draft hf model path (empty string = no draft model) */ char draft_model[XLLM_META_STRING_FIELD_MAX_LEN]; - /** - * Devices to run the draft model on (e.g. npu:0, npu:0,npu:1). - * Empty string = use the same devices as main model - */ - char draft_devices[XLLM_META_STRING_FIELD_MAX_LEN]; - // CPU affinity settings, format be like: // 1,2,3 // meaing ith thread attach to logical processor 1,2,3 diff --git a/xllm/cc_api/examples/multiple_llm_instances.cpp b/xllm/cc_api/examples/multiple_llm_instances.cpp index eb460e8fc4..49814cb563 100644 --- a/xllm/cc_api/examples/multiple_llm_instances.cpp +++ b/xllm/cc_api/examples/multiple_llm_instances.cpp @@ -26,7 +26,6 @@ limitations under the License. * multiple models needs to be supported, then you can follow this example. */ -std::string devices = "npu:0"; std::string model_path = "/export/home/models/Qwen3-4B"; int main(int argc, char** argv) { @@ -35,7 +34,7 @@ int main(int argc, char** argv) { std::shared_ptr llm_instance_01 = std::make_shared(); xllm::XLLM_InitLLMOptions options_01; options_01.max_memory_utilization = 0.45; - bool ret = llm_instance_01->Initialize(model_path, devices, options_01); + bool ret = llm_instance_01->Initialize(model_path, options_01); if (!ret) { std::cout << "LLM instance init failed." << std::endl; return -1; @@ -60,7 +59,7 @@ int main(int argc, char** argv) { options_02.server_idx = 1; options_02.max_memory_utilization = 0.9; - ret = llm_instance_02->Initialize(model_path, devices, options_02); + ret = llm_instance_02->Initialize(model_path, options_02); if (!ret) { std::cout << "LLM instance init failed." << std::endl; return -1; diff --git a/xllm/cc_api/examples/single_llm_instance.cpp b/xllm/cc_api/examples/single_llm_instance.cpp index 8490d4fdd3..b3bbf45013 100644 --- a/xllm/cc_api/examples/single_llm_instance.cpp +++ b/xllm/cc_api/examples/single_llm_instance.cpp @@ -25,14 +25,13 @@ limitations under the License. * inference engine. */ -std::string devices = "npu:0"; std::string model_path = "/export/home/models/Qwen3-4B"; int main(int argc, char** argv) { xllm::LLM llm_instance; xllm::XLLM_InitLLMOptions options; options.cp_size = 1; - bool ret = llm_instance.Initialize(model_path, devices, options); + bool ret = llm_instance.Initialize(model_path, options); if (!ret) { std::cout << "LLM init failed." << std::endl; return -1; diff --git a/xllm/cc_api/llm.cpp b/xllm/cc_api/llm.cpp index e6c45dc8f6..37d59a3b6d 100644 --- a/xllm/cc_api/llm.cpp +++ b/xllm/cc_api/llm.cpp @@ -59,7 +59,6 @@ LLM::~LLM() { } bool LLM::Initialize(const std::string& model_path, - const std::string& devices, const XLLM_InitLLMOptions& init_options) { if (!init_options.log_dir.empty()) { InitGlog(init_options.log_dir); @@ -74,9 +73,7 @@ bool LLM::Initialize(const std::string& model_path, Options options; options.model_path(model_path) .task_type(init_options.task) - .devices(devices) .draft_model_path(init_options.draft_model) - .draft_devices(init_options.draft_devices) .backend("llm") .block_size(init_options.block_size) .max_cache_size(init_options.max_cache_size) diff --git a/xllm/cc_api/llm.h b/xllm/cc_api/llm.h index 541ab3c110..9e6ab32646 100644 --- a/xllm/cc_api/llm.h +++ b/xllm/cc_api/llm.h @@ -44,16 +44,14 @@ class XLLM_CAPI_EXPORT LLM { * @brief Initialize the model: Load model files and configure runtime * environment * @param model_path Path to model files - * @param devices Device configuration (format: "npu:1" for specific NPU, - * "auto" for auto-selection) * @param init_options Advanced initialization options, Provided default * configuration * @return bool true if initialization succeeds; false if fails * @note Must be called before Completions/ChatCompletions, and only needs to - * be called once + * be called once. The devices to run on are auto-detected from the visible + * device mask (e.g. ASCEND_RT_VISIBLE_DEVICES / CUDA_VISIBLE_DEVICES). */ bool Initialize(const std::string& model_path, - const std::string& devices, const XLLM_InitLLMOptions& init_options); /** diff --git a/xllm/cc_api/types.h b/xllm/cc_api/types.h index f690c82a38..d8ea2b9145 100644 --- a/xllm/cc_api/types.h +++ b/xllm/cc_api/types.h @@ -137,9 +137,6 @@ struct XLLM_CAPI_EXPORT XLLM_InitLLMOptions { // draft hf model path to the model file std::optional draft_model = std::nullopt; - // Devices to run the draft model on, e.g. npu:0, npu:0,npu:1 - std::optional draft_devices = std::nullopt; - // Index ID for internal server ID, which must be set different values // if the model supports multiple version or there are multiple models. int64_t server_idx = 0; diff --git a/xllm/core/common/global_flags.h b/xllm/core/common/global_flags.h index ecc6a13d03..a9d65171fd 100755 --- a/xllm/core/common/global_flags.h +++ b/xllm/core/common/global_flags.h @@ -43,10 +43,6 @@ DECLARE_string(python_model_path); DECLARE_string(task); -DECLARE_string(devices); - -DECLARE_int32(device_id); - DECLARE_int32(limit_image_per_prompt); DECLARE_string(mm_download_headers); @@ -76,8 +72,6 @@ DECLARE_int32(max_tokens_per_chunk_for_prefill); // --- speculative inference config --- DECLARE_string(draft_model); -DECLARE_string(draft_devices); - DECLARE_int32(num_speculative_tokens); DECLARE_string(speculative_algorithm); diff --git a/xllm/core/common/options.cpp b/xllm/core/common/options.cpp index 7bbc65a030..bfe237b1f3 100644 --- a/xllm/core/common/options.cpp +++ b/xllm/core/common/options.cpp @@ -20,9 +20,7 @@ std::string Options::to_string() const { std::stringstream ss; ss << "Options: ["; ss << "model_path: " << model_path() - << ", devices: " << devices().value_or("null") << ", draft_model_path: " << draft_model_path().value_or("null") - << ", draft_devices: " << draft_devices().value_or("null") << ", backend: " << backend() << ", limit_image_per_prompt: " << limit_image_per_prompt() << ", block_size: " << block_size() diff --git a/xllm/core/common/options.h b/xllm/core/common/options.h index 63deb6116a..e1c56661fc 100644 --- a/xllm/core/common/options.h +++ b/xllm/core/common/options.h @@ -39,12 +39,8 @@ class Options { PROPERTY(std::string, model_id); - PROPERTY(std::optional, devices); - PROPERTY(std::optional, draft_model_path); - PROPERTY(std::optional, draft_devices); - // model backend PROPERTY(std::string, backend); diff --git a/xllm/core/distributed_runtime/master.cpp b/xllm/core/distributed_runtime/master.cpp index 0012e6b15e..b1743ddde2 100644 --- a/xllm/core/distributed_runtime/master.cpp +++ b/xllm/core/distributed_runtime/master.cpp @@ -199,10 +199,19 @@ Master::Master(const Options& options, EngineType type) master_status_(options.master_status()) { const auto model_path = std::filesystem::path(options_.model_path()).lexically_normal(); - const auto devices = - DeviceNameUtils::parse_devices(options_.devices().value_or("auto")); - const int32_t global_world_size = - options_.nnodes() * static_cast(devices.size()); + // Multi-process serving runs one worker per process. Enumerate the visible + // cards (honoring *_VISIBLE_DEVICES) and select the single card this process + // owns by its node_rank, so `devices` holds exactly the card this process + // uses -- mirroring the historical single-element devices semantics. + const auto visible_devices = DeviceNameUtils::parse_devices("auto"); + CHECK_LT(options_.node_rank(), static_cast(visible_devices.size())) + << "node_rank " << options_.node_rank() + << " exceeds the number of visible devices " << visible_devices.size() + << ". Ensure *_VISIBLE_DEVICES exposes all cards used across processes."; + const std::vector devices = { + visible_devices[options_.node_rank()]}; + // World size is the node count (one worker per process). + const int32_t global_world_size = options_.nnodes(); std::string cp_model_type; if (options_.cp_size() > 1 && Platform::uses_model_cp_partition()) { cp_model_type = util::get_model_type(model_path, options_.backend()); @@ -341,8 +350,8 @@ Master::Master(const Options& options, EngineType type) const bool use_suffix_spec = options_.speculative_algorithm() == "Suffix"; CHECK(use_suffix_spec || !draft_model_path.empty()) << "draft model path is required unless --speculative_algorithm=Suffix"; - const auto draft_devices = DeviceNameUtils::parse_devices( - options_.draft_devices().value_or("auto")); + // Draft model shares the same devices as the target model. + const auto& draft_devices = devices; LOG(INFO) << "Using draft devices: " << DeviceNameUtils::to_string(draft_devices); runtime::Options spec_options; diff --git a/xllm/core/framework/config/model_config.cpp b/xllm/core/framework/config/model_config.cpp index 66bd75a3e3..c4719a75c0 100644 --- a/xllm/core/framework/config/model_config.cpp +++ b/xllm/core/framework/config/model_config.cpp @@ -19,7 +19,6 @@ limitations under the License. #include "core/common/global_flags.h" #include "core/framework/config/config_utils.h" -#include "core/platform/device_name_utils.h" DEFINE_string(model_id, "", "hf model name."); @@ -48,13 +47,6 @@ DEFINE_string(task, "generate", "The task to use the model for(e.g. generate, embed, mm_embed)."); -DEFINE_string(devices, - "", - "Deprecated. Use --device_id instead. Devices to run the model " - "on, e.g. npu:0, npu:0,npu:1."); - -DEFINE_int32(device_id, -1, "Device id to run the model on, e.g. 0."); - DEFINE_int32(limit_image_per_prompt, 8, "Maximum number of image per prompt. Only applicable for " @@ -123,19 +115,6 @@ void ModelConfig::from_flags() { XLLM_CONFIG_ASSIGN_FROM_FLAG(python_model_path); XLLM_CONFIG_ASSIGN_FROM_FLAG(backend); XLLM_CONFIG_ASSIGN_FROM_FLAG(task); - XLLM_CONFIG_ASSIGN_FROM_FLAG(device_id); - const bool devices_specified = config::is_flag_specified("devices"); - const bool device_id_specified = config::is_flag_specified("device_id"); - if (devices_specified) { - LOG(WARNING) << "--devices is deprecated and will be removed in a future " - "release. Use --device_id instead."; - } - if (devices_specified && !device_id_specified) { - XLLM_CONFIG_ASSIGN_FROM_FLAG(devices); - } else { - CHECK(device_id() >= 0) << "--device_id must be >= 0."; - devices(DeviceNameUtils::to_device_string(device_id())); - } XLLM_CONFIG_ASSIGN_FROM_FLAG(limit_image_per_prompt); XLLM_CONFIG_ASSIGN_FROM_FLAG(max_encoder_cache_size); XLLM_CONFIG_ASSIGN_FROM_FLAG(reasoning_parser); @@ -176,9 +155,6 @@ void ModelConfig::from_json(const JsonReader& json) { XLLM_CONFIG_ASSIGN_FROM_JSON(python_model_path); XLLM_CONFIG_ASSIGN_FROM_JSON(backend); XLLM_CONFIG_ASSIGN_FROM_JSON(task); - // don't read rank-related config - // XLLM_CONFIG_ASSIGN_FROM_JSON(device_id); - // XLLM_CONFIG_ASSIGN_FROM_JSON(devices); XLLM_CONFIG_ASSIGN_FROM_JSON(limit_image_per_prompt); XLLM_CONFIG_ASSIGN_FROM_JSON(max_encoder_cache_size); XLLM_CONFIG_ASSIGN_FROM_JSON(reasoning_parser); @@ -203,11 +179,6 @@ void ModelConfig::append_config_json( config_json, default_config, python_model_path); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, backend); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, task); - // don't dump rank-related config - // APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, - // device_id); - // APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT(config_json, default_config, - // devices); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, limit_image_per_prompt); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( diff --git a/xllm/core/framework/config/model_config.h b/xllm/core/framework/config/model_config.h index f2a4f29b42..8566de744c 100644 --- a/xllm/core/framework/config/model_config.h +++ b/xllm/core/framework/config/model_config.h @@ -50,8 +50,6 @@ class ModelConfig final { "model_impl", "backend", "task", - "device_id", - "devices", "python_model_path", "limit_image_per_prompt", "max_encoder_cache_size", @@ -78,10 +76,6 @@ class ModelConfig final { PROPERTY(std::string, task) = "generate"; - PROPERTY(int32_t, device_id) = -1; - - PROPERTY(std::string, devices) = ""; - PROPERTY(int32_t, limit_image_per_prompt) = 8; PROPERTY(int64_t, max_encoder_cache_size) = 0; diff --git a/xllm/core/framework/config/speculative_config.cpp b/xllm/core/framework/config/speculative_config.cpp index d74b498e0b..beb20e93ce 100644 --- a/xllm/core/framework/config/speculative_config.cpp +++ b/xllm/core/framework/config/speculative_config.cpp @@ -22,12 +22,6 @@ limitations under the License. DEFINE_string(draft_model, "", "draft hf model path to the model file."); -DEFINE_string(draft_devices, - "", - "Devices to run the draft model on, e.g. npu:0, npu:0,npu:1. " - "If omitted, uses the target model devices when speculative " - "decoding is enabled."); - DEFINE_int32(num_speculative_tokens, 0, "Number of speculative tokens."); DEFINE_string(speculative_algorithm, @@ -80,11 +74,6 @@ namespace xllm { void SpeculativeConfig::from_flags() { XLLM_CONFIG_ASSIGN_FROM_FLAG(draft_model); - if (config::is_flag_specified("draft_devices")) { - LOG(WARNING) << "--draft_devices is deprecated and will be removed in a " - "future release. Because it's same as --devices."; - } - XLLM_CONFIG_ASSIGN_FROM_FLAG(draft_devices); XLLM_CONFIG_ASSIGN_FROM_FLAG(num_speculative_tokens); XLLM_CONFIG_ASSIGN_FROM_FLAG(speculative_algorithm); XLLM_CONFIG_ASSIGN_FROM_FLAG(speculative_suffix_cache_max_depth); @@ -100,8 +89,6 @@ void SpeculativeConfig::from_flags() { void SpeculativeConfig::from_json(const JsonReader& json) { XLLM_CONFIG_ASSIGN_FROM_JSON(draft_model); - // don't read rank-related config - // XLLM_CONFIG_ASSIGN_FROM_JSON(draft_devices); XLLM_CONFIG_ASSIGN_FROM_JSON(num_speculative_tokens); XLLM_CONFIG_ASSIGN_FROM_JSON(speculative_algorithm); XLLM_CONFIG_ASSIGN_FROM_JSON(speculative_suffix_cache_max_depth); @@ -120,9 +107,6 @@ void SpeculativeConfig::append_config_json( const SpeculativeConfig default_config; APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, draft_model); - // don't dump rank-related config - // APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( - // config_json, default_config, draft_devices); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( config_json, default_config, num_speculative_tokens); APPEND_CONFIG_JSON_VALUE_IF_NOT_DEFAULT( diff --git a/xllm/core/framework/config/speculative_config.h b/xllm/core/framework/config/speculative_config.h index 54c98b9db9..b259236b52 100644 --- a/xllm/core/framework/config/speculative_config.h +++ b/xllm/core/framework/config/speculative_config.h @@ -55,7 +55,6 @@ class SpeculativeConfig final { static const OptionCategory kOptionCategory = { "SPECULATIVE OPTIONS", {"draft_model", - "draft_devices", "num_speculative_tokens", "speculative_algorithm", "speculative_suffix_cache_max_depth", @@ -72,8 +71,6 @@ class SpeculativeConfig final { PROPERTY(std::string, draft_model); - PROPERTY(std::string, draft_devices) = ""; - PROPERTY(int32_t, num_speculative_tokens) = 0; PROPERTY(std::string, speculative_algorithm) = "MTP"; diff --git a/xllm/core/platform/device_name_utils.cpp b/xllm/core/platform/device_name_utils.cpp index 6efdb523dc..6b06324875 100644 --- a/xllm/core/platform/device_name_utils.cpp +++ b/xllm/core/platform/device_name_utils.cpp @@ -29,10 +29,6 @@ limitations under the License. namespace xllm { -std::string DeviceNameUtils::to_device_string(int32_t device_id) { - return Platform::type_str() + ":" + std::to_string(device_id); -} - std::vector DeviceNameUtils::parse_devices( const std::string& device_str) { std::vector devices; diff --git a/xllm/core/platform/device_name_utils.h b/xllm/core/platform/device_name_utils.h index ca4c0aa872..3adc3849ab 100644 --- a/xllm/core/platform/device_name_utils.h +++ b/xllm/core/platform/device_name_utils.h @@ -27,8 +27,6 @@ namespace xllm { class DeviceNameUtils { public: - static std::string to_device_string(int32_t device_id); - static std::vector parse_devices( const std::string& device_str); diff --git a/xllm/core/platform/platform.cpp b/xllm/core/platform/platform.cpp index 6503a60ebc..a12d3910e4 100644 --- a/xllm/core/platform/platform.cpp +++ b/xllm/core/platform/platform.cpp @@ -84,6 +84,20 @@ int32_t Platform::device_count() { #endif } +int32_t Platform::current_device() { +#if defined(USE_NPU) + return static_cast(c10_npu::current_device()); +#elif defined(USE_MLU) + return static_cast(torch_mlu::current_device()); +#elif defined(USE_CUDA) || defined(USE_ILU) + return static_cast(c10::cuda::current_device()); +#elif defined(USE_MUSA) + return static_cast(c10::musa::current_device()); +#elif defined(USE_DCU) + return static_cast(c10::hip::current_device()); +#endif +} + bool Platform::is_enable_pdl() { return enable_pdl_; } int32_t Platform::sm_count() { return sm_count_; } diff --git a/xllm/core/platform/platform.h b/xllm/core/platform/platform.h index c2a29ea7e3..67c2ed77c6 100644 --- a/xllm/core/platform/platform.h +++ b/xllm/core/platform/platform.h @@ -81,6 +81,9 @@ class Platform final { } static int32_t device_count(); + // Returns the logical index of the device bound to the current thread. + // Valid only after the device has been set (e.g. via Device::set_device). + static int32_t current_device(); static int32_t sm_count(); static bool is_enable_pdl(); static bool is_support_sm90a(); diff --git a/xllm/launch_server.py b/xllm/launch_server.py index 142b47beb7..c4c5c0c888 100644 --- a/xllm/launch_server.py +++ b/xllm/launch_server.py @@ -100,14 +100,6 @@ def _build_parser() -> argparse.ArgumentParser: default=None, help="Launch only this rank. If omitted, local ranks 0..nnodes-1 are launched.", ) - parser.add_argument( - "--start-device-id", - "--start_device_id", - dest="start_device_id", - type=int, - default=0, - help="Base logical device id. Local multi-rank launch uses id + rank.", - ) parser.add_argument( "--log-dir", "--log_dir", @@ -195,8 +187,6 @@ def _validate_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> parser.error("--nnodes must be greater than 0") if args.start_port < 1 or args.start_port > 65535: parser.error("--port/--start-port must be in range [1, 65535]") - if args.start_device_id < 0: - parser.error("--start-device-id must be greater than or equal to 0") if args.node_rank is not None and ( args.node_rank < 0 or args.node_rank >= args.nnodes ): @@ -224,15 +214,6 @@ def _resolve_port( return args.start_port -def _resolve_device_id( - args: argparse.Namespace, - rank: int, - launches_all_local_ranks: bool, -) -> int: - rank_offset = rank if launches_all_local_ranks else 0 - return args.start_device_id + rank_offset - - def _build_command( binary_path: str, args: argparse.Namespace, @@ -241,7 +222,6 @@ def _build_command( launches_all_local_ranks: bool, ) -> list[str]: port = _resolve_port(args, rank, launches_all_local_ranks) - device_id = _resolve_device_id(args, rank, launches_all_local_ranks) command = [binary_path] if args.config_json_file is not None: @@ -249,7 +229,6 @@ def _build_command( command.append(f"--port={port}") command.append(f"--nnodes={args.nnodes}") command.append(f"--node_rank={rank}") - command.append(f"--device_id={device_id}") command.extend(extra_args) return command diff --git a/xllm/pybind/args.py b/xllm/pybind/args.py index 7e4767c6a7..6dd34a4936 100644 --- a/xllm/pybind/args.py +++ b/xllm/pybind/args.py @@ -34,9 +34,7 @@ def __init__(self) -> None: self.parser.add_argument('--model', type=str, help='"Name or path of the huggingface model to use."') self.parser.add_argument('--task', type=str, default="generate", help='The task to use the model for. generate/embed.') self.parser.add_argument('--runner', type=str, choices=['pooling'], default=None, help='Optional runner mode for LLM. Currently supports: pooling.') - self.parser.add_argument('--devices', type=str, default='npu:0', help='Devices to run the model on, e.g. npu:0,npu:1.') self.parser.add_argument('--draft_model', type=str, default='', help='draft hf model path to the model file.') - self.parser.add_argument('--draft_devices', type=str, default='npu:0', help='Devices to run the draft model on, e.g. npu:0,npu:1.') self.parser.add_argument('--limit_image_per_prompt', type=int, default=8, help='Maximum number of images per prompt.') self.parser.add_argument('--block_size', type=int, default=128, help='Number of slots per kv cache block. Default is 128.') self.parser.add_argument('--max_cache_size', type=int, default=0, help='Max gpu memory size for kv cache. Default is 0, which means cache size is calculated by available memory.') diff --git a/xllm/pybind/bind.cpp b/xllm/pybind/bind.cpp index 412535cd24..aab3abf560 100644 --- a/xllm/pybind/bind.cpp +++ b/xllm/pybind/bind.cpp @@ -40,9 +40,7 @@ PYBIND11_MODULE(xllm_export, m) { py::class_(m, "Options") .def(py::init()) .def_readwrite("model_path", &Options::model_path_) - .def_readwrite("devices", &Options::devices_) .def_readwrite("draft_model_path", &Options::draft_model_path_) - .def_readwrite("draft_devices", &Options::draft_devices_) .def_readwrite("backend", &Options::backend_) .def_readwrite("limit_image_per_prompt", &Options::limit_image_per_prompt_) diff --git a/xllm/pybind/embedding.py b/xllm/pybind/embedding.py index 024f7f88b3..3593b7d41d 100644 --- a/xllm/pybind/embedding.py +++ b/xllm/pybind/embedding.py @@ -28,7 +28,6 @@ class Embedding: def __init__( self, model: str, - devices: str = 'npu:0', limit_image_per_prompt: int = 8, block_size: int = 128, max_cache_size: int = 0, @@ -76,9 +75,7 @@ def __init__( options = Options() options.model_path = model options.task_type = "embed" - options.devices = devices options.draft_model_path = None - options.draft_devices = None options.backend = "llm" options.limit_image_per_prompt = limit_image_per_prompt options.block_size = block_size diff --git a/xllm/pybind/llm.py b/xllm/pybind/llm.py index c15fdf51f7..969fb7f3b1 100644 --- a/xllm/pybind/llm.py +++ b/xllm/pybind/llm.py @@ -93,9 +93,7 @@ def __init__( model: str, task: str = "generate", runner: Optional[str] = None, - devices: str = 'npu:0', draft_model: Optional[str] = '', - draft_devices: Optional[str] = 'npu:0', limit_image_per_prompt: int = 8, block_size: int = 128, max_cache_size: int = 0, @@ -165,9 +163,7 @@ def __init__( options = Options() options.model_path = model options.task_type = task - options.devices = devices options.draft_model_path = draft_model - options.draft_devices = draft_devices options.backend = backend options.limit_image_per_prompt = limit_image_per_prompt options.block_size = block_size diff --git a/xllm/pybind/vlm.py b/xllm/pybind/vlm.py index 2f52ea5560..84c85da03b 100644 --- a/xllm/pybind/vlm.py +++ b/xllm/pybind/vlm.py @@ -50,9 +50,7 @@ def __init__( self, model: str, task: str = "generate", - devices: str = 'npu:0', draft_model: Optional[str] = '', - draft_devices: Optional[str] = 'npu:0', limit_image_per_prompt: int = 8, block_size: int = 128, max_cache_size: int = 0, @@ -107,9 +105,7 @@ def __init__( options = Options() options.model_path = model options.task_type = task - options.devices = devices options.draft_model_path = draft_model - options.draft_devices = draft_devices options.backend ="vlm" options.limit_image_per_prompt = limit_image_per_prompt options.block_size = block_size diff --git a/xllm/xllm.cpp b/xllm/xllm.cpp index e39787d061..d0ebb126fe 100644 --- a/xllm/xllm.cpp +++ b/xllm/xllm.cpp @@ -119,7 +119,6 @@ Options create_options(const std::string& instance_name, bool is_local) { options.model_path(model_config.model()) .model_id(model_config.model_id()) .task_type(model_config.task()) - .devices(model_config.devices()) .draft_model_path(speculative_config.draft_model()) .backend(model_config.backend()) .limit_image_per_prompt(model_config.limit_image_per_prompt()) @@ -224,13 +223,6 @@ Options create_options(const std::string& instance_name, bool is_local) { static_cast(rec_config.rec_worker_max_concurrency())) .is_local(is_local); - if (speculative_config.num_speculative_tokens() > 0) { - const std::string draft_devices = speculative_config.draft_devices().empty() - ? model_config.devices() - : speculative_config.draft_devices(); - options.draft_devices(draft_devices); - } - return options; } @@ -427,8 +419,7 @@ int run() { // init XTensor allocator and PhyPagePool for xtensor mode if (kv_cache_config.enable_xtensor()) { // Parse devices - const auto devices = - DeviceNameUtils::parse_devices(options.devices().value_or("auto")); + const auto devices = DeviceNameUtils::parse_devices("auto"); // Initialize XTensorAllocator with first device auto& allocator = XTensorAllocator::get_instance();