DeepSeek-V3.2 多机推理部署指南：vLLM-Ascend 与 TIONE 在线服务

1) vLLM 原生双节点部署 → 验证通信与基础功能
2) 固化 HCCL / DP / TP / EP 配置
3) TIONE 在线服务配置等效资源与数据源
4) 以 master/worker 脚本复刻原生命令
5) 功能与性能验证
6) 按长上下文场景调优或切换至 P/D 分离部署

export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1

docker run --rm \
  --name vllm-ascend \
  --shm-size=1g \
  --net=host \
  --device /dev/davinci0 \
  --device /dev/davinci1 \
  --device /dev/davinci2 \
  --device /dev/davinci3 \
  --device /dev/davinci4 \
  --device /dev/davinci5 \
  --device /dev/davinci6 \
  --device /dev/davinci7 \
  --device /dev/davinci_manager \
  --device /dev/devmm_svm \
  --device /dev/hisi_hdc \
  -v /usr/local/dcmi:/usr/local/dcmi \
  -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
  -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
  -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
  -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
  -v /etc/ascend_install.info:/etc/ascend_install.info \
  -v /root/.cache:/root/.cache \
  -it $IMAGE bash

# 节点本地参数，必须按实际环境修改
nic_name="eth0"                 # local_ip 对应的网卡名
local_ip="<current_node_ip>"    # 当前节点 IP
node0_ip="<node0_ip>"           # Node0 的 local_ip，所有节点保持一致

# 通用 HCCL / Gloo / TP 配置
export HCCL_OP_EXPANSION_MODE="AIV"
export HCCL_IF_IP=$local_ip
export GLOO_SOCKET_IFNAME=$nic_name
export TP_SOCKET_IFNAME=$nic_name
export HCCL_SOCKET_IFNAME=$nic_name

# 运行时通用环境变量
export OMP_PROC_BIND=false
export VLLM_USE_V1=1
export HCCL_BUFFSIZE=200
export VLLM_ASCEND_ENABLE_MLAPO=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export VLLM_ASCEND_ENABLE_FLASHCOMM1=1

export OMP_NUM_THREADS=100
export HCCL_CONNECT_TIMEOUT=120
export HCCL_INTRA_PCIE_ENABLE=1
export HCCL_INTRA_ROCE_ENABLE=0

export OMP_NUM_THREADS=10

vllm serve /root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V3.2-W8A8 \
  --host 0.0.0.0 \
  --port 8077 \
  --data-parallel-size 2 \
  --data-parallel-size-local 1 \
  --data-parallel-address $node0_ip \
  --data-parallel-rpc-port 13389 \
  --tensor-parallel-size 8 \
  --quantization ascend \
  --seed 1024 \
  --served-model-name deepseek_v3_2 \
  --enable-expert-parallel \
  --max-num-seqs 16 \
  --max-model-len 8192 \
  --max-num-batched-tokens 4096 \
  --trust-remote-code \
  --no-enable-prefix-caching \
  --gpu-memory-utilization 0.92 \
  --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[8, 16, 24, 32, 40, 48]}' \
  --additional-config '{"layer_sharding": ["q_b_proj", "o_proj"]}' \
  --speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}'

vllm serve /root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V3.2-W8A8 \
  --host 0.0.0.0 \
  --port 8077 \
  --headless \
  --data-parallel-size 2 \
  --data-parallel-size-local 1 \
  --data-parallel-start-rank 1 \
  --data-parallel-address $node0_ip \
  --data-parallel-rpc-port 13389 \
  --tensor-parallel-size 8 \
  --quantization ascend \
  --seed 1024 \
  --served-model-name deepseek_v3_2 \
  --enable-expert-parallel \
  --max-num-seqs 16 \
  --max-model-len 8192 \
  --max-num-batched-tokens 4096 \
  --trust-remote-code \
  --no-enable-prefix-caching \
  --gpu-memory-utilization 0.92 \
  --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[8, 16, 24, 32, 40, 48]}' \
  --additional-config '{"layer_sharding": ["q_b_proj", "o_proj"]}' \
  --speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}'

/bin/bash /data/model/start_$(echo $HOSTNAME | awk -F'-' '{print $5}').sh \
  >> /data/model/logs/DeepSeekV32-mtp_$(echo $HOSTNAME | awk -F'-' '{print $5}').log 2>&1

#!/bin/bash

# 推理和函数调用相关的附加参数
EXTRA_PARAMS=${EXTRA_PARAMS:-"--reasoning-parser deepseek_v3 --enable-auto-tool-choice --tool-call-parser deepseek_v32 --tokenizer-mode deepseek_v32"}

# nic_name 为当前节点 local_ip 对应的网卡名
nic_name=${NIC_NAME:-"eth0"}
local_ip=$(grep $HOSTNAME /etc/hosts | awk '{print $1}')

# node0_ip 必须与 master 节点的 local_ip 一致
node0_ip=${MASTER_ADDR:-$local_ip}

# HCCL / Gloo / TP 通信配置
export HCCL_OP_EXPANSION_MODE="AIV"
export HCCL_IF_IP=$local_ip
export GLOO_SOCKET_IFNAME=$nic_name
export TP_SOCKET_IFNAME=$nic_name
export HCCL_SOCKET_IFNAME=$nic_name

# 运行时环境变量
export OMP_PROC_BIND=false
export OMP_NUM_THREADS=100
export VLLM_USE_V1=1
export HCCL_BUFFSIZE=200
export VLLM_ASCEND_ENABLE_MLAPO=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export VLLM_ASCEND_ENABLE_FLASHCOMM1=1
export HCCL_CONNECT_TIMEOUT=120
export HCCL_INTRA_PCIE_ENABLE=1
export HCCL_INTRA_ROCE_ENABLE=0

vllm serve /data/model \
  --host 0.0.0.0 \
  --port 8000 \
  --data-parallel-size 2 \
  --data-parallel-size-local 1 \
  --data-parallel-address $node0_ip \
  --data-parallel-rpc-port 13389 \
  --tensor-parallel-size 8 \
  --quantization ascend \
  --seed 1024 \
  --served-model-name DeepSeek-V3_2-mtp \
  --enable-expert-parallel \
  --max-num-seqs 8 \
  --max-model-len 60000 \
  --max-num-batched-tokens 4096 \
  --trust-remote-code \
  --gpu-memory-utilization 0.95 \
  --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[8, 16, 24]}' \
  --additional-config '{"layer_sharding": ["q_b_proj", "o_proj"]}' \
  --speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}' \
  $EXTRA_PARAMS

#!/bin/bash

EXTRA_PARAMS=${EXTRA_PARAMS:-"--reasoning-parser deepseek_v3 --enable-auto-tool-choice --tool-call-parser deepseek_v32 --tokenizer-mode deepseek_v32"}

nic_name=${NIC_NAME:-"eth0"}
local_ip=$(grep $HOSTNAME /etc/hosts | awk '{print $1}')
node0_ip=${MASTER_ADDR:-$local_ip}

export HCCL_OP_EXPANSION_MODE="AIV"
export HCCL_IF_IP=$local_ip
export GLOO_SOCKET_IFNAME=$nic_name
export TP_SOCKET_IFNAME=$nic_name
export HCCL_SOCKET_IFNAME=$nic_name

export OMP_PROC_BIND=false
export OMP_NUM_THREADS=100
export VLLM_USE_V1=1
export HCCL_BUFFSIZE=200
export VLLM_ASCEND_ENABLE_MLAPO=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export VLLM_ASCEND_ENABLE_FLASHCOMM1=1
export HCCL_CONNECT_TIMEOUT=120
export HCCL_INTRA_PCIE_ENABLE=1
export HCCL_INTRA_ROCE_ENABLE=0

vllm serve /data/model \
  --host 0.0.0.0 \
  --port 8000 \
  --headless \
  --data-parallel-size 2 \
  --data-parallel-size-local 1 \
  --data-parallel-start-rank 1 \
  --data-parallel-address $node0_ip \
  --data-parallel-rpc-port 13389 \
  --tensor-parallel-size 8 \
  --quantization ascend \
  --seed 1024 \
  --served-model-name DeepSeek-V3_2-mtp \
  --enable-expert-parallel \
  --max-num-seqs 8 \
  --max-model-len 60000 \
  --max-num-batched-tokens 4096 \
  --trust-remote-code \
  --gpu-memory-utilization 0.95 \
  --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes":[8, 16, 24]}' \
  --additional-config '{"layer_sharding": ["q_b_proj", "o_proj"]}' \
  --speculative-config '{"num_speculative_tokens": 3, "method": "deepseek_mtp"}' \
  $EXTRA_PARAMS

{
  "kv_connector": "MooncakeLayerwiseConnector",
  "kv_role": "kv_producer",
  "kv_port": "30000",
  "engine_id": "0",
  "kv_connector_extra_config": {
    "prefill": {"dp_size": 2, "tp_size": 16},
    "decode":  {"dp_size": 8, "tp_size": 4}
  }
}

{
  "kv_connector": "MooncakeLayerwiseConnector",
  "kv_role": "kv_consumer",
  "kv_port": "30100",
  "engine_id": "1",
  "kv_connector_extra_config": {
    "prefill": {"dp_size": 2, "tp_size": 16},
    "decode":  {"dp_size": 8, "tp_size": 4}
  }
}

# Prefill Node0
python launch_online_dp.py \
  --dp-size 2 --tp-size 16 --dp-size-local 1 \
  --dp-rank-start 0 \
  --dp-address <prefill_node0_ip> --dp-rpc-port 12890 \
  --vllm-start-port 9100

# Prefill Node1
python launch_online_dp.py \
  --dp-size 2 --tp-size 16 --dp-size-local 1 \
  --dp-rank-start 1 \
  --dp-address <prefill_node0_ip> --dp-rpc-port 12890 \
  --vllm-start-port 9100

# Decode Node0
python launch_online_dp.py \
  --dp-size 8 --tp-size 4 --dp-size-local 4 \
  --dp-rank-start 0 \
  --dp-address <decode_node0_ip> --dp-rpc-port 12777 \
  --vllm-start-port 9100

# Decode Node1
python launch_online_dp.py \
  --dp-size 8 --tp-size 4 --dp-size-local 4 \
  --dp-rank-start 4 \
  --dp-address <decode_node0_ip> --dp-rpc-port 12777 \
  --vllm-start-port 9100

curl http://<node0_ip>:8077/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "deepseek_v3_2",
    "prompt": "The future of AI is",
    "max_completion_tokens": 50,
    "temperature": 0
  }'

curl http://<master_ip>:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "DeepSeek-V3_2-mtp",
    "prompt": "请用三句话解释 vLLM 多机推理中的 DP 和 TP。",
    "max_completion_tokens": 128,
    "temperature": 0
  }'

export VLLM_USE_MODELSCOPE=True

vllm bench serve \
  --model /root/.cache/Eco-Tech/DeepSeek-V3.2-w8a8-mtp-QuaRot \
  --dataset-name random \
  --random-input 200 \
  --num-prompts 200 \
  --request-rate 1 \
  --save-result \
  --result-dir ./