vllm

pip install vllm
## Qwen2.5-7B
vllm serve qwen/Qwen2.5-7B  --api-key token-abc123 --host 127.0.0.1 --port  8999 --gpu-memory-utilization 0.3 --max-model-len 5000 --max_num_seqs 2
## Qwen2.5-72B-AWQ
vllm serve /zengdaojian/wuhaodong/MODELS/Qwen2.5-72B-Instruct-AWQ/ --served-model-name Qwen2.5-72B-Instruct-AWQ --quantization awq --api-key token-abc123   --host 0.0.0.0     --port  8999 --tensor-parallel-size 2
vllm serve /zengdaojian/wuhaodong/MODELS/Qwen2.5-72B-Instruct-AWQ/ --served-model-name Qwen2.5-72B-Instruct-AWQ --quantization awq --api-key token-abc123   --host 0.0.0.0     --port  8999 --gpu-memory-utilization 0.8
## Qwen2-VL
vllm serve /zengdaojian/wuhaodong/MODELS/Qwen2-VL-72B-Instruct-AWQ/ --served-model-name Qwen2-VL-72B-Instruct-AWQ --quantization awq --api-key token-abc123   --host 0.0.0.0     --port  8999
## Qwen2.5-Coder-32B多卡
vllm serve /zengdaojian/wuhaodong/MODELS/Qwen2.5-Coder-32B-Instruct/ --served-model-name Qwen2.5-Coder-32B-Instruct --api-key token-abc123   --host 0.0.0.0     --port  8999 --tensor-parallel-size 2

sglang

## Qwen2.5-Coder-32B
python -m sglang.launch_server --model-path /zengdaojian/wuhaodong/MODELS/Qwen2.5-Coder-32B-Instruct/ --served-model-name Qwen2.5-Coder-32B-Instruct --api-key token-abc123   --host 0.0.0.0     --port  8999 --tp-size 4
## Qwen2.5-72B-AWQ
python -m sglang.launch_server --model-path /zengdaojian/wuhaodong/MODELS/Qwen2.5-72B-Instruct-AWQ/ --served-model-name Qwen2.5-72B-Instruct-AWQ --api-key token-abc123   --host 0.0.0.0     --port  8999 --tp-size 4
## Qwen2.5-72B 非量化
python -m sglang.launch_server --model-path /zengdaojian/wuhaodong/MODELS/Qwen2.5-72B-Instruct/ --served-model-name Qwen2.5-72B-Instruct --api-key token-abc123 --host 0.0.0.0 --port  8999 --tp-size 4
## Qwen-QwQ-32B
python -m sglang.launch_server --model-path /zengdaojian/wuhaodong/MODELS/QwQ-32B-Preview/ --served-model-name QwQ-32B-Preview --api-key token-abc123 --host 0.0.0.0 --port  8999 --tp-size 4

LMDEPLOY

pip install timm
pip install flash-attn
# InternVL2_5-78B
lmdeploy serve api_server /zengdaojian/wuhaodong/MODELS/InternVL2_5-78B/ --server-port 8080 --server-name 0.0.0.0 --model-name internvl-2.5 --backend turbomind --tp 4

# Qwen2-VL-72B
lmdeploy serve api_server /zengdaojian/wuhaodong/MODELS/Qwen2-VL-72B-Instruct --server-port 8999 --server-name 0.0.0.0 --api-keys token-abc123 --model-name internvl-2.5 --backend pytorch --tp 4