It took me months to get this right for my RTX 6000 ADA 48GB, so I'm sharing it here. Obviously, this is on Linux - Ubuntu to be specific. I get ~60 tokens per second, full context and can run ~16 concurrent sessions.
Helper commands
sudo systemctl enable vllm-27b-fp8
sudo systemctl start vllm-27b-fp8
sudo systemctl stop vllm-27b-fp8
sudo systemctl status vllm-27b-fp8
sudo systemctl disable vllm-27b-fp8
journalctl -u vllm-27b-fp8 -f
sudo nano /etc/systemd/system/vllm-27b-fp8.service
BEGIN SERVICE ENTRY
[Unit]
Description=vLLM Qwen3.8-27B FP8
After=network.target
[Service]
User=MY_USER_NAME
WorkingDirectory=/home/MY_USER_NAME
Environment="PATH=/home/MY_USER_NAME/.venv-0.29.0/bin:/usr/local/cuda/bin:/usr/bin:/usr/local/bin:/bin"
Environment="CUDA_HOME=/usr/local/cuda"
Environment="LD_LIBRARY_PATH=/usr/local/cuda/lib64"
Environment="CUDA_DEVICE_ORDER=PCI_BUS_ID"
Environment="CUDA_VISIBLE_DEVICES=0"
Environment="HF_TOKEN=HUGGINGFACE_TOKEN"
ExecStart=/home/MY_USER_NAME/.venv-0.29.0/bin/python -m vllm.entrypoints.openai.api_server \
--model Qwen/Qwen3.8-27B-FP8 \
--port 5001 \
--enable-chunked-prefill \
--limit-mm-per-prompt "{\"image\": 1}" \
--kv-cache-dtype fp8 \
--enable-prefix-caching \
--max-model-len 262144 \
--async-scheduling \
--max-num-batched-tokens 8192 \
--gpu-memory-utilization 0.96 \
--compilation-config '{"cudagraph_capture_sizes": [1, 2, 4, 8]}' \
--speculative-config '{"method":"mtp","num_speculative_tokens":3}' \
--enable-auto-tool-choice \
--tool-call-parser qwen3_xml \
--reasoning-parser qwen3 \
--chat-template /home/MY_USER_NAME/chat-templates/qwen-fixed.jinja
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target