Qwen3 通义千问腾讯云本地部署完整指南

1# 安装依赖
2apt-get update
3apt-get install -y ca-certificates curl gnupg lsb-release
4
5# 添加腾讯云内网 Docker GPG key
6curl -fsSL https://mirrors.cloud.tencent.com/docker-ce/linux/ubuntu/gpg -o docker-keyring.gpg
7gpg --dearmor -o /etc/apt/keyrings/docker.gpg docker-keyring.gpg
8
9# 添加腾讯云内网 Docker 软件源
10echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://mirrors.cloud.tencent.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
11
12# 安装 Docker Engine 与 compose 插件
13apt-get update
14apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
15
16# 配置 Docker Hub 镜像加速器（关键，否则拉镜像会慢到超时）
17mkdir -p /etc/docker
18tee /etc/docker/daemon.json <<-'EOF'
19{
20  "registry-mirrors": ["https://mirror.ccs.tencentyun.com"]
21}
22EOF
23systemctl daemon-reload
24systemctl restart docker
25usermod -aG docker $USER

1fallocate -l 8G /swapfile
2chmod 600 /swapfile
3mkswap /swapfile
4swapon /swapfile
5
6cp /etc/fstab /etc/fstab.bak
7echo '/swapfile none swap sw 0 0' | tee -a /etc/fstab
8
9free -h

1mkdir -p /opt/qwen3 && cd /opt/qwen3
2
3cat > docker-compose.yml <<-'EOF'
4version: '3.8'
5
6services:
7  ollama:
8    image: ollama/ollama:latest
9    container_name: ollama
10    restart: unless-stopped
11    ports:
12      - "127.0.0.1:11434:11434"
13    volumes:
14      - ./ollama:/root/.ollama
15    environment:
16      - OLLAMA_HOST=0.0.0.0
17      - OLLAMA_MAX_LOADED_MODELS=2
18      - OLLAMA_KEEP_ALIVE=10m
19      - OLLAMA_FLASH_ATTENTION=1
20    healthcheck:
21      test: ["CMD", "ollama", "list"]
22      interval: 30s
23      timeout: 10s
24      retries: 3
25
26  open-webui:
27    image: ghcr.io/open-webui/open-webui:main
28    container_name: open-webui
29    restart: unless-stopped
30    ports:
31      - "127.0.0.1:3000:8080"
32    volumes:
33      - ./webui:/app/backend/data
34    environment:
35      - OLLAMA_BASE_URL=http://ollama:11434
36      - WEBUI_AUTH=true
37    depends_on:
38      ollama:
39        condition: service_healthy
40EOF
41
42docker compose up -d
43docker compose ps

1# 8B 版本：内存友好，CPU 也能跑，推荐起步
2docker exec -it ollama ollama pull qwen3:8b
3
4# 14B 版本：效果更好，建议 GPU 实例
5docker exec -it ollama ollama pull qwen3:14b
6
7# 4B 版本：低配尝鲜或边缘设备
8docker exec -it ollama ollama pull qwen3:4b

1# 一次性问答验证
2docker exec -it ollama ollama run qwen3:8b "用三句话解释什么是向量数据库"
3
4# 验证 OpenAI 兼容 API 是否可用
5curl -s http://127.0.0.1:11434/v1/chat/completions \
6  -H "Content-Type: application/json" \
7  -d '{
8    "model": "qwen3:8b",
9    "messages": [{"role": "user", "content": "你好，介绍一下你自己"}],
10    "stream": false
11  }' | head -40

1# 复杂推理（默认开启思考）：不加标记
2docker exec -it ollama ollama run qwen3:8b "一个水池有两个进水管和一个出水管，单独开甲管 6 小时注满，单独开乙管 4 小时注满，单独开丙管 12 小时排空。三管齐开几小时注满？"
3
4# 简单问答 / 闲聊（关闭思考省时间）：提示词末尾加 <arg_key:opensource>
5docker exec -it ollama ollama run qwen3:8b "把这句话翻译成英文：今天天气不错 <arg_key:opensource>"

1# API 方式也可以直接拼在 content 后面
2curl -s http://127.0.0.1:11434/v1/chat/completions \
3  -H "Content-Type: application/json" \
4  -d '{
5    "model": "qwen3:8b",
6    "messages": [{"role": "user", "content": "深圳有什么好玩的地方？<arg_key:opensource>"}]
7  }'

1# 安装 ModelScope 客户端
2pip3 install -y modelscope
3
4# 下载 Qwen3-8B 的 GGUF 量化版（约 5GB）
5mkdir -p /opt/qwen3/weights && cd /opt/qwen3/weights
6modelscope download --model Qwen/Qwen3-8B-GGUF qwen3-8b-q4_k_m.gguf --local_dir .
7
8# 查看下载结果
9ls -lh

1cat > /opt/qwen3/Modelfile <<-'EOF'
2FROM /weights/qwen3-8b-q4_k_m.gguf
3
4PARAMETER num_ctx 32768
5PARAMETER num_predict 2048
6PARAMETER temperature 0.7
7PARAMETER top_p 0.8
8PARAMETER repeat_penalty 1.05
9
10SYSTEM 你是 Qwen，一个由阿里云开发的国产大语言模型。请用简洁清晰的中文回答用户问题。
11EOF
12
13docker cp /opt/qwen3/Modelfile ollama:/tmp/Modelfile
14docker exec -it ollama ollama create qwen3-8b-local -f /tmp/Modelfile
15docker exec -it ollama ollama run qwen3-8b-local "做个自我介绍"

1# 确认驱动与显卡可见
2nvidia-smi
3
4# 安装 NVIDIA Container Toolkit
5distribution=$(. /etc/os-release; echo $ID$VERSION_ID)
6curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
7curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
8  sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
9  tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
10
11apt-get update
12apt-get install -y nvidia-container-toolkit
13nvidia-ctk runtime configure --runtime=docker
14systemctl restart docker

1docker run --rm --gpus all nvidia/cuda:12.2.0-base-ubuntu22.04 nvidia-smi

1mkdir -p /opt/qwen3-vllm && cd /opt/qwen3-vllm
2
3cat > docker-compose.yml <<-'EOF'
4version: '3.8'
5
6services:
7  vllm:
8    image: vllm/vllm-openai:latest
9    container_name: vllm
10    restart: unless-stopped
11    ipc: host
12    ports:
13      - "127.0.0.1:8000:8000"
14    volumes:
15      - ./models:/root/.models
16      - $HOME/.cache/huggingface:/root/.cache/huggingface
17    environment:
18      - HF_ENDPOINT=https://www.modelscope.cn
19      - VLLM_WORKER_MULTIPROC_METHOD=spawn
20    command: >
21      --model Qwen/Qwen3-8B
22      --served-model-name qwen3-8b
23      --max-model-len 32768
24      --gpu-memory-utilization 0.85
25      --max-num-seqs 64
26      --dtype auto
27      --api-key your-strong-api-key
28    deploy:
29      resources:
30        reservations:
31          devices:
32            - driver: nvidia
33              count: all
34              capabilities: [gpu]
35EOF
36
37docker compose up -d
38docker compose logs -f vllm

1# 健康检查
2curl -s http://127.0.0.1:8000/health
3
4# 实际推理（注意带上 api-key）
5curl -s http://127.0.0.1:8000/v1/chat/completions \
6  -H "Content-Type: application/json" \
7  -H "Authorization: Bearer your-strong-api-key" \
8  -d '{
9    "model": "qwen3-8b",
10    "messages": [{"role": "user", "content": "用 Python 写一个快速排序"}],
11    "max_tokens": 512
12  }'

1from openai import OpenAI
2
3client = OpenAI(
4    base_url="http://127.0.0.1:11434/v1",  # 换成 vLLM 就是 :8000/v1
5    api_key="ollama"  # vLLM 时填你设置的 --api-key
6)
7
8resp = client.chat.completions.create(
9    model="qwen3:8b",
10    messages=[
11        {"role": "system", "content": "你是客服助手，只回答与产品相关的问题。"},
12        {"role": "user", "content": "你们的退款政策是什么？"}
13    ],
14    temperature=0.7,
15    max_tokens=1024
16)
17print(resp.choices[0].message.content)

1apt-get install -y nginx certbot python3-certbot-nginx
2
3cat > /etc/nginx/sites-available/qwen3 <<-'EOF'
4server {
5    listen 80;
6    server_name ai.yourcompany.com;
7
8    location / {
9        proxy_pass http://127.0.0.1:3000;
10        proxy_set_header Host $host;
11        proxy_set_header X-Real-IP $remote_addr;
12        proxy_http_version 1.1;
13        proxy_set_header Upgrade $http_upgrade;
14        proxy_set_header Connection "upgrade";
15        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
16        proxy_set_header X-Forwarded-Proto $scheme;
17        # 流式输出必须关掉缓冲，否则前端会等很久才出字
18        proxy_buffering off;
19        proxy_read_timeout 300s;
20    }
21}
22EOF
23
24ln -s /etc/nginx/sites-available/qwen3 /etc/nginx/sites-enabled/
25nginx -t && systemctl reload nginx
26
27certbot --nginx -d ai.yourcompany.com

1# 确认端口只监听在本机（应看到 127.0.0.1:11434）
2ss -tlnp | grep -E '11434|8000|3000'
3
4# 腾讯云两层防火墙都要检查：控制台安全组 + 系统 ufw
5ufw status