dgx-llm-serve/backends/trtllm/compose.yml at main · toku345/dgx-llm-serve · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
x-common: &common
  image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc4
  restart: unless-stopped
  ipc: host
  ports:
    - "127.0.0.1:8000:8000"
  deploy:
    resources:
      reservations:
        devices:
          - driver: nvidia
            count: all
            capabilities: [gpu]
  ulimits:
    memlock: -1
    stack: 67108864
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
    interval: 30s
    timeout: 10s
    retries: 3
    start_period: 300s

services:
  trtllm-qwen:
    <<: *common
    profiles: ["qwen"]
    command:
      - trtllm-serve
      - /app/model/nvidia/Qwen3-30B-A3B-FP4
      - --backend
      - _autodeploy
      - --extra_llm_api_options
      - /app/config/qwen.yaml
      - --host
      - "0.0.0.0"
    volumes:
      - ~/model_weights:/app/model:ro
      - ./qwen.yaml:/app/config/qwen.yaml:ro

  trtllm-nemotron:
    <<: *common
    profiles: ["nemotron"]
    command:
      - trtllm-serve
      - /app/model/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4
      - --backend
      - _autodeploy
      - --trust_remote_code
      - --reasoning_parser
      - deepseek-r1
      - --extra_llm_api_options
      - /app/config/nano_v3.yaml
      - --host
      - "0.0.0.0"
    volumes:
      - ~/model_weights:/app/model:ro
      - ./nano_v3.yaml:/app/config/nano_v3.yaml:ro

  # --- multi プロファイル: Qwen3-FP4 + Nemotron-NVFP4 同時起動 ---

  trtllm-multi-proxy:
    image: openresty/openresty:alpine
    profiles: ["multi"]
    restart: unless-stopped
    ports:
      - "127.0.0.1:8000:8000"
    volumes:
      - ./nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro
    depends_on:
      trtllm-multi-qwen:
        condition: service_healthy
      trtllm-multi-nemotron:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget -qO- http://localhost:8000/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

  trtllm-multi-qwen:
    <<: *common
    profiles: ["multi"]
    ports: []
    command:
      - trtllm-serve
      - /app/model/nvidia/Qwen3-30B-A3B-FP4
      - --backend
      - _autodeploy
      - --extra_llm_api_options
      - /app/config/qwen_multi.yaml
      - --host
      - "0.0.0.0"
    volumes:
      - ~/model_weights:/app/model:ro
      - ./qwen_multi.yaml:/app/config/qwen_multi.yaml:ro

  trtllm-multi-nemotron:
    <<: *common
    profiles: ["multi"]
    ports: []
    depends_on:
      trtllm-multi-qwen:
        condition: service_healthy
    command:
      - trtllm-serve
      - /app/model/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4
      - --backend
      - _autodeploy
      - --trust_remote_code
      - --reasoning_parser
      - deepseek-r1
      - --extra_llm_api_options
      - /app/config/nano_v3.yaml
      - --host
      - "0.0.0.0"
    volumes:
      - ~/model_weights:/app/model:ro
      - ./nano_v3.yaml:/app/config/nano_v3.yaml:ro