trtllm-serving/docker-compose.yml at master · BEEB0SS/trtllm-serving · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
# Observability stack + gateway. The engine is deliberately NOT a compose service; scripts/04_serve.sh runs Triton on the host network.

name: trtllm-serving

services:
  # ---------------------------------------------------------------------------
  gateway:
    build:
      context: .
      dockerfile: Dockerfile
    image: trtllm-serving/gateway:local
    container_name: trtllm-gateway
    restart: unless-stopped
    command:
      - uvicorn
      - gateway.main:app
      - --host
      - "0.0.0.0"
      - --port
      - "8080"
      # 1 worker on purpose: multiple uvicorn workers mean separate prometheus registries.
      - --workers
      - "1"
    ports:
      - "8080:8080"
    environment:
      TRTLLM_GW_ENGINE__BASE_URL: "${ENGINE_URL:-http://host.docker.internal:9000}"
      TRTLLM_GW_ENGINE__SYSTEM_METRICS_URL: "${ENGINE_METRICS_URL:-http://host.docker.internal:9000/metrics}"
      TRTLLM_GW_GATEWAY__LOG_LEVEL: "info"
    extra_hosts:
      - "host.docker.internal:host-gateway"
    healthcheck:
      test: ["CMD", "python", "-c",
             "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health',timeout=3).status==200 else 1)"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 10s

  # ---------------------------------------------------------------------------
  gpu-exporter:
    build:
      context: .
      dockerfile: Dockerfile
    image: trtllm-serving/gateway:local
    container_name: trtllm-gpu-exporter
    restart: unless-stopped
    command: ["python", "exporter/gpu_exporter.py", "--port", "9835", "--interval", "2"]
    ports:
      - "9835:9835"
    environment:
      NVIDIA_VISIBLE_DEVICES: "all"
      NVIDIA_DRIVER_CAPABILITIES: "utility"
    # pid: host so /proc/meminfo shows the real unified pool, not a cgroup-filtered view.
    pid: host
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [utility]

  # ---------------------------------------------------------------------------
  prometheus:
    image: prom/prometheus:v3.1.0
    container_name: trtllm-prometheus
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=15d
      - --web.enable-lifecycle
    ports:
      - "9090:9090"
    volumes:
      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    extra_hosts:
      - "host.docker.internal:host-gateway"
    depends_on:
      - gateway
      - gpu-exporter

  # ---------------------------------------------------------------------------
  grafana:
    image: grafana/grafana:11.6.0
    container_name: trtllm-grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      GF_SECURITY_ADMIN_USER: "admin"
      GF_SECURITY_ADMIN_PASSWORD: "admin"
      # Anonymous access for a single-user box; do not copy to anything internet-facing.
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer"
      GF_USERS_DEFAULT_THEME: "dark"
    volumes:
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
      - grafana-data:/var/lib/grafana
    depends_on:
      - prometheus

volumes:
  prometheus-data:
  grafana-data: