llm-d-inference-scheduler/deploy/components/vllm-decode/deployment.yaml at f74e4661bcb001463a389d41ddd29ec9c5e4c309 · mayabar/llm-d-inference-scheduler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-d
  labels:
    app: ${POOL_NAME}
spec:
  replicas: ${VLLM_REPLICA_COUNT_D}
  selector:
    matchLabels:
      app: ${POOL_NAME}
      llm-d.ai/component: decode
  template:
    metadata:
      labels:
        app: ${POOL_NAME}
        llm-d.ai/component: decode
        llm-d.ai/role: ${DECODE_ROLE}
    spec:
      initContainers:
      # Routing sidecar - proxies requests from port 8000 to vllm on port 8200
      # Scenario patches add --kv-connector and/or --ec-connector args as needed
      - name: routing-sidecar
        image: ${SIDECAR_IMAGE}
        imagePullPolicy: IfNotPresent
        args:
        - "--port=8000"
        - "--secure-proxy=false"
        - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}"
        ports:
        - name: sidecar-http
          containerPort: 8000
          protocol: TCP
        - name: sidecar-rank1
          containerPort: 8001
          protocol: TCP
        - name: sidecar-rank2
          containerPort: 8002
          protocol: TCP
        - name: sidecar-rank3
          containerPort: 8003
          protocol: TCP
        - name: sidecar-rank4
          containerPort: 8004
          protocol: TCP
        - name: sidecar-rank5
          containerPort: 8005
          protocol: TCP
        - name: sidecar-rank6
          containerPort: 8006
          protocol: TCP
        - name: sidecar-rank7
          containerPort: 8007
          protocol: TCP
        restartPolicy: Always
        env:
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
      containers:
      - name: vllm
        # VLLM_IMAGE can be a simulator (e.g. llm-d-inference-sim) or a real vLLM image (e.g. vllm/vllm-openai:v0.16.0)
        image: ${VLLM_IMAGE}
        imagePullPolicy: IfNotPresent
        args:
        - "--port=8200"
        - "--model=${MODEL_NAME}"
        - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}"
        - "--render-url=${VLLM_RENDER_URL}"
        - "${VLLM_EXTRA_ARGS_D}"
        ports:
        - name: http
          containerPort: 8200
          protocol: TCP
        - name: rank1
          containerPort: 8201
          protocol: TCP
        - name: rank2
          containerPort: 8202
          protocol: TCP
        - name: rank3
          containerPort: 8203
          protocol: TCP
        - name: rank4
          containerPort: 8204
          protocol: TCP
        - name: rank5
          containerPort: 8205
          protocol: TCP
        - name: rank6
          containerPort: 8206
          protocol: TCP
        - name: rank7
          containerPort: 8207
          protocol: TCP
        # Startup Probe: Wait for model to load during initialization
        # Protects liveness/readiness probes from firing too early
        startupProbe:
          httpGet:
            path: /health
            port: 8200
          failureThreshold: 60  # Allow up to 10 minutes (60 * 10s)
          periodSeconds: 10
          timeoutSeconds: 5
        # Readiness Probe: Is the model loaded and ready?
        # Controls traffic routing, removes from service if failing
        readinessProbe:
          httpGet:
            path: /health # Use /health/ready if available in your version
            port: 8200
          periodSeconds: 10
          timeoutSeconds: 2
          failureThreshold: 3
        # Liveness Probe: Is the server process alive?
        # Simple health check, restarts container if failing
        livenessProbe:
          httpGet:
            path: /health
            port: 8200
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        env:
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: PORT
          value: "8200"
        - name: PYTHONHASHSEED
          value: "42"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
      volumes: []