llm-d-inference-scheduler/deploy/components/vllm-prefill/deployment.yaml at f74e4661bcb001463a389d41ddd29ec9c5e4c309 · mayabar/llm-d-inference-scheduler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-p
  labels:
    app: ${POOL_NAME}
spec:
  replicas: ${VLLM_REPLICA_COUNT_P}
  selector:
    matchLabels:
      app: ${POOL_NAME}
      llm-d.ai/component: prefill
  template:
    metadata:
      labels:
        app: ${POOL_NAME}
        llm-d.ai/component: prefill
        llm-d.ai/role: prefill
    spec:
      initContainers: []
      containers:
      - name: vllm
        # VLLM_IMAGE can be a simulator (e.g. llm-d-inference-sim) or a real
        # vLLM image (e.g. vllm/vllm-openai:v0.16.0)
        image: ${VLLM_IMAGE}
        imagePullPolicy: IfNotPresent
        args:
        - "--model=${MODEL_NAME}"
        - "--port=8000"
        - "--data-parallel-size=${VLLM_DATA_PARALLEL_SIZE}"
        - "--render-url=${VLLM_RENDER_URL}"
        - "${VLLM_EXTRA_ARGS_P}"
        ports:
        - name: prefill-http
          containerPort: 8000
          protocol: TCP
        - name: prefill-rank1
          containerPort: 8001
          protocol: TCP
        - name: prefill-rank2
          containerPort: 8002
          protocol: TCP
        - name: prefill-rank3
          containerPort: 8003
          protocol: TCP
        - name: prefill-rank4
          containerPort: 8004
          protocol: TCP
        - name: prefill-rank5
          containerPort: 8005
          protocol: TCP
        - name: prefill-rank6
          containerPort: 8006
          protocol: TCP
        - name: prefill-rank7
          containerPort: 8007
          protocol: TCP
        # Startup Probe: Wait for model to load during initialization
        # Protects liveness/readiness probes from firing too early
        startupProbe:
          httpGet:
            path: /health
            port: 8000
          failureThreshold: 60  # Allow up to 10 minutes (60 * 10s)
          periodSeconds: 10
          timeoutSeconds: 5
        # Readiness Probe: Is the model loaded and ready?
        # Controls traffic routing, removes from service if failing
        readinessProbe:
          httpGet:
            path: /health # Use /health/ready if available in your version
            port: 8000
          periodSeconds: 10
          timeoutSeconds: 2
          failureThreshold: 3
        # Liveness Probe: Is the server process alive?
        # Simple health check, restarts container if failing
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        env:
        - name: PORT
          value: "8000"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      volumes: []