llm-d-inference-scheduler/deploy/components/inference-gateway/deployment.yaml at f74e4661bcb001463a389d41ddd29ec9c5e4c309 · mayabar/llm-d-inference-scheduler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${EPP_NAME}
  labels:
    app: ${EPP_NAME}
spec:
  replicas: ${EPP_REPLICA_COUNT}
  selector:
    matchLabels:
      app: ${EPP_NAME}
  template:
    metadata:
      labels:
        app: ${EPP_NAME}
    spec:
      serviceAccountName: ${EPP_NAME}
      terminationGracePeriodSeconds: 130
      containers:
      - name: epp
        image: ${EPP_IMAGE}
        imagePullPolicy: IfNotPresent
        args:
        - --pool-name
        - "${POOL_NAME}"
        - --pool-namespace
        - "${NAMESPACE}"
        - --v
        - "4"
        - --zap-encoder
        - "json"
        - --grpc-port
        - "9002"
        - --grpc-health-port
        - "9003"
        - --config-file
        - "/etc/epp/epp-config.yaml"
        - --metrics-endpoint-auth=${METRICS_ENDPOINT_AUTH}
        - --ha-enable-leader-election=${ENABLE_LEADER_ELECTION}
        ports:
        - containerPort: 5557
        - containerPort: 9002
        - containerPort: 9003
        - name: metrics
          containerPort: 9090
        livenessProbe:
          grpc:
            port: 9003
            service: envoy.service.ext_proc.v3.ExternalProcessor
          initialDelaySeconds: 5
          periodSeconds: 10
        readinessProbe:
          grpc:
            port: 9003
            service: envoy.service.ext_proc.v3.ExternalProcessor
          initialDelaySeconds: 5
          periodSeconds: 10
        volumeMounts:
        - name: epp-config
          mountPath: /etc/epp
        - name: cache
          mountPath: /cache
      volumes:
      - name: epp-config
        configMap:
          name: epp-config
      - name: cache
        emptyDir: {}