llm-d-modelservice/examples/output-cpu.yaml at main · llm-d-incubation/llm-d-modelservice · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
# generated by generate-example-output.sh
---
# Source: llm-d-modelservice/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: cpu-sim-llm-d-modelservice
  labels:
    helm.sh/chart: llm-d-modelservice-v0.4.15
    app.kubernetes.io/version: "v0.4.0"
    app.kubernetes.io/managed-by: Helm
---
# Source: llm-d-modelservice/templates/decode-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cpu-sim-llm-d-modelservice-decode
  labels:
    helm.sh/chart: llm-d-modelservice-v0.4.15
    app.kubernetes.io/version: "v0.4.0"
    app.kubernetes.io/managed-by: Helm
spec:
  replicas: 1
  selector:
    matchLabels:
      llm-d.ai/inference-serving: "true"
      llm-d.ai/model: random-model
      llm-d.ai/role: decode
  template:
    metadata:
      labels:
        llm-d.ai/inference-serving: "true"
        llm-d.ai/model: random-model
        llm-d.ai/role: decode
    spec:
      initContainers:
        - name: routing-proxy
          args:
            - --port=8000
            - --vllm-port=8200
            - --connector=nixlv2
            - --zap-encoder=json
            - --zap-log-level=debug
            - --secure-proxy=false
          image: ghcr.io/llm-d/llm-d-routing-sidecar:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 8000
          resources: {}
          restartPolicy: Always
          securityContext:
            allowPrivilegeEscalation: false
            runAsNonRoot: true

      serviceAccountName: cpu-sim-llm-d-modelservice

      volumes:
        - emptyDir: {}
          name: metrics-volume

        - name: model-storage
          emptyDir:
            sizeLimit: 5Mi


      containers:
        - name: vllm
          image: ghcr.io/llm-d/llm-d-inference-sim:latest

          args:
            - --model
            - random/model
            - --port
            - "8200"
            - --served-model-name
            - "random/model"

          env:
          - name: DP_SIZE
            value: "1"
          - name: TP_SIZE
            value: "1"
          - name: DP_SIZE_LOCAL
            value: "1"

          - name: HF_HOME
            value: /model-cache

          ports:
          - containerPort: 8200
            protocol: TCP
          resources:
            limits:
              {}
            requests:
              {}

          volumeMounts:
            - name: model-storage
              mountPath: /model-cache
---
# Source: llm-d-modelservice/templates/prefill-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cpu-sim-llm-d-modelservice-prefill
  labels:
    helm.sh/chart: llm-d-modelservice-v0.4.15
    app.kubernetes.io/version: "v0.4.0"
    app.kubernetes.io/managed-by: Helm
spec:
  replicas: 1
  selector:
    matchLabels:
      llm-d.ai/inference-serving: "true"
      llm-d.ai/model: random-model
      llm-d.ai/role: prefill
  template:
    metadata:
      labels:
        llm-d.ai/inference-serving: "true"
        llm-d.ai/model: random-model
        llm-d.ai/role: prefill
    spec:

      serviceAccountName: cpu-sim-llm-d-modelservice

      volumes:
        - emptyDir: {}
          name: metrics-volume

        - name: model-storage
          emptyDir:
            sizeLimit: 5Mi


      containers:
        - name: vllm
          image: ghcr.io/llm-d/llm-d-inference-sim:latest

          args:
            - --model
            - random/model
            - --port
            - "8000"
            - --served-model-name
            - "random/model"

          env:
          - name: DP_SIZE
            value: "1"
          - name: TP_SIZE
            value: "1"
          - name: DP_SIZE_LOCAL
            value: "1"

          - name: HF_HOME
            value: /model-cache

          ports:
          - containerPort: 8000
            protocol: TCP
          resources:
            limits:
              {}
            requests:
              {}

          volumeMounts:
            - name: model-storage
              mountPath: /model-cache