Skip to content

Commit e7d83f0

Browse files
rlakhtakiarlakhtakia
committed
update helm chart to use preferred backends for gke
Signed-off-by: Radhika Lakhtakia <rlakhtakia@google.com>
1 parent 8fa2015 commit e7d83f0

6 files changed

Lines changed: 209 additions & 29 deletions

File tree

‎config/charts/llm-d-router-gateway/templates/gke.yaml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,43 @@ spec:
5858
grpcHealthCheck:
5959
portSpecification: "USE_FIXED_PORT"
6060
port: {{ $eppHealthPort }}
61+
{{- if .Values.router.epp.enablePreferredBackends }}
62+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
63+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
64+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
65+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
66+
{{- $serviceName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
67+
{{- if eq (int $i) 1 }}
68+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
69+
{{- end }}
70+
{{- $policyName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
71+
{{- if eq (int $i) 1 }}
72+
{{- $policyName = printf "%s-backup" (include "llm-d-router.name" $) }}
73+
{{- end }}
74+
---
75+
kind: HealthCheckPolicy
76+
apiVersion: networking.gke.io/v1
77+
metadata:
78+
name: {{ $policyName }}
79+
namespace: {{ $.Release.Namespace }}
80+
labels:
81+
{{- include "llm-d-router.labels" $ | nindent 4 }}
82+
spec:
83+
targetRef:
84+
group: ""
85+
kind: Service
86+
name: {{ $serviceName }}
87+
default:
88+
config:
89+
type: GRPC
90+
grpcHealthCheck:
91+
portSpecification: "USE_FIXED_PORT"
92+
port: {{ $eppHealthPort }}
93+
{{- end }}
94+
{{- end }}
6195
{{- end }}
6296
---
97+
{{- if not .Values.router.epp.enablePreferredBackends }}
6398
apiVersion: networking.gke.io/v1
6499
kind: GCPBackendPolicy
65100
metadata:
@@ -78,4 +113,55 @@ spec:
78113
enabled: true # log all requests by default
79114
---
80115
{{- end }}
116+
{{- if .Values.router.epp.enablePreferredBackends }}
117+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
118+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
119+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
120+
apiVersion: networking.gke.io/v1
121+
kind: GCPBackendPolicy
122+
metadata:
123+
name: {{ .Release.Name }}
124+
namespace: {{ .Release.Namespace }}
125+
spec:
126+
targetRef:
127+
group: ""
128+
kind: Service
129+
name: {{ include "llm-d-router.name" . }}
130+
default:
131+
timeoutSec: 300
132+
backendPreference: PREFERRED
133+
{{- if .Values.router.epp.balancingMode }}
134+
balancingMode: {{ .Values.router.epp.balancingMode }}
135+
{{- end }}
136+
{{- if .Values.router.epp.maxRatePerEndpoint }}
137+
maxRatePerEndpoint: {{ .Values.router.epp.maxRatePerEndpoint }}
138+
{{- end }}
139+
{{- if .Values.router.epp.capacityScalerPercent }}
140+
capacityScalerPercent: {{ .Values.router.epp.capacityScalerPercent }}
141+
{{- end }}
142+
---
143+
apiVersion: networking.gke.io/v1
144+
kind: GCPBackendPolicy
145+
metadata:
146+
name: {{ printf "%s-backup" .Release.Name }}
147+
namespace: {{ .Release.Namespace }}
148+
spec:
149+
targetRef:
150+
group: ""
151+
kind: Service
152+
name: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
153+
default:
154+
timeoutSec: 300
155+
backendPreference: DEFAULT
156+
{{- if .Values.router.epp.balancingMode }}
157+
balancingMode: {{ .Values.router.epp.balancingMode }}
158+
{{- end }}
159+
{{- if .Values.router.epp.maxRatePerEndpoint }}
160+
maxRatePerEndpoint: {{ .Values.router.epp.maxRatePerEndpoint }}
161+
{{- end }}
162+
{{- if .Values.router.epp.capacityScalerPercent }}
163+
capacityScalerPercent: {{ .Values.router.epp.capacityScalerPercent }}
164+
{{- end }}
165+
{{- end }}
166+
{{- end }}
81167
{{- include "llm-d-router.gke" . -}}

‎config/charts/routerlib/templates/_deployment.yaml‎

Lines changed: 64 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,4 @@
1-
{{- define "llm-d-epp.deployment" -}}
2-
apiVersion: apps/v1
3-
kind: Deployment
4-
metadata:
5-
name: {{ include "llm-d-router.name" . }}
6-
namespace: {{ .Release.Namespace }}
7-
labels:
8-
{{- include "llm-d-router.labels" . | nindent 4 }}
9-
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
10-
spec:
11-
replicas: {{ .Values.router.epp.replicas | default 1 }}
12-
strategy:
13-
# The current recommended EPP deployment pattern is to have a single active replica. This ensures
14-
# optimal performance of the stateful operations such prefix cache aware scorer.
15-
# The Recreate strategy the old replica is killed immediately, and allow the new replica(s) to
16-
# quickly take over. This is particularly important in the high availability set up with leader
17-
# election, as the rolling update strategy would prevent the old leader being killed because
18-
# otherwise the maxUnavailable would be 100%.
19-
type: Recreate
20-
selector:
21-
matchLabels:
22-
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
23-
template:
24-
metadata:
25-
labels:
26-
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
27-
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
28-
spec:
1+
{{- define "llm-d-epp.deployment-pod-spec" -}}
292
{{- $proxy := include "llm-d-router.proxy" . | fromYaml | default dict }}
303
{{- $proxyType := include "llm-d-router.proxyType" . | trim }}
314
{{- $proxyMode := include "llm-d-router.proxyMode" . | trim }}
@@ -129,7 +102,7 @@ spec:
129102
- "json"
130103
- --config-file
131104
- "/config/{{ .Values.router.epp.pluginsConfigFile }}"
132-
{{- if gt (.Values.router.epp.replicas | int) 1 }}
105+
{{- if and (gt (.Values.router.epp.replicas | int) 1) (not .Values.router.epp.enablePreferredBackends) }}
133106
- --ha-enable-leader-election
134107
{{- end }}
135108
{{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }}
@@ -305,5 +278,67 @@ spec:
305278
tolerations:
306279
{{- toYaml .Values.router.epp.tolerations | nindent 8 }}
307280
{{- end }}
281+
{{- end }}
282+
283+
{{- define "llm-d-epp.deployment" -}}
284+
apiVersion: apps/v1
285+
kind: Deployment
286+
metadata:
287+
name: {{ include "llm-d-router.name" . }}
288+
namespace: {{ .Release.Namespace }}
289+
labels:
290+
{{- include "llm-d-router.labels" . | nindent 4 }}
291+
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
292+
spec:
293+
{{- $replicas := .Values.router.epp.replicas | default 1 }}
294+
{{- if .Values.router.epp.enablePreferredBackends }}
295+
{{- $replicas = .Values.router.epp.preferredReplicas | default 1 }}
296+
{{- end }}
297+
replicas: {{ $replicas }}
298+
strategy:
299+
type: Recreate
300+
selector:
301+
matchLabels:
302+
{{- if .Values.router.epp.enablePreferredBackends }}
303+
llm-d-router-gateway: {{ include "llm-d-router.name" . }}
304+
{{- else }}
305+
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
306+
{{- end }}
307+
template:
308+
metadata:
309+
labels:
310+
{{- if .Values.router.epp.enablePreferredBackends }}
311+
llm-d-router-gateway: {{ include "llm-d-router.name" . }}
312+
{{- else }}
313+
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
314+
{{- end }}
315+
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
316+
spec:
317+
{{ include "llm-d-epp.deployment-pod-spec" . }}
318+
{{- if .Values.router.epp.enablePreferredBackends }}
319+
---
320+
apiVersion: apps/v1
321+
kind: Deployment
322+
metadata:
323+
name: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
324+
namespace: {{ .Release.Namespace }}
325+
labels:
326+
{{- include "llm-d-router.labels" . | nindent 4 }}
327+
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
328+
spec:
329+
replicas: {{ .Values.router.epp.defaultReplicas | default 1 }}
330+
strategy:
331+
type: Recreate
332+
selector:
333+
matchLabels:
334+
llm-d-router-gateway: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
335+
template:
336+
metadata:
337+
labels:
338+
llm-d-router-gateway: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
339+
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
340+
spec:
341+
{{ include "llm-d-epp.deployment-pod-spec" . }}
342+
{{- end }}
308343
---
309344
{{- end }}

‎config/charts/routerlib/templates/_helpers.tpl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,11 +397,25 @@ EPP resource validations
397397
{{/*
398398
EPP generic validations
399399
*/}}
400+
{{- define "llm-d-router.validations.epp.preferredBackends" -}}
401+
{{- if .Values.router.epp.enablePreferredBackends }}
402+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
403+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
404+
{{- if lt $preferredReplicas 1 }}
405+
{{- fail ".Values.router.epp.preferredReplicas must be at least 1 when enablePreferredBackends is true" }}
406+
{{- end }}
407+
{{- if lt $defaultReplicas 1 }}
408+
{{- fail ".Values.router.epp.defaultReplicas must be at least 1 when enablePreferredBackends is true" }}
409+
{{- end }}
410+
{{- end }}
411+
{{- end -}}
412+
400413
{{- define "llm-d-router.validations.epp" -}}
401414
{{- include "llm-d-router.validations.deprecations" . }}
402415
{{- include "llm-d-router.validations.epp.resources" . }}
403416
{{- include "llm-d-router.validations.epp.inferenceObjectives" . }}
404417
{{- include "llm-d-router.validations.epp.tokenizer" . }}
418+
{{- include "llm-d-router.validations.epp.preferredBackends" . }}
405419
{{- end -}}
406420

407421
{{/*

‎config/charts/routerlib/templates/_inferencepool.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@ spec:
2626
{{- end }}
2727
{{- end }}
2828
endpointPickerRef:
29+
{{- if .Values.router.epp.enablePreferredBackends }}
30+
name: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
31+
{{- else }}
2932
name: {{ include "llm-d-router.name" . }}
33+
{{- end }}
3034
port:
3135
number: {{ .Values.router.epp.extProcPort | default 9002 }}
3236
failureMode: {{ .Values.router.inferencePool.failureMode | default "FailOpen" }}

‎config/charts/routerlib/templates/_service.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,22 @@ metadata:
66
namespace: {{ .Release.Namespace }}
77
labels:
88
{{- include "llm-d-router.labels" . | nindent 4 }}
9+
{{- if .Values.router.epp.enablePreferredBackends }}
10+
annotations:
11+
cloud.google.com/neg: '{"exposed_ports":{"{{ .Values.router.epp.extProcPort | default 9002 }}":{}}}'
12+
{{- end }}
913
spec:
1014
selector:
15+
{{- if .Values.router.epp.enablePreferredBackends }}
16+
llm-d-router-gateway: {{ include "llm-d-router.name" . }}
17+
{{- else }}
1118
{{- include "llm-d-router.selectorLabels" . | nindent 4 }}
19+
{{- end }}
1220
ports:
1321
- name: grpc-ext-proc
1422
protocol: TCP
1523
port: {{ .Values.router.epp.extProcPort | default 9002 }}
24+
appProtocol: kubernetes.io/h2c
1625
- name: http-metrics
1726
protocol: TCP
1827
port: {{ .Values.router.metricsPort | default 9090 }}
@@ -21,4 +30,24 @@ spec:
2130
{{- end }}
2231
type: ClusterIP
2332
---
33+
{{- if .Values.router.epp.enablePreferredBackends }}
34+
apiVersion: v1
35+
kind: Service
36+
metadata:
37+
name: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
38+
namespace: {{ .Release.Namespace }}
39+
labels:
40+
{{- include "llm-d-router.labels" . | nindent 4 }}
41+
annotations:
42+
cloud.google.com/neg: '{"exposed_ports":{"{{ .Values.router.epp.extProcPort | default 9002 }}":{}}}'
43+
spec:
44+
selector:
45+
llm-d-router-gateway: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
46+
ports:
47+
- name: grpc-ext-proc
48+
protocol: TCP
49+
port: {{ .Values.router.epp.extProcPort | default 9002 }}
50+
appProtocol: kubernetes.io/h2c
51+
type: ClusterIP
52+
{{- end }}
2453
{{- end }}

‎config/charts/routerlib/values.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,18 @@ clusterDomain: cluster.local
55

66
epp:
77
replicas: 1
8+
enablePreferredBackends: false
9+
# Number of primary active pod instances pinned to the PREFERRED load balancing tier.
10+
# Setting >1 scales concurrent active routing.
11+
preferredReplicas: 1
12+
# Number of standby pod instances pinned to the DEFAULT load balancing tier.
13+
# Setting >1 scales warm standby failover capacity.
14+
defaultReplicas: 1
15+
# Benchmark capacity and threshold settings for PREFERRED load balancing tier.
16+
# When incoming traffic exceeds capacityScalerPercent of maxRatePerEndpoint, excess traffic spills over to DEFAULT standby pods.
17+
balancingMode: RATE
18+
maxRatePerEndpoint: 100
19+
capacityScalerPercent: 100
820
image:
921
registry: ghcr.io/llm-d
1022
repository: llm-d-router-endpoint-picker-dev

0 commit comments

Comments
 (0)