Skip to content

Commit 08fd8ca

Browse files
rlakhtakiarlakhtakia
committed
update helm chart to use preferred backends for gke
Signed-off-by: Radhika Lakhtakia <rlakhtakia@google.com>
1 parent 8fa2015 commit 08fd8ca

6 files changed

Lines changed: 213 additions & 30 deletions

File tree

‎config/charts/llm-d-router-gateway/templates/gke.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,42 @@ spec:
5959
portSpecification: "USE_FIXED_PORT"
6060
port: {{ $eppHealthPort }}
6161
{{- end }}
62+
{{- if .Values.router.epp.gkePreferredBackends.enabled }}
63+
{{- $preferredReplicas := .Values.router.epp.gkePreferredBackends.preferredReplicas | default 1 | int }}
64+
{{- $defaultReplicas := .Values.router.epp.gkePreferredBackends.defaultReplicas | default 1 | int }}
65+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
66+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
67+
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
68+
{{- if ge (int $i) (int $preferredReplicas) }}
69+
{{- if eq (int $i) (int $preferredReplicas) }}
70+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
71+
{{- else }}
72+
{{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
73+
{{- end }}
74+
{{- end }}
75+
---
76+
kind: HealthCheckPolicy
77+
apiVersion: networking.gke.io/v1
78+
metadata:
79+
name: {{ $serviceName }}
80+
namespace: {{ $.Release.Namespace }}
81+
labels:
82+
{{- include "llm-d-router.labels" $ | nindent 4 }}
83+
spec:
84+
targetRef:
85+
group: ""
86+
kind: Service
87+
name: {{ $serviceName }}
88+
default:
89+
config:
90+
type: GRPC
91+
grpcHealthCheck:
92+
portSpecification: "USE_FIXED_PORT"
93+
port: {{ $eppHealthPort }}
94+
{{- end }}
95+
{{- end }}
6296
---
97+
{{- if not .Values.router.epp.gkePreferredBackends.enabled }}
6398
apiVersion: networking.gke.io/v1
6499
kind: GCPBackendPolicy
65100
metadata:
@@ -78,4 +113,51 @@ spec:
78113
enabled: true # log all requests by default
79114
---
80115
{{- end }}
116+
{{- if .Values.router.epp.gkePreferredBackends.enabled }}
117+
{{- $preferredReplicas := .Values.router.epp.gkePreferredBackends.preferredReplicas | default 1 | int }}
118+
{{- $defaultReplicas := .Values.router.epp.gkePreferredBackends.defaultReplicas | default 1 | int }}
119+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
120+
{{- range $i := untilStep 0 (int $totalReplicas) 1 }}
121+
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
122+
{{- $policyName := printf "%s-preferred-%d" $.Release.Name (int $i) }}
123+
{{- $pref := "PREFERRED" }}
124+
{{- if eq (int $i) 0 }}
125+
{{- $serviceName = include "llm-d-router.name" $ }}
126+
{{- $policyName = $.Release.Name }}
127+
{{- else if ge (int $i) (int $preferredReplicas) }}
128+
{{- $pref = "DEFAULT" }}
129+
{{- if eq (int $i) (int $preferredReplicas) }}
130+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
131+
{{- $policyName = printf "%s-backup" $.Release.Name }}
132+
{{- else }}
133+
{{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
134+
{{- $policyName = printf "%s-backup-%d" $.Release.Name (sub (int $i) (int $preferredReplicas)) }}
135+
{{- end }}
136+
{{- end }}
137+
apiVersion: networking.gke.io/v1
138+
kind: GCPBackendPolicy
139+
metadata:
140+
name: {{ $policyName }}
141+
namespace: {{ $.Release.Namespace }}
142+
spec:
143+
targetRef:
144+
group: ""
145+
kind: Service
146+
name: {{ $serviceName }}
147+
default:
148+
timeoutSec: 300
149+
backendPreference: {{ $pref }}
150+
{{- if $.Values.router.epp.gkePreferredBackends.balancingMode }}
151+
balancingMode: {{ $.Values.router.epp.gkePreferredBackends.balancingMode }}
152+
{{- end }}
153+
{{- if $.Values.router.epp.gkePreferredBackends.maxRatePerEndpoint }}
154+
maxRatePerEndpoint: {{ $.Values.router.epp.gkePreferredBackends.maxRatePerEndpoint }}
155+
{{- end }}
156+
{{- if $.Values.router.epp.gkePreferredBackends.capacityScalerPercent }}
157+
capacityScalerPercent: {{ $.Values.router.epp.gkePreferredBackends.capacityScalerPercent }}
158+
{{- end }}
159+
---
160+
{{- end }}
161+
{{- end }}
81162
{{- include "llm-d-router.gke" . -}}
163+
{{- end }}

‎config/charts/routerlib/templates/_deployment.yaml‎

Lines changed: 56 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,4 @@
1-
{{- define "llm-d-epp.deployment" -}}
2-
apiVersion: apps/v1
3-
kind: Deployment
4-
metadata:
5-
name: {{ include "llm-d-router.name" . }}
6-
namespace: {{ .Release.Namespace }}
7-
labels:
8-
{{- include "llm-d-router.labels" . | nindent 4 }}
9-
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
10-
spec:
11-
replicas: {{ .Values.router.epp.replicas | default 1 }}
12-
strategy:
13-
# The current recommended EPP deployment pattern is to have a single active replica. This ensures
14-
# optimal performance of the stateful operations such prefix cache aware scorer.
15-
# The Recreate strategy the old replica is killed immediately, and allow the new replica(s) to
16-
# quickly take over. This is particularly important in the high availability set up with leader
17-
# election, as the rolling update strategy would prevent the old leader being killed because
18-
# otherwise the maxUnavailable would be 100%.
19-
type: Recreate
20-
selector:
21-
matchLabels:
22-
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
23-
template:
24-
metadata:
25-
labels:
26-
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
27-
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
28-
spec:
1+
{{- define "llm-d-epp.deployment-pod-spec" -}}
292
{{- $proxy := include "llm-d-router.proxy" . | fromYaml | default dict }}
303
{{- $proxyType := include "llm-d-router.proxyType" . | trim }}
314
{{- $proxyMode := include "llm-d-router.proxyMode" . | trim }}
@@ -129,7 +102,7 @@ spec:
129102
- "json"
130103
- --config-file
131104
- "/config/{{ .Values.router.epp.pluginsConfigFile }}"
132-
{{- if gt (.Values.router.epp.replicas | int) 1 }}
105+
{{- if and (gt (.Values.router.epp.replicas | int) 1) (not .Values.router.epp.gkePreferredBackends.enabled) }}
133106
- --ha-enable-leader-election
134107
{{- end }}
135108
{{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }}
@@ -305,5 +278,59 @@ spec:
305278
tolerations:
306279
{{- toYaml .Values.router.epp.tolerations | nindent 8 }}
307280
{{- end }}
281+
{{- end }}
282+
283+
{{- define "llm-d-epp.deployment" -}}
284+
{{- if .Values.router.epp.gkePreferredBackends.enabled }}
285+
{{- $preferredReplicas := .Values.router.epp.gkePreferredBackends.preferredReplicas | default 1 | int }}
286+
{{- $defaultReplicas := .Values.router.epp.gkePreferredBackends.defaultReplicas | default 1 | int }}
287+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
288+
apiVersion: apps/v1
289+
kind: StatefulSet
290+
metadata:
291+
name: {{ include "llm-d-router.name" . }}
292+
namespace: {{ .Release.Namespace }}
293+
labels:
294+
{{- include "llm-d-router.labels" . | nindent 4 }}
295+
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
296+
spec:
297+
replicas: {{ $totalReplicas }}
298+
serviceName: {{ include "llm-d-router.name" . }}
299+
podManagementPolicy: Parallel
300+
selector:
301+
matchLabels:
302+
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
303+
template:
304+
metadata:
305+
labels:
306+
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
307+
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
308+
spec:
309+
{{ include "llm-d-epp.deployment-pod-spec" . }}
310+
---
311+
{{- else }}
312+
apiVersion: apps/v1
313+
kind: Deployment
314+
metadata:
315+
name: {{ include "llm-d-router.name" . }}
316+
namespace: {{ .Release.Namespace }}
317+
labels:
318+
{{- include "llm-d-router.labels" . | nindent 4 }}
319+
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
320+
spec:
321+
replicas: {{ .Values.router.epp.replicas | default 1 }}
322+
strategy:
323+
type: Recreate
324+
selector:
325+
matchLabels:
326+
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
327+
template:
328+
metadata:
329+
labels:
330+
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
331+
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
332+
spec:
333+
{{ include "llm-d-epp.deployment-pod-spec" . }}
308334
---
309335
{{- end }}
336+
{{- end }}

‎config/charts/routerlib/templates/_helpers.tpl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,11 +397,25 @@ EPP resource validations
397397
{{/*
398398
EPP generic validations
399399
*/}}
400+
{{- define "llm-d-router.validations.epp.preferredBackends" -}}
401+
{{- if .Values.router.epp.gkePreferredBackends.enabled }}
402+
{{- $preferredReplicas := .Values.router.epp.gkePreferredBackends.preferredReplicas | default 1 | int }}
403+
{{- $defaultReplicas := .Values.router.epp.gkePreferredBackends.defaultReplicas | default 1 | int }}
404+
{{- if lt $preferredReplicas 1 }}
405+
{{- fail ".Values.router.epp.gkePreferredBackends.preferredReplicas must be at least 1 when gkePreferredBackends.enabled is true" }}
406+
{{- end }}
407+
{{- if lt $defaultReplicas 1 }}
408+
{{- fail ".Values.router.epp.gkePreferredBackends.defaultReplicas must be at least 1 when gkePreferredBackends.enabled is true" }}
409+
{{- end }}
410+
{{- end }}
411+
{{- end -}}
412+
400413
{{- define "llm-d-router.validations.epp" -}}
401414
{{- include "llm-d-router.validations.deprecations" . }}
402415
{{- include "llm-d-router.validations.epp.resources" . }}
403416
{{- include "llm-d-router.validations.epp.inferenceObjectives" . }}
404417
{{- include "llm-d-router.validations.epp.tokenizer" . }}
418+
{{- include "llm-d-router.validations.epp.preferredBackends" . }}
405419
{{- end -}}
406420

407421
{{/*

‎config/charts/routerlib/templates/_inferencepool.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@ spec:
2626
{{- end }}
2727
{{- end }}
2828
endpointPickerRef:
29+
{{- if .Values.router.epp.gkePreferredBackends.enabled }}
30+
name: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
31+
{{- else }}
2932
name: {{ include "llm-d-router.name" . }}
33+
{{- end }}
3034
port:
3135
number: {{ .Values.router.epp.extProcPort | default 9002 }}
3236
failureMode: {{ .Values.router.inferencePool.failureMode | default "FailOpen" }}

‎config/charts/routerlib/templates/_service.yaml‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,46 @@
11
{{- define "llm-d-epp.service" -}}
2+
{{- if .Values.router.epp.gkePreferredBackends.enabled }}
3+
{{- $preferredReplicas := .Values.router.epp.gkePreferredBackends.preferredReplicas | default 1 | int }}
4+
{{- $defaultReplicas := .Values.router.epp.gkePreferredBackends.defaultReplicas | default 1 | int }}
5+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
6+
{{- range $i := untilStep 0 (int $totalReplicas) 1 }}
7+
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
8+
{{- if eq (int $i) 0 }}
9+
{{- $serviceName = include "llm-d-router.name" $ }}
10+
{{- else if ge (int $i) (int $preferredReplicas) }}
11+
{{- if eq (int $i) (int $preferredReplicas) }}
12+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
13+
{{- else }}
14+
{{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
15+
{{- end }}
16+
{{- end }}
17+
apiVersion: v1
18+
kind: Service
19+
metadata:
20+
name: {{ $serviceName }}
21+
namespace: {{ $.Release.Namespace }}
22+
labels:
23+
{{- include "llm-d-router.labels" $ | nindent 4 }}
24+
annotations:
25+
cloud.google.com/neg: '{"exposed_ports":{"{{ $.Values.router.epp.extProcPort | default 9002 }}":{}}}'
26+
spec:
27+
selector:
28+
statefulset.kubernetes.io/pod-name: {{ printf "%s-%d" (include "llm-d-router.name" $) (int $i) }}
29+
ports:
30+
- name: grpc-ext-proc
31+
protocol: TCP
32+
port: {{ $.Values.router.epp.extProcPort | default 9002 }}
33+
appProtocol: kubernetes.io/h2c
34+
- name: http-metrics
35+
protocol: TCP
36+
port: {{ $.Values.router.metricsPort | default 9090 }}
37+
{{- with $.Values.router.extraServicePorts }}
38+
{{- . | toYaml | nindent 4 }}
39+
{{- end }}
40+
type: ClusterIP
41+
---
42+
{{- end }}
43+
{{- else }}
244
apiVersion: v1
345
kind: Service
446
metadata:
@@ -13,12 +55,13 @@ spec:
1355
- name: grpc-ext-proc
1456
protocol: TCP
1557
port: {{ .Values.router.epp.extProcPort | default 9002 }}
58+
appProtocol: kubernetes.io/h2c
1659
- name: http-metrics
1760
protocol: TCP
1861
port: {{ .Values.router.metricsPort | default 9090 }}
1962
{{- with .Values.router.extraServicePorts }}
2063
{{- . | toYaml | nindent 4 }}
2164
{{- end }}
2265
type: ClusterIP
23-
---
66+
{{- end }}
2467
{{- end }}

‎config/charts/routerlib/values.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,19 @@ clusterDomain: cluster.local
55

66
epp:
77
replicas: 1
8+
gkePreferredBackends:
9+
enabled: false
10+
# Number of primary active pod instances pinned to the PREFERRED load balancing tier.
11+
# Setting >1 scales concurrent active routing.
12+
preferredReplicas: 1
13+
# Number of standby pod instances pinned to the DEFAULT load balancing tier.
14+
# Setting >1 scales warm standby failover capacity.
15+
defaultReplicas: 1
16+
# Benchmark capacity and threshold settings for PREFERRED load balancing tier.
17+
# When incoming traffic exceeds capacityScalerPercent of maxRatePerEndpoint, excess traffic spills over to DEFAULT standby pods.
18+
balancingMode: RATE
19+
maxRatePerEndpoint: 100
20+
capacityScalerPercent: 100
821
image:
922
registry: ghcr.io/llm-d
1023
repository: llm-d-router-endpoint-picker-dev

0 commit comments

Comments
 (0)