Skip to content

Commit 763a600

Browse files
rlakhtakiarlakhtakia
committed
update helm chart to use preferred backends for gke
Signed-off-by: Radhika Lakhtakia <rlakhtakia@google.com>
1 parent 8fa2015 commit 763a600

5 files changed

Lines changed: 155 additions & 2 deletions

File tree

‎config/charts/llm-d-router-gateway/templates/gke.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,43 @@ spec:
5858
grpcHealthCheck:
5959
portSpecification: "USE_FIXED_PORT"
6060
port: {{ $eppHealthPort }}
61+
{{- if .Values.router.epp.enablePreferredBackends }}
62+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
63+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
64+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
65+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
66+
{{- $serviceName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
67+
{{- if eq (int $i) 1 }}
68+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
69+
{{- end }}
70+
{{- $policyName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
71+
{{- if eq (int $i) 1 }}
72+
{{- $policyName = printf "%s-backup" (include "llm-d-router.name" $) }}
6173
{{- end }}
6274
---
75+
kind: HealthCheckPolicy
76+
apiVersion: networking.gke.io/v1
77+
metadata:
78+
name: {{ $policyName }}
79+
namespace: {{ $.Release.Namespace }}
80+
labels:
81+
{{- include "llm-d-router.labels" $ | nindent 4 }}
82+
spec:
83+
targetRef:
84+
group: ""
85+
kind: Service
86+
name: {{ $serviceName }}
87+
default:
88+
config:
89+
type: GRPC
90+
grpcHealthCheck:
91+
portSpecification: "USE_FIXED_PORT"
92+
port: {{ $eppHealthPort }}
93+
{{- end }}
94+
{{- end }}
95+
{{- end }}
96+
---
97+
{{- if not .Values.router.epp.enablePreferredBackends }}
6398
apiVersion: networking.gke.io/v1
6499
kind: GCPBackendPolicy
65100
metadata:
@@ -78,4 +113,51 @@ spec:
78113
enabled: true # log all requests by default
79114
---
80115
{{- end }}
116+
{{- if .Values.router.epp.enablePreferredBackends }}
117+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
118+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
119+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
120+
apiVersion: networking.gke.io/v1
121+
kind: GCPBackendPolicy
122+
metadata:
123+
name: {{ .Release.Name }}
124+
namespace: {{ .Release.Namespace }}
125+
spec:
126+
targetRef:
127+
group: ""
128+
kind: Service
129+
name: {{ include "llm-d-router.name" . }}
130+
default:
131+
timeoutSec: 300
132+
backendPreference: PREFERRED
133+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
134+
{{- $serviceName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
135+
{{- if eq (int $i) 1 }}
136+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
137+
{{- end }}
138+
{{- $policyName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
139+
{{- if eq (int $i) 1 }}
140+
{{- $policyName = printf "%s-backup" (include "llm-d-router.name" $) }}
141+
{{- end }}
142+
---
143+
apiVersion: networking.gke.io/v1
144+
kind: GCPBackendPolicy
145+
metadata:
146+
name: {{ $policyName }}
147+
namespace: {{ $.Release.Namespace }}
148+
spec:
149+
targetRef:
150+
group: ""
151+
kind: Service
152+
name: {{ $serviceName }}
153+
default:
154+
timeoutSec: 300
155+
{{- if lt (int $i) $preferredReplicas }}
156+
backendPreference: PREFERRED
157+
{{- else }}
158+
backendPreference: DEFAULT
159+
{{- end }}
160+
{{- end }}
161+
{{- end }}
162+
{{- end }}
81163
{{- include "llm-d-router.gke" . -}}

‎config/charts/routerlib/templates/_deployment.yaml‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,28 @@
11
{{- define "llm-d-epp.deployment" -}}
22
apiVersion: apps/v1
3+
{{- if .Values.router.epp.enablePreferredBackends }}
4+
kind: StatefulSet
5+
{{- else }}
36
kind: Deployment
7+
{{- end }}
48
metadata:
59
name: {{ include "llm-d-router.name" . }}
610
namespace: {{ .Release.Namespace }}
711
labels:
812
{{- include "llm-d-router.labels" . | nindent 4 }}
913
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
1014
spec:
11-
replicas: {{ .Values.router.epp.replicas | default 1 }}
15+
{{- if .Values.router.epp.enablePreferredBackends }}
16+
serviceName: {{ include "llm-d-router.name" . }}
17+
{{- end }}
18+
{{- $replicas := .Values.router.epp.replicas | default 1 }}
19+
{{- if .Values.router.epp.enablePreferredBackends }}
20+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
21+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
22+
{{- $replicas = add $preferredReplicas $defaultReplicas }}
23+
{{- end }}
24+
replicas: {{ $replicas }}
25+
{{- if not .Values.router.epp.enablePreferredBackends }}
1226
strategy:
1327
# The current recommended EPP deployment pattern is to have a single active replica. This ensures
1428
# optimal performance of the stateful operations such prefix cache aware scorer.
@@ -17,6 +31,7 @@ spec:
1731
# election, as the rolling update strategy would prevent the old leader being killed because
1832
# otherwise the maxUnavailable would be 100%.
1933
type: Recreate
34+
{{- end }}
2035
selector:
2136
matchLabels:
2237
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
@@ -129,7 +144,7 @@ spec:
129144
- "json"
130145
- --config-file
131146
- "/config/{{ .Values.router.epp.pluginsConfigFile }}"
132-
{{- if gt (.Values.router.epp.replicas | int) 1 }}
147+
{{- if and (gt (.Values.router.epp.replicas | int) 1) (not .Values.router.epp.enablePreferredBackends) }}
133148
- --ha-enable-leader-election
134149
{{- end }}
135150
{{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }}

‎config/charts/routerlib/templates/_helpers.tpl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,11 +397,25 @@ EPP resource validations
397397
{{/*
398398
EPP generic validations
399399
*/}}
400+
{{- define "llm-d-router.validations.epp.preferredBackends" -}}
401+
{{- if .Values.router.epp.enablePreferredBackends }}
402+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
403+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
404+
{{- if lt $preferredReplicas 1 }}
405+
{{- fail ".Values.router.epp.preferredReplicas must be at least 1 when enablePreferredBackends is true" }}
406+
{{- end }}
407+
{{- if lt $defaultReplicas 1 }}
408+
{{- fail ".Values.router.epp.defaultReplicas must be at least 1 when enablePreferredBackends is true" }}
409+
{{- end }}
410+
{{- end }}
411+
{{- end -}}
412+
400413
{{- define "llm-d-router.validations.epp" -}}
401414
{{- include "llm-d-router.validations.deprecations" . }}
402415
{{- include "llm-d-router.validations.epp.resources" . }}
403416
{{- include "llm-d-router.validations.epp.inferenceObjectives" . }}
404417
{{- include "llm-d-router.validations.epp.tokenizer" . }}
418+
{{- include "llm-d-router.validations.epp.preferredBackends" . }}
405419
{{- end -}}
406420

407421
{{/*

‎config/charts/routerlib/templates/_service.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,16 @@ metadata:
88
{{- include "llm-d-router.labels" . | nindent 4 }}
99
spec:
1010
selector:
11+
{{- if .Values.router.epp.enablePreferredBackends }}
12+
statefulset.kubernetes.io/pod-name: {{ include "llm-d-router.name" . }}-0
13+
{{- else }}
1114
{{- include "llm-d-router.selectorLabels" . | nindent 4 }}
15+
{{- end }}
1216
ports:
1317
- name: grpc-ext-proc
1418
protocol: TCP
1519
port: {{ .Values.router.epp.extProcPort | default 9002 }}
20+
appProtocol: kubernetes.io/h2c
1621
- name: http-metrics
1722
protocol: TCP
1823
port: {{ .Values.router.metricsPort | default 9090 }}
@@ -21,4 +26,34 @@ spec:
2126
{{- end }}
2227
type: ClusterIP
2328
---
29+
{{- if .Values.router.epp.enablePreferredBackends }}
30+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
31+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
32+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
33+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
34+
{{- $serviceName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
35+
{{- if eq (int $i) 1 }}
36+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
37+
{{- end }}
38+
---
39+
apiVersion: v1
40+
kind: Service
41+
metadata:
42+
name: {{ $serviceName }}
43+
namespace: {{ $.Release.Namespace }}
44+
labels:
45+
{{- include "llm-d-router.labels" $ | nindent 4 }}
46+
annotations:
47+
cloud.google.com/neg: '{"exposed_ports":{"{{ $.Values.router.epp.extProcPort | default 9002 }}":{}}}'
48+
spec:
49+
selector:
50+
statefulset.kubernetes.io/pod-name: {{ include "llm-d-router.name" $ }}-{{ $i }}
51+
ports:
52+
- name: grpc-ext-proc
53+
protocol: TCP
54+
port: {{ $.Values.router.epp.extProcPort | default 9002 }}
55+
appProtocol: kubernetes.io/h2c
56+
type: ClusterIP
57+
{{- end }}
58+
{{- end }}
2459
{{- end }}

‎config/charts/routerlib/values.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ clusterDomain: cluster.local
55

66
epp:
77
replicas: 1
8+
enablePreferredBackends: false
9+
# Number of primary active pod instances pinned to the PREFERRED load balancing tier.
10+
# Setting >1 scales concurrent active routing.
11+
preferredReplicas: 1
12+
# Number of standby pod instances pinned to the DEFAULT load balancing tier.
13+
# Setting >1 scales warm standby failover capacity.
14+
defaultReplicas: 1
815
image:
916
registry: ghcr.io/llm-d
1017
repository: llm-d-router-endpoint-picker-dev

0 commit comments

Comments
 (0)