Skip to content

Commit 2acf74b

Browse files
rlakhtakiarlakhtakia
committed
update helm chart to use preferred backends for gke
Signed-off-by: Radhika Lakhtakia <rlakhtakia@google.com>
1 parent 71c234f commit 2acf74b

5 files changed

Lines changed: 117 additions & 2 deletions

File tree

‎config/charts/llm-d-router-gateway/templates/gke.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ spec:
6060
port: {{ $eppHealthPort }}
6161
{{- end }}
6262
---
63+
{{- if not .Values.router.epp.enablePreferredBackends }}
6364
apiVersion: networking.gke.io/v1
6465
kind: GCPBackendPolicy
6566
metadata:
@@ -78,4 +79,51 @@ spec:
7879
enabled: true # log all requests by default
7980
---
8081
{{- end }}
82+
{{- if .Values.router.epp.enablePreferredBackends }}
83+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
84+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
85+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
86+
apiVersion: networking.gke.io/v1
87+
kind: GCPBackendPolicy
88+
metadata:
89+
name: {{ .Release.Name }}
90+
namespace: {{ .Release.Namespace }}
91+
spec:
92+
targetRef:
93+
group: ""
94+
kind: Service
95+
name: {{ include "llm-d-router.name" . }}
96+
default:
97+
timeoutSec: 300
98+
backendPreference: PREFERRED
99+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
100+
{{- $serviceName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
101+
{{- if eq (int $i) 1 }}
102+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
103+
{{- end }}
104+
{{- $policyName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
105+
{{- if eq (int $i) 1 }}
106+
{{- $policyName = printf "%s-backup" (include "llm-d-router.name" $) }}
107+
{{- end }}
108+
---
109+
apiVersion: networking.gke.io/v1
110+
kind: GCPBackendPolicy
111+
metadata:
112+
name: {{ $policyName }}
113+
namespace: {{ $.Release.Namespace }}
114+
spec:
115+
targetRef:
116+
group: ""
117+
kind: Service
118+
name: {{ $serviceName }}
119+
default:
120+
timeoutSec: 300
121+
{{- if lt (int $i) $preferredReplicas }}
122+
backendPreference: PREFERRED
123+
{{- else }}
124+
backendPreference: DEFAULT
125+
{{- end }}
126+
{{- end }}
127+
{{- end }}
128+
{{- end }}
81129
{{- include "llm-d-router.gke" . -}}

‎config/charts/routerlib/templates/_deployment.yaml‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,28 @@
11
{{- define "llm-d-epp.deployment" -}}
22
apiVersion: apps/v1
3+
{{- if .Values.router.epp.enablePreferredBackends }}
4+
kind: StatefulSet
5+
{{- else }}
36
kind: Deployment
7+
{{- end }}
48
metadata:
59
name: {{ include "llm-d-router.name" . }}
610
namespace: {{ .Release.Namespace }}
711
labels:
812
{{- include "llm-d-router.labels" . | nindent 4 }}
913
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
1014
spec:
11-
replicas: {{ .Values.router.epp.replicas | default 1 }}
15+
{{- if .Values.router.epp.enablePreferredBackends }}
16+
serviceName: {{ include "llm-d-router.name" . }}
17+
{{- end }}
18+
{{- $replicas := .Values.router.epp.replicas | default 1 }}
19+
{{- if .Values.router.epp.enablePreferredBackends }}
20+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
21+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
22+
{{- $replicas = add $preferredReplicas $defaultReplicas }}
23+
{{- end }}
24+
replicas: {{ $replicas }}
25+
{{- if not .Values.router.epp.enablePreferredBackends }}
1226
strategy:
1327
# The current recommended EPP deployment pattern is to have a single active replica. This ensures
1428
# optimal performance of the stateful operations such prefix cache aware scorer.
@@ -17,6 +31,7 @@ spec:
1731
# election, as the rolling update strategy would prevent the old leader being killed because
1832
# otherwise the maxUnavailable would be 100%.
1933
type: Recreate
34+
{{- end }}
2035
selector:
2136
matchLabels:
2237
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
@@ -129,7 +144,7 @@ spec:
129144
- "json"
130145
- --config-file
131146
- "/config/{{ .Values.router.epp.pluginsConfigFile }}"
132-
{{- if gt (.Values.router.epp.replicas | int) 1 }}
147+
{{- if and (gt (.Values.router.epp.replicas | int) 1) (not .Values.router.epp.enablePreferredBackends) }}
133148
- --ha-enable-leader-election
134149
{{- end }}
135150
{{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }}

‎config/charts/routerlib/templates/_helpers.tpl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -405,11 +405,25 @@ EPP resource validations
405405
{{/*
406406
EPP generic validations
407407
*/}}
408+
{{- define "llm-d-router.validations.epp.preferredBackends" -}}
409+
{{- if .Values.router.epp.enablePreferredBackends }}
410+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
411+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
412+
{{- if lt $preferredReplicas 1 }}
413+
{{- fail ".Values.router.epp.preferredReplicas must be at least 1 when enablePreferredBackends is true" }}
414+
{{- end }}
415+
{{- if lt $defaultReplicas 1 }}
416+
{{- fail ".Values.router.epp.defaultReplicas must be at least 1 when enablePreferredBackends is true" }}
417+
{{- end }}
418+
{{- end }}
419+
{{- end -}}
420+
408421
{{- define "llm-d-router.validations.epp" -}}
409422
{{- include "llm-d-router.validations.deprecations" . }}
410423
{{- include "llm-d-router.validations.epp.resources" . }}
411424
{{- include "llm-d-router.validations.epp.inferenceObjectives" . }}
412425
{{- include "llm-d-router.validations.epp.tokenizer" . }}
426+
{{- include "llm-d-router.validations.epp.preferredBackends" . }}
413427
{{- end -}}
414428

415429
{{/*

‎config/charts/routerlib/templates/_service.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,16 @@ metadata:
88
{{- include "llm-d-router.labels" . | nindent 4 }}
99
spec:
1010
selector:
11+
{{- if .Values.router.epp.enablePreferredBackends }}
12+
statefulset.kubernetes.io/pod-name: {{ include "llm-d-router.name" . }}-0
13+
{{- else }}
1114
{{- include "llm-d-router.selectorLabels" . | nindent 4 }}
15+
{{- end }}
1216
ports:
1317
- name: grpc-ext-proc
1418
protocol: TCP
1519
port: {{ .Values.router.epp.extProcPort | default 9002 }}
20+
appProtocol: kubernetes.io/h2c
1621
- name: http-metrics
1722
protocol: TCP
1823
port: {{ .Values.router.metricsPort | default 9090 }}
@@ -21,4 +26,34 @@ spec:
2126
{{- end }}
2227
type: ClusterIP
2328
---
29+
{{- if .Values.router.epp.enablePreferredBackends }}
30+
{{- $preferredReplicas := .Values.router.epp.preferredReplicas | default 1 | int }}
31+
{{- $defaultReplicas := .Values.router.epp.defaultReplicas | default 1 | int }}
32+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
33+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
34+
{{- $serviceName := printf "%s-backend-%d" (include "llm-d-router.name" $) (int $i) }}
35+
{{- if eq (int $i) 1 }}
36+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
37+
{{- end }}
38+
---
39+
apiVersion: v1
40+
kind: Service
41+
metadata:
42+
name: {{ $serviceName }}
43+
namespace: {{ $.Release.Namespace }}
44+
labels:
45+
{{- include "llm-d-router.labels" $ | nindent 4 }}
46+
annotations:
47+
cloud.google.com/neg: '{"exposed_ports":{"{{ $.Values.router.epp.extProcPort | default 9002 }}":{}}}'
48+
spec:
49+
selector:
50+
statefulset.kubernetes.io/pod-name: {{ include "llm-d-router.name" $ }}-{{ $i }}
51+
ports:
52+
- name: grpc-ext-proc
53+
protocol: TCP
54+
port: {{ $.Values.router.epp.extProcPort | default 9002 }}
55+
appProtocol: kubernetes.io/h2c
56+
type: ClusterIP
57+
{{- end }}
58+
{{- end }}
2459
{{- end }}

‎config/charts/routerlib/values.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ clusterDomain: cluster.local
55

66
epp:
77
replicas: 1
8+
enablePreferredBackends: false
9+
preferredReplicas: 1
10+
defaultReplicas: 1
811
image:
912
registry: ghcr.io/llm-d
1013
repository: llm-d-router-endpoint-picker-dev

0 commit comments

Comments
 (0)