Skip to content

Commit 9263898

Browse files
rlakhtakiarlakhtakia
committed
update helm chart to use preferred backends for gke
Signed-off-by: Radhika Lakhtakia <rlakhtakia@google.com>
1 parent d09c9c5 commit 9263898

7 files changed

Lines changed: 246 additions & 31 deletions

File tree

‎config/charts/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,12 @@ Configures routing policies, target Gateway interfaces, and priority objectives
491491
| `provider.name` | Name of Gateway implementation. Options: `[none, gke, istio]`. | `none` |
492492
| `provider.istio.destinationRule.host` | Custom host value for Istio DestinationRule. | `""` |
493493
| `provider.istio.destinationRule.trafficPolicy.connectionPool` | Connection pool settings for Istio DestinationRule. | `{}` |
494+
| `provider.gke.preferredBackends.enabled` | Enable Preferred Backends high availability routing for GKE. | `false` |
495+
| `provider.gke.preferredBackends.preferredReplicas` | Replica count for primary active pod ordinals pinned to PREFERRED tier. | `1` |
496+
| `provider.gke.preferredBackends.defaultReplicas` | Replica count for standby backup pod ordinals pinned to DEFAULT tier. | `1` |
497+
| `provider.gke.preferredBackends.balancingMode` | Load balancing calculation mode (`RATE`, `UTILIZATION`, `CONNECTION`). | `RATE` |
498+
| `provider.gke.preferredBackends.maxRatePerEndpoint` | Maximum requests per second per pod instance before spilling over. | `100` |
499+
| `provider.gke.preferredBackends.capacityScalerPercent` | Effective capacity ceiling percentage (`0` to `100`). | `100` |
494500
| `httpRoute.create` | Deploy an `HTTPRoute` resource as part of the gateway chart. | `false` |
495501
| `httpRoute.inferenceGatewayName` | Target Gateway name for the `HTTPRoute`. | `inference-gateway` |
496502
| `httpRoute.inferenceGatewayNamespace` | Target Gateway namespace for the `HTTPRoute`. | `""` |
@@ -501,6 +507,14 @@ Configures routing policies, target Gateway interfaces, and priority objectives
501507
```yaml
502508
provider:
503509
name: gke # Use GKE gateway implementation
510+
gke:
511+
preferredBackends:
512+
enabled: true
513+
preferredReplicas: 1
514+
defaultReplicas: 1
515+
balancingMode: RATE
516+
maxRatePerEndpoint: 100
517+
capacityScalerPercent: 100
504518
505519
httpRoute:
506520
create: true

‎config/charts/llm-d-router-gateway/templates/gke.yaml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,43 @@ spec:
5959
portSpecification: "USE_FIXED_PORT"
6060
port: {{ $eppHealthPort }}
6161
{{- end }}
62+
{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }}
63+
{{- if $gkePB.enabled }}
64+
{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
65+
{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
66+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
67+
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
68+
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
69+
{{- if ge (int $i) (int $preferredReplicas) }}
70+
{{- if eq (int $i) (int $preferredReplicas) }}
71+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
72+
{{- else }}
73+
{{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
74+
{{- end }}
75+
{{- end }}
76+
---
77+
kind: HealthCheckPolicy
78+
apiVersion: networking.gke.io/v1
79+
metadata:
80+
name: {{ $serviceName }}
81+
namespace: {{ $.Release.Namespace }}
82+
labels:
83+
{{- include "llm-d-router.labels" $ | nindent 4 }}
84+
spec:
85+
targetRef:
86+
group: ""
87+
kind: Service
88+
name: {{ $serviceName }}
89+
default:
90+
config:
91+
type: GRPC
92+
grpcHealthCheck:
93+
portSpecification: "USE_FIXED_PORT"
94+
port: {{ $eppHealthPort }}
95+
{{- end }}
96+
{{- end }}
6297
---
98+
{{- if not $gkePB.enabled }}
6399
apiVersion: networking.gke.io/v1
64100
kind: GCPBackendPolicy
65101
metadata:
@@ -78,4 +114,51 @@ spec:
78114
enabled: true # log all requests by default
79115
---
80116
{{- end }}
117+
{{- if $gkePB.enabled }}
118+
{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
119+
{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
120+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
121+
{{- range $i := untilStep 0 (int $totalReplicas) 1 }}
122+
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
123+
{{- $policyName := printf "%s-preferred-%d" $.Release.Name (int $i) }}
124+
{{- $pref := "PREFERRED" }}
125+
{{- if eq (int $i) 0 }}
126+
{{- $serviceName = include "llm-d-router.name" $ }}
127+
{{- $policyName = $.Release.Name }}
128+
{{- else if ge (int $i) (int $preferredReplicas) }}
129+
{{- $pref = "DEFAULT" }}
130+
{{- if eq (int $i) (int $preferredReplicas) }}
131+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
132+
{{- $policyName = printf "%s-backup" $.Release.Name }}
133+
{{- else }}
134+
{{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
135+
{{- $policyName = printf "%s-backup-%d" $.Release.Name (sub (int $i) (int $preferredReplicas)) }}
136+
{{- end }}
137+
{{- end }}
138+
apiVersion: networking.gke.io/v1
139+
kind: GCPBackendPolicy
140+
metadata:
141+
name: {{ $policyName }}
142+
namespace: {{ $.Release.Namespace }}
143+
spec:
144+
targetRef:
145+
group: ""
146+
kind: Service
147+
name: {{ $serviceName }}
148+
default:
149+
timeoutSec: 300
150+
backendPreference: {{ $pref }}
151+
{{- if $gkePB.balancingMode }}
152+
balancingMode: {{ $gkePB.balancingMode }}
153+
{{- end }}
154+
{{- if $gkePB.maxRatePerEndpoint }}
155+
maxRatePerEndpoint: {{ $gkePB.maxRatePerEndpoint }}
156+
{{- end }}
157+
{{- if $gkePB.capacityScalerPercent }}
158+
capacityScalerPercent: {{ $gkePB.capacityScalerPercent }}
159+
{{- end }}
160+
---
161+
{{- end }}
162+
{{- end }}
81163
{{- include "llm-d-router.gke" . -}}
164+
{{- end }}

‎config/charts/llm-d-router-gateway/values.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,23 @@ provider:
2121
# http:
2222
# maxRequestsPerConnection: 256000
2323

24+
# GKE-specific configuration.
25+
# This block is only used if name is "gke".
26+
gke:
27+
preferredBackends:
28+
enabled: false
29+
# Number of primary active pod instances pinned to the PREFERRED load balancing tier.
30+
# Setting >1 scales concurrent active routing.
31+
preferredReplicas: 1
32+
# Number of standby pod instances pinned to the DEFAULT load balancing tier.
33+
# Setting >1 scales warm standby failover capacity.
34+
defaultReplicas: 1
35+
# Benchmark capacity and threshold settings for PREFERRED load balancing tier.
36+
# When incoming traffic exceeds capacityScalerPercent of maxRatePerEndpoint, excess traffic spills over to DEFAULT standby pods.
37+
balancingMode: RATE
38+
maxRatePerEndpoint: 100
39+
capacityScalerPercent: 100
40+
2441
# httpRoute section is used to deploy an HTTPRoute resource as part of the gateway chart.
2542
httpRoute:
2643
create: false # a flag to indicate whether to create the httproute as part of the chart or not.

‎config/charts/routerlib/templates/_deployment.yaml‎

Lines changed: 58 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,4 @@
1-
{{- define "llm-d-epp.deployment" -}}
2-
apiVersion: apps/v1
3-
kind: Deployment
4-
metadata:
5-
name: {{ include "llm-d-router.name" . }}
6-
namespace: {{ .Release.Namespace }}
7-
labels:
8-
{{- include "llm-d-router.labels" . | nindent 4 }}
9-
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
10-
spec:
11-
replicas: {{ .Values.router.epp.replicas | default 1 }}
12-
strategy:
13-
# The current recommended EPP deployment pattern is to have a single active replica. This ensures
14-
# optimal performance of the stateful operations such prefix cache aware scorer.
15-
# The Recreate strategy the old replica is killed immediately, and allow the new replica(s) to
16-
# quickly take over. This is particularly important in the high availability set up with leader
17-
# election, as the rolling update strategy would prevent the old leader being killed because
18-
# otherwise the maxUnavailable would be 100%.
19-
type: Recreate
20-
selector:
21-
matchLabels:
22-
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
23-
template:
24-
metadata:
25-
labels:
26-
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
27-
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
28-
spec:
1+
{{- define "llm-d-epp.deployment-pod-spec" -}}
292
{{- $proxy := include "llm-d-router.proxy" . | fromYaml | default dict }}
303
{{- $proxyType := include "llm-d-router.proxyType" . | trim }}
314
{{- $proxyMode := include "llm-d-router.proxyMode" . | trim }}
@@ -128,8 +101,8 @@ spec:
128101
- --zap-encoder
129102
- "json"
130103
- --config-file
131-
- "/config/{{ .Values.router.epp.pluginsConfigFile }}"
132-
{{- if gt (.Values.router.epp.replicas | int) 1 }}
104+
{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }}
105+
{{- if and (gt (.Values.router.epp.replicas | int) 1) (not $gkePB.enabled) }}
133106
- --ha-enable-leader-election
134107
{{- end }}
135108
{{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }}
@@ -305,5 +278,60 @@ spec:
305278
tolerations:
306279
{{- toYaml .Values.router.epp.tolerations | nindent 8 }}
307280
{{- end }}
281+
{{- end }}
282+
283+
{{- define "llm-d-epp.deployment" -}}
284+
{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}}
285+
{{- if $gkePB.enabled }}
286+
{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
287+
{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
288+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
289+
apiVersion: apps/v1
290+
kind: StatefulSet
291+
metadata:
292+
name: {{ include "llm-d-router.name" . }}
293+
namespace: {{ .Release.Namespace }}
294+
labels:
295+
{{- include "llm-d-router.labels" . | nindent 4 }}
296+
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
297+
spec:
298+
replicas: {{ $totalReplicas }}
299+
serviceName: {{ include "llm-d-router.name" . }}
300+
podManagementPolicy: Parallel
301+
selector:
302+
matchLabels:
303+
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
304+
template:
305+
metadata:
306+
labels:
307+
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
308+
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
309+
spec:
310+
{{ include "llm-d-epp.deployment-pod-spec" . }}
311+
---
312+
{{- else }}
313+
apiVersion: apps/v1
314+
kind: Deployment
315+
metadata:
316+
name: {{ include "llm-d-router.name" . }}
317+
namespace: {{ .Release.Namespace }}
318+
labels:
319+
{{- include "llm-d-router.labels" . | nindent 4 }}
320+
{{- include "llm-d-router.modeLabels" . | nindent 4 }}
321+
spec:
322+
replicas: {{ .Values.router.epp.replicas | default 1 }}
323+
strategy:
324+
type: Recreate
325+
selector:
326+
matchLabels:
327+
{{- include "llm-d-router.selectorLabels" . | nindent 6 }}
328+
template:
329+
metadata:
330+
labels:
331+
{{- include "llm-d-router.selectorLabels" . | nindent 8 }}
332+
{{- include "llm-d-router.modeLabels" . | nindent 8 }}
333+
spec:
334+
{{ include "llm-d-epp.deployment-pod-spec" . }}
308335
---
309336
{{- end }}
337+
{{- end }}

‎config/charts/routerlib/templates/_helpers.tpl‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,14 +394,38 @@ EPP resource validations
394394
{{- $_ := required ".Values.router.epp.resources.requests.memory is required. EPP is a critical component that must have memory requests set." .Values.router.epp.resources.requests.memory }}
395395
{{- end -}}
396396

397+
{{/*
398+
Helper to retrieve GKE preferredBackends configuration safely across chart contexts.
399+
*/}}
400+
{{- define "llm-d-router.gkePreferredBackends" -}}
401+
{{- $provider := .Values.provider | default dict -}}
402+
{{- $gke := index $provider "gke" | default dict -}}
403+
{{- index $gke "preferredBackends" | default dict | toYaml -}}
404+
{{- end -}}
405+
397406
{{/*
398407
EPP generic validations
399408
*/}}
409+
{{- define "llm-d-router.validations.epp.preferredBackends" -}}
410+
{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}}
411+
{{- if $gkePB.enabled }}
412+
{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
413+
{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
414+
{{- if lt $preferredReplicas 1 }}
415+
{{- fail ".Values.provider.gke.preferredBackends.preferredReplicas must be at least 1 when preferredBackends.enabled is true" }}
416+
{{- end }}
417+
{{- if lt $defaultReplicas 1 }}
418+
{{- fail ".Values.provider.gke.preferredBackends.defaultReplicas must be at least 1 when preferredBackends.enabled is true" }}
419+
{{- end }}
420+
{{- end }}
421+
{{- end -}}
422+
400423
{{- define "llm-d-router.validations.epp" -}}
401424
{{- include "llm-d-router.validations.deprecations" . }}
402425
{{- include "llm-d-router.validations.epp.resources" . }}
403426
{{- include "llm-d-router.validations.epp.inferenceObjectives" . }}
404427
{{- include "llm-d-router.validations.epp.tokenizer" . }}
428+
{{- include "llm-d-router.validations.epp.preferredBackends" . }}
405429
{{- end -}}
406430

407431
{{/*

‎config/charts/routerlib/templates/_inferencepool.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@ spec:
2626
{{- end }}
2727
{{- end }}
2828
endpointPickerRef:
29+
{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }}
30+
{{- if $gkePB.enabled }}
31+
name: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
32+
{{- else }}
2933
name: {{ include "llm-d-router.name" . }}
34+
{{- end }}
3035
port:
3136
number: {{ .Values.router.epp.extProcPort | default 9002 }}
3237
failureMode: {{ .Values.router.inferencePool.failureMode | default "FailOpen" }}

‎config/charts/routerlib/templates/_service.yaml‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,47 @@
11
{{- define "llm-d-epp.service" -}}
2+
{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}}
3+
{{- if $gkePB.enabled }}
4+
{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
5+
{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
6+
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
7+
{{- range $i := untilStep 0 (int $totalReplicas) 1 }}
8+
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
9+
{{- if eq (int $i) 0 }}
10+
{{- $serviceName = include "llm-d-router.name" $ }}
11+
{{- else if ge (int $i) (int $preferredReplicas) }}
12+
{{- if eq (int $i) (int $preferredReplicas) }}
13+
{{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
14+
{{- else }}
15+
{{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
16+
{{- end }}
17+
{{- end }}
18+
apiVersion: v1
19+
kind: Service
20+
metadata:
21+
name: {{ $serviceName }}
22+
namespace: {{ $.Release.Namespace }}
23+
labels:
24+
{{- include "llm-d-router.labels" $ | nindent 4 }}
25+
annotations:
26+
cloud.google.com/neg: '{"exposed_ports":{"{{ $.Values.router.epp.extProcPort | default 9002 }}":{}}}'
27+
spec:
28+
selector:
29+
statefulset.kubernetes.io/pod-name: {{ printf "%s-%d" (include "llm-d-router.name" $) (int $i) }}
30+
ports:
31+
- name: grpc-ext-proc
32+
protocol: TCP
33+
port: {{ $.Values.router.epp.extProcPort | default 9002 }}
34+
appProtocol: kubernetes.io/h2c
35+
- name: http-metrics
36+
protocol: TCP
37+
port: {{ $.Values.router.metricsPort | default 9090 }}
38+
{{- with $.Values.router.extraServicePorts }}
39+
{{- . | toYaml | nindent 4 }}
40+
{{- end }}
41+
type: ClusterIP
42+
---
43+
{{- end }}
44+
{{- else }}
245
apiVersion: v1
346
kind: Service
447
metadata:
@@ -13,12 +56,13 @@ spec:
1356
- name: grpc-ext-proc
1457
protocol: TCP
1558
port: {{ .Values.router.epp.extProcPort | default 9002 }}
59+
appProtocol: kubernetes.io/h2c
1660
- name: http-metrics
1761
protocol: TCP
1862
port: {{ .Values.router.metricsPort | default 9090 }}
1963
{{- with .Values.router.extraServicePorts }}
2064
{{- . | toYaml | nindent 4 }}
2165
{{- end }}
2266
type: ClusterIP
23-
---
67+
{{- end }}
2468
{{- end }}

0 commit comments

Comments
 (0)