update helm chart to use preferred backends for gke

config/charts/README.md

@@ -491,6 +491,12 @@ Configures routing policies, target Gateway interfaces, and priority objectives
 | `provider.name` | Name of Gateway implementation. Options: `[none, gke, istio]`. | `none` |
 | `provider.istio.destinationRule.host` | Custom host value for Istio DestinationRule. | `""` |
 | `provider.istio.destinationRule.trafficPolicy.connectionPool` | Connection pool settings for Istio DestinationRule. | `{}` |
+| `provider.gke.preferredBackends.enabled` | Enable Preferred Backends high availability routing for GKE. | `false` |
+| `provider.gke.preferredBackends.preferredReplicas` | Replica count for primary active pod ordinals pinned to PREFERRED tier. | `1` |
+| `provider.gke.preferredBackends.defaultReplicas` | Replica count for standby backup pod ordinals pinned to DEFAULT tier. | `1` |
+| `provider.gke.preferredBackends.balancingMode` | Load balancing calculation mode (`RATE`, `UTILIZATION`, `CONNECTION`). | `RATE` |
+| `provider.gke.preferredBackends.maxRatePerEndpoint` | Maximum requests per second per pod instance before spilling over. | `100` |
+| `provider.gke.preferredBackends.capacityScalerPercent` | Effective capacity ceiling percentage (`0` to `100`). | `100` |
 | `httpRoute.create` | Deploy an `HTTPRoute` resource as part of the gateway chart. | `false` |
 | `httpRoute.inferenceGatewayName` | Target Gateway name for the `HTTPRoute`. | `inference-gateway` |
 | `httpRoute.inferenceGatewayNamespace` | Target Gateway namespace for the `HTTPRoute`. | `""` |
@@ -501,6 +507,14 @@ Configures routing policies, target Gateway interfaces, and priority objectives
 ```yaml
 provider:
   name: gke # Use GKE gateway implementation
+  gke:
+    preferredBackends:
+      enabled: true
+      preferredReplicas: 1
+      defaultReplicas: 1
+      balancingMode: RATE
+      maxRatePerEndpoint: 100
+      capacityScalerPercent: 100
 
 httpRoute:
   create: true

config/charts/llm-d-router-gateway/templates/gke.yaml

@@ -59,7 +59,43 @@ spec:
         portSpecification: "USE_FIXED_PORT"
         port: {{ $eppHealthPort }}
 {{- end }}
+{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }}
+{{- if $gkePB.enabled }}
+{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
+{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
+{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
+{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
+{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
+{{- if ge (int $i) (int $preferredReplicas) }}
+  {{- if eq (int $i) (int $preferredReplicas) }}
+    {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
+  {{- else }}
+    {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
+  {{- end }}
+{{- end }}
+---
+kind: HealthCheckPolicy
+apiVersion: networking.gke.io/v1
+metadata:
+  name: {{ $serviceName }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "llm-d-router.labels" $ | nindent 4 }}
+spec:
+  targetRef:
+    group: ""
+    kind: Service
+    name: {{ $serviceName }}
+  default:
+    config:
+      type: GRPC
+      grpcHealthCheck:
+        portSpecification: "USE_FIXED_PORT"
+        port: {{ $eppHealthPort }}
+{{- end }}
+{{- end }}
 ---
+{{- if not $gkePB.enabled }}
 apiVersion: networking.gke.io/v1
 kind: GCPBackendPolicy
 metadata:
@@ -78,4 +114,51 @@ spec:
       enabled: true    # log all requests by default
 ---
 {{- end }}
+{{- if $gkePB.enabled }}
+{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
+{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
+{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
+{{- range $i := untilStep 0 (int $totalReplicas) 1 }}
+{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
+{{- $policyName := printf "%s-preferred-%d" $.Release.Name (int $i) }}
+{{- $pref := "PREFERRED" }}
+{{- if eq (int $i) 0 }}
+  {{- $serviceName = include "llm-d-router.name" $ }}
+  {{- $policyName = $.Release.Name }}
+{{- else if ge (int $i) (int $preferredReplicas) }}
+  {{- $pref = "DEFAULT" }}
+  {{- if eq (int $i) (int $preferredReplicas) }}
+    {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
+    {{- $policyName = printf "%s-backup" $.Release.Name }}
+  {{- else }}
+    {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
+    {{- $policyName = printf "%s-backup-%d" $.Release.Name (sub (int $i) (int $preferredReplicas)) }}
+  {{- end }}
+{{- end }}
+apiVersion: networking.gke.io/v1
+kind: GCPBackendPolicy
+metadata:
+  name: {{ $policyName }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  targetRef:
+    group: ""
+    kind: Service
+    name: {{ $serviceName }}
+  default:
+    timeoutSec: 300
+    backendPreference: {{ $pref }}
+    {{- if $gkePB.balancingMode }}
+    balancingMode: {{ $gkePB.balancingMode }}
+    {{- end }}
+    {{- if $gkePB.maxRatePerEndpoint }}
+    maxRatePerEndpoint: {{ $gkePB.maxRatePerEndpoint }}
+    {{- end }}
+    {{- if $gkePB.capacityScalerPercent }}
+    capacityScalerPercent: {{ $gkePB.capacityScalerPercent }}
+    {{- end }}
+---
+{{- end }}
+{{- end }}
 {{- include "llm-d-router.gke" . -}}
+{{- end }}

config/charts/llm-d-router-gateway/values.yaml

@@ -21,6 +21,23 @@ provider:
         #   http:
         #     maxRequestsPerConnection: 256000
 
+  # GKE-specific configuration.
+  # This block is only used if name is "gke".
+  gke:
+    preferredBackends:
+      enabled: false
+      # Number of primary active pod instances pinned to the PREFERRED load balancing tier.
+      # Setting >1 scales concurrent active routing.
+      preferredReplicas: 1
+      # Number of standby pod instances pinned to the DEFAULT load balancing tier.
+      # Setting >1 scales warm standby failover capacity.
+      defaultReplicas: 1
+      # Benchmark capacity and threshold settings for PREFERRED load balancing tier.
+      # When incoming traffic exceeds capacityScalerPercent of maxRatePerEndpoint, excess traffic spills over to DEFAULT standby pods.
+      balancingMode: RATE
+      maxRatePerEndpoint: 100
+      capacityScalerPercent: 100
+
 # httpRoute section is used to deploy an HTTPRoute resource as part of the gateway chart.
 httpRoute:
   create: false # a flag to indicate whether to create the httproute as part of the chart or not.

config/charts/routerlib/templates/_deployment.yaml

@@ -1,31 +1,4 @@
-{{- define "llm-d-epp.deployment" -}}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "llm-d-router.name" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "llm-d-router.labels" . | nindent 4 }}
-    {{- include "llm-d-router.modeLabels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.router.epp.replicas | default 1 }}
-  strategy:
-    # The current recommended EPP deployment pattern is to have a single active replica. This ensures
-    # optimal performance of the stateful operations such prefix cache aware scorer.
-    # The Recreate strategy the old replica is killed immediately, and allow the new replica(s) to
-    # quickly take over. This is particularly important in the high availability set up with leader
-    # election, as the rolling update strategy would prevent the old leader being killed because
-    # otherwise the maxUnavailable would be 100%.
-    type: Recreate
-  selector:
-    matchLabels:
-      {{- include "llm-d-router.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "llm-d-router.selectorLabels" . | nindent 8 }}
-        {{- include "llm-d-router.modeLabels" . | nindent 8 }}
-    spec:
+{{- define "llm-d-epp.deployment-pod-spec" -}}
       {{- $proxy := include "llm-d-router.proxy" . | fromYaml | default dict }}
       {{- $proxyType := include "llm-d-router.proxyType" . | trim }}
       {{- $proxyMode := include "llm-d-router.proxyMode" . | trim }}
@@ -128,8 +101,8 @@ spec:
               - --zap-encoder
               - "json"
               - --config-file
-              - "/config/{{ .Values.router.epp.pluginsConfigFile }}"
-          {{- if gt (.Values.router.epp.replicas | int) 1 }}
+          {{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }}
+          {{- if and (gt (.Values.router.epp.replicas | int) 1) (not $gkePB.enabled) }}
               - --ha-enable-leader-election
           {{- end }}
           {{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }}
@@ -305,5 +278,60 @@ spec:
       tolerations:
         {{- toYaml .Values.router.epp.tolerations | nindent 8 }}
       {{- end }}
+{{- end }}
+
+{{- define "llm-d-epp.deployment" -}}
+{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}}
+{{- if $gkePB.enabled }}
+{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
+{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
+{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "llm-d-router.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "llm-d-router.labels" . | nindent 4 }}
+    {{- include "llm-d-router.modeLabels" . | nindent 4 }}
+spec:
+  replicas: {{ $totalReplicas }}
+  serviceName: {{ include "llm-d-router.name" . }}
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      {{- include "llm-d-router.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "llm-d-router.selectorLabels" . | nindent 8 }}
+        {{- include "llm-d-router.modeLabels" . | nindent 8 }}
+    spec:
+{{ include "llm-d-epp.deployment-pod-spec" . }}
+---
+{{- else }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "llm-d-router.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "llm-d-router.labels" . | nindent 4 }}
+    {{- include "llm-d-router.modeLabels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.router.epp.replicas | default 1 }}
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      {{- include "llm-d-router.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "llm-d-router.selectorLabels" . | nindent 8 }}
+        {{- include "llm-d-router.modeLabels" . | nindent 8 }}
+    spec:
+{{ include "llm-d-epp.deployment-pod-spec" . }}
 ---
 {{- end }}
+{{- end }}

config/charts/routerlib/templates/_helpers.tpl

@@ -394,14 +394,38 @@ EPP resource validations
 {{- $_ := required ".Values.router.epp.resources.requests.memory is required. EPP is a critical component that must have memory requests set." .Values.router.epp.resources.requests.memory }}
 {{- end -}}
 
+{{/*
+Helper to retrieve GKE preferredBackends configuration safely across chart contexts.
+*/}}
+{{- define "llm-d-router.gkePreferredBackends" -}}
+{{- $provider := .Values.provider | default dict -}}
+{{- $gke := index $provider "gke" | default dict -}}
+{{- index $gke "preferredBackends" | default dict | toYaml -}}
+{{- end -}}
+
 {{/*
 EPP generic validations
 */}}
+{{- define "llm-d-router.validations.epp.preferredBackends" -}}
+{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}}
+{{- if $gkePB.enabled }}
+  {{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
+  {{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
+  {{- if lt $preferredReplicas 1 }}
+    {{- fail ".Values.provider.gke.preferredBackends.preferredReplicas must be at least 1 when preferredBackends.enabled is true" }}
+  {{- end }}
+  {{- if lt $defaultReplicas 1 }}
+    {{- fail ".Values.provider.gke.preferredBackends.defaultReplicas must be at least 1 when preferredBackends.enabled is true" }}
+  {{- end }}
+{{- end }}
+{{- end -}}
+
 {{- define "llm-d-router.validations.epp" -}}
 {{- include "llm-d-router.validations.deprecations" . }}
 {{- include "llm-d-router.validations.epp.resources" . }}
 {{- include "llm-d-router.validations.epp.inferenceObjectives" . }}
 {{- include "llm-d-router.validations.epp.tokenizer" . }}
+{{- include "llm-d-router.validations.epp.preferredBackends" . }}
 {{- end -}}
 
 {{/*

config/charts/routerlib/templates/_inferencepool.yaml

@@ -26,7 +26,12 @@ spec:
       {{- end }}
       {{- end }}
   endpointPickerRef:
+    {{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }}
+    {{- if $gkePB.enabled }}
+    name: {{ printf "%s-backup" (include "llm-d-router.name" .) }}
+    {{- else }}
     name: {{ include "llm-d-router.name" . }}
+    {{- end }}
     port:
       number: {{ .Values.router.epp.extProcPort | default 9002 }}
     failureMode: {{ .Values.router.inferencePool.failureMode | default "FailOpen" }}

config/charts/routerlib/templates/_service.yaml

@@ -1,4 +1,47 @@
 {{- define "llm-d-epp.service" -}}
+{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}}
+{{- if $gkePB.enabled }}
+{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
+{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
+{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
+{{- range $i := untilStep 0 (int $totalReplicas) 1 }}
+{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
+{{- if eq (int $i) 0 }}
+  {{- $serviceName = include "llm-d-router.name" $ }}
+{{- else if ge (int $i) (int $preferredReplicas) }}
+  {{- if eq (int $i) (int $preferredReplicas) }}
+    {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
+  {{- else }}
+    {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
+  {{- end }}
+{{- end }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $serviceName }}
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    {{- include "llm-d-router.labels" $ | nindent 4 }}
+  annotations:
+    cloud.google.com/neg: '{"exposed_ports":{"{{ $.Values.router.epp.extProcPort | default 9002 }}":{}}}'
+spec:
+  selector:
+    statefulset.kubernetes.io/pod-name: {{ printf "%s-%d" (include "llm-d-router.name" $) (int $i) }}
+  ports:
+    - name: grpc-ext-proc
+      protocol: TCP
+      port: {{ $.Values.router.epp.extProcPort | default 9002 }}
+      appProtocol: kubernetes.io/h2c
+    - name: http-metrics
+      protocol: TCP
+      port: {{ $.Values.router.metricsPort | default 9090 }}
+    {{- with $.Values.router.extraServicePorts }}
+    {{- . | toYaml | nindent 4 }}
+    {{- end }}
+  type: ClusterIP
+---
+{{- end }}
+{{- else }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -13,12 +56,13 @@ spec:
     - name: grpc-ext-proc
       protocol: TCP
       port: {{ .Values.router.epp.extProcPort | default 9002 }}
+      appProtocol: kubernetes.io/h2c
     - name: http-metrics
       protocol: TCP
       port: {{ .Values.router.metricsPort | default 9090 }}
     {{- with .Values.router.extraServicePorts }}
     {{- . | toYaml | nindent 4 }}
     {{- end }}
   type: ClusterIP
----
+{{- end }}
 {{- end }}