From 926389825e9e865f13ce5c564307175a9e5ccbc2 Mon Sep 17 00:00:00 2001 From: Radhika Lakhtakia Date: Tue, 13 May 2025 10:00:00 +0000 Subject: [PATCH] update helm chart to use preferred backends for gke Signed-off-by: Radhika Lakhtakia --- config/charts/README.md | 14 +++ .../llm-d-router-gateway/templates/gke.yaml | 83 +++++++++++++++++ .../charts/llm-d-router-gateway/values.yaml | 17 ++++ .../routerlib/templates/_deployment.yaml | 88 ++++++++++++------- .../charts/routerlib/templates/_helpers.tpl | 24 +++++ .../routerlib/templates/_inferencepool.yaml | 5 ++ .../charts/routerlib/templates/_service.yaml | 46 +++++++++- 7 files changed, 246 insertions(+), 31 deletions(-) diff --git a/config/charts/README.md b/config/charts/README.md index c30ef6441a..e1f0c07f31 100644 --- a/config/charts/README.md +++ b/config/charts/README.md @@ -491,6 +491,12 @@ Configures routing policies, target Gateway interfaces, and priority objectives | `provider.name` | Name of Gateway implementation. Options: `[none, gke, istio]`. | `none` | | `provider.istio.destinationRule.host` | Custom host value for Istio DestinationRule. | `""` | | `provider.istio.destinationRule.trafficPolicy.connectionPool` | Connection pool settings for Istio DestinationRule. | `{}` | +| `provider.gke.preferredBackends.enabled` | Enable Preferred Backends high availability routing for GKE. | `false` | +| `provider.gke.preferredBackends.preferredReplicas` | Replica count for primary active pod ordinals pinned to PREFERRED tier. | `1` | +| `provider.gke.preferredBackends.defaultReplicas` | Replica count for standby backup pod ordinals pinned to DEFAULT tier. | `1` | +| `provider.gke.preferredBackends.balancingMode` | Load balancing calculation mode (`RATE`, `UTILIZATION`, `CONNECTION`). | `RATE` | +| `provider.gke.preferredBackends.maxRatePerEndpoint` | Maximum requests per second per pod instance before spilling over. | `100` | +| `provider.gke.preferredBackends.capacityScalerPercent` | Effective capacity ceiling percentage (`0` to `100`). | `100` | | `httpRoute.create` | Deploy an `HTTPRoute` resource as part of the gateway chart. | `false` | | `httpRoute.inferenceGatewayName` | Target Gateway name for the `HTTPRoute`. | `inference-gateway` | | `httpRoute.inferenceGatewayNamespace` | Target Gateway namespace for the `HTTPRoute`. | `""` | @@ -501,6 +507,14 @@ Configures routing policies, target Gateway interfaces, and priority objectives ```yaml provider: name: gke # Use GKE gateway implementation + gke: + preferredBackends: + enabled: true + preferredReplicas: 1 + defaultReplicas: 1 + balancingMode: RATE + maxRatePerEndpoint: 100 + capacityScalerPercent: 100 httpRoute: create: true diff --git a/config/charts/llm-d-router-gateway/templates/gke.yaml b/config/charts/llm-d-router-gateway/templates/gke.yaml index 82b3a6eada..db528e3d3c 100644 --- a/config/charts/llm-d-router-gateway/templates/gke.yaml +++ b/config/charts/llm-d-router-gateway/templates/gke.yaml @@ -59,7 +59,43 @@ spec: portSpecification: "USE_FIXED_PORT" port: {{ $eppHealthPort }} {{- end }} +{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }} +{{- if $gkePB.enabled }} +{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }} +{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }} +{{- $totalReplicas := add $preferredReplicas $defaultReplicas }} +{{- range $i := untilStep 1 (int $totalReplicas) 1 }} +{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }} +{{- if ge (int $i) (int $preferredReplicas) }} + {{- if eq (int $i) (int $preferredReplicas) }} + {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }} + {{- else }} + {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }} + {{- end }} +{{- end }} +--- +kind: HealthCheckPolicy +apiVersion: networking.gke.io/v1 +metadata: + name: {{ $serviceName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "llm-d-router.labels" $ | nindent 4 }} +spec: + targetRef: + group: "" + kind: Service + name: {{ $serviceName }} + default: + config: + type: GRPC + grpcHealthCheck: + portSpecification: "USE_FIXED_PORT" + port: {{ $eppHealthPort }} +{{- end }} +{{- end }} --- +{{- if not $gkePB.enabled }} apiVersion: networking.gke.io/v1 kind: GCPBackendPolicy metadata: @@ -78,4 +114,51 @@ spec: enabled: true # log all requests by default --- {{- end }} +{{- if $gkePB.enabled }} +{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }} +{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }} +{{- $totalReplicas := add $preferredReplicas $defaultReplicas }} +{{- range $i := untilStep 0 (int $totalReplicas) 1 }} +{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }} +{{- $policyName := printf "%s-preferred-%d" $.Release.Name (int $i) }} +{{- $pref := "PREFERRED" }} +{{- if eq (int $i) 0 }} + {{- $serviceName = include "llm-d-router.name" $ }} + {{- $policyName = $.Release.Name }} +{{- else if ge (int $i) (int $preferredReplicas) }} + {{- $pref = "DEFAULT" }} + {{- if eq (int $i) (int $preferredReplicas) }} + {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }} + {{- $policyName = printf "%s-backup" $.Release.Name }} + {{- else }} + {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }} + {{- $policyName = printf "%s-backup-%d" $.Release.Name (sub (int $i) (int $preferredReplicas)) }} + {{- end }} +{{- end }} +apiVersion: networking.gke.io/v1 +kind: GCPBackendPolicy +metadata: + name: {{ $policyName }} + namespace: {{ $.Release.Namespace }} +spec: + targetRef: + group: "" + kind: Service + name: {{ $serviceName }} + default: + timeoutSec: 300 + backendPreference: {{ $pref }} + {{- if $gkePB.balancingMode }} + balancingMode: {{ $gkePB.balancingMode }} + {{- end }} + {{- if $gkePB.maxRatePerEndpoint }} + maxRatePerEndpoint: {{ $gkePB.maxRatePerEndpoint }} + {{- end }} + {{- if $gkePB.capacityScalerPercent }} + capacityScalerPercent: {{ $gkePB.capacityScalerPercent }} + {{- end }} +--- +{{- end }} +{{- end }} {{- include "llm-d-router.gke" . -}} +{{- end }} diff --git a/config/charts/llm-d-router-gateway/values.yaml b/config/charts/llm-d-router-gateway/values.yaml index fc28c8c46a..6309a2ef06 100644 --- a/config/charts/llm-d-router-gateway/values.yaml +++ b/config/charts/llm-d-router-gateway/values.yaml @@ -21,6 +21,23 @@ provider: # http: # maxRequestsPerConnection: 256000 + # GKE-specific configuration. + # This block is only used if name is "gke". + gke: + preferredBackends: + enabled: false + # Number of primary active pod instances pinned to the PREFERRED load balancing tier. + # Setting >1 scales concurrent active routing. + preferredReplicas: 1 + # Number of standby pod instances pinned to the DEFAULT load balancing tier. + # Setting >1 scales warm standby failover capacity. + defaultReplicas: 1 + # Benchmark capacity and threshold settings for PREFERRED load balancing tier. + # When incoming traffic exceeds capacityScalerPercent of maxRatePerEndpoint, excess traffic spills over to DEFAULT standby pods. + balancingMode: RATE + maxRatePerEndpoint: 100 + capacityScalerPercent: 100 + # httpRoute section is used to deploy an HTTPRoute resource as part of the gateway chart. httpRoute: create: false # a flag to indicate whether to create the httproute as part of the chart or not. diff --git a/config/charts/routerlib/templates/_deployment.yaml b/config/charts/routerlib/templates/_deployment.yaml index 6a98249073..2a0b02c7b8 100644 --- a/config/charts/routerlib/templates/_deployment.yaml +++ b/config/charts/routerlib/templates/_deployment.yaml @@ -1,31 +1,4 @@ -{{- define "llm-d-epp.deployment" -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "llm-d-router.name" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "llm-d-router.labels" . | nindent 4 }} - {{- include "llm-d-router.modeLabels" . | nindent 4 }} -spec: - replicas: {{ .Values.router.epp.replicas | default 1 }} - strategy: - # The current recommended EPP deployment pattern is to have a single active replica. This ensures - # optimal performance of the stateful operations such prefix cache aware scorer. - # The Recreate strategy the old replica is killed immediately, and allow the new replica(s) to - # quickly take over. This is particularly important in the high availability set up with leader - # election, as the rolling update strategy would prevent the old leader being killed because - # otherwise the maxUnavailable would be 100%. - type: Recreate - selector: - matchLabels: - {{- include "llm-d-router.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "llm-d-router.selectorLabels" . | nindent 8 }} - {{- include "llm-d-router.modeLabels" . | nindent 8 }} - spec: +{{- define "llm-d-epp.deployment-pod-spec" -}} {{- $proxy := include "llm-d-router.proxy" . | fromYaml | default dict }} {{- $proxyType := include "llm-d-router.proxyType" . | trim }} {{- $proxyMode := include "llm-d-router.proxyMode" . | trim }} @@ -128,8 +101,8 @@ spec: - --zap-encoder - "json" - --config-file - - "/config/{{ .Values.router.epp.pluginsConfigFile }}" - {{- if gt (.Values.router.epp.replicas | int) 1 }} + {{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }} + {{- if and (gt (.Values.router.epp.replicas | int) 1) (not $gkePB.enabled) }} - --ha-enable-leader-election {{- end }} {{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }} @@ -305,5 +278,60 @@ spec: tolerations: {{- toYaml .Values.router.epp.tolerations | nindent 8 }} {{- end }} +{{- end }} + +{{- define "llm-d-epp.deployment" -}} +{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}} +{{- if $gkePB.enabled }} +{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }} +{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }} +{{- $totalReplicas := add $preferredReplicas $defaultReplicas }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "llm-d-router.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "llm-d-router.labels" . | nindent 4 }} + {{- include "llm-d-router.modeLabels" . | nindent 4 }} +spec: + replicas: {{ $totalReplicas }} + serviceName: {{ include "llm-d-router.name" . }} + podManagementPolicy: Parallel + selector: + matchLabels: + {{- include "llm-d-router.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "llm-d-router.selectorLabels" . | nindent 8 }} + {{- include "llm-d-router.modeLabels" . | nindent 8 }} + spec: +{{ include "llm-d-epp.deployment-pod-spec" . }} +--- +{{- else }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "llm-d-router.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "llm-d-router.labels" . | nindent 4 }} + {{- include "llm-d-router.modeLabels" . | nindent 4 }} +spec: + replicas: {{ .Values.router.epp.replicas | default 1 }} + strategy: + type: Recreate + selector: + matchLabels: + {{- include "llm-d-router.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "llm-d-router.selectorLabels" . | nindent 8 }} + {{- include "llm-d-router.modeLabels" . | nindent 8 }} + spec: +{{ include "llm-d-epp.deployment-pod-spec" . }} --- {{- end }} +{{- end }} diff --git a/config/charts/routerlib/templates/_helpers.tpl b/config/charts/routerlib/templates/_helpers.tpl index 86870f48e7..c02e60d9ed 100644 --- a/config/charts/routerlib/templates/_helpers.tpl +++ b/config/charts/routerlib/templates/_helpers.tpl @@ -394,14 +394,38 @@ EPP resource validations {{- $_ := required ".Values.router.epp.resources.requests.memory is required. EPP is a critical component that must have memory requests set." .Values.router.epp.resources.requests.memory }} {{- end -}} +{{/* +Helper to retrieve GKE preferredBackends configuration safely across chart contexts. +*/}} +{{- define "llm-d-router.gkePreferredBackends" -}} +{{- $provider := .Values.provider | default dict -}} +{{- $gke := index $provider "gke" | default dict -}} +{{- index $gke "preferredBackends" | default dict | toYaml -}} +{{- end -}} + {{/* EPP generic validations */}} +{{- define "llm-d-router.validations.epp.preferredBackends" -}} +{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}} +{{- if $gkePB.enabled }} + {{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }} + {{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }} + {{- if lt $preferredReplicas 1 }} + {{- fail ".Values.provider.gke.preferredBackends.preferredReplicas must be at least 1 when preferredBackends.enabled is true" }} + {{- end }} + {{- if lt $defaultReplicas 1 }} + {{- fail ".Values.provider.gke.preferredBackends.defaultReplicas must be at least 1 when preferredBackends.enabled is true" }} + {{- end }} +{{- end }} +{{- end -}} + {{- define "llm-d-router.validations.epp" -}} {{- include "llm-d-router.validations.deprecations" . }} {{- include "llm-d-router.validations.epp.resources" . }} {{- include "llm-d-router.validations.epp.inferenceObjectives" . }} {{- include "llm-d-router.validations.epp.tokenizer" . }} +{{- include "llm-d-router.validations.epp.preferredBackends" . }} {{- end -}} {{/* diff --git a/config/charts/routerlib/templates/_inferencepool.yaml b/config/charts/routerlib/templates/_inferencepool.yaml index 8aace3618e..bcd41e9ecc 100644 --- a/config/charts/routerlib/templates/_inferencepool.yaml +++ b/config/charts/routerlib/templates/_inferencepool.yaml @@ -26,7 +26,12 @@ spec: {{- end }} {{- end }} endpointPickerRef: + {{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }} + {{- if $gkePB.enabled }} + name: {{ printf "%s-backup" (include "llm-d-router.name" .) }} + {{- else }} name: {{ include "llm-d-router.name" . }} + {{- end }} port: number: {{ .Values.router.epp.extProcPort | default 9002 }} failureMode: {{ .Values.router.inferencePool.failureMode | default "FailOpen" }} diff --git a/config/charts/routerlib/templates/_service.yaml b/config/charts/routerlib/templates/_service.yaml index 574e41ef0c..261874941c 100644 --- a/config/charts/routerlib/templates/_service.yaml +++ b/config/charts/routerlib/templates/_service.yaml @@ -1,4 +1,47 @@ {{- define "llm-d-epp.service" -}} +{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict -}} +{{- if $gkePB.enabled }} +{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }} +{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }} +{{- $totalReplicas := add $preferredReplicas $defaultReplicas }} +{{- range $i := untilStep 0 (int $totalReplicas) 1 }} +{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }} +{{- if eq (int $i) 0 }} + {{- $serviceName = include "llm-d-router.name" $ }} +{{- else if ge (int $i) (int $preferredReplicas) }} + {{- if eq (int $i) (int $preferredReplicas) }} + {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }} + {{- else }} + {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }} + {{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $serviceName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "llm-d-router.labels" $ | nindent 4 }} + annotations: + cloud.google.com/neg: '{"exposed_ports":{"{{ $.Values.router.epp.extProcPort | default 9002 }}":{}}}' +spec: + selector: + statefulset.kubernetes.io/pod-name: {{ printf "%s-%d" (include "llm-d-router.name" $) (int $i) }} + ports: + - name: grpc-ext-proc + protocol: TCP + port: {{ $.Values.router.epp.extProcPort | default 9002 }} + appProtocol: kubernetes.io/h2c + - name: http-metrics + protocol: TCP + port: {{ $.Values.router.metricsPort | default 9090 }} + {{- with $.Values.router.extraServicePorts }} + {{- . | toYaml | nindent 4 }} + {{- end }} + type: ClusterIP +--- +{{- end }} +{{- else }} apiVersion: v1 kind: Service metadata: @@ -13,6 +56,7 @@ spec: - name: grpc-ext-proc protocol: TCP port: {{ .Values.router.epp.extProcPort | default 9002 }} + appProtocol: kubernetes.io/h2c - name: http-metrics protocol: TCP port: {{ .Values.router.metricsPort | default 9090 }} @@ -20,5 +64,5 @@ spec: {{- . | toYaml | nindent 4 }} {{- end }} type: ClusterIP ---- +{{- end }} {{- end }}