gke.yaml

{{- if eq (lower .Values.provider.name) "gke" }}
---
kind: HealthCheckPolicy
apiVersion: networking.gke.io/v1
metadata:
  name: {{ .Release.Name }}
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "llm-d-router.labels" . | nindent 4 }}
spec:
  targetRef:
    group: inference.networking.k8s.io
    kind: InferencePool
    name: {{ .Release.Name }}
  default:
    # Set a more aggressive health check than the default 5s for faster switch
    # over during EPP rollout.
    timeoutSec: 2
    checkIntervalSec: 2
    {{- $port := .Values.router.modelServers.targetPortNumber }}
    {{- if .Values.router.modelServers.targetPorts }}
      {{- $port = (index .Values.router.modelServers.targetPorts 0).number }}
    {{- end }}
    {{- if eq .Values.router.modelServers.protocol "http" }}
    config:
      type: HTTP
      httpHealthCheck:
        requestPath: /health
        port: {{ $port }}
    {{- else if and (eq (lower .Values.router.modelServers.type) "vllm") (eq .Values.router.modelServers.protocol "grpc") }}
    config:
      type: GRPC
      grpcHealthCheck:
        portSpecification: "USE_FIXED_PORT"
        port: {{ $port }}
        grpcServiceName: "/vllm.grpc.engine.VllmEngine/HealthCheck"
    {{- end }}
{{- $eppHealthPort := int (.Values.router.epp.grpcHealthPort | default 9003) }}
{{- /* The GKE gateway controller health-checks the EPP backend on gRPC port
9003 by default, so an explicit HealthCheckPolicy is only needed for other ports. */}}
{{- if ne $eppHealthPort 9003 }}
---
kind: HealthCheckPolicy
apiVersion: networking.gke.io/v1
metadata:
  name: {{ .Release.Name }}-epp
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "llm-d-router.labels" . | nindent 4 }}
spec:
  targetRef:
    group: ""
    kind: Service
    name: {{ include "llm-d-router.name" . }}
  default:
    config:
      type: GRPC
      grpcHealthCheck:
        portSpecification: "USE_FIXED_PORT"
        port: {{ $eppHealthPort }}
{{- end }}
{{- $gkePB := include "llm-d-router.gkePreferredBackends" . | fromYaml | default dict }}
{{- if $gkePB.enabled }}
{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
{{- range $i := untilStep 1 (int $totalReplicas) 1 }}
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
{{- if ge (int $i) (int $preferredReplicas) }}
  {{- if eq (int $i) (int $preferredReplicas) }}
    {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
  {{- else }}
    {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
  {{- end }}
{{- end }}
---
kind: HealthCheckPolicy
apiVersion: networking.gke.io/v1
metadata:
  name: {{ $serviceName }}
  namespace: {{ $.Release.Namespace }}
  labels:
    {{- include "llm-d-router.labels" $ | nindent 4 }}
spec:
  targetRef:
    group: ""
    kind: Service
    name: {{ $serviceName }}
  default:
    config:
      type: GRPC
      grpcHealthCheck:
        portSpecification: "USE_FIXED_PORT"
        port: {{ $eppHealthPort }}
{{- end }}
{{- end }}
---
{{- if not $gkePB.enabled }}
apiVersion: networking.gke.io/v1
kind: GCPBackendPolicy
metadata:
  name: {{ .Release.Name }}
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "llm-d-router.labels" . | nindent 4 }}
spec:
  targetRef:
    group: inference.networking.k8s.io
    kind: InferencePool
    name: {{ .Release.Name }}
  default:
    timeoutSec: 300    # 5-minute timeout (adjust as needed)
    logging:
      enabled: true    # log all requests by default
---
{{- end }}
{{- if $gkePB.enabled }}
{{- $preferredReplicas := $gkePB.preferredReplicas | default 1 | int }}
{{- $defaultReplicas := $gkePB.defaultReplicas | default 1 | int }}
{{- $totalReplicas := add $preferredReplicas $defaultReplicas }}
{{- range $i := untilStep 0 (int $totalReplicas) 1 }}
{{- $serviceName := printf "%s-preferred-%d" (include "llm-d-router.name" $) (int $i) }}
{{- $policyName := printf "%s-preferred-%d" $.Release.Name (int $i) }}
{{- $pref := "PREFERRED" }}
{{- if eq (int $i) 0 }}
  {{- $serviceName = include "llm-d-router.name" $ }}
  {{- $policyName = $.Release.Name }}
{{- else if ge (int $i) (int $preferredReplicas) }}
  {{- $pref = "DEFAULT" }}
  {{- if eq (int $i) (int $preferredReplicas) }}
    {{- $serviceName = printf "%s-backup" (include "llm-d-router.name" $) }}
    {{- $policyName = printf "%s-backup" $.Release.Name }}
  {{- else }}
    {{- $serviceName = printf "%s-backup-%d" (include "llm-d-router.name" $) (sub (int $i) (int $preferredReplicas)) }}
    {{- $policyName = printf "%s-backup-%d" $.Release.Name (sub (int $i) (int $preferredReplicas)) }}
  {{- end }}
{{- end }}
apiVersion: networking.gke.io/v1
kind: GCPBackendPolicy
metadata:
  name: {{ $policyName }}
  namespace: {{ $.Release.Namespace }}
spec:
  targetRef:
    group: ""
    kind: Service
    name: {{ $serviceName }}
  default:
    timeoutSec: 300
    backendPreference: {{ $pref }}
    {{- if $gkePB.balancingMode }}
    balancingMode: {{ $gkePB.balancingMode }}
    {{- end }}
    {{- if $gkePB.maxRatePerEndpoint }}
    maxRatePerEndpoint: {{ $gkePB.maxRatePerEndpoint }}
    {{- end }}
    {{- if $gkePB.capacityScalerPercent }}
    capacityScalerPercent: {{ $gkePB.capacityScalerPercent }}
    {{- end }}
---
{{- end }}
{{- end }}
{{- include "llm-d-router.gke" . -}}
{{- end }}