_deployment.yaml

{{- define "llm-d-epp.deployment" -}}
apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "llm-d-router.name" . }}
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "llm-d-router.labels" . | nindent 4 }}
    {{- include "llm-d-router.modeLabels" . | nindent 4 }}
spec:
  replicas: {{ .Values.router.epp.replicas | default 1 }}
  strategy:
    # The current recommended EPP deployment pattern is to have a single active replica. This ensures
    # optimal performance of the stateful operations such prefix cache aware scorer.
    # The Recreate strategy the old replica is killed immediately, and allow the new replica(s) to
    # quickly take over. This is particularly important in the high availability set up with leader
    # election, as the rolling update strategy would prevent the old leader being killed because
    # otherwise the maxUnavailable would be 100%.
    type: Recreate
  selector:
    matchLabels:
      {{- include "llm-d-router.selectorLabels" . | nindent 6 }}
  template:
    metadata:
      labels:
        {{- include "llm-d-router.selectorLabels" . | nindent 8 }}
        {{- include "llm-d-router.modeLabels" . | nindent 8 }}
      {{- with .Values.router.epp.podAnnotations }}
      annotations:
        {{- toYaml . | nindent 8 }}
      {{- end }}
    spec:
      {{- $proxy := include "llm-d-router.proxy" . | fromYaml | default dict }}
      {{- $proxyType := include "llm-d-router.proxyType" . | trim }}
      {{- $proxyMode := include "llm-d-router.proxyMode" . | trim }}
      {{- $proxySidecar := and $proxy.enabled (eq $proxyMode "sidecar") }}
      {{- $proxyConfigMap := index $proxy "configMap" | default dict }}
      {{- $proxyConfigMapName := index $proxyConfigMap "name" | default "" }}
      {{- $eppFlags := deepCopy (.Values.router.epp.flags | default dict) }}
      {{- if and (eq $proxyType "agentgateway") (not (hasKey $eppFlags "secure-serving")) }}
        {{- $_ := set $eppFlags "secure-serving" false }}
      {{- end }}
      serviceAccountName: {{ include "llm-d-router.name" . }}
      # Conservatively, this timeout should mirror the longest grace period of the pods within the pool
      terminationGracePeriodSeconds: 130
      {{- if and .Values.router.tokenizer.enabled .Values.router.tokenizer.initContainers }}
      initContainers:
        {{- toYaml .Values.router.tokenizer.initContainers | nindent 8 }}
      {{- end }}
      containers:
      {{- if $proxySidecar }}
        - name: {{ $proxy.name }}
          image: {{ $proxy.image }}
          imagePullPolicy: {{ $proxy.imagePullPolicy | default "IfNotPresent" }}
        {{- with $proxy.command }}
          command:
            - {{ . | quote }}
        {{- end }}
        {{- with $proxy.args }}
          args:
          {{- range . }}
            - {{ . | quote }}
          {{- end }}
        {{- end }}
        {{- with $proxy.env }}
          env:
          {{- toYaml . | nindent 10 }}
          {{- end }}
        {{- with $proxy.ports }}
          ports:
          {{- toYaml . | nindent 10 }}
        {{- end }}
        {{- with $proxy.livenessProbe }}
          livenessProbe:
          {{- toYaml . | nindent 12 }}
        {{- end }}
        {{- with $proxy.readinessProbe }}
          readinessProbe:
          {{- toYaml . | nindent 12 }}
        {{- end }}
        {{- with $proxy.startupProbe }}
          startupProbe:
          {{- toYaml . | nindent 12 }}
        {{- end }}
        {{- with $proxy.securityContext }}
          securityContext:
          {{- toYaml . | nindent 12 }}
        {{- end }}
        {{- with $proxy.resources }}
          resources:
          {{- toYaml . | nindent 12 }}
        {{- end }}
        {{- if or (and (eq $proxyType "agentgateway") $proxyConfigMapName) $proxy.volumeMounts }}
          volumeMounts:
          {{- if and (eq $proxyType "agentgateway") $proxyConfigMapName }}
            - name: agentgateway-config-template
              mountPath: /config
              readOnly: true
          {{- end }}
          {{- with $proxy.volumeMounts }}
          {{- toYaml . | nindent 12 }}
          {{- end }}
        {{- end }}
        {{- end }}
        - name: epp
          image: {{ .Values.router.epp.image.registry }}/{{ .Values.router.epp.image.repository }}:{{ .Values.router.epp.image.tag }}
          imagePullPolicy: {{ .Values.router.epp.image.pullPolicy | default "IfNotPresent" }}
          args:
          {{- /* 1. Determine Mode Logic */ -}}
          {{- $useInferencePool := ne .Values.router.inferencePool.create false }}
          {{- /* 2. Determine Model Server Type */ -}}
          {{- $modelServerType := .Values.router.modelServers.type | default "vllm" }}
          {{- /* 3. Mode Specific Flags */ -}}
          {{- if not $useInferencePool }}
              - --endpoint-selector
              {{- $selector := "" }}
              {{- if and .Values.router.modelServers .Values.router.modelServers.matchLabels }}
                {{- $labels := list }}
                {{- range $k, $v := .Values.router.modelServers.matchLabels }}
                  {{- $labels = append $labels (printf "%s=%s" $k $v) }}
                {{- end }}
                {{- $selector = join "," $labels }}
              {{- end }}
              - {{ $selector | quote }}
              - --endpoint-target-ports
              - {{ include "llm-d-router.standaloneEndpointTargetPorts" . | quote }}
          {{- else }}
              - --pool-name
              - {{ .Release.Name }}
              # The pool namespace is optional because EPP can default to the NAMESPACE env var.
              - --pool-namespace
              - {{ .Release.Namespace }}
              - --pool-group
              - "inference.networking.k8s.io"
          {{- end }}
              - --zap-encoder
              - "json"
              - --config-file
              - "/config/{{ .Values.router.epp.pluginsConfigFile }}"
          {{- if gt (.Values.router.epp.replicas | int) 1 }}
              - --ha-enable-leader-election
          {{- end }}
          {{- $grpcHealthPort := .Values.router.epp.grpcHealthPort | default 9003 }}
              - --grpc-health-port
              - "{{ $grpcHealthPort }}"
              # Pass additional flags via the router.epp.flags field in values.yaml.
              # Render them as --flag=value so boolean values are parsed correctly.
          {{- range $key, $value := $eppFlags }}
              - {{ printf "--%s=%v" $key $value | quote }}
          {{- end }}
          {{- if .Values.router.tracing.enabled }}
              - --tracing=true
          {{- else }}
              - --tracing=false
          {{- end }}
          {{- if not .Values.router.monitoring.prometheus.auth.enabled }}
              - --metrics-endpoint-auth=false
          {{- end }}
          {{- with .Values.router.epp.resources }}
          resources:
            {{ toYaml . | nindent 12 }}
          {{- end }}
          ports:
            - name: grpc
              containerPort: 9002
            - name: grpc-health
              containerPort: {{ $grpcHealthPort }}
            - name: metrics
              containerPort: 9090
        {{- if .Values.router.epp.extraContainerPorts }}
            {{- toYaml .Values.router.epp.extraContainerPorts | nindent 12 }}
        {{- end }}
          livenessProbe:
          {{- if gt (.Values.router.epp.replicas | int) 1 }}
            grpc:
              port: {{ $grpcHealthPort }}
              service: liveness
          {{- else }}
            grpc:
              port: {{ $grpcHealthPort }}
              service: inference-extension
          {{- end }}
            initialDelaySeconds: 5
            periodSeconds: 10
          readinessProbe:
          {{- if gt (.Values.router.epp.replicas | int) 1 }}
            grpc:
              port: {{ $grpcHealthPort }}
              service: readiness
          {{- else }}
            grpc:
              port: {{ $grpcHealthPort }}
              service: inference-extension
          {{- end }}
            periodSeconds: 2
          env:
            - name: NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
        {{- include "llm-d-router.latencyPredictor.env" . | nindent 12 }}
        {{- if .Values.router.tracing.enabled }}
            - name: OTEL_SERVICE_NAME
              value: "llm-d-router/epp"
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: {{ .Values.router.tracing.otelExporterEndpoint | quote }}
            - name: OTEL_TRACES_EXPORTER
              value: "otlp"
            - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
            - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: OTEL_RESOURCE_ATTRIBUTES
              value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)'
            - name: OTEL_TRACES_SAMPLER
              value: {{ .Values.router.tracing.sampling.sampler | quote }}
            - name: OTEL_TRACES_SAMPLER_ARG
              value: {{ .Values.router.tracing.sampling.samplerArg | quote }}
        {{- end }}
        {{- if .Values.router.epp.env }}
        {{- toYaml .Values.router.epp.env | nindent 12 }}
        {{- end }}
          volumeMounts:
            - name: plugins-config-volume
              mountPath: "/config"
        {{- if .Values.router.epp.volumeMounts }}
        {{- toYaml .Values.router.epp.volumeMounts | nindent 12 }}
        {{- end }}
        {{- if .Values.router.tokenizer.enabled }}
        {{- $tokenizer := .Values.router.tokenizer }}
        {{- $renderPort := $tokenizer.port | default 8000 | int }}
        - name: vllm-render
          image: {{ $tokenizer.image.registry }}/{{ $tokenizer.image.repository }}:{{ $tokenizer.image.tag }}
          imagePullPolicy: {{ $tokenizer.image.pullPolicy | default "IfNotPresent" }}
          command:
          {{- if $tokenizer.command }}
          {{- toYaml $tokenizer.command | nindent 12 }}
          {{- else }}
            - vllm
            - launch
            - render
          {{- end }}
          args:
          {{- if $tokenizer.args }}
          {{- toYaml $tokenizer.args | nindent 12 }}
          {{- else }}
            - {{ $tokenizer.modelName | quote }}
            - {{ printf "--port=%d" $renderPort | quote }}
          {{- end }}
          {{- with $tokenizer.extraArgs }}
          {{- toYaml . | nindent 12 }}
          {{- end }}
          ports:
            - name: render-http
              containerPort: {{ $renderPort }}
          {{- with $tokenizer.readinessProbe }}
          readinessProbe:
            {{- toYaml . | nindent 12 }}
          {{- end }}
          {{- with $tokenizer.env }}
          env:
            {{- toYaml . | nindent 12 }}
          {{- end }}
          {{- with $tokenizer.resources }}
          resources:
            {{- toYaml . | nindent 12 }}
          {{- end }}
          volumeMounts:
            - mountPath: /root/.cache/huggingface
              name: model-cache
            {{- with $tokenizer.volumeMounts }}
            {{- toYaml . | nindent 12 }}
            {{- end }}
        {{- end }}
      {{- include "llm-d-router.latencyPredictor.containers" . | nindent 8 }}
      volumes:
      {{- if .Values.router.epp.volumes }}
      {{- toYaml .Values.router.epp.volumes | nindent 8 }}
      {{- end }}
      {{- if .Values.router.tokenizer.enabled }}
        - name: model-cache
          emptyDir: {}
        {{- with .Values.router.tokenizer.volumes }}
        {{- toYaml . | nindent 8 }}
        {{- end }}
      {{- end }}
      {{- if and $proxySidecar (eq $proxyType "agentgateway") $proxyConfigMapName }}
        - name: agentgateway-config-template
          configMap:
            name: {{ $proxyConfigMapName }}
            items:
              - key: config.yaml
                path: config.yaml
      {{- end }}
      {{- if $proxySidecar }}
      {{- with $proxy.volumes }}
      {{- tpl (toYaml .) $ | nindent 8 }}
      {{- end }}
      {{- end }}
        - name: plugins-config-volume
          configMap:
            name: {{ include "llm-d-router.name" . }}
      {{- include "llm-d-router.latencyPredictor.volumes" . | nindent 8 }}
      {{- if .Values.router.epp.affinity }}
      affinity:
        {{- toYaml .Values.router.epp.affinity | nindent 8 }}
      {{- end }}
      {{- if .Values.router.epp.tolerations }}
      tolerations:
        {{- toYaml .Values.router.epp.tolerations | nindent 8 }}
      {{- end }}
---
{{- end }}