diff --git a/config/charts/llm-d-router-gateway/templates/epp.yaml b/config/charts/llm-d-router-gateway/templates/epp.yaml new file mode 100644 index 0000000000..95483ef637 --- /dev/null +++ b/config/charts/llm-d-router-gateway/templates/epp.yaml @@ -0,0 +1,8 @@ +{{ include "llm-d-router.validations.epp" $ }} +{{ include "llm-d-router.inferencepool-resource" . }} +{{ include "llm-d-epp.config" . }} +{{ include "llm-d-epp.deployment" . }} +{{ include "llm-d-epp.leader-election-rbac" . }} +{{ include "llm-d-epp.sa-token-secret" . }} +{{ include "llm-d-epp.service" . }} +{{ include "llm-d-epp.service-monitor" . }} diff --git a/config/charts/llm-d-router-gateway/templates/gke.yaml b/config/charts/llm-d-router-gateway/templates/gke.yaml index 7af64c58b6..68528ba082 100644 --- a/config/charts/llm-d-router-gateway/templates/gke.yaml +++ b/config/charts/llm-d-router-gateway/templates/gke.yaml @@ -54,4 +54,4 @@ spec: enabled: true # log all requests by default --- {{- end }} -{{- include "inference-extension.gke" . -}} +{{- include "llm-d-router.gke" . -}} diff --git a/config/charts/llm-d-router-gateway/templates/inferenceextension.yaml b/config/charts/llm-d-router-gateway/templates/inferenceextension.yaml deleted file mode 100644 index 5e984ca7a7..0000000000 --- a/config/charts/llm-d-router-gateway/templates/inferenceextension.yaml +++ /dev/null @@ -1,8 +0,0 @@ -{{ include "llm-d-router.validations.epp" $ }} -{{ include "inference-extension.config" . }} -{{ include "inference-extension.deployment" . }} -{{ include "inference-extension.lead-election-rbac" . }} -{{ include "inference-extension.sa-token-secret" . }} -{{ include "inference-extension.service" . }} -{{ include "inference-extension.service-monitor" . }} -{{ include "inference-extension.inferencepool-resource" . }} diff --git a/config/charts/llm-d-router-gateway/templates/rbac.yaml b/config/charts/llm-d-router-gateway/templates/rbac.yaml index 9fc3bf223c..d232e92625 100644 --- a/config/charts/llm-d-router-gateway/templates/rbac.yaml +++ b/config/charts/llm-d-router-gateway/templates/rbac.yaml @@ -30,4 +30,4 @@ roleRef: kind: Role name: {{ printf "%s-non-sa" (include "llm-d-router.name" .) }} --- -{{- include "inference-extension.rbac" . -}} +{{- include "llm-d-router.rbac" . -}} diff --git a/config/charts/llm-d-router-standalone/templates/epp.yaml b/config/charts/llm-d-router-standalone/templates/epp.yaml new file mode 100644 index 0000000000..ef002d5e35 --- /dev/null +++ b/config/charts/llm-d-router-standalone/templates/epp.yaml @@ -0,0 +1,11 @@ +{{ include "llm-d-router.validations.epp" $ }} +{{ include "llm-d-router.validations.standalone" . }} +{{ include "llm-d-epp.config" . }} +{{ include "llm-d-epp.deployment" . }} +{{ include "llm-d-router.gke" . }} +{{ include "llm-d-epp.leader-election-rbac" . }} +{{ include "llm-d-epp.sa-token-secret" . }} +{{ include "llm-d-epp.service" . }} +{{ include "llm-d-epp.service-monitor" . }} +{{ include "llm-d-router.rbac" . }} +{{ include "llm-d-router.inferencepool-resource" . }} diff --git a/config/charts/llm-d-router-standalone/templates/inferenceextension.yaml b/config/charts/llm-d-router-standalone/templates/inferenceextension.yaml deleted file mode 100644 index 1b64e8003c..0000000000 --- a/config/charts/llm-d-router-standalone/templates/inferenceextension.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{ include "llm-d-router.validations.epp" $ }} -{{ include "llm-d-router.validations.standalone" . }} -{{ include "inference-extension.config" . }} -{{ include "inference-extension.deployment" . }} -{{ include "inference-extension.gke" . }} -{{ include "inference-extension.lead-election-rbac" . }} -{{ include "inference-extension.sa-token-secret" . }} -{{ include "inference-extension.service" . }} -{{ include "inference-extension.service-monitor" . }} -{{ include "inference-extension.rbac" . }} -{{ include "inference-extension.inferencepool-resource" . }} diff --git a/config/charts/routerlib/templates/_config.yaml b/config/charts/routerlib/templates/_config.yaml index 441fdbc99c..39cb211b09 100644 --- a/config/charts/routerlib/templates/_config.yaml +++ b/config/charts/routerlib/templates/_config.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.config" -}} +{{- define "llm-d-epp.config" -}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/config/charts/routerlib/templates/_deployment.yaml b/config/charts/routerlib/templates/_deployment.yaml index e3aa7530ea..dce08293f6 100644 --- a/config/charts/routerlib/templates/_deployment.yaml +++ b/config/charts/routerlib/templates/_deployment.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.deployment" -}} +{{- define "llm-d-epp.deployment" -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -30,9 +30,9 @@ spec: {{- $proxyType := include "llm-d-router.proxyType" . | trim }} {{- $proxyConfigMap := index $proxy "configMap" | default dict }} {{- $proxyConfigMapName := index $proxyConfigMap "name" | default "" }} - {{- $inferenceExtensionFlags := deepCopy (.Values.router.epp.flags | default dict) }} - {{- if and (eq $proxyType "agentgateway") (not (hasKey $inferenceExtensionFlags "secure-serving")) }} - {{- $_ := set $inferenceExtensionFlags "secure-serving" false }} + {{- $eppFlags := deepCopy (.Values.router.epp.flags | default dict) }} + {{- if and (eq $proxyType "agentgateway") (not (hasKey $eppFlags "secure-serving")) }} + {{- $_ := set $eppFlags "secure-serving" false }} {{- end }} serviceAccountName: {{ include "llm-d-router.name" . }} # Conservatively, this timeout should mirror the longest grace period of the pods within the pool @@ -130,9 +130,9 @@ spec: {{- if gt (.Values.router.epp.replicas | int) 1 }} - --ha-enable-leader-election {{- end }} - # Pass additional flags via the inferenceExtension.flags field in values.yaml. + # Pass additional flags via the router.epp.flags field in values.yaml. # Render them as --flag=value so boolean values are parsed correctly. - {{- range $key, $value := $inferenceExtensionFlags }} + {{- range $key, $value := $eppFlags }} - {{ printf "--%s=%v" $key $value | quote }} {{- end }} {{- if .Values.router.tracing.enabled }} diff --git a/config/charts/routerlib/templates/_gke.yaml b/config/charts/routerlib/templates/_gke.yaml index f4588c97fa..cb0f5dd2db 100644 --- a/config/charts/routerlib/templates/_gke.yaml +++ b/config/charts/routerlib/templates/_gke.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.gke" -}} +{{- define "llm-d-router.gke" -}} {{- $monitoringProvider := include "llm-d-router.monitoring.provider" . | fromYaml | default dict -}} {{- if and $monitoringProvider (eq (lower (index $monitoringProvider "name" | default "")) "gmp") }} {{- if and .Values.router.monitoring.prometheus.enabled .Values.router.monitoring.prometheus.auth.enabled }} diff --git a/config/charts/routerlib/templates/_helpers.tpl b/config/charts/routerlib/templates/_helpers.tpl index 16aeddec78..06e05e628b 100644 --- a/config/charts/routerlib/templates/_helpers.tpl +++ b/config/charts/routerlib/templates/_helpers.tpl @@ -9,7 +9,7 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} {{/* -Inference extension name +Router deployment name */}} {{- define "llm-d-router.name" -}} {{- $base := .Release.Name | default "default-pool" | lower | trim | trunc 40 -}} @@ -52,7 +52,7 @@ llm-d.ai/igw-mode: llm-d-router-gateway {{/* Return the monitoring provider name. -If inferenceExtension.monitoring.provider.name is unset/empty, default to +If router.monitoring.provider.name is unset/empty, default to prometheusoperator. For backwards compatibility, provider.name=gke still maps to gmp when no monitoring provider is explicitly set. */}} @@ -74,7 +74,7 @@ prometheusoperator {{/* Return the monitoring provider config object. -When inferenceExtension.monitoring.provider.name is unset/empty, use defaults. +When router.monitoring.provider.name is unset/empty, use defaults. For backwards compatibility, provider.gke.autopilot is still honored when provider.name=gke and no monitoring provider is explicitly set. */}} diff --git a/config/charts/routerlib/templates/_inferenceobjective.yaml b/config/charts/routerlib/templates/_inferenceobjective.yaml index d01891a81f..85c381b29f 100644 --- a/config/charts/routerlib/templates/_inferenceobjective.yaml +++ b/config/charts/routerlib/templates/_inferenceobjective.yaml @@ -1,6 +1,5 @@ -{{- define "inference-extension.inferenceobjective-resource" -}} +{{- define "llm-d-router.inferenceobjective-resource" -}} {{- range .Values.router.inferenceObjectives }} ---- apiVersion: llm-d.ai/v1alpha2 kind: InferenceObjective metadata: @@ -13,5 +12,6 @@ spec: poolRef: group: "inference.networking.k8s.io" name: {{ $.Release.Name }} +--- {{- end }} {{- end }} diff --git a/config/charts/routerlib/templates/_inferencepool.yaml b/config/charts/routerlib/templates/_inferencepool.yaml index 9e4cb41302..8aace3618e 100644 --- a/config/charts/routerlib/templates/_inferencepool.yaml +++ b/config/charts/routerlib/templates/_inferencepool.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.inferencepool" -}} +{{- define "llm-d-router.inferencepool" -}} apiVersion: "inference.networking.k8s.io/v1" kind: InferencePool metadata: @@ -30,12 +30,13 @@ spec: port: number: {{ .Values.router.epp.extProcPort | default 9002 }} failureMode: {{ .Values.router.inferencePool.failureMode | default "FailOpen" }} +--- {{- end }} -{{- define "inference-extension.inferencepool-resource" -}} +{{- define "llm-d-router.inferencepool-resource" -}} {{- if ne .Values.router.inferencePool.create false }} {{- include "llm-d-router.validations.gateway.common" . }} -{{- include "inference-extension.inferencepool" . }} -{{- include "inference-extension.inferenceobjective-resource" . }} +{{- include "llm-d-router.inferencepool" . }} +{{- include "llm-d-router.inferenceobjective-resource" . }} {{- end }} {{- end }} diff --git a/config/charts/routerlib/templates/_leader-election-rbac.yaml b/config/charts/routerlib/templates/_leader-election-rbac.yaml index 7700170220..7f8d629647 100644 --- a/config/charts/routerlib/templates/_leader-election-rbac.yaml +++ b/config/charts/routerlib/templates/_leader-election-rbac.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.lead-election-rbac" -}} +{{- define "llm-d-epp.leader-election-rbac" -}} {{- if gt (.Values.router.epp.replicas | int) 1 }} --- kind: Role diff --git a/config/charts/routerlib/templates/_rbac.yaml b/config/charts/routerlib/templates/_rbac.yaml index c016c60b0b..cd710fb08d 100644 --- a/config/charts/routerlib/templates/_rbac.yaml +++ b/config/charts/routerlib/templates/_rbac.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.rbac" -}} +{{- define "llm-d-router.rbac" -}} {{- if .Values.router.monitoring.prometheus.enabled }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 diff --git a/config/charts/routerlib/templates/_sa-token-secret.yaml b/config/charts/routerlib/templates/_sa-token-secret.yaml index ba080477da..b0ee0176fe 100644 --- a/config/charts/routerlib/templates/_sa-token-secret.yaml +++ b/config/charts/routerlib/templates/_sa-token-secret.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.sa-token-secret" -}} +{{- define "llm-d-epp.sa-token-secret" -}} {{- $monitoringProviderName := include "llm-d-router.monitoring.provider.name" . | lower -}} {{- if and .Values.router.monitoring.prometheus.enabled .Values.router.monitoring.prometheus.auth.enabled (eq $monitoringProviderName "prometheusoperator") }} apiVersion: v1 diff --git a/config/charts/routerlib/templates/_service.yaml b/config/charts/routerlib/templates/_service.yaml index 6b8f647f46..574e41ef0c 100644 --- a/config/charts/routerlib/templates/_service.yaml +++ b/config/charts/routerlib/templates/_service.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.service" -}} +{{- define "llm-d-epp.service" -}} apiVersion: v1 kind: Service metadata: diff --git a/config/charts/routerlib/templates/_servicemonitor.yaml b/config/charts/routerlib/templates/_servicemonitor.yaml index c40a39ab33..0aee9e0f65 100644 --- a/config/charts/routerlib/templates/_servicemonitor.yaml +++ b/config/charts/routerlib/templates/_servicemonitor.yaml @@ -1,4 +1,4 @@ -{{- define "inference-extension.service-monitor" -}} +{{- define "llm-d-epp.service-monitor" -}} {{- $monitoringProviderName := include "llm-d-router.monitoring.provider.name" . | lower -}} {{- if and .Values.router.monitoring.prometheus.enabled (eq $monitoringProviderName "prometheusoperator") }} apiVersion: monitoring.coreos.com/v1