deploy operator in chart with pod group controller (NVIDIA#467)

enoodle · web-flow · commit ee5bee05e220 · 2025-09-05T13:12:39.000+02:00
* deploy kai operator with pg controller operand

* disable operator admission and qcontroller operands
diff --git a/deployments/kai-scheduler/templates/kai-config.yaml b/deployments/kai-scheduler/templates/kai-config.yaml
@@ -0,0 +1,51 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kai.scheduler/v1
+kind: Config
+metadata:
+  name: kai-config
+  namespace: {{ .Release.Namespace }}
+spec:
+  namespace: {{ .Release.Namespace }}
+  global:
+    schedulerName: "kai-scheduler"
+    queueLabelKey: "kai.scheduler/queue"
+    nodePoolLabelKey: "kai.scheduler/nodepool"
+    {{- if .Values.global.namespaceLabelSelector }}
+    namespaceLabelSelector:
+      {{- toYaml .Values.global.namespaceLabelSelector | nindent 6 }}
+    {{- end }}
+    {{- if .Values.global.podLabelSelector }}
+    podLabelSelector:
+      {{- toYaml .Values.global.podLabelSelector | nindent 6 }}
+    {{- end }}
+    {{- if .Values.global.affinity }}
+    affinity:
+      {{- toYaml .Values.global.affinity | nindent 6 }}
+    {{- end }}
+    {{- if .Values.global.tolerations }}
+    tolerations:
+      {{- toYaml .Values.global.tolerations | nindent 6 }}
+    {{- end }}
+    {{- if .Values.global.securityContext }}
+    securityContext:
+      {{- toYaml .Values.global.securityContext | nindent 6 }}
+    {{- end }}
+    {{- if .Values.global.imagePullSecrets }}
+    imagesPullSecret: {{ index .Values.global.imagePullSecrets 0 | default "" }}
+    {{- end }}
+    replicaCount: {{ .Values.operator.replicaCount | default 1 }}
+
+  podGroupController:
+    service:
+        enabled: true
+        image:
+          name: {{ .Values.podgroupcontroller.image.name }}
+          repository: {{ .Values.global.registry }}
+          tag: {{ .Chart.Version }}
+          pullPolicy: {{ .Values.podgroupcontroller.image.pullPolicy | default .Values.global.imagePullPolicy }}
+        {{- if .Values.podgroupcontroller.resources }}
+        resources:
+          {{- toYaml .Values.podgroupcontroller.resources | nindent 6 }}
+        {{- end }}
diff --git a/deployments/kai-scheduler/templates/rbac/operator-binding.yaml b/deployments/kai-scheduler/templates/rbac/operator-binding.yaml
@@ -0,0 +1,16 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kai-operator
+subjects:
+  - kind: ServiceAccount
+    name: kai-operator
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: kai-operator
+  apiGroup: rbac.authorization.k8s.io
+
diff --git a/deployments/kai-scheduler/templates/services/operator-serviceaccount.yaml b/deployments/kai-scheduler/templates/services/operator-serviceaccount.yaml
@@ -4,4 +4,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: podgroup-controller
+  name: kai-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: kai-operator
diff --git a/deployments/kai-scheduler/templates/services/operator.yaml b/deployments/kai-scheduler/templates/services/operator.yaml
@@ -0,0 +1,80 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kai-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: kai-operator
+spec:
+  replicas: {{ .Values.operator.replicaCount }}
+  selector:
+    matchLabels:
+      app: kai-operator
+  template:
+    metadata:
+      labels:
+        app: kai-operator
+    spec:
+      serviceAccountName: kai-operator
+      containers:
+      - name: operator
+        image: {{ .Values.global.registry }}/{{ .Values.operator.image.name }}:{{ .Chart.Version }}
+        imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
+        resources:
+          {{- toYaml .Values.operator.resources | nindent 12 }}
+        args:
+          - --metrics-bind-address={{ .Values.operator.metricsBindAddress | default ":8080" }}
+          - --health-probe-bind-address={{ .Values.operator.probeBindAddress | default ":8081" }}
+          - --leader-elect={{ .Values.global.leaderElection | default false }}
+          - --qps={{ .Values.operator.qps | default 50 }}
+          - --burst={{ .Values.operator.burst | default 300 }}
+          - --namespace={{ .Release.Namespace }}
+        env:
+        - name: WATCH_NAMESPACE
+          value: {{ .Release.Namespace }}
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: OPERATOR_NAME
+          value: "kai-operator"
+        - name: MS_REPOSITORY
+          value: {{ .Values.global.registry }}
+        - name: MS_TAG
+          value: {{ .Chart.Version }}
+        ports:
+        - containerPort: 8080
+          name: metrics
+        - containerPort: 8081
+          name: probe
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml .Values.global.imagePullSecrets | nindent 8 }}
+      {{- end }}
+      {{- if .Values.global.affinity }}
+      affinity:
+        {{- toYaml .Values.global.affinity | nindent 8 }}
+      {{- end }}
+      {{- if .Values.global.tolerations }}
+      tolerations:
+        {{- toYaml .Values.global.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.global.nodeSelector }}
+      nodeSelector:
+        {{- toYaml .Values.global.nodeSelector | nindent 8 }}
+      {{- end }}
diff --git a/deployments/kai-scheduler/templates/services/podgroup-controller.yaml b/deployments/kai-scheduler/templates/services/podgroup-controller.yaml
diff --git a/deployments/kai-scheduler/values.yaml b/deployments/kai-scheduler/values.yaml
@@ -3,6 +3,8 @@
 
 global:
   registry: registry/local/kai-scheduler
+  tag: latest
+  imagePullPolicy: IfNotPresent
   securityContext: {}
   imagePullSecrets: []
   leaderElection: false
@@ -18,33 +20,27 @@ global:
     serviceAccount: kai-resource-reservation
     appLabel: kai-resource-reservation
 
+operator:
+  image:
+    name: operator
+    pullPolicy: IfNotPresent
+  replicaCount: 1
+  metricsBindAddress: ":8080"
+  probeBindAddress: ":8081"
+  qps: 50
+  burst: 300
 
 podgrouper:
   image:
     name: podgrouper
     pullPolicy: IfNotPresent
   additionalArgs: []
-  resources:
-    limits:
-      cpu: "500m"
-      memory: "256Mi"
-    requests:
-      cpu: "250m"
-      memory: "128Mi"
   queueLabelKey: "kai.scheduler/queue"
 
 podgroupcontroller:
   image:
     name: podgroupcontroller
     pullPolicy: IfNotPresent
-  additionalArgs: []
-  resources:
-    limits:
-      cpu: "500m"
-      memory: "256Mi"
-    requests:
-      cpu: "250m"
-      memory: "128Mi"
 
 binder:
   name: binder
@@ -56,14 +52,6 @@ binder:
   additionalArgs: []
   ports:
     metricsPort: 8080
-    probePort: 8081
-  resources:
-    limits:
-      cpu: "500m"
-      memory: "256Mi"
-    requests:
-      cpu: "250m"
-      memory: "128Mi"
   cdi: false
 
 scheduler:
@@ -74,59 +62,13 @@ scheduler:
   placementStrategy: binpack
   ports:
     metricsPort: 8080
-  resources:
-    limits:
-      cpu: "500m"
-      memory: "256Mi"
-    requests:
-      cpu: "250m"
-      memory: "128Mi"
-
-webhookmanager:
-  image:
-    name: webhookmanager
-    pullPolicy: IfNotPresent
-  resources:
-    limits:
-      cpu: "500m"
-      memory: "256Mi"
-    requests:
-      cpu: "250m"
-      memory: "128Mi"
-
-nodescaleadjuster:
-  image:
-    name: nodescaleadjuster
-    pullPolicy: IfNotPresent
-  scalingPodImage:
-    name: scalingpod
-  additionalArgs: []
-  resources:
-    limits:
-      cpu: "500m"
-      memory: "256Mi"
-    requests:
-      cpu: "250m"
-      memory: "128Mi"
 
 queuecontroller:
   image:
     name: queuecontroller
     pullPolicy: IfNotPresent
   additionalArgs: []
   certSecretName: queuecontroller-webhook-tls-secret
-  resources:
-    limits:
-      cpu: "500m"
-      memory: "256Mi"
-    requests:
-      cpu: "250m"
-      memory: "128Mi"
-
-crdupgrader:
-  image:
-    name: crd-upgrader
-    pullPolicy: IfNotPresent
 
 admission:
   name: kai-admission
@@ -139,10 +81,30 @@ admission:
     webhookPort: 9443
     metricsPort: 8080
     probePort: 8081
+  cdi: false
+
+nodescaleadjuster:
+  image:
+    name: nodescaleadjuster
+    pullPolicy: IfNotPresent
+  scalingPodImage:
+    name: scalingpod
+  additionalArgs: []
+
+crdupgrader:
+  image:
+    name: crd-upgrader
+    pullPolicy: IfNotPresent
+
+webhookmanager:
+  image:
+    name: webhookmanager
+    pullPolicy: IfNotPresent
   resources:
     limits:
       cpu: "500m"
       memory: "256Mi"
     requests:
       cpu: "250m"
       memory: "128Mi"
+
diff --git a/pkg/apis/kai/v1/admission/admission.go b/pkg/apis/kai/v1/admission/admission.go
@@ -36,6 +36,7 @@ type Admission struct {
 
 func (b *Admission) SetDefaultsWhereNeeded(replicaCount *int32) {
 	b.Service = common.SetDefault(b.Service, &common.Service{})
+	b.Service.Enabled = common.SetDefault(b.Service.Enabled, ptr.To(false))
 	b.Service.SetDefaultsWhereNeeded(imageName)
 
 	b.Service.Resources = common.SetDefault(b.Service.Resources, &common.Resources{})
diff --git a/pkg/apis/kai/v1/admission/admission_test.go b/pkg/apis/kai/v1/admission/admission_test.go
@@ -22,7 +22,7 @@ var _ = Describe("Admission", func() {
 		var replicaCount int32
 		replicaCount = 1
 		Admission.SetDefaultsWhereNeeded(&replicaCount)
-		Expect(*Admission.Service.Enabled).To(Equal(true))
+		Expect(*Admission.Service.Enabled).To(Equal(false))
 		Expect(*Admission.Service.Image.Name).To(Equal("admission"))
 		Expect(*Admission.Replicas).To(Equal(int32(1)))
 	})
diff --git a/pkg/apis/kai/v1/pod_group_controller/pod_group_controller.go b/pkg/apis/kai/v1/pod_group_controller/pod_group_controller.go
@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	imageName = "pod-group-controller"
+	imageName = "podgroupcontroller"
 )
 
 type PodGroupController struct {
diff --git a/pkg/apis/kai/v1/queue_controller/queue_controller.go b/pkg/apis/kai/v1/queue_controller/queue_controller.go
diff --git a/pkg/apis/kai/v1/queue_controller/queue_controller_test.go b/pkg/apis/kai/v1/queue_controller/queue_controller_test.go
diff --git a/pkg/operator/operands/pod_group_controller/resources.go b/pkg/operator/operands/pod_group_controller/resources.go

Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ import (`
`12`	`12`	`)`
`13`	`13`
`14`	`14`	`const (`
`15`		`- imageName = "pod-group-controller"`
	`15`	`+ imageName = "podgroupcontroller"`
`16`	`16`	`)`
`17`	`17`
`18`	`18`	`type PodGroupController struct {`