DROP: Remove deprecated kube-rbac-proxy. (kubernetes-sigs#3760)

mbobrovskyi · ChristianZaccaria · commit ce056030e8c1 · 2025-01-15T16:03:06.000Z
* Remove deprecated kube-rbac-proxy.

* Bump limit for controller manager.
diff --git a/apis/config/v1beta1/defaults.go b/apis/config/v1beta1/defaults.go
@@ -33,7 +33,7 @@ const (
 	DefaultWebhookSecretName                            = "kueue-webhook-server-cert"
 	DefaultWebhookPort                                  = 9443
 	DefaultHealthProbeBindAddress                       = ":8081"
-	DefaultMetricsBindAddress                           = ":8080"
+	DefaultMetricsBindAddress                           = ":8443"
 	DefaultLeaderElectionID                             = "c1f6bfd2.kueue.x-k8s.io"
 	DefaultLeaderElectionLeaseDuration                  = 15 * time.Second
 	DefaultLeaderElectionRenewDeadline                  = 10 * time.Second
diff --git a/charts/kueue/README.md b/charts/kueue/README.md
@@ -57,7 +57,6 @@ The following table lists the configurable parameters of the kueue chart and the
 | `enablePrometheus`                                     | enable Prometheus                                      | `false`                                     |
 | `enableCertManager`                                    | enable CertManager                                     | `false`                                     |
 | `enableVisibilityAPF`                                  | enable APF for the visibility API                      | `false`                                     |
-| `controllerManager.kubeRbacProxy.image`                | controllerManager.kubeRbacProxy's image                | `registry.k8s.io/kubebuilder/kube-rbac-proxy:v0.8.0` |
 | `controllerManager.manager.image.repository`           | controllerManager.manager's repository and image       | `us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue` |
 | `controllerManager.manager.image.tag`                  | controllerManager.manager's tag                        | `main`                                      |
 | `controllerManager.manager.resources`                  | controllerManager.manager's resources                  | abbr.                                       |
diff --git a/charts/kueue/templates/manager/manager.yaml b/charts/kueue/templates/manager/manager.yaml
@@ -67,19 +67,6 @@ spec:
         - mountPath: /controller_manager_config.yaml
           name: manager-config
           subPath: controller_manager_config.yaml
-      - args:
-        - --secure-listen-address=0.0.0.0:8443
-        - --upstream=http://127.0.0.1:8080/
-        - --logtostderr=true
-        - --v=10
-        image: "{{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag }}"
-        imagePullPolicy: {{ .Values.controllerManager.kubeRbacProxy.image.pullPolicy }}
-        name: kube-rbac-proxy
-        ports:
-        - containerPort: 8443
-          name: https
-          protocol: TCP
-        resources: {}
       securityContext:
         {{- toYaml .Values.controllerManager.manager.podSecurityContext | nindent 8 }}
       serviceAccountName: {{ include "kueue.fullname" . }}-controller-manager
diff --git a/charts/kueue/templates/rbac/metrics_auth_role.yaml b/charts/kueue/templates/rbac/metrics_auth_role.yaml
@@ -3,7 +3,7 @@ kind: ClusterRole
 metadata:
   labels:
   {{- include "kueue.labels" . | nindent 4 }}
-  name: '{{ include "kueue.fullname" . }}-proxy-role'
+  name: '{{ include "kueue.fullname" . }}-metrics-auth-role'
 rules:
   - apiGroups:
       - authentication.k8s.io
diff --git a/charts/kueue/templates/rbac/metrics_auth_role_binding.yaml b/charts/kueue/templates/rbac/metrics_auth_role_binding.yaml
@@ -3,11 +3,11 @@ kind: ClusterRoleBinding
 metadata:
   labels:
   {{- include "kueue.labels" . | nindent 4 }}
-  name: '{{ include "kueue.fullname" . }}-proxy-rolebinding'
+  name: '{{ include "kueue.fullname" . }}-metrics-auth-rolebinding'
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: '{{ include "kueue.fullname" . }}-proxy-role'
+  name: '{{ include "kueue.fullname" . }}-metrics-auth-role'
 subjects:
   - kind: ServiceAccount
     name: '{{ include "kueue.fullname" . }}-controller-manager'
diff --git a/charts/kueue/templates/rbac/metrics_reader_role.yaml b/charts/kueue/templates/rbac/metrics_reader_role.yaml
diff --git a/charts/kueue/values.yaml b/charts/kueue/values.yaml
@@ -14,13 +14,6 @@ controllerManager:
   #featureGates:
   #  - name: PartialAdmission
   #    enabled: true
-  kubeRbacProxy:
-    image:
-      repository: registry.k8s.io/kubebuilder/kube-rbac-proxy
-      # tag
-      tag: v0.16.0
-      # This should be set to 'IfNotPresent' for released version
-      pullPolicy: IfNotPresent
   manager:
     image:
       repository: us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue
@@ -29,7 +22,7 @@ controllerManager:
     podAnnotations: {}
     resources:
       limits:
-        cpu: 500m
+        cpu: "2"
         memory: 512Mi
       requests:
         cpu: 500m
@@ -65,7 +58,7 @@ managerConfig:
     health:
       healthProbeBindAddress: :8081
     metrics:
-      bindAddress: :8080
+      bindAddress: :8443
     # enableClusterQueueResources: true
     webhook:
       port: 9443
diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go
@@ -41,6 +41,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	configapi "sigs.k8s.io/kueue/apis/config/v1beta1"
 	kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1"
@@ -142,6 +144,17 @@ func main() {
 
 	features.LogFeatureGates(setupLog)
 
+	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
+	// More info:
+	// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server
+	// - https://book.kubebuilder.io/reference/metrics.html
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:    cfg.Metrics.BindAddress,
+		SecureServing:  true,
+		FilterProvider: filters.WithAuthenticationAndAuthorization,
+	}
+	options.Metrics = metricsServerOptions
+
 	metrics.Register()
 
 	kubeConfig := ctrl.GetConfigOrDie()
diff --git a/config/components/manager/auth_proxy_service.yaml b/config/components/manager/auth_proxy_service.yaml
diff --git a/config/components/manager/controller_manager_config.yaml b/config/components/manager/controller_manager_config.yaml
@@ -3,7 +3,7 @@ kind: Configuration
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
   enableClusterQueueResources: true
 webhook:
   port: 9443
diff --git a/config/components/manager/kustomization.yaml b/config/components/manager/kustomization.yaml
@@ -1,8 +1,5 @@
-# Comment the "auth_proxy_service.yaml" entry if you want to disable the service
-# for auth proxy (https://github.com/brancz/kube-rbac-proxy)
 resources:
 - manager.yaml
-- auth_proxy_service.yaml
 
 generatorOptions:
   disableNameSuffixHash: true
diff --git a/config/components/manager/manager.yaml b/config/components/manager/manager.yaml
@@ -43,7 +43,7 @@ spec:
         # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           limits:
-            cpu: 500m
+            cpu: "2"
             memory: 512Mi
           requests:
             cpu: 500m
diff --git a/config/components/rbac/kustomization.yaml b/config/components/rbac/kustomization.yaml
@@ -9,12 +9,15 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
-# Comment the following 3 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml
 # ClusterRoles for Kueue APIs
 - batch_admin_role.yaml
 - batch_user_role.yaml
diff --git a/config/components/rbac/metrics_auth_role.yaml b/config/components/rbac/metrics_auth_role.yaml
@@ -1,7 +1,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: proxy-role
+  name: metrics-auth-role
 rules:
 - apiGroups:
   - authentication.k8s.io
diff --git a/config/components/rbac/metrics_auth_role_binding.yaml b/config/components/rbac/metrics_auth_role_binding.yaml
@@ -1,11 +1,11 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-auth-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-auth-role
 subjects:
 - kind: ServiceAccount
   name: controller-manager
diff --git a/config/components/rbac/metrics_reader_role.yaml b/config/components/rbac/metrics_reader_role.yaml
diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -30,17 +30,14 @@ resources:
 # - ../components/certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../components/prometheus
+# [METRICS] Expose the controller manager metrics service.
+- metrics_service.yaml
 
 transformers:
 # Sets the namespace for the role binding as kube-system instead of default kueue-system
 - role_binding_visibility_transformer.yaml
 
 patches:
-# Protect the /metrics endpoint by putting it behind auth.
-# If you want your controller-manager to expose the /metrics
-# endpoint w/o any authn/z, please comment the following line.
-- path: manager_auth_proxy_patch.yaml
-
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type
 - path: manager_config_patch.yaml
diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: controller-manager
+    app.kubernetes.io/name: kueue
+    app.kubernetes.io/managed-by: kustomize
+  name: controller-manager-metrics-service
+  namespace: system
+spec:
+  ports:
+    - name: https
+      port: 8443
+      protocol: TCP
+      targetPort: 8443
+  selector:
+    control-plane: controller-manager
diff --git a/hack/e2e-common.sh b/hack/e2e-common.sh
@@ -41,10 +41,13 @@ if [[ -n ${KUBEFLOW_MPI_VERSION:-} ]]; then
 fi
 
 # sleep image to use for testing.
-export E2E_TEST_IMAGE_OLD=gcr.io/k8s-staging-perf-tests/sleep:v0.0.3@sha256:00ae8e01dd4439edfb7eb9f1960ac28eba16e952956320cce7f2ac08e3446e6b
-E2E_TEST_IMAGE_OLD_WITHOUT_SHA=${E2E_TEST_IMAGE_OLD%%@*}
-export E2E_TEST_IMAGE=gcr.io/k8s-staging-perf-tests/sleep:v0.1.0@sha256:8d91ddf9f145b66475efda1a1b52269be542292891b5de2a7fad944052bab6ea
-E2E_TEST_IMAGE_WITHOUT_SHA=${E2E_TEST_IMAGE%%@*}
+export E2E_TEST_SLEEP_IMAGE_OLD=gcr.io/k8s-staging-perf-tests/sleep:v0.0.3@sha256:00ae8e01dd4439edfb7eb9f1960ac28eba16e952956320cce7f2ac08e3446e6b
+E2E_TEST_SLEEP_IMAGE_OLD_WITHOUT_SHA=${E2E_TEST_SLEEP_IMAGE_OLD%%@*}
+export E2E_TEST_SLEEP_IMAGE=gcr.io/k8s-staging-perf-tests/sleep:v0.1.0@sha256:8d91ddf9f145b66475efda1a1b52269be542292891b5de2a7fad944052bab6ea
+E2E_TEST_SLEEP_IMAGE_WITHOUT_SHA=${E2E_TEST_SLEEP_IMAGE%%@*}
+export E2E_TEST_CURL_IMAGE=curlimages/curl:8.11.0@sha256:6324a8b41a7f9d80db93c7cf65f025411f55956c6b248037738df3bfca32410c
+E2E_TEST_CURL_IMAGE_WITHOUT_SHA=${E2E_TEST_CURL_IMAGE%%@*}
+
 
 # $1 - cluster name
 function cluster_cleanup {
@@ -66,14 +69,16 @@ function cluster_create {
 }
 
 function prepare_docker_images {
-    docker pull "$E2E_TEST_IMAGE_OLD"
-    docker pull "$E2E_TEST_IMAGE"
+    docker pull "$E2E_TEST_SLEEP_IMAGE_OLD"
+    docker pull "$E2E_TEST_SLEEP_IMAGE"
+    docker pull "$E2E_TEST_CURL_IMAGE"
 
     # We can load image by a digest but we cannot reference it by the digest that we pulled.
     # For more information https://github.com/kubernetes-sigs/kind/issues/2394#issuecomment-888713831.
     # Manually create tag for image with digest which is already pulled
-    docker tag $E2E_TEST_IMAGE_OLD "$E2E_TEST_IMAGE_OLD_WITHOUT_SHA"
-    docker tag $E2E_TEST_IMAGE "$E2E_TEST_IMAGE_WITHOUT_SHA"
+    docker tag $E2E_TEST_SLEEP_IMAGE_OLD "$E2E_TEST_SLEEP_IMAGE_OLD_WITHOUT_SHA"
+    docker tag $E2E_TEST_SLEEP_IMAGE "$E2E_TEST_SLEEP_IMAGE_WITHOUT_SHA"
+    docker tag $E2E_TEST_CURL_IMAGE "$E2E_TEST_CURL_IMAGE_WITHOUT_SHA"
 
     if [[ -n ${JOBSET_VERSION:-} ]]; then
         docker pull "${JOBSET_IMAGE}"
@@ -88,8 +93,9 @@ function prepare_docker_images {
 
 # $1 cluster
 function cluster_kind_load {
-    cluster_kind_load_image "$1" "${E2E_TEST_IMAGE_OLD_WITHOUT_SHA}"
-    cluster_kind_load_image "$1" "${E2E_TEST_IMAGE_WITHOUT_SHA}"
+    cluster_kind_load_image "$1" "${E2E_TEST_SLEEP_IMAGE_OLD_WITHOUT_SHA}"
+    cluster_kind_load_image "$1" "${E2E_TEST_SLEEP_IMAGE_WITHOUT_SHA}"
+    cluster_kind_load_image "$1" "${E2E_TEST_CURL_IMAGE_WITHOUT_SHA}"
     cluster_kind_load_image "$1" "$IMAGE_TAG"
 }
 
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -63,7 +63,7 @@ namespace: kueue-tenant-a
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
 leaderElection:
   leaderElect: true
   resourceName: c1f6bfd2.kueue.x-k8s.io
@@ -99,7 +99,7 @@ namespace: kueue-system
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
 leaderElection:
   leaderElect: true
   resourceName: c1f6bfd2.kueue.x-k8s.io
@@ -121,7 +121,7 @@ namespace: kueue-system
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
 leaderElection:
   leaderElect: true
   resourceName: c1f6bfd2.kueue.x-k8s.io
@@ -141,7 +141,7 @@ namespace: kueue-system
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
 leaderElection:
   leaderElect: false
 webhook:
@@ -175,7 +175,7 @@ namespace: kueue-system
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
 leaderElection:
   leaderElect: true
   resourceName: c1f6bfd2.kueue.x-k8s.io
@@ -198,7 +198,7 @@ health:
   readinessEndpointName: ready
   livenessEndpointName: live
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
 pprofBindAddress: :8083
 leaderElection:
   leaderElect: true
@@ -317,7 +317,7 @@ invalidField: invalidValue
 health:
   healthProbeBindAddress: :8081
 metrics:
-  bindAddress: :8080
+  bindAddress: :8443
 leaderElection:
   leaderElect: true
   resourceName: c1f6bfd2.kueue.x-k8s.io
diff --git a/pkg/util/testingjobs/pod/wrappers.go b/pkg/util/testingjobs/pod/wrappers.go
@@ -203,6 +203,11 @@ func (p *PodWrapper) Request(r corev1.ResourceName, v string) *PodWrapper {
 	return p
 }
 
+func (p *PodWrapper) ServiceAccountName(serviceAccountName string) *PodWrapper {
+	p.Spec.ServiceAccountName = serviceAccountName
+	return p
+}
+
 func (p *PodWrapper) Image(image string, args []string) *PodWrapper {
 	p.Spec.Containers[0].Image = image
 	p.Spec.Containers[0].Args = args
diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md
@@ -119,7 +119,7 @@ data:
     health:
       healthProbeBindAddress: :8081
     metrics:
-      bindAddress: :8080
+      bindAddress: :8443
       # enableClusterQueueResources: true
     webhook:
       port: 9443
diff --git a/test/e2e/singlecluster/metrics_test.go b/test/e2e/singlecluster/metrics_test.go
diff --git a/test/util/e2e.go b/test/util/e2e.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go b/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go
