Merge pull request #1952 from Aryan-sharma11/fix-pod-deletion

fix: handle pod deletion in kubearmor controller

Author: daemon1024

.github/workflows/ci-test-ginkgo.yml

@@ -133,7 +133,7 @@ jobs:
           fi
           docker system prune -a -f 
           docker buildx prune -a -f
-          helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest 
+          helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest  --set kubearmorOperator.annotateExisting=true
           kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
           kubectl get pods -A
           if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then

deployments/controller/kubearmor-controller-mutating-webhook-config.yaml

@@ -47,6 +47,7 @@ webhooks:
     - UPDATE
     resources:
     - pods
+    - pods/binding
   sideEffects: NoneOnDryRun
   objectSelector:
     matchExpressions:

deployments/get/objects.go

@@ -526,6 +526,7 @@ func GetKubeArmorControllerDeployment(namespace string) *appsv1.Deployment {
 							Args: []string{
 								"--leader-elect",
 								"--health-probe-bind-address=:8081",
+								"--annotateExisting=false",
 							},
 							Command: []string{"/manager"},
 							Ports: []corev1.ContainerPort{
@@ -769,7 +770,7 @@ func GetKubeArmorControllerMutationAdmissionConfiguration(namespace string, caCe
 						Rule: admissionregistrationv1.Rule{
 							APIGroups:   []string{""},
 							APIVersions: []string{"v1"},
-							Resources:   []string{"pods"},
+							Resources:   []string{"pods", "pods/binding"},
 						},
 						Operations: []admissionregistrationv1.OperationType{
 							admissionregistrationv1.Create,

deployments/helm/KubeArmor/templates/RBAC/roles.yaml

@@ -97,7 +97,17 @@ rules:
   verbs:
   - get
   - list
-  - watch   
+  - watch 
+- apiGroups:
+  - "apps"
+  resources:
+  - deployments
+  - statefulsets
+  - daemonsets
+  - replicasets
+  verbs:
+  - get
+  - update  
 - apiGroups:
   - security.kubearmor.com
   resources:

deployments/helm/KubeArmor/templates/deployment.yaml

@@ -82,6 +82,7 @@ spec:
       - args:
         - --health-probe-bind-address=:8081
         - --leader-elect
+        - --anotateExisting=false
         command:
         - /manager
         image: {{printf "%s:%s" .Values.kubearmorController.image.repository .Values.kubearmorController.image.tag}}

deployments/helm/KubeArmor/templates/secrets.yaml

@@ -43,6 +43,7 @@ webhooks:
     - CREATE
     - UPDATE
     resources:
-    - pods
+    - pods 
+    - pods/binding
     scope: '*'
   sideEffects: NoneOnDryRun

deployments/helm/KubeArmorOperator/templates/NOTES.txt

@@ -0,0 +1,12 @@
+{{- if not .Values.kubearmorOperator.annotateExisting }}
+⚠️  WARNING: Pre-existing pods will not be annotated. Policy enforcement for already existing pods on Apparmor nodes will not work.  
+  • To check enforcer present on nodes use:
+    ➤ kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.kubearmor\.io/enforcer}{"\n"}{end}' 
+  • To annotate existing pods use: 
+    ➤ helm upgrade --install {{ .Values.kubearmorOperator.name }} kubearmor/kubearmor-operator -n kubearmor --create-namespace --set annotateExisting=true  
+    Our controller will automatically rollout restart deployments during the Helm upgrade to force the admission controller to add annotations.  
+  • Alternatively, if you prefer manual control, you can restart your deployments yourself:
+    ➤ kubectl rollout restart deployment <deployment> -n <namespace>
+{{- end }}
+ℹ️  Your release is named {{ .Release.Name }}.
+💙 Thank you for installing KubeArmor.

deployments/helm/KubeArmorOperator/templates/clusterrole-rbac.yaml

@@ -28,6 +28,7 @@ rules:
   verbs:
   - create
   - get
+  - update
 - apiGroups:
   - operator.kubearmor.com
   resources:
@@ -155,6 +156,18 @@ rules:
   - list
   - watch
   - update
+{{- if .Values.kubearmorOperator.annotateExisting }}
+- apiGroups:
+  - "apps"
+  resources:
+  - deployments
+  - statefulsets
+  - daemonsets
+  - replicasets
+  verbs:
+  - get
+  - update
+{{- end }}
 - apiGroups:
   - ""
   resources:

deployments/helm/KubeArmorOperator/templates/deployment.yaml

@@ -46,6 +46,8 @@ spec:
         image: {{ include "operatorImage" . }}
         imagePullPolicy: {{ .Values.kubearmorOperator.imagePullPolicy }}
         args:
+        - --annotateExisting={{ .Values.kubearmorOperator.annotateExisting }}
+        - --annotateResource={{ .Values.kubearmorOperator.annotateResource }}
         {{- if .Values.kubearmorOperator.args -}}
           {{- toYaml .Values.kubearmorOperator.args | trim | nindent 8 }}
         {{- end }}

deployments/helm/KubeArmorOperator/values.yaml

@@ -29,6 +29,7 @@ oci_meta:
 # in case if image pinning is disabled
 kubearmorOperator:
   annotateResource: false
+  annotateExisting: false
   name: kubearmor-operator
   image:
     repository: kubearmor/kubearmor-operator

pkg/KubeArmorController/cmd/main.go

@@ -49,6 +49,7 @@ func main() {
 	var probeAddr string
 	var secureMetrics bool
 	var enableHTTP2 bool
+	var annotateExisting bool
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -60,6 +61,8 @@ func main() {
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
+	flag.BoolVar(&annotateExisting, "annotateExisting", false,
+		"If 'true', controller will restart and annotate existing resources with required annotations")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -157,24 +160,24 @@ func main() {
 	cluster := informer.InitCluster()
 	setupLog.Info("Starting node watcher")
 	go informer.NodeWatcher(client, &cluster, ctrl.Log.WithName("informer").WithName("NodeWatcher"))
-	setupLog.Info("Starting pod watcher")
-	go informer.PodWatcher(client, &cluster, ctrl.Log.WithName("informer").WithName("PodWatcher"))
 
 	setupLog.Info("Adding mutation webhook")
 	mgr.GetWebhookServer().Register("/mutate-pods", &webhook.Admission{
 		Handler: &handlers.PodAnnotator{
-			Client:  mgr.GetClient(),
-			Logger:  setupLog,
-			Decoder: admission.NewDecoder(mgr.GetScheme()),
-			Cluster: &cluster,
+			Client:    mgr.GetClient(),
+			Logger:    setupLog,
+			Decoder:   admission.NewDecoder(mgr.GetScheme()),
+			Cluster:   &cluster,
+			ClientSet: client,
 		},
 	})
-
 	setupLog.Info("Adding pod refresher controller")
 	if err = (&controllers.PodRefresherReconciler{
-		Client:  mgr.GetClient(),
-		Scheme:  mgr.GetScheme(),
-		Cluster: &cluster,
+		Client:           mgr.GetClient(),
+		Scheme:           mgr.GetScheme(),
+		Cluster:          &cluster,
+		ClientSet:        client,
+		AnnotateExisting: annotateExisting,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Pod")
 		os.Exit(1)

pkg/KubeArmorController/common/common.go

@@ -4,19 +4,21 @@
 package common
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 )
 
 const k8sVisibility = "process,file,network,capabilities"
 const appArmorAnnotation = "container.apparmor.security.beta.kubernetes.io/"
-const KubeArmorRestartedAnnotation = "kubearmor.io/restarted"
-const KubeArmorForceAppArmorAnnotation = "kubearmor.io/force-apparmor"
+const KubeArmorRestartedAnnotation = "kubearmor.kubernetes.io/restartedAt"
 
 // == Add AppArmor annotations == //
-func AppArmorAnnotator(pod *corev1.Pod) {
+func AppArmorAnnotator(pod *corev1.Pod, binding *corev1.Binding, isBinding bool) {
 	podAnnotations := map[string]string{}
 	var podOwnerName string
 
@@ -64,52 +66,57 @@ func AppArmorAnnotator(pod *corev1.Pod) {
 		if v == "unconfined" {
 			continue
 		}
-		pod.Annotations[appArmorAnnotation+k] = "localhost/" + v
+		if isBinding {
+			binding.Annotations[appArmorAnnotation+k] = "localhost/" + v
+		} else {
+			pod.Annotations[appArmorAnnotation+k] = "localhost/" + v
+		}
 	}
 }
-func AddCommonAnnotations(pod *corev1.Pod) {
-	if pod.Annotations == nil {
-		pod.Annotations = map[string]string{}
+func AddCommonAnnotations(obj *metav1.ObjectMeta) {
+
+	if obj.Annotations == nil {
+		obj.Annotations = map[string]string{}
 	}
 
 	// == Policy == //
 
-	if _, ok := pod.Annotations["kubearmor-policy"]; !ok {
+	if _, ok := obj.Annotations["kubearmor-policy"]; !ok {
 		// if no annotation is set enable kubearmor by default
-		pod.Annotations["kubearmor-policy"] = "enabled"
-	} else if pod.Annotations["kubearmor-policy"] != "enabled" && pod.Annotations["kubearmor-policy"] != "disabled" && pod.Annotations["kubearmor-policy"] != "audited" {
+		obj.Annotations["kubearmor-policy"] = "enabled"
+	} else if obj.Annotations["kubearmor-policy"] != "enabled" && obj.Annotations["kubearmor-policy"] != "disabled" && obj.Annotations["kubearmor-policy"] != "audited" {
 		// if kubearmor policy is not set correctly, default it to enabled
-		pod.Annotations["kubearmor-policy"] = "enabled"
+		obj.Annotations["kubearmor-policy"] = "enabled"
 	}
 	// == Exception == //
 
 	// exception: kubernetes app
-	if pod.Namespace == "kube-system" {
-		if _, ok := pod.Labels["k8s-app"]; ok {
-			pod.Annotations["kubearmor-policy"] = "audited"
+	if obj.Namespace == "kube-system" {
+		if _, ok := obj.Labels["k8s-app"]; ok {
+			obj.Annotations["kubearmor-policy"] = "audited"
 		}
 
-		if value, ok := pod.Labels["component"]; ok {
+		if value, ok := obj.Labels["component"]; ok {
 			if value == "etcd" || value == "kube-apiserver" || value == "kube-controller-manager" || value == "kube-scheduler" || value == "kube-proxy" {
-				pod.Annotations["kubearmor-policy"] = "audited"
+				obj.Annotations["kubearmor-policy"] = "audited"
 			}
 		}
 	}
 
 	// exception: cilium-operator
-	if _, ok := pod.Labels["io.cilium/app"]; ok {
-		pod.Annotations["kubearmor-policy"] = "audited"
+	if _, ok := obj.Labels["io.cilium/app"]; ok {
+		obj.Annotations["kubearmor-policy"] = "audited"
 	}
 
 	// exception: kubearmor
-	if _, ok := pod.Labels["kubearmor-app"]; ok {
-		pod.Annotations["kubearmor-policy"] = "audited"
+	if _, ok := obj.Labels["kubearmor-app"]; ok {
+		obj.Annotations["kubearmor-policy"] = "audited"
 	}
 
 	// == Visibility == //
 
-	if _, ok := pod.Annotations["kubearmor-visibility"]; !ok {
-		pod.Annotations["kubearmor-visibility"] = k8sVisibility
+	if _, ok := obj.Annotations["kubearmor-visibility"]; !ok {
+		obj.Annotations["kubearmor-visibility"] = k8sVisibility
 	}
 }
 
@@ -125,3 +132,64 @@ func RemoveApparmorAnnotation(pod *corev1.Pod) {
 		delete(pod.Annotations, key)
 	}
 }
+
+func CheckKubearmorStatus(nodeName string, c *kubernetes.Clientset) (bool, error) {
+	pods, err := c.CoreV1().Pods("kubearmor").List(context.TODO(), metav1.ListOptions{
+		LabelSelector: "kubearmor-app=kubearmor",
+	})
+	if err != nil {
+		return false, fmt.Errorf("failed to list pods: %v", err)
+	}
+	// Filter Pods by nodeName and return their status.phase
+	for _, pod := range pods.Items {
+		if pod.Spec.NodeName == nodeName {
+			return true, nil
+		}
+	}
+
+	return false, nil
+
+}
+func hasApparmorAnnotation(annotations map[string]string) bool {
+	for key := range annotations {
+		if strings.HasPrefix(key, "container.apparmor.security.beta.kubernetes.io/") {
+			return true
+		}
+	}
+	return false
+}
+
+func HandleAppArmor(annotations map[string]string) bool {
+	return !hasApparmorAnnotation(annotations)
+}
+
+func HandleBPF(annotations map[string]string) bool {
+	return hasApparmorAnnotation(annotations)
+}
+
+func IsAppArmorExempt(labels map[string]string, namespace string) bool {
+
+	// exception: kubernetes app
+	if namespace == "kube-system" {
+		if _, ok := labels["k8s-app"]; ok {
+			return true
+		}
+
+		if value, ok := labels["component"]; ok {
+			if value == "etcd" || value == "kube-apiserver" || value == "kube-controller-manager" || value == "kube-scheduler" || value == "kube-proxy" {
+				return true
+			}
+		}
+	}
+
+	// exception: cilium-operator
+	if _, ok := labels["io.cilium/app"]; ok {
+		return true
+	}
+
+	// exception: kubearmor
+	if _, ok := labels["kubearmor-app"]; ok {
+		return true
+	}
+	return false
+}

pkg/KubeArmorController/config/webhook/manifests.yaml

@@ -23,4 +23,5 @@ webhooks:
     - UPDATE
     resources:
     - pods
+    - pods/binding
   sideEffects: NoneOnDryRun