feat: auto tolerate daemonsets with MAP

pehlicd · pehlicd · commit 24fde7b0b8e4 · 2026-02-02T12:24:36.000+01:00
Signed-off-by: pehlicd &lt;furkanpehlivan34@gmail.com&gt;
diff --git a/config/admission-policy/binding.yaml b/config/admission-policy/binding.yaml
@@ -0,0 +1,15 @@
+---
+# MutatingAdmissionPolicyBinding binds the policy to the ConfigMap parameter
+apiVersion: admissionregistration.k8s.io/v1alpha1
+kind: MutatingAdmissionPolicyBinding
+metadata:
+  name: inject-daemonset-readiness-tolerations-binding
+spec:
+  policyName: inject-daemonset-readiness-tolerations
+  # Reference the ConfigMap containing toleration data
+  paramRef:
+    name: readiness-taints
+    namespace: nrr-system
+    parameterNotFoundAction: Deny
+  matchResources:
+    namespaceSelector: {}
diff --git a/config/admission-policy/configmap.yaml b/config/admission-policy/configmap.yaml
@@ -0,0 +1,12 @@
+---
+# ConfigMap that stores readiness tolerations
+# This will be populated/updated by the NodeReadinessRule controller
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: readiness-taints
+  namespace: nrr-system
+data:
+  # Store each toleration key separately for easier CEL access
+  # Format: key1=readiness.k8s.io/NetworkReady,key2=readiness.k8s.io/StorageReady
+  taint-keys: ""
diff --git a/config/admission-policy/kustomization.yaml b/config/admission-policy/kustomization.yaml
@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- configmap.yaml
+- policy.yaml
+- binding.yaml
+
+labels:
+- pairs:
+    app.kubernetes.io/name: nrrcontroller
+    app.kubernetes.io/component: admission-policy
diff --git a/config/admission-policy/policy.yaml b/config/admission-policy/policy.yaml
@@ -0,0 +1,71 @@
+---
+# MutatingAdmissionPolicy for automatic DaemonSet toleration injection
+# Reads taint keys from a ConfigMap parameter resource
+# Requires: MutatingAdmissionPolicy feature enabled in the cluster
+apiVersion: admissionregistration.k8s.io/v1alpha1
+kind: MutatingAdmissionPolicy
+metadata:
+  name: inject-daemonset-readiness-tolerations
+spec:
+  failurePolicy: Fail
+  
+  # Define what this policy watches
+  matchConstraints:
+    resourceRules:
+    - apiGroups: ["apps"]
+      apiVersions: ["v1"]
+      operations: ["CREATE", "UPDATE"]
+      resources: ["daemonsets"]
+  
+  # Reference the ConfigMap that contains toleration data
+  paramKind:
+    apiVersion: v1
+    kind: ConfigMap
+  
+  # Variables for CEL expressions
+  variables:
+  # Check if opt-out annotation is set
+  - name: optedOut
+    expression: |
+      has(object.metadata.annotations) && 
+      object.metadata.annotations.exists(k, k == "readiness.k8s.io/auto-tolerate" && object.metadata.annotations[k] == "false")
+  
+  # Get existing tolerations (empty array if none)
+  - name: existingTolerations
+    expression: |
+      has(object.spec.template.spec.tolerations) ? 
+        object.spec.template.spec.tolerations : []
+  
+  # Get taint keys from ConfigMap and parse to array
+  - name: taintKeys
+    expression: |
+      ("taint-keys" in params.data) && params.data["taint-keys"] != "" ?
+        params.data["taint-keys"].split(",") : []
+  
+  # Create tolerations from taint keys (as plain maps since CEL has issues with complex types)
+  - name: tolerationsToInject
+    expression: |
+      variables.taintKeys
+        .filter(key, !variables.existingTolerations.exists(t, t.key == key))
+        .map(key, {
+          "key": key,
+          "operator": "Exists",
+          "effect": "NoSchedule"
+        })
+  
+  # Apply mutations
+  mutations:
+  - patchType: JSONPatch
+    jsonPatch:
+      expression: |
+        !variables.optedOut && size(variables.tolerationsToInject) > 0 ?
+        [
+          JSONPatch{
+            op: has(object.spec.template.spec.tolerations) ? "replace" : "add",
+            path: "/spec/template/spec/tolerations",
+            value: variables.existingTolerations + variables.tolerationsToInject
+          }
+        ] : []
+  
+  # Never reinvoke this policy
+  reinvocationPolicy: Never
diff --git a/docs/admission-policy.md b/docs/admission-policy.md
@@ -0,0 +1,54 @@
+# MutatingAdmissionPolicy for DaemonSet Toleration Injection
+
+This document describes how to deploy and use the MutatingAdmissionPolicy-based approach for automatically injecting readiness tolerations into DaemonSets.
+
+## Overview
+
+The MutatingAdmissionPolicy approach uses Kubernetes's native admission control mechanism with CEL (Common Expression Language) to inject tolerations **without running a webhook server**. This provides a simpler, more declarative alternative to the webhook-based approach.
+
+## Requirements
+
+> [!IMPORTANT]
+> MutatingAdmissionPolicy is needed to be enabled in the cluster.
+
+- Feature gate: `MutatingAdmissionPolicy=true`
+- Runtime config: `admissionregistration.k8s.io/v1alpha1=true`
+- `kubectl` configured to access your cluster
+- NodeReadinessRule CRDs installed
+
+## Architecture
+
+```
+User applies DaemonSet
+    ↓
+API Server evaluates CEL policy
+    ↓
+Fetches Tolerations ConfigMap which contains the tolerations to be injected
+    ↓
+Injects tolerations (if applicable)
+    ↓
+DaemonSet created with tolerations
+```
+
+## Deployment
+
+### Option 1: Using kustomize
+
+```bash
+# Install CRDs first
+make install
+
+# Deploy the admission policy
+kubectl apply -k config/admission-policy
+```
+
+### Option 2: Direct kubectl apply
+
+```bash
+# Install CRDs first
+make install
+
+# Deploy policy and binding
+kubectl apply -f config/admission-policy/policy.yaml
+kubectl apply -f config/admission-policy/binding.yaml
+```
diff --git a/go.mod b/go.mod
@@ -10,6 +10,7 @@ require (
 	k8s.io/api v0.34.0
 	k8s.io/apimachinery v0.34.0
 	k8s.io/client-go v0.34.0
+	k8s.io/klog/v2 v2.130.1
 	sigs.k8s.io/controller-runtime v0.22.1
 )
 
@@ -91,7 +92,6 @@ require (
 	k8s.io/apiextensions-apiserver v0.34.0 // indirect
 	k8s.io/apiserver v0.34.0 // indirect
 	k8s.io/component-base v0.34.0 // indirect
-	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
 	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
diff --git a/internal/controller/nodereadinessrule_controller.go b/internal/controller/nodereadinessrule_controller.go
@@ -166,6 +166,12 @@ func (r *RuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 		return ctrl.Result{RequeueAfter: time.Minute}, err
 	}
 
+	// Sync taints to ConfigMap for MutatingAdmissionPolicy
+	if err := r.Controller.syncTaintsConfigMap(ctx); err != nil {
+		log.Error(err, "Failed to sync taints configmap", "rule", rule.Name)
+		// Don't fail reconciliation for this - log and continue
+	}
+
 	return ctrl.Result{}, nil
 }
 
@@ -671,6 +677,75 @@ func (r *RuleReadinessController) cleanupNodesAfterSelectorChange(ctx context.Co
 	return nil
 }
 
+// syncTaintsConfigMap synchronizes readiness taints to a ConfigMap for admission policy.
+func (r *RuleReadinessController) syncTaintsConfigMap(ctx context.Context) error {
+	log := ctrl.LoggerFrom(ctx)
+
+	// List all NodeReadinessRules
+	var ruleList readinessv1alpha1.NodeReadinessRuleList
+	if err := r.List(ctx, &ruleList); err != nil {
+		return fmt.Errorf("failed to list NodeReadinessRules: %w", err)
+	}
+
+	// Extract unique taint keys with readiness.k8s.io/ prefix and NoSchedule effect
+	taintKeysSet := make(map[string]struct{})
+	for _, rule := range ruleList.Items {
+		if rule.Spec.Taint.Key != "" &&
+			strings.HasPrefix(rule.Spec.Taint.Key, "readiness.k8s.io/") &&
+			rule.Spec.Taint.Effect == corev1.TaintEffectNoSchedule {
+			taintKeysSet[rule.Spec.Taint.Key] = struct{}{}
+		}
+	}
+
+	// Convert set to comma-separated string
+	taintKeys := make([]string, 0, len(taintKeysSet))
+	for key := range taintKeysSet {
+		taintKeys = append(taintKeys, key)
+	}
+	taintKeysStr := strings.Join(taintKeys, ",")
+
+	// Update or create ConfigMap
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "readiness-taints",
+			Namespace: "nrr-system",
+		},
+	}
+
+	// Try to get existing ConfigMap
+	existingCM := &corev1.ConfigMap{}
+	err := r.Get(ctx, client.ObjectKey{Name: "readiness-taints", Namespace: "nrr-system"}, existingCM)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return fmt.Errorf("failed to get configmap: %w", err)
+	}
+
+	// Set data
+	cm.Data = map[string]string{
+		"taint-keys": taintKeysStr,
+	}
+
+	if apierrors.IsNotFound(err) {
+		// Create new ConfigMap
+		log.Info("Creating readiness-taints ConfigMap", "taintCount", len(taintKeys))
+		if err := r.Create(ctx, cm); err != nil {
+			return fmt.Errorf("failed to create configmap: %w", err)
+		}
+	} else {
+		// Update existing ConfigMap
+		log.V(1).Info("Updating readiness-taints ConfigMap", "taintCount", len(taintKeys))
+		patch := client.MergeFrom(existingCM)
+		existingCM.Data = cm.Data
+		if err := r.Patch(ctx, existingCM, patch); err != nil {
+			return fmt.Errorf("failed to update configmap: %w", err)
+		}
+	}
+
+	log.V(2).Info("Successfully synced taints to ConfigMap",
+		"totalRules", len(ruleList.Items),
+		"readinessTaints", len(taintKeys))
+	return nil
+}
+
 func (r *RuleReconciler) ensureFinalizer(ctx context.Context, rule *readinessv1alpha1.NodeReadinessRule, finalizer string) (finalizerAdded bool, err error) {
 	// Finalizers can only be added when the deletionTimestamp is not set.
 	if !rule.GetDeletionTimestamp().IsZero() {
