docs: document NoExecute taint risks and add admission warning

ketanjani21 · ketanjani21 · commit 890f9700a0d3 · 2026-02-10T14:41:02.000+05:30
restructure docs

update taint key naming instructions
diff --git a/api/v1alpha1/nodereadinessrule_types.go b/api/v1alpha1/nodereadinessrule_types.go
@@ -69,6 +69,10 @@ type NodeReadinessRuleSpec struct {
 	// taint defines the specific Taint (Key, Value, and Effect) to be managed
 	// on Nodes that meet the defined condition criteria.
 	//
+	// Supported effects: NoSchedule, PreferNoSchedule, NoExecute.
+	// Caution: NoExecute evicts existing pods and can cause significant disruption
+	// when combined with continuous enforcement mode. Prefer NoSchedule for most use cases.
+	//
 	// +required
 	Taint corev1.Taint `json:"taint,omitempty,omitzero"`
 
diff --git a/config/crd/bases/readiness.node.x-k8s.io_nodereadinessrules.yaml b/config/crd/bases/readiness.node.x-k8s.io_nodereadinessrules.yaml
@@ -157,6 +157,10 @@ spec:
                 description: |-
                   taint defines the specific Taint (Key, Value, and Effect) to be managed
                   on Nodes that meet the defined condition criteria.
+
+                  Supported effects: NoSchedule, PreferNoSchedule, NoExecute.
+                  Caution: NoExecute evicts existing pods and can cause significant disruption
+                  when combined with continuous enforcement mode. Prefer NoSchedule for most use cases.
                 properties:
                   effect:
                     description: |-
diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md
@@ -6,7 +6,7 @@
 
 - [Core Concepts](./user-guide/concepts.md)
 - [Installation](./user-guide/installation.md)
-<!-- - [Creating Rules](./user-guide/creating-rules.md) -->
+- [Getting Started](./user-guide/getting-started.md)
 <!-- - [Quickstart](./user-guide/quickstart.md) -- [TODO] How to run the Calico example in KIND cluster -->
 
 # Examples
diff --git a/docs/book/src/user-guide/getting-started.md b/docs/book/src/user-guide/getting-started.md
@@ -1,6 +1,10 @@
 
 ## Getting Started
 
+This guide covers creating and configuring `NodeReadinessRule` resources.
+
+> **Prerequisites**: Node Readiness Controller must be installed. See [Installation](./installation.md).
+
 ### API Spec
 
 #### Example: Storage Readiness Rule (Bootstrap-only)
@@ -43,82 +47,19 @@ spec:
 | `nodeSelector` | Label selector to target specific nodes | No |
 | `dryRun` | Preview changes without applying them | No |
 
-### Deployment
-
-#### Option 1: Install official release images
-
-Node-Readiness Controller offers two variants of the container image to support different cluster architectures.
-
-Released container images are available for:
-* **x86_64** (AMD64)
-* **Arm64** (AArch64)
-
-The controller image is available in the Kubernetes staging registry:
-
-```sh
-REPO="us-central1-docker.pkg.dev/k8s-staging-images/node-readiness-controller/node-readiness-controller"
-
-TAG=$(skopeo list-tags docker://$REPO | jq .Tags[-1] | tr -d '"')
-
-docker pull $REPO:$TAG
-```
-
-#### Option 2: Deploy Using Make Commands
-
-**Build and push your image to the location specified by `IMG_PREFIX`:`IMG_TAG` :**
-
-```sh
-make docker-build docker-push IMG_PREFIX=<some-registry>/nrr-controller IMG_TAG=tag
-```
-
-```sh
-# Install the CRDs
-make install
-
-# Deploy the controller
-make deploy IMG_PREFIX=<some-registry>/nrr-controller IMG_TAG=tag
-
-# Create sample rules
-kubectl apply -k examples/network-readiness-rule.yaml
-```
-
-#### Option 3: Deploy Using Kustomize Directly
-
-```sh
-# Install CRDs
-kubectl apply -k config/crd
-
-# Deploy controller and RBAC
-kubectl apply -k config/default
-
-# Create sample rules
-kubectl apply -f examples/network-readiness-rule.yaml
-```
-
-### Uninstallation
-
-> **Important**: Follow this order to avoid stuck resources due to finalizers.
-
-The controller adds a finalizer (`readiness.node.x-k8s.io/cleanup-taints`) to each `NodeReadinessRule` to ensure node taints are cleaned up before the rule is deleted. This means you must delete CRs **while the controller is still running**.
-
-```sh
-# 1. Delete all rule instances first (while controller is running)
-kubectl delete nodereadinessrules --all
-
-# 2. Delete the controller
-make undeploy
-
-# 3. Delete the CRDs
-make uninstall
-```
-
-#### Recovering from Stuck Resources
+### Enforcement Modes
 
-If you deleted the controller before removing the CRs, the finalizer will block CR deletion. To recover, manually remove the finalizer:
+#### Bootstrap-only Mode
+- Removes bootstrap taint when conditions are first satisfied
+- Marks completion with node annotation
+- Stops monitoring after successful removal (fail-safe)
+- Ideal for one-time setup conditions (storage, installing node daemons e.g: security agent or kernel-module update)
 
-```sh
-kubectl patch nodereadinessrule <rule-name> -p '{"metadata":{"finalizers":[]}}' --type=merge
-```
+#### Continuous Mode
+- Continuously monitors conditions
+- Adds taint when any condition becomes unsatisfied
+- Removes taint when all conditions become satisfied
+- Ideal for ongoing health monitoring (network connectivity, resource availability)
 
 ## Operations
 
@@ -162,19 +103,65 @@ Check dry run results:
 kubectl get nodereadinessrule <rule-name> -o jsonpath='{.status.dryRunResults}'
 ```
 
-### Enforcement Modes
+### Rule Validation and Constraints
 
-#### Bootstrap-only Mode
-- Removes bootstrap taint when conditions are first satisfied
-- Marks completion with node annotation
-- Stops monitoring after successful removal (fail-safe)
-- Ideal for one-time setup conditions (storage, installing node daemons e.g: security agent or kernel-module update)
+#### NoExecute Taint Effect Warning
+
+**`NoExecute` with `continuous` enforcement mode will evict existing workloads when conditions fail.**
+
+If a critical component becomes temporarily unavailable (e.g., CNI daemon restart), all pods without matching tolerations are immediately evicted from the node. Use `NoSchedule` to prevent new scheduling without disrupting running workloads.
+
+The admission webhook warns when using `NoExecute`. 
+
+See [Kubernetes taints documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for taint behavior details.
+
+#### Avoiding Taint Key Conflicts
+
+The admission webhook prevents multiple rules from using the same `taint.key` and `taint.effect` on overlapping node selectors.
+
+**Example conflict:**
+```yaml
+# Rule 1
+spec:
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/worker: ""
+  taint:
+    key: "readiness.k8s.io/network"
+    effect: "NoSchedule"
+
+# Rule 2 - This will be REJECTED
+spec:
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/worker: ""
+  taint:
+    key: "readiness.k8s.io/network"  # Same key + effect = conflict
+    effect: "NoSchedule"
+```
+
+Use unique, descriptive taint keys for different readiness checks.
+
+#### Taint Key Naming
+
+Follow [Kubernetes naming conventions](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/).
+
+Taint keys must have the `readiness.k8s.io/` prefix to clearly identify readiness-related taints and avoid conflicts with other controllers
+
+**Valid:**
+```yaml
+taint:
+  key: "readiness.k8s.io/NetworkReady"
+  key: "readiness.k8s.io/StorageReady"
+```
+
+**Invalid:**
+```yaml
+taint:
+  key: "network-ready"              # Missing prefix
+  key: "node.kubernetes.io/ready"   # Wrong prefix
+```
 
-#### Continuous Mode
-- Continuously monitors conditions
-- Adds taint when any condition becomes unsatisfied
-- Removes taint when all conditions become satisfied
-- Ideal for ongoing health monitoring (network connectivity, resource availability)
 
 ## Configuration
 
diff --git a/docs/book/src/user-guide/installation.md b/docs/book/src/user-guide/installation.md
@@ -34,9 +34,30 @@ If it gets evicted during resource pressure, nodes can't transition to Ready sta
 
 This is the priority class used by other critical cluster components (eg: core-dns).
 
-**Images**: The official releases use multi-arch images (AMD64, Arm64).
+**Images**: The official releases use multi-arch images (AMD64, Arm64) available in the Kubernetes staging registry:
 
-### Option 2: Deploy Using Kustomize
+```sh
+REPO="us-central1-docker.pkg.dev/k8s-staging-images/node-readiness-controller/node-readiness-controller"
+TAG=$(skopeo list-tags docker://$REPO | jq .Tags[-1] | tr -d '"')
+docker pull $REPO:$TAG
+```
+
+### Option 2: Deploy Using Make Commands
+
+**Build and push your image to the location specified by `IMG_PREFIX`:`IMG_TAG` :**
+
+```sh
+# Build and push your image
+make docker-build docker-push IMG_PREFIX=<some-registry>/nrr-controller IMG_TAG=tag
+
+# Install the CRDs
+make install
+
+# Deploy the controller
+make deploy IMG_PREFIX=<some-registry>/nrr-controller IMG_TAG=tag
+```
+
+### Option 3: Deploy Using Kustomize
 
 If you have cloned the repository and want to deploy from source, you can use Kustomize.
 
diff --git a/internal/webhook/nodereadinessgaterule_webhook.go b/internal/webhook/nodereadinessgaterule_webhook.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
@@ -165,6 +166,35 @@ func (w *NodeReadinessRuleWebhook) nodSelectorsOverlap(selector1, selector2 meta
 	return sel1.String() == sel2.String()
 }
 
+// generateNoExecuteWarnings generates admission warnings for NoExecute taint usage.
+// NoExecute taints cause immediate pod eviction, which can be disruptive when
+// used with continuous enforcement mode.
+func (w *NodeReadinessRuleWebhook) generateNoExecuteWarnings(spec readinessv1alpha1.NodeReadinessRuleSpec) admission.Warnings {
+	var warnings admission.Warnings
+
+	if spec.Taint.Effect != corev1.TaintEffectNoExecute {
+		return warnings
+	}
+
+	// NoExecute with continuous mode is particularly risky
+	if spec.EnforcementMode == readinessv1alpha1.EnforcementModeContinuous {
+		warnings = append(warnings,
+			"CAUTION: Using NoExecute taint effect with continuous enforcement mode. "+
+				"This configuration will evict existing pods when conditions fail, which may cause "+
+				"significant workload disruption if conditions are unstable. Consider using NoSchedule "+
+				"effect instead, or bootstrap-only enforcement mode. "+
+				"See: https://node-readiness-controller.sigs.k8s.io/user-guide/getting-started.html")
+	} else {
+		// NoExecute with bootstrap-only is less risky but still worth noting
+		warnings = append(warnings,
+			"NOTE: Using NoExecute taint effect. This will evict existing pods that do not "+
+				"tolerate this taint when applied. Ensure critical system pods have appropriate tolerations. "+
+				"See: https://node-readiness-controller.sigs.k8s.io/user-guide/getting-started.html")
+	}
+
+	return warnings
+}
+
 // SetupWithManager sets up the webhook with the manager.
 func (w *NodeReadinessRuleWebhook) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).
@@ -185,7 +215,10 @@ func (w *NodeReadinessRuleWebhook) ValidateCreate(ctx context.Context, obj runti
 	if allErrs := w.validateNodeReadinessRule(ctx, rule, false); len(allErrs) > 0 {
 		return nil, fmt.Errorf("validation failed: %v", allErrs)
 	}
-	return nil, nil
+
+	// Generate warnings for NoExecute taint usage
+	warnings := w.generateNoExecuteWarnings(rule.Spec)
+	return warnings, nil
 }
 
 func (w *NodeReadinessRuleWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) {
@@ -197,7 +230,10 @@ func (w *NodeReadinessRuleWebhook) ValidateUpdate(ctx context.Context, oldObj, n
 	if allErrs := w.validateNodeReadinessRule(ctx, rule, true); len(allErrs) > 0 {
 		return nil, fmt.Errorf("validation failed: %v", allErrs)
 	}
-	return nil, nil
+
+	// Generate warnings for NoExecute taint usage
+	warnings := w.generateNoExecuteWarnings(rule.Spec)
+	return warnings, nil
 }
 
 func (w *NodeReadinessRuleWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) {
diff --git a/internal/webhook/nodereadinessgaterule_webhook_test.go b/internal/webhook/nodereadinessgaterule_webhook_test.go
