Disable webhook admissions enforcer in AKS (#350)

Author: jefchien

charts/amazon-cloudwatch-observability/templates/admission-webhooks/operator-webhook-with-cert-manager.yaml

@@ -4,6 +4,12 @@ kind: MutatingWebhookConfiguration
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: {{ printf "%s/%s-serving-cert" .Release.Namespace (include "amazon-cloudwatch-observability.name" .) }}
+    {{- if eq .Values.k8sMode "AKS" }}
+    # Opt out of the AKS admissionsenforcer, which otherwise injects its own namespaceSelector into
+    # each webhook and takes server-side-apply ownership of the field, causing helm upgrade to
+    # conflict. See https://learn.microsoft.com/en-us/azure/aks/faq (admissions enforcer).
+    admissions.enforcer/disabled: "true"
+    {{- end }}
   labels:
     {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}}
   name: {{ template "amazon-cloudwatch-observability.name" . }}-mutating-webhook-configuration
@@ -156,6 +162,12 @@ kind: ValidatingWebhookConfiguration
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: {{ printf "%s/%s-serving-cert" .Release.Namespace (include "amazon-cloudwatch-observability.name" .) }}
+    {{- if eq .Values.k8sMode "AKS" }}
+    # Opt out of the AKS admissionsenforcer, which otherwise injects its own namespaceSelector into
+    # each webhook and takes server-side-apply ownership of the field, causing helm upgrade to
+    # conflict. See https://learn.microsoft.com/en-us/azure/aks/faq (admissions enforcer).
+    admissions.enforcer/disabled: "true"
+    {{- end }}
   labels:
     {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}}
   name: {{ template "amazon-cloudwatch-observability.name" . }}-validating-webhook-configuration

charts/amazon-cloudwatch-observability/templates/admission-webhooks/operator-webhook.yaml

@@ -15,6 +15,13 @@ data:
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
+  {{- if eq .Values.k8sMode "AKS" }}
+  annotations:
+    # Opt out of the AKS admissionsenforcer, which otherwise injects its own namespaceSelector into
+    # each webhook and takes server-side-apply ownership of the field, causing helm upgrade to
+    # conflict. See https://learn.microsoft.com/en-us/azure/aks/faq (admissions enforcer).
+    admissions.enforcer/disabled: "true"
+  {{- end }}
   labels:
     {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}}
   name: {{ template "amazon-cloudwatch-observability.name" . }}-mutating-webhook-configuration
@@ -170,6 +177,13 @@ webhooks:
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
+  {{- if eq .Values.k8sMode "AKS" }}
+  annotations:
+    # Opt out of the AKS admissionsenforcer, which otherwise injects its own namespaceSelector into
+    # each webhook and takes server-side-apply ownership of the field, causing helm upgrade to
+    # conflict. See https://learn.microsoft.com/en-us/azure/aks/faq (admissions enforcer).
+    admissions.enforcer/disabled: "true"
+  {{- end }}
   labels:
     {{- include "amazon-cloudwatch-observability.labels" . | nindent 4}}
   name: {{ template "amazon-cloudwatch-observability.name" . }}-validating-webhook-configuration

integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/aks_otel_container_insights_test.go

@@ -73,6 +73,9 @@ func TestAKSOtelContainerInsights(t *testing.T) {
 	t.Run("OTELConfigRouting", func(t *testing.T) {
 		validateOTELConfigRoutingAKS(t, agentMap)
 	})
+	t.Run("WebhookEnforcerDisabled", func(t *testing.T) {
+		validateAKSWebhookEnforcerDisabled(t, k8sClient)
+	})
 
 	t.Log("AKS OTEL Container Insights scenario validation passed")
 }
@@ -197,3 +200,39 @@ func validateAKSHostAttributes(t *testing.T, agentMap map[string]unstructured.Un
 	assertEnabled("node agent", otelConfigOf(t, agentMap, "cloudwatch-agent"), true)
 	assertEnabled("cluster-scraper", otelConfigOf(t, agentMap, "cloudwatch-agent-cluster-scraper"), false)
 }
+
+// validateAKSWebhookEnforcerDisabled checks the webhook configurations carry the
+// admissions.enforcer/disabled annotation on AKS. Without it, the AKS admissionsenforcer rewrites each
+// webhook's namespaceSelector and takes server-side-apply ownership of the field, which makes the next
+// helm upgrade fail with an apply conflict.
+func validateAKSWebhookEnforcerDisabled(t *testing.T, k8sClient *util.K8sClient) {
+	const enforcerDisabled = "admissions.enforcer/disabled"
+
+	mwc, err := k8sClient.ListMutatingWebhookConfigurations()
+	require.NoError(t, err, "failed to list MutatingWebhookConfigurations")
+	assertEnforcerDisabled := func(name string, annotations map[string]string) {
+		assert.Equal(t, "true", annotations[enforcerDisabled],
+			"%s should set %s on AKS", name, enforcerDisabled)
+	}
+
+	foundMutating := false
+	for _, wh := range mwc.Items {
+		if wh.Name == minikube.WebhookName {
+			foundMutating = true
+			assertEnforcerDisabled(wh.Name, wh.Annotations)
+		}
+	}
+	assert.True(t, foundMutating, "mutating webhook configuration %s should exist", minikube.WebhookName)
+
+	vwc, err := k8sClient.ListValidatingWebhookConfigurations()
+	require.NoError(t, err, "failed to list ValidatingWebhookConfigurations")
+	validatingName := "amazon-cloudwatch-observability-validating-webhook-configuration"
+	foundValidating := false
+	for _, wh := range vwc.Items {
+		if wh.Name == validatingName {
+			foundValidating = true
+			assertEnforcerDisabled(wh.Name, wh.Annotations)
+		}
+	}
+	assert.True(t, foundValidating, "validating webhook configuration %s should exist", validatingName)
+}