Skip to content

Commit 3292fbb

Browse files
dobrerazvandobrerazvan
committed
Comply with mutating webhook
1 parent e537b1c commit 3292fbb

File tree

4 files changed

+195
-11
lines changed

4 files changed

+195
-11
lines changed

‎controllers/kafkacluster_controller.go‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -385,15 +385,18 @@ func SetupKafkaClusterWithManager(mgr ctrl.Manager) *ctrl.Builder {
385385
}
386386
return false
387387
},
388-
UpdateFunc: func(e event.UpdateEvent) bool {
389-
switch newObj := e.ObjectNew.(type) {
390-
case *corev1.Pod, *corev1.ConfigMap, *corev1.PersistentVolumeClaim:
391-
patchResult, err := patch.DefaultPatchMaker.Calculate(e.ObjectOld, e.ObjectNew)
392-
if err != nil {
393-
log.Error(err, "could not match objects", "kind", e.ObjectOld.GetObjectKind())
394-
} else if patchResult.IsEmpty() {
395-
return false
396-
}
388+
UpdateFunc: func(e event.UpdateEvent) bool {
389+
switch newObj := e.ObjectNew.(type) {
390+
case *corev1.Pod, *corev1.ConfigMap, *corev1.PersistentVolumeClaim:
391+
opts := []patch.CalculateOption{
392+
k8sutil.IgnoreMutationWebhookFields(),
393+
}
394+
patchResult, err := patch.DefaultPatchMaker.Calculate(e.ObjectOld, e.ObjectNew, opts...)
395+
if err != nil {
396+
log.Error(err, "could not match objects", "kind", e.ObjectOld.GetObjectKind())
397+
} else if patchResult.IsEmpty() {
398+
return false
399+
}
397400
case *v1beta1.KafkaCluster:
398401
oldObj := e.ObjectOld.(*v1beta1.KafkaCluster)
399402
if !reflect.DeepEqual(oldObj.Spec, newObj.Spec) ||

‎pkg/k8sutil/patch_options.go‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
// Copyright © 2019 Cisco Systems, Inc. and/or its affiliates
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package k8sutil
16+
17+
import (
18+
"emperror.dev/errors"
19+
json "github.com/json-iterator/go"
20+
corev1 "k8s.io/api/core/v1"
21+
22+
"github.com/banzaicloud/k8s-objectmatcher/patch"
23+
)
24+
25+
// IgnoreMutationWebhookFields creates a CalculateOption that ignores fields commonly
26+
// modified by mutation webhooks like Gatekeeper, OPA, and Pod Security Policies
27+
func IgnoreMutationWebhookFields() patch.CalculateOption {
28+
return func(current, modified []byte) ([]byte, []byte, error) {
29+
currentPod := &corev1.Pod{}
30+
if err := json.Unmarshal(current, currentPod); err != nil {
31+
// Not a pod, return unchanged
32+
return current, modified, nil
33+
}
34+
35+
modifiedPod := &corev1.Pod{}
36+
if err := json.Unmarshal(modified, modifiedPod); err != nil {
37+
return current, modified, nil
38+
}
39+
40+
// Remove fields that mutation webhooks commonly modify
41+
currentPod = cleanMutationWebhookFields(currentPod)
42+
modifiedPod = cleanMutationWebhookFields(modifiedPod)
43+
44+
currentBytes, err := json.Marshal(currentPod)
45+
if err != nil {
46+
return []byte{}, []byte{}, errors.Wrap(err, "could not marshal cleaned current pod")
47+
}
48+
49+
modifiedBytes, err := json.Marshal(modifiedPod)
50+
if err != nil {
51+
return []byte{}, []byte{}, errors.Wrap(err, "could not marshal cleaned modified pod")
52+
}
53+
54+
return currentBytes, modifiedBytes, nil
55+
}
56+
}
57+
58+
func cleanMutationWebhookFields(pod *corev1.Pod) *corev1.Pod {
59+
// Create a copy to avoid modifying the original
60+
cleaned := pod.DeepCopy()
61+
62+
// Remove mutation webhook annotations that should not trigger reconciliation
63+
if cleaned.Annotations != nil {
64+
delete(cleaned.Annotations, "gatekeeper.sh/mutation-id")
65+
delete(cleaned.Annotations, "gatekeeper.sh/mutations")
66+
}
67+
68+
// Clean security context fields commonly set by PSPs/Gatekeeper
69+
for i := range cleaned.Spec.InitContainers {
70+
cleanSecurityContext(&cleaned.Spec.InitContainers[i])
71+
}
72+
for i := range cleaned.Spec.Containers {
73+
cleanSecurityContext(&cleaned.Spec.Containers[i])
74+
}
75+
76+
return cleaned
77+
}
78+
79+
func cleanSecurityContext(container *corev1.Container) {
80+
if container.SecurityContext == nil {
81+
return
82+
}
83+
84+
// Note: We intentionally do NOT clean security context fields here by default
85+
// because those are typically important security controls that should be reconciled.
86+
// If you need to ignore specific security context fields, uncomment the relevant lines below:
87+
88+
// AllowPrivilegeEscalation is often set by PSPs
89+
// container.SecurityContext.AllowPrivilegeEscalation = nil
90+
91+
// ReadOnlyRootFilesystem is often set by PSPs
92+
// container.SecurityContext.ReadOnlyRootFilesystem = nil
93+
94+
// Capabilities are often modified by PSPs
95+
// container.SecurityContext.Capabilities = nil
96+
}
97+
98+
// IgnorePodResourcesIfAnnotated creates a CalculateOption that ignores pod resource
99+
// requests/limits if the pod has specific annotations indicating it's managed by
100+
// an external system (e.g., ScaleOps, VPA)
101+
func IgnorePodResourcesIfAnnotated() patch.CalculateOption {
102+
return func(current, modified []byte) ([]byte, []byte, error) {
103+
currentMap := map[string]interface{}{}
104+
if err := json.Unmarshal(current, &currentMap); err != nil {
105+
return current, modified, nil
106+
}
107+
108+
// Check if this pod should ignore resource diffs (e.g., via annotation)
109+
if shouldIgnoreResources(currentMap) {
110+
// Remove resources from comparison
111+
current = removeResourcesFromPod(current)
112+
modified = removeResourcesFromPod(modified)
113+
}
114+
115+
return current, modified, nil
116+
}
117+
}
118+
119+
func shouldIgnoreResources(podMap map[string]interface{}) bool {
120+
metadata, ok := podMap["metadata"].(map[string]interface{})
121+
if !ok {
122+
return false
123+
}
124+
125+
annotations, ok := metadata["annotations"].(map[string]interface{})
126+
if !ok {
127+
return false
128+
}
129+
130+
// Check for annotations that indicate external resource management
131+
annotationsToCheck := []string{
132+
"scaleops.sh/pod-owner-grouping",
133+
"vpa.k8s.io/updateMode",
134+
"cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes",
135+
}
136+
137+
for _, ann := range annotationsToCheck {
138+
if _, exists := annotations[ann]; exists {
139+
return true
140+
}
141+
}
142+
143+
return false
144+
}
145+
146+
func removeResourcesFromPod(podBytes []byte) []byte {
147+
podMap := map[string]interface{}{}
148+
if err := json.Unmarshal(podBytes, &podMap); err != nil {
149+
return podBytes
150+
}
151+
152+
if spec, ok := podMap["spec"].(map[string]interface{}); ok {
153+
// Remove resources from all containers
154+
if containers, ok := spec["containers"].([]interface{}); ok {
155+
for _, c := range containers {
156+
if container, ok := c.(map[string]interface{}); ok {
157+
delete(container, "resources")
158+
}
159+
}
160+
}
161+
if initContainers, ok := spec["initContainers"].([]interface{}); ok {
162+
for _, c := range initContainers {
163+
if container, ok := c.(map[string]interface{}); ok {
164+
delete(container, "resources")
165+
}
166+
}
167+
}
168+
}
169+
170+
result, err := json.Marshal(podMap)
171+
if err != nil {
172+
return podBytes
173+
}
174+
return result
175+
}

‎pkg/k8sutil/resource.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,10 @@ func Reconcile(log logr.Logger, client runtimeClient.Client, desired runtime.Obj
164164

165165
// CheckIfObjectUpdated checks if the given object is updated using K8sObjectMatcher
166166
func CheckIfObjectUpdated(log logr.Logger, desiredType reflect.Type, current, desired runtime.Object) bool {
167-
patchResult, err := patch.DefaultPatchMaker.Calculate(current, desired)
167+
opts := []patch.CalculateOption{
168+
IgnoreMutationWebhookFields(),
169+
}
170+
patchResult, err := patch.DefaultPatchMaker.Calculate(current, desired, opts...)
168171
if err != nil {
169172
log.Error(err, "could not match objects", "kind", desiredType)
170173
return true

‎pkg/resources/kafka/kafka.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -953,7 +953,10 @@ func (r *Reconciler) handleRollingUpgrade(log logr.Logger, desiredPod, currentPo
953953
desiredPod.Spec.Tolerations = uniqueTolerations
954954
}
955955
// Check if the resource actually updated or if labels match TaintedBrokersSelector
956-
patchResult, err := patch.DefaultPatchMaker.Calculate(currentPod, desiredPod)
956+
opts := []patch.CalculateOption{
957+
k8sutil.IgnoreMutationWebhookFields(),
958+
}
959+
patchResult, err := patch.DefaultPatchMaker.Calculate(currentPod, desiredPod, opts...)
957960
switch {
958961
case err != nil:
959962
log.Error(err, "could not match objects", "kind", desiredType)

0 commit comments

Comments
 (0)