Skip to content

Commit 7f86c76

Browse files
authored
Merge pull request #1952 from Aryan-sharma11/fix-pod-deletion
fix: handle pod deletion in kubearmor controller
2 parents 349edd3 + 4687a1f commit 7f86c76

File tree

23 files changed

+424
-273
lines changed

23 files changed

+424
-273
lines changed

.github/workflows/ci-test-ginkgo.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ jobs:
133133
fi
134134
docker system prune -a -f
135135
docker buildx prune -a -f
136-
helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest
136+
helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest --set kubearmorOperator.annotateExisting=true
137137
kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
138138
kubectl get pods -A
139139
if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then

deployments/controller/kubearmor-controller-mutating-webhook-config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ webhooks:
4747
- UPDATE
4848
resources:
4949
- pods
50+
- pods/binding
5051
sideEffects: NoneOnDryRun
5152
objectSelector:
5253
matchExpressions:

deployments/get/objects.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,7 @@ func GetKubeArmorControllerDeployment(namespace string) *appsv1.Deployment {
526526
Args: []string{
527527
"--leader-elect",
528528
"--health-probe-bind-address=:8081",
529+
"--annotateExisting=false",
529530
},
530531
Command: []string{"/manager"},
531532
Ports: []corev1.ContainerPort{
@@ -769,7 +770,7 @@ func GetKubeArmorControllerMutationAdmissionConfiguration(namespace string, caCe
769770
Rule: admissionregistrationv1.Rule{
770771
APIGroups: []string{""},
771772
APIVersions: []string{"v1"},
772-
Resources: []string{"pods"},
773+
Resources: []string{"pods", "pods/binding"},
773774
},
774775
Operations: []admissionregistrationv1.OperationType{
775776
admissionregistrationv1.Create,

deployments/helm/KubeArmor/templates/RBAC/roles.yaml

+11-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,17 @@ rules:
9797
verbs:
9898
- get
9999
- list
100-
- watch
100+
- watch
101+
- apiGroups:
102+
- "apps"
103+
resources:
104+
- deployments
105+
- statefulsets
106+
- daemonsets
107+
- replicasets
108+
verbs:
109+
- get
110+
- update
101111
- apiGroups:
102112
- security.kubearmor.com
103113
resources:

deployments/helm/KubeArmor/templates/deployment.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ spec:
8282
- args:
8383
- --health-probe-bind-address=:8081
8484
- --leader-elect
85+
- --anotateExisting=false
8586
command:
8687
- /manager
8788
image: {{printf "%s:%s" .Values.kubearmorController.image.repository .Values.kubearmorController.image.tag}}

deployments/helm/KubeArmor/templates/secrets.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ webhooks:
4343
- CREATE
4444
- UPDATE
4545
resources:
46-
- pods
46+
- pods
47+
- pods/binding
4748
scope: '*'
4849
sideEffects: NoneOnDryRun
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{{- if not .Values.kubearmorOperator.annotateExisting }}
2+
⚠️ WARNING: Pre-existing pods will not be annotated. Policy enforcement for already existing pods on Apparmor nodes will not work.
3+
• To check enforcer present on nodes use:
4+
➤ kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.kubearmor\.io/enforcer}{"\n"}{end}'
5+
• To annotate existing pods use:
6+
➤ helm upgrade --install {{ .Values.kubearmorOperator.name }} kubearmor/kubearmor-operator -n kubearmor --create-namespace --set annotateExisting=true
7+
Our controller will automatically rollout restart deployments during the Helm upgrade to force the admission controller to add annotations.
8+
• Alternatively, if you prefer manual control, you can restart your deployments yourself:
9+
➤ kubectl rollout restart deployment <deployment> -n <namespace>
10+
{{- end }}
11+
ℹ️ Your release is named {{ .Release.Name }}.
12+
💙 Thank you for installing KubeArmor.

deployments/helm/KubeArmorOperator/templates/clusterrole-rbac.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ rules:
2828
verbs:
2929
- create
3030
- get
31+
- update
3132
- apiGroups:
3233
- operator.kubearmor.com
3334
resources:
@@ -155,6 +156,18 @@ rules:
155156
- list
156157
- watch
157158
- update
159+
{{- if .Values.kubearmorOperator.annotateExisting }}
160+
- apiGroups:
161+
- "apps"
162+
resources:
163+
- deployments
164+
- statefulsets
165+
- daemonsets
166+
- replicasets
167+
verbs:
168+
- get
169+
- update
170+
{{- end }}
158171
- apiGroups:
159172
- ""
160173
resources:

deployments/helm/KubeArmorOperator/templates/deployment.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ spec:
4646
image: {{ include "operatorImage" . }}
4747
imagePullPolicy: {{ .Values.kubearmorOperator.imagePullPolicy }}
4848
args:
49+
- --annotateExisting={{ .Values.kubearmorOperator.annotateExisting }}
50+
- --annotateResource={{ .Values.kubearmorOperator.annotateResource }}
4951
{{- if .Values.kubearmorOperator.args -}}
5052
{{- toYaml .Values.kubearmorOperator.args | trim | nindent 8 }}
5153
{{- end }}

deployments/helm/KubeArmorOperator/values.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ oci_meta:
2929
# in case if image pinning is disabled
3030
kubearmorOperator:
3131
annotateResource: false
32+
annotateExisting: false
3233
name: kubearmor-operator
3334
image:
3435
repository: kubearmor/kubearmor-operator

pkg/KubeArmorController/cmd/main.go

+13-10
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ func main() {
4949
var probeAddr string
5050
var secureMetrics bool
5151
var enableHTTP2 bool
52+
var annotateExisting bool
5253
var tlsOpts []func(*tls.Config)
5354
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
5455
"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -60,6 +61,8 @@ func main() {
6061
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
6162
flag.BoolVar(&enableHTTP2, "enable-http2", false,
6263
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
64+
flag.BoolVar(&annotateExisting, "annotateExisting", false,
65+
"If 'true', controller will restart and annotate existing resources with required annotations")
6366
opts := zap.Options{
6467
Development: true,
6568
}
@@ -157,24 +160,24 @@ func main() {
157160
cluster := informer.InitCluster()
158161
setupLog.Info("Starting node watcher")
159162
go informer.NodeWatcher(client, &cluster, ctrl.Log.WithName("informer").WithName("NodeWatcher"))
160-
setupLog.Info("Starting pod watcher")
161-
go informer.PodWatcher(client, &cluster, ctrl.Log.WithName("informer").WithName("PodWatcher"))
162163

163164
setupLog.Info("Adding mutation webhook")
164165
mgr.GetWebhookServer().Register("/mutate-pods", &webhook.Admission{
165166
Handler: &handlers.PodAnnotator{
166-
Client: mgr.GetClient(),
167-
Logger: setupLog,
168-
Decoder: admission.NewDecoder(mgr.GetScheme()),
169-
Cluster: &cluster,
167+
Client: mgr.GetClient(),
168+
Logger: setupLog,
169+
Decoder: admission.NewDecoder(mgr.GetScheme()),
170+
Cluster: &cluster,
171+
ClientSet: client,
170172
},
171173
})
172-
173174
setupLog.Info("Adding pod refresher controller")
174175
if err = (&controllers.PodRefresherReconciler{
175-
Client: mgr.GetClient(),
176-
Scheme: mgr.GetScheme(),
177-
Cluster: &cluster,
176+
Client: mgr.GetClient(),
177+
Scheme: mgr.GetScheme(),
178+
Cluster: &cluster,
179+
ClientSet: client,
180+
AnnotateExisting: annotateExisting,
178181
}).SetupWithManager(mgr); err != nil {
179182
setupLog.Error(err, "unable to create controller", "controller", "Pod")
180183
os.Exit(1)

pkg/KubeArmorController/common/common.go

+90-22
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,21 @@
44
package common
55

66
import (
7+
"context"
78
"fmt"
89
"strings"
910

1011
corev1 "k8s.io/api/core/v1"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/client-go/kubernetes"
1114
)
1215

1316
const k8sVisibility = "process,file,network,capabilities"
1417
const appArmorAnnotation = "container.apparmor.security.beta.kubernetes.io/"
15-
const KubeArmorRestartedAnnotation = "kubearmor.io/restarted"
16-
const KubeArmorForceAppArmorAnnotation = "kubearmor.io/force-apparmor"
18+
const KubeArmorRestartedAnnotation = "kubearmor.kubernetes.io/restartedAt"
1719

1820
// == Add AppArmor annotations == //
19-
func AppArmorAnnotator(pod *corev1.Pod) {
21+
func AppArmorAnnotator(pod *corev1.Pod, binding *corev1.Binding, isBinding bool) {
2022
podAnnotations := map[string]string{}
2123
var podOwnerName string
2224

@@ -64,52 +66,57 @@ func AppArmorAnnotator(pod *corev1.Pod) {
6466
if v == "unconfined" {
6567
continue
6668
}
67-
pod.Annotations[appArmorAnnotation+k] = "localhost/" + v
69+
if isBinding {
70+
binding.Annotations[appArmorAnnotation+k] = "localhost/" + v
71+
} else {
72+
pod.Annotations[appArmorAnnotation+k] = "localhost/" + v
73+
}
6874
}
6975
}
70-
func AddCommonAnnotations(pod *corev1.Pod) {
71-
if pod.Annotations == nil {
72-
pod.Annotations = map[string]string{}
76+
func AddCommonAnnotations(obj *metav1.ObjectMeta) {
77+
78+
if obj.Annotations == nil {
79+
obj.Annotations = map[string]string{}
7380
}
7481

7582
// == Policy == //
7683

77-
if _, ok := pod.Annotations["kubearmor-policy"]; !ok {
84+
if _, ok := obj.Annotations["kubearmor-policy"]; !ok {
7885
// if no annotation is set enable kubearmor by default
79-
pod.Annotations["kubearmor-policy"] = "enabled"
80-
} else if pod.Annotations["kubearmor-policy"] != "enabled" && pod.Annotations["kubearmor-policy"] != "disabled" && pod.Annotations["kubearmor-policy"] != "audited" {
86+
obj.Annotations["kubearmor-policy"] = "enabled"
87+
} else if obj.Annotations["kubearmor-policy"] != "enabled" && obj.Annotations["kubearmor-policy"] != "disabled" && obj.Annotations["kubearmor-policy"] != "audited" {
8188
// if kubearmor policy is not set correctly, default it to enabled
82-
pod.Annotations["kubearmor-policy"] = "enabled"
89+
obj.Annotations["kubearmor-policy"] = "enabled"
8390
}
8491
// == Exception == //
8592

8693
// exception: kubernetes app
87-
if pod.Namespace == "kube-system" {
88-
if _, ok := pod.Labels["k8s-app"]; ok {
89-
pod.Annotations["kubearmor-policy"] = "audited"
94+
if obj.Namespace == "kube-system" {
95+
if _, ok := obj.Labels["k8s-app"]; ok {
96+
obj.Annotations["kubearmor-policy"] = "audited"
9097
}
9198

92-
if value, ok := pod.Labels["component"]; ok {
99+
if value, ok := obj.Labels["component"]; ok {
93100
if value == "etcd" || value == "kube-apiserver" || value == "kube-controller-manager" || value == "kube-scheduler" || value == "kube-proxy" {
94-
pod.Annotations["kubearmor-policy"] = "audited"
101+
obj.Annotations["kubearmor-policy"] = "audited"
95102
}
96103
}
97104
}
98105

99106
// exception: cilium-operator
100-
if _, ok := pod.Labels["io.cilium/app"]; ok {
101-
pod.Annotations["kubearmor-policy"] = "audited"
107+
if _, ok := obj.Labels["io.cilium/app"]; ok {
108+
obj.Annotations["kubearmor-policy"] = "audited"
102109
}
103110

104111
// exception: kubearmor
105-
if _, ok := pod.Labels["kubearmor-app"]; ok {
106-
pod.Annotations["kubearmor-policy"] = "audited"
112+
if _, ok := obj.Labels["kubearmor-app"]; ok {
113+
obj.Annotations["kubearmor-policy"] = "audited"
107114
}
108115

109116
// == Visibility == //
110117

111-
if _, ok := pod.Annotations["kubearmor-visibility"]; !ok {
112-
pod.Annotations["kubearmor-visibility"] = k8sVisibility
118+
if _, ok := obj.Annotations["kubearmor-visibility"]; !ok {
119+
obj.Annotations["kubearmor-visibility"] = k8sVisibility
113120
}
114121
}
115122

@@ -125,3 +132,64 @@ func RemoveApparmorAnnotation(pod *corev1.Pod) {
125132
delete(pod.Annotations, key)
126133
}
127134
}
135+
136+
func CheckKubearmorStatus(nodeName string, c *kubernetes.Clientset) (bool, error) {
137+
pods, err := c.CoreV1().Pods("kubearmor").List(context.TODO(), metav1.ListOptions{
138+
LabelSelector: "kubearmor-app=kubearmor",
139+
})
140+
if err != nil {
141+
return false, fmt.Errorf("failed to list pods: %v", err)
142+
}
143+
// Filter Pods by nodeName and return their status.phase
144+
for _, pod := range pods.Items {
145+
if pod.Spec.NodeName == nodeName {
146+
return true, nil
147+
}
148+
}
149+
150+
return false, nil
151+
152+
}
153+
func hasApparmorAnnotation(annotations map[string]string) bool {
154+
for key := range annotations {
155+
if strings.HasPrefix(key, "container.apparmor.security.beta.kubernetes.io/") {
156+
return true
157+
}
158+
}
159+
return false
160+
}
161+
162+
func HandleAppArmor(annotations map[string]string) bool {
163+
return !hasApparmorAnnotation(annotations)
164+
}
165+
166+
func HandleBPF(annotations map[string]string) bool {
167+
return hasApparmorAnnotation(annotations)
168+
}
169+
170+
func IsAppArmorExempt(labels map[string]string, namespace string) bool {
171+
172+
// exception: kubernetes app
173+
if namespace == "kube-system" {
174+
if _, ok := labels["k8s-app"]; ok {
175+
return true
176+
}
177+
178+
if value, ok := labels["component"]; ok {
179+
if value == "etcd" || value == "kube-apiserver" || value == "kube-controller-manager" || value == "kube-scheduler" || value == "kube-proxy" {
180+
return true
181+
}
182+
}
183+
}
184+
185+
// exception: cilium-operator
186+
if _, ok := labels["io.cilium/app"]; ok {
187+
return true
188+
}
189+
190+
// exception: kubearmor
191+
if _, ok := labels["kubearmor-app"]; ok {
192+
return true
193+
}
194+
return false
195+
}

pkg/KubeArmorController/config/webhook/manifests.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,5 @@ webhooks:
2323
- UPDATE
2424
resources:
2525
- pods
26+
- pods/binding
2627
sideEffects: NoneOnDryRun

0 commit comments

Comments
 (0)