Skip to content

Commit c6c14fd

Browse files
kyledong-susekyledong-suse
committed
feat: add finalizer to WorkloadPolicy
Add a finalizer to WorkloadPolicy, prevent the deletion of policies are still in use by Pods. Signed-off-by: Kyle Dong <kyle.dong@suse.com>
1 parent 9b8204d commit c6c14fd

9 files changed

Lines changed: 518 additions & 3 deletions

File tree

‎api/v1alpha1/workloadpolicy_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ const (
1414

1515
DeployedState = "Deployed"
1616
ErrorState = "Error"
17+
18+
// WorkloadPolicyFinalizer is added to WorkloadPolicy resources to ensure
19+
// they are not deleted while still in use by Pods.
20+
WorkloadPolicyFinalizer = "workloadpolicy.security.rancher.io/finalizer"
1721
)
1822

1923
// todo!: we should support `AllowedPrefixes`.

‎charts/runtime-enforcer/templates/operator/role.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,14 @@ kind: ClusterRole
44
metadata:
55
name: {{ include "runtime-enforcer.fullname" . }}-operator
66
rules:
7+
- apiGroups:
8+
- ""
9+
resources:
10+
- pods
11+
verbs:
12+
- get
13+
- list
14+
- watch
715
- apiGroups:
816
- apps
917
resources:

‎charts/runtime-enforcer/templates/operator/webhook.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,23 @@ webhooks:
3333
expression: '!has(object.spec.selector) || object.spec.selector.size() == 0'
3434
- name: no-owner-reference-uid
3535
expression: '!has(object.metadata.ownerReferences) || object.metadata.ownerReferences.size() == 0 || object.metadata.ownerReferences[0].uid.size() == 0'
36+
- admissionReviewVersions:
37+
- v1
38+
clientConfig:
39+
service:
40+
name: {{ include "runtime-enforcer.fullname" . }}-mutating-webhook
41+
namespace: {{ .Release.Namespace }}
42+
path: /mutate-security-rancher-io-v1alpha1-workloadpolicy
43+
failurePolicy: Fail
44+
name: workloadpolicies.rancher.io
45+
rules:
46+
- apiGroups:
47+
- security.rancher.io
48+
apiVersions:
49+
- v1alpha1
50+
operations:
51+
- CREATE
52+
- UPDATE
53+
resources:
54+
- workloadpolicies
55+
sideEffects: None

‎cmd/operator/main.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,13 @@ func SetupControllers(logger logr.Logger,
9393
) error {
9494
var err error
9595

96+
if err = (&controller.WorkloadPolicyReconciler{
97+
Client: mgr.GetClient(),
98+
Scheme: mgr.GetScheme(),
99+
}).SetupWithManager(mgr); err != nil {
100+
return fmt.Errorf("unable to create WorkloadPolicyReconciler controller: %w", err)
101+
}
102+
96103
if err = (&controller.WorkloadPolicyProposalReconciler{
97104
Client: mgr.GetClient(),
98105
Scheme: mgr.GetScheme(),
@@ -250,7 +257,16 @@ func main() {
250257
WithDefaulter(&controller.ProposalWebhook{Client: mgr.GetClient()}).
251258
Complete()
252259
if err != nil {
253-
setupLog.Error(err, "unable to create a webhook")
260+
setupLog.Error(err, "unable to create WorkloadPolicyProposal webhook")
261+
os.Exit(1)
262+
}
263+
264+
err = builder.WebhookManagedBy(mgr).
265+
For(&securityv1alpha1.WorkloadPolicy{}).
266+
WithDefaulter(&controller.PolicyWebhook{}).
267+
Complete()
268+
if err != nil {
269+
setupLog.Error(err, "unable to create WorkloadPolicy webhook")
254270
os.Exit(1)
255271
}
256272

‎internal/controller/workloadpolicy_controller.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package controller
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
corev1 "k8s.io/api/core/v1"
9+
"k8s.io/apimachinery/pkg/api/errors"
10+
"k8s.io/apimachinery/pkg/runtime"
11+
"k8s.io/apimachinery/pkg/runtime/schema"
12+
ctrl "sigs.k8s.io/controller-runtime"
13+
"sigs.k8s.io/controller-runtime/pkg/builder"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
15+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
16+
"sigs.k8s.io/controller-runtime/pkg/handler"
17+
"sigs.k8s.io/controller-runtime/pkg/log"
18+
"sigs.k8s.io/controller-runtime/pkg/predicate"
19+
20+
"github.com/neuvector/runtime-enforcer/api/v1alpha1"
21+
"github.com/neuvector/runtime-enforcer/internal/policygenerator"
22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
)
24+
25+
const (
26+
PolicyDeletionRequeueDelay = 90 * time.Second
27+
)
28+
29+
// WorkloadPolicyReconciler reconciles a WorkloadPolicy object.
30+
type WorkloadPolicyReconciler struct {
31+
client.Client
32+
33+
Scheme *runtime.Scheme
34+
}
35+
36+
// +kubebuilder:rbac:groups=security.rancher.io,resources=workloadpolicies,verbs=get;list;watch;create;update;patch;delete
37+
// +kubebuilder:rbac:groups=security.rancher.io,resources=workloadpolicies/status,verbs=get;update;patch
38+
// +kubebuilder:rbac:groups=security.rancher.io,resources=workloadpolicies/finalizers,verbs=update
39+
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
40+
41+
func (r *WorkloadPolicyReconciler) Reconcile(
42+
ctx context.Context,
43+
req ctrl.Request,
44+
) (ctrl.Result, error) {
45+
policy := &v1alpha1.WorkloadPolicy{}
46+
if err := r.Get(ctx, req.NamespacedName, policy); err != nil {
47+
if errors.IsNotFound(err) {
48+
return ctrl.Result{}, nil
49+
}
50+
return ctrl.Result{}, fmt.Errorf("failed to get WorkloadPolicy '%s/%s': %w", req.Namespace, req.Name, err)
51+
}
52+
53+
// Check if the policy is being deleted
54+
if !policy.DeletionTimestamp.IsZero() {
55+
return r.handleDeletion(ctx, policy)
56+
}
57+
58+
return ctrl.Result{}, nil
59+
}
60+
61+
func (r *WorkloadPolicyReconciler) handleDeletion(
62+
ctx context.Context,
63+
policy *v1alpha1.WorkloadPolicy,
64+
) (ctrl.Result, error) {
65+
logger := log.FromContext(ctx)
66+
67+
if controllerutil.ContainsFinalizer(policy, v1alpha1.WorkloadPolicyFinalizer) {
68+
// Check if any pods are using this policy
69+
// Note, this uses a PartialObjectMetadataList because the Pod cache
70+
// is configured to only store metadata
71+
podList := &metav1.PartialObjectMetadataList{}
72+
podList.SetGroupVersionKind(schema.GroupVersionKind{
73+
Group: "",
74+
Version: "v1",
75+
Kind: "PodList",
76+
})
77+
78+
var err error
79+
err = r.List(ctx, podList,
80+
client.InNamespace(policy.Namespace),
81+
client.MatchingLabels{policygenerator.PolicyLabelKey: policy.Name},
82+
)
83+
if err != nil {
84+
logger.Error(err, "Failed to list pods using policy")
85+
return ctrl.Result{}, fmt.Errorf("failed to list pods using policy: %w", err)
86+
}
87+
88+
if len(podList.Items) > 0 {
89+
logger.Info("Cannot remove finalizer: policy still in use by pods",
90+
"policy", policy.Name,
91+
"podCount", len(podList.Items))
92+
// Requeue to check again later
93+
return ctrl.Result{RequeueAfter: PolicyDeletionRequeueDelay}, nil
94+
}
95+
96+
// No pods using this policy, safe to remove finalizer
97+
original := policy.DeepCopy()
98+
controllerutil.RemoveFinalizer(policy, v1alpha1.WorkloadPolicyFinalizer)
99+
if err = r.Patch(ctx, policy, client.MergeFrom(original)); err != nil {
100+
logger.Error(err, "Failed to remove finalizer")
101+
return ctrl.Result{}, fmt.Errorf(
102+
"failed to remove finalizer from WorkloadPolicy '%s/%s': %w",
103+
policy.Namespace, policy.Name, err,
104+
)
105+
}
106+
logger.Info("Removed finalizer from WorkloadPolicy", "policy", policy.Name)
107+
}
108+
return ctrl.Result{}, nil
109+
}
110+
111+
// SetupWithManager sets up the controller with the Manager.
112+
func (r *WorkloadPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
113+
err := ctrl.NewControllerManagedBy(mgr).
114+
For(&v1alpha1.WorkloadPolicy{}).
115+
Watches(
116+
&corev1.Pod{},
117+
handler.EnqueueRequestsFromMapFunc(r.findPoliciesForPod),
118+
builder.OnlyMetadata,
119+
builder.WithPredicates(predicate.LabelChangedPredicate{}),
120+
).
121+
Named("workloadpolicy").
122+
Complete(r)
123+
if err != nil {
124+
return fmt.Errorf("unable to set up WorkloadPolicy controller: %w", err)
125+
}
126+
return nil
127+
}
128+
129+
// findPoliciesForPod maps a Pod to the WorkloadPolicy(s) it references.
130+
func (r *WorkloadPolicyReconciler) findPoliciesForPod(_ context.Context, pod client.Object) []ctrl.Request {
131+
policyName, ok := pod.GetLabels()[policygenerator.PolicyLabelKey]
132+
if !ok {
133+
return nil
134+
}
135+
136+
return []ctrl.Request{
137+
{
138+
NamespacedName: client.ObjectKey{
139+
Name: policyName,
140+
Namespace: pod.GetNamespace(),
141+
},
142+
},
143+
}
144+
}

‎internal/controller/workloadpolicy_controller_test.go‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
package controller_test
2+
3+
import (
4+
"context"
5+
6+
"github.com/neuvector/runtime-enforcer/api/v1alpha1"
7+
"github.com/neuvector/runtime-enforcer/internal/controller"
8+
"github.com/neuvector/runtime-enforcer/internal/policygenerator"
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
corev1 "k8s.io/api/core/v1"
12+
"k8s.io/apimachinery/pkg/api/errors"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/apimachinery/pkg/types"
15+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
16+
)
17+
18+
var _ = Describe("WorkloadPolicy Controller", func() {
19+
Context("When reconciling a resource", func() {
20+
const policyName = "test-policy"
21+
const testNamespace = "default"
22+
23+
ctx = context.Background()
24+
25+
typeNamespacedName := types.NamespacedName{
26+
Name: policyName,
27+
Namespace: testNamespace,
28+
}
29+
policy := &v1alpha1.WorkloadPolicy{}
30+
31+
BeforeEach(func() {
32+
By("Creating a new WorkloadPolicy that is ")
33+
err := k8sClient.Get(ctx, typeNamespacedName, policy)
34+
if err != nil && errors.IsNotFound(err) {
35+
resource :=
36+
&v1alpha1.WorkloadPolicy{
37+
ObjectMeta: metav1.ObjectMeta{
38+
Name: policyName,
39+
Namespace: testNamespace,
40+
Finalizers: []string{v1alpha1.WorkloadPolicyFinalizer},
41+
},
42+
Spec: v1alpha1.WorkloadPolicySpec{
43+
Mode: "monitor",
44+
RulesByContainer: map[string]*v1alpha1.WorkloadPolicyRules{
45+
"main": {
46+
Executables: v1alpha1.WorkloadPolicyExecutables{
47+
Allowed: []string{"/usr/bin/sleep"},
48+
},
49+
},
50+
},
51+
Severity: 10,
52+
Tags: []string{
53+
"tag",
54+
},
55+
Message: "TEST_RULE",
56+
},
57+
}
58+
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
59+
}
60+
})
61+
62+
AfterEach(func() {
63+
resource := &v1alpha1.WorkloadPolicy{}
64+
err := k8sClient.Get(ctx, typeNamespacedName, resource)
65+
if errors.IsNotFound(err) {
66+
// Resource already deleted, nothing to clean up
67+
return
68+
}
69+
Expect(err).NotTo(HaveOccurred())
70+
71+
By("Cleanup the specific resource instance WorkloadPolicy")
72+
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
73+
})
74+
75+
It("Should delete a WorkloadPolicy that is not referenced by any Pod", func() {
76+
By("Deleting the created WorkloadPolicy")
77+
policy = &v1alpha1.WorkloadPolicy{
78+
ObjectMeta: metav1.ObjectMeta{
79+
Name: policyName,
80+
Namespace: testNamespace,
81+
},
82+
}
83+
Expect(k8sClient.Delete(ctx, policy)).To(Succeed())
84+
85+
By("Reconciling the created resource")
86+
controllerReconciler := &controller.WorkloadPolicyReconciler{
87+
Client: k8sClient,
88+
Scheme: k8sClient.Scheme(),
89+
}
90+
91+
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
92+
NamespacedName: typeNamespacedName,
93+
})
94+
Expect(err).NotTo(HaveOccurred())
95+
96+
By("Verifying the WorkloadPolicy has been deleted")
97+
err = k8sClient.Get(ctx, typeNamespacedName, policy)
98+
Expect(errors.IsNotFound(err)).To(BeTrue())
99+
})
100+
101+
It("Should not delete a WorkloadPolicy that is referenced by a Pod", func() {
102+
By("Associating the WorkloadPolicy with a Pod")
103+
podName := "test-pod"
104+
pod := &corev1.Pod{
105+
ObjectMeta: metav1.ObjectMeta{
106+
Name: podName,
107+
Namespace: testNamespace,
108+
Labels: map[string]string{
109+
policygenerator.PolicyLabelKey: policyName,
110+
},
111+
},
112+
Spec: corev1.PodSpec{
113+
Containers: []corev1.Container{
114+
{
115+
Name: "pause",
116+
Image: "registry.k8s.io/pause",
117+
},
118+
},
119+
},
120+
}
121+
Expect(k8sClient.Create(ctx, pod)).To(Succeed())
122+
123+
By("Deleting the referenced WorkloadPolicy")
124+
policy = &v1alpha1.WorkloadPolicy{
125+
ObjectMeta: metav1.ObjectMeta{
126+
Name: policyName,
127+
Namespace: testNamespace,
128+
},
129+
}
130+
Expect(k8sClient.Delete(ctx, policy)).To(Succeed())
131+
132+
By("Reconciling the deleted WorkloadPolicy")
133+
controllerReconciler := &controller.WorkloadPolicyReconciler{
134+
Client: k8sClient,
135+
Scheme: k8sClient.Scheme(),
136+
}
137+
138+
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
139+
NamespacedName: typeNamespacedName,
140+
})
141+
Expect(err).NotTo(HaveOccurred())
142+
143+
By("Verifying the WorkloadPolicy has not been deleted")
144+
err = k8sClient.Get(ctx, typeNamespacedName, policy)
145+
Expect(err).NotTo(HaveOccurred())
146+
147+
By("Cleaning up the created Pod")
148+
Expect(k8sClient.Delete(ctx, pod)).To(Succeed())
149+
150+
By("Reconciling the WorkloadPolicy deletion again")
151+
_, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
152+
NamespacedName: typeNamespacedName,
153+
})
154+
Expect(err).NotTo(HaveOccurred())
155+
156+
By("Verifying the WorkloadPolicy has been deleted after Pod removal")
157+
err = k8sClient.Get(ctx, typeNamespacedName, policy)
158+
Expect(errors.IsNotFound(err)).To(BeTrue())
159+
})
160+
})
161+
162+
})

0 commit comments

Comments
 (0)