Merge pull request #166 from appuio/agent-unmanaged

Allow unmanaged (no RBAC, LimitRange, Quotas, ...) organization namespaces (lbl: `appuio.io/unmanaged-namespace`)

Author: bastjan

config.go

@@ -15,6 +15,11 @@ import (
 type Config struct {
 	// OrganizationLabel is the label used to mark namespaces to belong to an organization
 	OrganizationLabel string
+	// UnmanagedOrganizationNamespaceLabel is the label used to mark namespaces that should not be
+	// managed by this controller or its webhooks and this should not be restricted by resource quotas
+	// and limit ranges, even if they belong to an organization.
+	// The controller doesn't look at the value of this label, it only looks for the label and skips any namespaces that have both the organization label and this label.
+	UnmanagedOrganizationNamespaceLabel string
 
 	// UserDefaultOrganizationAnnotation is the annotation the default organization setting for a user is stored in.
 	UserDefaultOrganizationAnnotation string
@@ -106,6 +111,9 @@ func (c Config) Validate() error {
 	if c.OrganizationLabel == "" {
 		errs = append(errs, errors.New("OrganizationLabel must not be empty"))
 	}
+	if c.UnmanagedOrganizationNamespaceLabel == "" {
+		errs = append(errs, errors.New("UnmanagedOrganizationNamespaceLabel must not be empty"))
+	}
 
 	return multierr.Combine(errs...)
 }

config.yaml

@@ -1,6 +1,8 @@
 ---
 # The label used to mark namespaces to belong to an organization
 OrganizationLabel: appuio.io/organization
+# The label used to mark namespaces that are not managed by the controller or its webhooks, but still belong to an organization.
+UnmanagedOrganizationNamespaceLabel: appuio.io/unmanaged-namespace
 # UserDefaultOrganizationAnnotation is the annotation the default organization setting for a user is stored in.
 UserDefaultOrganizationAnnotation: appuio.io/default-organization
 

controllers/legacy_resource_quota_controller.go

@@ -9,6 +9,7 @@ import (
 	"go.uber.org/multierr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -24,7 +25,7 @@ type LegacyResourceQuotaReconciler struct {
 	Scheme   *runtime.Scheme
 	Recorder record.EventRecorder
 
-	OrganizationLabel string
+	NamespaceSelector labels.Selector
 
 	ResourceQuotaAnnotationBase string
 	DefaultResourceQuotas       map[string]corev1.ResourceQuotaSpec
@@ -46,8 +47,8 @@ func (r *LegacyResourceQuotaReconciler) Reconcile(ctx context.Context, req ctrl.
 		return ctrl.Result{}, nil
 	}
 
-	if _, ok := ns.Labels[r.OrganizationLabel]; !ok {
-		l.Info("Namespace does not have organization label, skipping reconciliation")
+	if !r.NamespaceSelector.Matches(labels.Set(ns.Labels)) {
+		l.Info("Namespace does not match namespace selector, skipping reconciliation")
 		return ctrl.Result{}, nil
 	}
 
@@ -127,7 +128,7 @@ func (r *LegacyResourceQuotaReconciler) Reconcile(ctx context.Context, req ctrl.
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *LegacyResourceQuotaReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	orgPredicate, err := labelExistsPredicate(r.OrganizationLabel)
+	orgPredicate, err := labelSelectorPredicate(r.NamespaceSelector)
 	if err != nil {
 		return fmt.Errorf("failed to create organization label predicate: %w", err)
 	}

controllers/legacy_resource_quota_controller_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -28,7 +29,7 @@ func Test_LegacyResourceQuotaReconciler_Reconcile(t *testing.T) {
 		Scheme:   scheme,
 		Recorder: recorder,
 
-		OrganizationLabel: "organization",
+		NamespaceSelector: labels.SelectorFromValidatedSet(labels.Set{"organization": "testorg"}),
 
 		ResourceQuotaAnnotationBase: "resourcequota.example.com",
 		DefaultResourceQuotas: map[string]corev1.ResourceQuotaSpec{

controllers/org_rbac_controller.go

@@ -2,16 +2,19 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	"go.uber.org/multierr"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/tools/record"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -23,6 +26,8 @@ type OrganizationRBACReconciler struct {
 	Recorder record.EventRecorder
 	Scheme   *runtime.Scheme
 
+	NamespaceSelector labels.Selector
+
 	// OrganizationLabel is the label that marks to what organization (if any) the namespace belongs to
 	OrganizationLabel string
 	// DefaultClusterRoles is a map where the keys are the name of default rolebindings to create and the values are the names of the clusterroles they bind to
@@ -60,6 +65,11 @@ func (r *OrganizationRBACReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{}, nil
 	}
 
+	if !r.NamespaceSelector.Matches(labels.Set(ns.Labels)) {
+		l.Info("Namespace does not match namespace selector, skipping reconciliation")
+		return ctrl.Result{}, nil
+	}
+
 	org := r.getOrganization(ns)
 	if org == "" {
 		return ctrl.Result{}, nil
@@ -143,8 +153,12 @@ func rolebindingIsUninitialized(rolebinding *rbacv1.RoleBinding) bool {
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *OrganizationRBACReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	p, err := labelSelectorPredicate(r.NamespaceSelector)
+	if err != nil {
+		return fmt.Errorf("failed to create predicate: %w", err)
+	}
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&corev1.Namespace{}).
+		For(&corev1.Namespace{}, builder.WithPredicates(p)).
 		Owns(&rbacv1.RoleBinding{}).
 		Complete(r)
 }

controllers/org_rbac_controller_test.go

@@ -11,6 +11,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -407,11 +408,15 @@ func prepareOranizationRBACTest(t *testing.T, cfg testOrganizationRBACfg) *Organ
 		cfg.recorder = &record.FakeRecorder{}
 	}
 
+	sel, err := labels.Parse(cfg.organizationLabel)
+	require.NoError(t, err)
+
 	return &OrganizationRBACReconciler{
 		Client:              client,
 		Recorder:            cfg.recorder,
 		Scheme:              scheme,
 		OrganizationLabel:   cfg.organizationLabel,
+		NamespaceSelector:   sel,
 		DefaultClusterRoles: cfg.clusterRoles,
 	}
 }

controllers/zoneusageprofileapply_controller.go

@@ -8,8 +8,8 @@ import (
 
 	"go.uber.org/multierr"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/tools/record"
@@ -46,7 +46,7 @@ type ZoneUsageProfileApplyReconciler struct {
 	// watchTracker keeps track of which resources are already being watched.
 	watchTracker sync.Map
 
-	OrganizationLabel string
+	NamespaceSelector labels.Selector
 	Transformers      []transformers.Transformer
 
 	// SelectedProfile applies only selected profile, if set. Dynamic selection is in future tickets.
@@ -79,7 +79,7 @@ func (r *ZoneUsageProfileApplyReconciler) Reconcile(ctx context.Context, req ctr
 	}
 
 	var orgNsl corev1.NamespaceList
-	if err := r.Client.List(ctx, &orgNsl, client.HasLabels{r.OrganizationLabel}); err != nil {
+	if err := r.Client.List(ctx, &orgNsl, client.MatchingLabelsSelector{Selector: r.NamespaceSelector}); err != nil {
 		l.Error(err, "unable to list Namespaces")
 		return ctrl.Result{}, err
 	}
@@ -180,7 +180,7 @@ func (r *ZoneUsageProfileApplyReconciler) ensureWatch(ctx context.Context, gvk s
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *ZoneUsageProfileApplyReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	orgPredicate, err := labelExistsPredicate(r.OrganizationLabel)
+	orgPredicate, err := labelSelectorPredicate(r.NamespaceSelector)
 	if err != nil {
 		return fmt.Errorf("unable to create LabelSelectorPredicate: %w", err)
 	}
@@ -201,13 +201,11 @@ func (r *ZoneUsageProfileApplyReconciler) SetupWithManager(mgr ctrl.Manager) err
 	return nil
 }
 
-// labelExistsPredicate returns a predicate that matches objects with the given label.
-func labelExistsPredicate(label string) (predicate.Predicate, error) {
-	return predicate.LabelSelectorPredicate(metav1.LabelSelector{
-		MatchExpressions: []metav1.LabelSelectorRequirement{{
-			Key:      label,
-			Operator: metav1.LabelSelectorOpExists,
-		}}})
+// labelSelectorPredicate returns a predicate that matches objects with the given label.
+func labelSelectorPredicate(sel labels.Selector) (predicate.Predicate, error) {
+	return predicate.NewPredicateFuncs(func(o client.Object) bool {
+		return sel.Matches(labels.Set(o.GetLabels()))
+	}), nil
 }
 
 // mapToAllUsageProfiles returns a MapFunc that enqueues reconcile requests for all ZoneUsageProfiles on every event.

controllers/zoneusageprofileapply_controller_test.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/event"
@@ -53,13 +54,16 @@ func Test_ZoneUsageProfileApplyReconciler_Reconcile(t *testing.T) {
 	org2NS := newNamespace("org2", map[string]string{orgLbl: "bar"}, nil)
 	require.NoError(t, c.Create(context.Background(), org2NS))
 
+	nsSel, err := labels.Parse(orgLbl)
+	require.NoError(t, err)
+
 	subject := &ZoneUsageProfileApplyReconciler{
 		Client:   c,
 		Scheme:   scheme,
 		Recorder: recorder,
 		Cache:    mgr.GetCache(),
 
-		OrganizationLabel: orgLbl,
+		NamespaceSelector: nsSel,
 
 		Transformers: []transformers.Transformer{
 			addTestAnnotationTransformer{},
@@ -121,9 +125,11 @@ func requireEventually(t *testing.T, f func(collect *assert.CollectT), msgAndArg
 	require.EventuallyWithT(t, f, 10*time.Second, time.Second/10, msgAndArgs...)
 }
 
-func Test_labelExistsPredicate(t *testing.T) {
+func Test_labelSelectorPredicate(t *testing.T) {
 	lbl := "test.com/organization"
-	subject, err := labelExistsPredicate(lbl)
+	sel, err := labels.Parse(lbl)
+	require.NoError(t, err)
+	subject, err := labelSelectorPredicate(sel)
 	require.NoError(t, err)
 
 	assert.True(t, subject.Generic(event.GenericEvent{

main.go

@@ -13,7 +13,9 @@ import (
 	userv1 "github.com/openshift/api/user/v1"
 	"go.uber.org/multierr"
 	authenticationv1 "k8s.io/api/authentication/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/selection"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -184,8 +186,14 @@ func main() {
 		os.Exit(1)
 	}
 
-	registerRatioController(mgr, conf, conf.OrganizationLabel)
-	registerOrganizationRBACController(mgr, conf.OrganizationLabel, conf.DefaultOrganizationClusterRoles)
+	nsSel, err := newNamespaceLabelSelector(conf.OrganizationLabel, conf.UnmanagedOrganizationNamespaceLabel)
+	if err != nil {
+		setupLog.Error(err, "unable to create namespace label selector")
+		os.Exit(1)
+	}
+
+	registerRatioController(mgr, conf, nsSel)
+	registerOrganizationRBACController(mgr, conf.OrganizationLabel, nsSel, conf.DefaultOrganizationClusterRoles)
 	registerZoneK8sVersionController(mgr, controlAPICluster, upstreamZoneIdentifier)
 
 	if !disableUserAttributeSync {
@@ -232,7 +240,7 @@ func main() {
 			Recorder: mgr.GetEventRecorderFor("usage-profile-apply-controller"),
 			Cache:    mgr.GetCache(),
 
-			OrganizationLabel: conf.OrganizationLabel,
+			NamespaceSelector: nsSel,
 			Transformers: []transformers.Transformer{
 				transformers.NewResourceQuotaTransformer("resourcequota.appuio.io"),
 			},
@@ -249,7 +257,7 @@ func main() {
 			Scheme:   mgr.GetScheme(),
 			Recorder: mgr.GetEventRecorderFor("legacy-resource-quota-controller"),
 
-			OrganizationLabel:           conf.OrganizationLabel,
+			NamespaceSelector:           nsSel,
 			ResourceQuotaAnnotationBase: conf.LegacyResourceQuotaAnnotationBase,
 			DefaultResourceQuotas:       conf.LegacyDefaultResourceQuotas,
 			LimitRangeName:              conf.LegacyLimitRangeName,
@@ -280,6 +288,7 @@ func main() {
 			SkipValidateQuota: disableUsageProfiles && !legacyNamespaceQuotaEnabled,
 
 			OrganizationLabel:                 conf.OrganizationLabel,
+			NamespaceSelector:                 nsSel,
 			UserDefaultOrganizationAnnotation: conf.UserDefaultOrganizationAnnotation,
 
 			SelectedProfile:        selectedUsageProfile,
@@ -412,21 +421,22 @@ func registerNodeSelectorValidationWebhooks(mgr ctrl.Manager, conf Config) {
 	})
 }
 
-func registerOrganizationRBACController(mgr ctrl.Manager, orgLabel string, defaultClusterRoles map[string]string) {
+func registerOrganizationRBACController(mgr ctrl.Manager, orgLabel string, nsSel labels.Selector, defaultClusterRoles map[string]string) {
 	if err := (&controllers.OrganizationRBACReconciler{
 		Client:   mgr.GetClient(),
 		Recorder: mgr.GetEventRecorderFor("organization-rbac-controller"),
 		Scheme:   mgr.GetScheme(),
 
 		OrganizationLabel:   orgLabel,
+		NamespaceSelector:   nsSel,
 		DefaultClusterRoles: defaultClusterRoles,
 	}).SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "ratio")
+		setupLog.Error(err, "unable to create controller", "controller", "organization-rbac")
 		os.Exit(1)
 	}
 }
 
-func registerRatioController(mgr ctrl.Manager, conf Config, orgLabel string) {
+func registerRatioController(mgr ctrl.Manager, conf Config, nsSel labels.Selector) {
 	mgr.GetWebhookServer().Register("/validate-request-ratio", &webhook.Admission{
 		Handler: &webhooks.RatioValidator{
 			DefaultNodeSelector:                    conf.DefaultNodeSelector,
@@ -450,7 +460,7 @@ func registerRatioController(mgr ctrl.Manager, conf Config, orgLabel string) {
 		RatioLimits: conf.MemoryPerCoreLimits,
 		Ratio: &ratio.Fetcher{
 			Client:            mgr.GetClient(),
-			OrganizationLabel: orgLabel,
+			NamespaceSelector: nsSel,
 		},
 		RatioWarnThreshold: conf.MemoryPerCoreWarnThreshold,
 	}).SetupWithManager(mgr); err != nil {
@@ -479,3 +489,15 @@ func registerZoneK8sVersionController(mgr ctrl.Manager, controlAPICluster cluste
 		os.Exit(1)
 	}
 }
+
+func newNamespaceLabelSelector(orgLabel, unmanagedOrgNsLabel string) (labels.Selector, error) {
+	orgReq, err := labels.NewRequirement(orgLabel, selection.Exists, nil)
+	if err != nil {
+		return nil, err
+	}
+	unmanagedReq, err := labels.NewRequirement(unmanagedOrgNsLabel, selection.DoesNotExist, nil)
+	if err != nil {
+		return nil, err
+	}
+	return labels.NewSelector().Add(*orgReq, *unmanagedReq), nil
+}

ratio/fetcher.go

@@ -20,7 +20,7 @@ var ErrorDisabled error = errors.New("request ratio validation disabled")
 type Fetcher struct {
 	Client client.Client
 
-	OrganizationLabel string
+	NamespaceSelector labels.Selector
 }
 
 // FetchRatios collects the CPU to memory request ratio for the given namespace grouped by `.spec.nodeSelector`.
@@ -41,8 +41,8 @@ func (f Fetcher) FetchRatios(ctx context.Context, name string) (map[string]*Rati
 		}
 	}
 
-	if f.OrganizationLabel != "" {
-		if _, isOrgNs := ns.Labels[f.OrganizationLabel]; !isOrgNs {
+	if f.NamespaceSelector != nil {
+		if !f.NamespaceSelector.Matches(labels.Set(ns.Labels)) {
 			return nil, ErrorDisabled
 		}
 	}