feat: migrating to validating admission policies for our validating logic (2/): enable the VAP manager

charts/hub-agent/README.md

@@ -122,6 +122,10 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen
 | `resourceSnapshotCreationMinimumInterval` | The minimum interval at which resource snapshots could be created.                         | `30s`                                            |
 | `resourceChangesCollectionDuration`       | The duration for collecting resource changes into one snapshot.                            | `15s`                                            |
 | `enableWorkload`                          | Enable kubernetes builtin workload to run in hub cluster.                                  | `false`                                          |
+| `additionalConfigData`                    | Additional key-value data to include in the hub-agent ConfigMap.                           | `{}`                                             |
+| `additionalConfigDataMountPath`           | Mount path for the additional config data volume.                                          | `/etc/kubefleet/additional-config`               |
+| `enableAdmissionPolicyManager`            | Enable the admission policy manager to enforce VAP-based policies on the hub cluster.      | `false`                                          |
+| `admissionPolicyManagerConfigName`        | Name of the ConfigMap containing the admission policy manager configuration.               | `""`                                             |
 
 ## Certificate Management
 

charts/hub-agent/templates/config.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.additionalConfigData }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "hub-agent.fullname" . }}-config
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- include "hub-agent.labels" . | nindent 4 }}
+data:
+  {{- .Values.additionalConfigData | toYaml | nindent 2 }}
+{{- end }}

charts/hub-agent/templates/deployment.yaml

@@ -62,6 +62,10 @@ spec:
             - --cluster-unhealthy-threshold={{ .Values.clusterUnhealthyThreshold }}
             - --resource-snapshot-creation-minimum-interval={{ .Values.resourceSnapshotCreationMinimumInterval }}
             - --resource-changes-collection-duration={{ .Values.resourceChangesCollectionDuration }}
+            - --enable-admission-policy-manager={{ .Values.enableAdmissionPolicyManager }}
+            {{- if and .Values.additionalConfigData .Values.admissionPolicyManagerConfigName }}
+            - --admission-policy-manager-config={{ .Values.additionalConfigDataMountPath }}/{{ .Values.admissionPolicyManagerConfigName }}
+            {{- end }}
           ports:
             - name: metrics
               containerPort: 8080
@@ -96,6 +100,11 @@ spec:
             # This path must match FleetWebhookCertDir in pkg/webhook/webhook.go
             mountPath: /tmp/k8s-webhook-server/serving-certs
             readOnly: true
+          {{- if .Values.additionalConfigData }}
+          - name: additional-config
+            mountPath: {{ .Values.additionalConfigDataMountPath }}
+            readOnly: true
+          {{- end }}
           {{- else }}
           volumeMounts:
           - name: webhook-cert
@@ -104,6 +113,11 @@ spec:
             # the read only root filesystem setup would block the agent from attempting to
             # clear the directory.
             mountPath: /tmp/k8s-webhook-server/
+          {{- if .Values.additionalConfigData }}
+          - name: additional-config
+            mountPath: {{ .Values.additionalConfigDataMountPath }}
+            readOnly: true
+          {{- end }}
           {{- end }}
       volumes:
       - name: webhook-cert
@@ -116,6 +130,11 @@ spec:
         {{- else }}
         emptyDir: {}
         {{- end }}
+      {{- if .Values.additionalConfigData }}
+      - name: additional-config
+        configMap:
+          name: {{ include "hub-agent.fullname" . }}-config
+      {{- end }}
       {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

charts/hub-agent/templates/rbac.yaml

@@ -139,6 +139,14 @@ rules:
       - fleet-validating-webhook-configuration
       - fleet-guard-rail-webhook-configuration
     verbs: ["delete"]
+
+  # Admission policies management. If the admission policy manager is enabled, the hub agent
+  # will install/uninstall validating admission policies and bindings based on user-provided configuration.
+  {{- if .Values.enableAdmissionPolicyManager }}
+  - apiGroups: ["admissionregistration.k8s.io"]
+    resources: ["validatingadmissionpolicies", "validatingadmissionpolicybindings"]
+    verbs: ["get", "list", "create", "update", "patch", "delete"]
+  {{- end }}
 
   # Leader election. Split into two rules so that only the hub-agent's own
   # election lease can be mutated; a compromised hub-agent cannot hijack

charts/hub-agent/values.yaml

@@ -57,3 +57,9 @@ hubAPIBurst: 1000
 MaxConcurrentClusterPlacement: 100
 ConcurrentResourceChangeSyncs: 20
 MaxFleetSizeSupported: 100
+
+additionalConfigData: {}
+additionalConfigDataMountPath: /etc/kubefleet/additional-config
+
+enableAdmissionPolicyManager: false
+admissionPolicyManagerConfigName: ""

cmd/hubagent/main.go

@@ -27,12 +27,14 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/yaml"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	clusterinventory "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -46,6 +48,7 @@ import (
 	placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
 	"github.com/kubefleet-dev/kubefleet/cmd/hubagent/options"
 	"github.com/kubefleet-dev/kubefleet/cmd/hubagent/workload"
+	"github.com/kubefleet-dev/kubefleet/pkg/admissionpolicymanager"
 	mcv1beta1 "github.com/kubefleet-dev/kubefleet/pkg/controllers/membercluster/v1beta1"
 	readiness "github.com/kubefleet-dev/kubefleet/pkg/utils/informer/readiness"
 	"github.com/kubefleet-dev/kubefleet/pkg/utils/validator"
@@ -160,7 +163,7 @@ func main() {
 		exitWithErrorFunc()
 	}
 
-	klog.V(2).InfoS("starting hubagent")
+	klog.V(2).InfoS("starting hub agent")
 	if opts.FeatureFlags.EnableV1Beta1APIs {
 		klog.Info("Setting up memberCluster v1beta1 controller")
 		if err = (&mcv1beta1.Reconciler{
@@ -183,7 +186,7 @@ func main() {
 		exitWithErrorFunc()
 	}
 
-	if opts.WebhookOpts.EnableWebhooks {
+	if opts.WebhookAndAdmissionPolicyOpts.EnableWebhooks {
 		// Generate webhook configuration with certificates
 		webhookConfig, err := webhook.NewWebhookConfigFromOptions(mgr, opts, FleetWebhookPort)
 		if err != nil {
@@ -208,7 +211,7 @@ func main() {
 		// When using cert-manager, add a readiness check to ensure CA bundles are injected before marking ready.
 		// This prevents the pod from accepting traffic before cert-manager has populated the webhook CA bundles,
 		// which would cause webhook calls to fail.
-		if opts.WebhookOpts.UseCertManager {
+		if opts.WebhookAndAdmissionPolicyOpts.UseCertManager {
 			if err := mgr.AddReadyzCheck("cert-manager-ca-injection", func(req *http.Request) error {
 				return webhookConfig.CheckCAInjection(req.Context())
 			}); err != nil {
@@ -220,6 +223,45 @@ func main() {
 	}
 
 	ctx := ctrl.SetupSignalHandler()
+
+	if opts.WebhookAndAdmissionPolicyOpts.EnableAdmissionPolicyManager {
+		policyManagerCfgs := admissionpolicymanager.DefaultPolicyGeneratorConfigs
+		if len(opts.WebhookAndAdmissionPolicyOpts.AdmissionPolicyManagerConfig) != 0 {
+			// Read user-provided admission policy manager config from given path.
+			cfgData, err := os.ReadFile(opts.WebhookAndAdmissionPolicyOpts.AdmissionPolicyManagerConfig)
+			if err != nil {
+				klog.ErrorS(err, "failed to read the admission policy manager config file")
+				exitWithErrorFunc()
+			}
+
+			policyManagerCfgs = &admissionpolicymanager.PolicyGeneratorConfigs{}
+			if err := yaml.Unmarshal(cfgData, policyManagerCfgs); err != nil {
+				klog.ErrorS(err, "failed to unmarshal the admission policy manager config file")
+				exitWithErrorFunc()
+			}
+
+			// Note that validation has been performed when the flags are parsed.
+		}
+
+		// Create a separate client for the admission policy manager to use, as the cached client from
+		// the controller manager side is not initialized yet at this point.
+		hubUncachedClient, err := client.New(defaultCfg, client.Options{Scheme: scheme})
+		if err != nil {
+			klog.ErrorS(err, "failed to create uncached client for the admission policy manager")
+			exitWithErrorFunc()
+		}
+		policyMgr, err := admissionpolicymanager.New(hubUncachedClient, policyManagerCfgs)
+		if err != nil {
+			klog.ErrorS(err, "failed to create the admission policy manager")
+			exitWithErrorFunc()
+		}
+
+		if err := policyMgr.Start(ctx); err != nil {
+			klog.ErrorS(err, "failed to start the admission policy manager")
+			exitWithErrorFunc()
+		}
+	}
+
 	if err := workload.SetupControllers(ctx, &wg, mgr, defaultCfg, opts); err != nil {
 		klog.ErrorS(err, "unable to set up controllers")
 		exitWithErrorFunc()

cmd/hubagent/options/options.go

@@ -28,8 +28,8 @@ type Options struct {
 	// Options that concern the setup of the controller manager instance in use by the KubeFleet hub agent.
 	CtrlMgrOpts ControllerManagerOptions
 
-	// KubeFleet webhook related options.
-	WebhookOpts WebhookOptions
+	// KubeFleet webhook and admission policy related options.
+	WebhookAndAdmissionPolicyOpts WebhookAndAdmissionPolicyOptions
 
 	// Feature flags that control the enabling of certain features in the hub agent.
 	FeatureFlags FeatureFlags
@@ -48,7 +48,7 @@ func NewOptions() *Options {
 func (o *Options) AddFlags(flags *flag.FlagSet) {
 	o.LeaderElectionOpts.AddFlags(flags)
 	o.CtrlMgrOpts.AddFlags(flags)
-	o.WebhookOpts.AddFlags(flags)
+	o.WebhookAndAdmissionPolicyOpts.AddFlags(flags)
 	o.FeatureFlags.AddFlags(flags)
 	o.ClusterMgmtOpts.AddFlags(flags)
 	o.PlacementMgmtOpts.AddFlags(flags)

cmd/hubagent/options/options_test.go

@@ -706,21 +706,22 @@ func TestPlacementManagementOptions(t *testing.T) {
 	}
 }
 
-// TestWebhookOptions tests the parsing and validation logic of the webhook options defined in WebhookOptions.
+// TestWebhookAndAdmissionPolicyOptions tests the parsing and validation logic of the webhook
+// options defined in WebhookAndAdmissionPolicyOptions.
 func TestWebhookOptions(t *testing.T) {
 	testCases := []struct {
 		name             string
 		flagSetName      string
 		args             []string
-		wantWebhookOpts  WebhookOptions
+		wantWebhookOpts  WebhookAndAdmissionPolicyOptions
 		wantErred        bool
 		wantErrMsgSubStr string
 	}{
 		{
 			name:        "all default",
 			flagSetName: "allDefault",
 			args:        []string{},
-			wantWebhookOpts: WebhookOptions{
+			wantWebhookOpts: WebhookAndAdmissionPolicyOptions{
 				EnableWebhooks:                         true,
 				ClientConnectionType:                   "url",
 				ServiceName:                            "fleetwebhook",
@@ -730,6 +731,8 @@ func TestWebhookOptions(t *testing.T) {
 				EnableWorkload:                         false,
 				EnablePDBs:                             true,
 				UseCertManager:                         false,
+				EnableAdmissionPolicyManager:           false,
+				AdmissionPolicyManagerConfig:           "",
 			},
 		},
 		{
@@ -744,8 +747,10 @@ func TestWebhookOptions(t *testing.T) {
 				"--deny-modify-member-cluster-labels=true",
 				"--enable-workload=true",
 				"--use-cert-manager=true",
+				"--enable-admission-policy-manager=true",
+				"--admission-policy-manager-config=/etc/fleet/policy-config.json",
 			},
-			wantWebhookOpts: WebhookOptions{
+			wantWebhookOpts: WebhookAndAdmissionPolicyOptions{
 				EnableWebhooks:                         false,
 				ClientConnectionType:                   "service",
 				ServiceName:                            "customwebhook",
@@ -755,13 +760,15 @@ func TestWebhookOptions(t *testing.T) {
 				EnableWorkload:                         true,
 				EnablePDBs:                             true,
 				UseCertManager:                         true,
+				EnableAdmissionPolicyManager:           true,
+				AdmissionPolicyManagerConfig:           "/etc/fleet/policy-config.json",
 			},
 		},
 		{
 			name:        "webhook client connection type URL (case-insensitive)",
 			flagSetName: "webhookClientConnTypeURL",
 			args:        []string{"--webhook-client-connection-type=URL"},
-			wantWebhookOpts: WebhookOptions{
+			wantWebhookOpts: WebhookAndAdmissionPolicyOptions{
 				EnableWebhooks:                         true,
 				ClientConnectionType:                   "url",
 				ServiceName:                            "fleetwebhook",
@@ -771,13 +778,15 @@ func TestWebhookOptions(t *testing.T) {
 				EnableWorkload:                         false,
 				EnablePDBs:                             true,
 				UseCertManager:                         false,
+				EnableAdmissionPolicyManager:           false,
+				AdmissionPolicyManagerConfig:           "",
 			},
 		},
 		{
 			name:        "webhook client connection type service (case-insensitive)",
 			flagSetName: "webhookClientConnTypeService",
 			args:        []string{"--webhook-client-connection-type=Service"},
-			wantWebhookOpts: WebhookOptions{
+			wantWebhookOpts: WebhookAndAdmissionPolicyOptions{
 				EnableWebhooks:                         true,
 				ClientConnectionType:                   "service",
 				ServiceName:                            "fleetwebhook",
@@ -787,6 +796,8 @@ func TestWebhookOptions(t *testing.T) {
 				EnableWorkload:                         false,
 				EnablePDBs:                             true,
 				UseCertManager:                         false,
+				EnableAdmissionPolicyManager:           false,
+				AdmissionPolicyManagerConfig:           "",
 			},
 		},
 		{
@@ -801,8 +812,8 @@ func TestWebhookOptions(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			flags := flag.NewFlagSet(tc.flagSetName, flag.ContinueOnError)
-			webhookOpts := WebhookOptions{}
-			webhookOpts.AddFlags(flags)
+			webhookAndAdmissionPolicyOpts := WebhookAndAdmissionPolicyOptions{}
+			webhookAndAdmissionPolicyOpts.AddFlags(flags)
 
 			err := flags.Parse(tc.args)
 			if tc.wantErred {
@@ -820,7 +831,7 @@ func TestWebhookOptions(t *testing.T) {
 				t.Fatalf("flag Parse() = %v, want nil", err)
 			}
 
-			if diff := cmp.Diff(webhookOpts, tc.wantWebhookOpts); diff != "" {
+			if diff := cmp.Diff(webhookAndAdmissionPolicyOpts, tc.wantWebhookOpts); diff != "" {
 				t.Errorf("webhook options diff (-got, +want):\n%s", diff)
 			}
 		})

cmd/hubagent/options/validation.go

@@ -17,7 +17,12 @@ limitations under the License.
 package options
 
 import (
+	"os"
+
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apimachinery/pkg/util/yaml"
+
+	"github.com/kubefleet-dev/kubefleet/pkg/admissionpolicymanager"
 )
 
 // Validate checks Options and return a slice of found errs.
@@ -45,12 +50,12 @@ func (o *Options) Validate() field.ErrorList {
 	// but here the logic enforces that a service name must be provided. The way we handle
 	// URLs is also problematic as the code will always format a service-targeted URL using
 	// the input. We keep this logic for now for compatibility reasons.
-	if o.WebhookOpts.EnableWebhooks && o.WebhookOpts.ServiceName == "" {
-		errs = append(errs, field.Invalid(newPath.Child("WebhookServiceName"), o.WebhookOpts.ServiceName, "A webhook service name is required when webhooks are enabled"))
+	if o.WebhookAndAdmissionPolicyOpts.EnableWebhooks && o.WebhookAndAdmissionPolicyOpts.ServiceName == "" {
+		errs = append(errs, field.Invalid(newPath.Child("WebhookServiceName"), o.WebhookAndAdmissionPolicyOpts.ServiceName, "A webhook service name is required when webhooks are enabled"))
 	}
 
-	if o.WebhookOpts.UseCertManager && !o.WebhookOpts.EnableWorkload {
-		errs = append(errs, field.Invalid(newPath.Child("UseCertManager"), o.WebhookOpts.UseCertManager, "If cert manager is used for securing webhook connections, the EnableWorkload option must be set to true, so that cert manager pods can run in the hub cluster."))
+	if o.WebhookAndAdmissionPolicyOpts.UseCertManager && !o.WebhookAndAdmissionPolicyOpts.EnableWorkload {
+		errs = append(errs, field.Invalid(newPath.Child("UseCertManager"), o.WebhookAndAdmissionPolicyOpts.UseCertManager, "If cert manager is used for securing webhook connections, the EnableWorkload option must be set to true, so that cert manager pods can run in the hub cluster."))
 	}
 
 	if o.PlacementMgmtOpts.AllowedPropagatingAPIs != "" && o.PlacementMgmtOpts.SkippedPropagatingAPIs != "" {
@@ -66,5 +71,33 @@ func (o *Options) Validate() field.ErrorList {
 		errs = append(errs, field.Invalid(newPath.Child("PlacementControllerWorkQueueRateLimiterOpts").Child("RateLimiterQPS"), o.PlacementMgmtOpts.PlacementControllerWorkQueueRateLimiterOpts.RateLimiterQPS, "the QPS for the placement controller set rate limiter must be less than its bucket size"))
 	}
 
+	// Validate admission policy manager setup (if enabled).
+	if err := o.validateAdmissionPolicyManagerConfig(newPath); err != nil {
+		errs = append(errs, err)
+	}
+
 	return errs
 }
+
+func (o *Options) validateAdmissionPolicyManagerConfig(newPath *field.Path) *field.Error {
+	if o.WebhookAndAdmissionPolicyOpts.EnableAdmissionPolicyManager {
+		managerConfigPath := o.WebhookAndAdmissionPolicyOpts.AdmissionPolicyManagerConfig
+		if len(managerConfigPath) != 0 {
+			// Read the file from the path.
+			data, err := os.ReadFile(managerConfigPath)
+			if err != nil {
+				return field.Invalid(newPath.Child("AdmissionPolicyManagerConfig"), managerConfigPath, "failed to read the admission policy manager config file: "+err.Error())
+			}
+
+			policyGeneratorConfigs := &admissionpolicymanager.PolicyGeneratorConfigs{}
+			if err := yaml.Unmarshal(data, policyGeneratorConfigs); err != nil {
+				return field.Invalid(newPath.Child("AdmissionPolicyManagerConfig"), managerConfigPath, "failed to unmarshal the admission policy manager config file: "+err.Error())
+			}
+
+			if err := policyGeneratorConfigs.Validate(); err != nil {
+				return field.Invalid(newPath.Child("AdmissionPolicyManagerConfig"), managerConfigPath, "invalid admission policy manager config: "+err.Error())
+			}
+		}
+	}
+	return nil
+}