feat: implement federated deployment scheduling across POP cells

Workloads targeting a city location are now automatically routed to the
correct physical site via a Karmada-based federation layer. Each POP cell
operates independently, instance health is surfaced back to the control
plane in real time, and the platform remains available even when parts of
the control plane are temporarily unreachable.

Controllers added:
- WorkloadDeploymentFederator: replicates WDs into Karmada and manages
  PropagationPolicies per city code
- InstanceProjector: mirrors Instance write-backs from Karmada into the
  project namespace on the control plane

ResourceInterpreterCustomization deployed at config time teaches Karmada
how to aggregate replica counts and conditions across POP cells.

Operator flags --enable-management-controllers and --enable-cell-controllers
allow each deployment to opt into only the controllers it needs.

Includes a 6-test Chainsaw e2e suite covering federation, deletion cascade,
propagation policy lifecycle, instance projection, instance write-back, and
the full end-to-end chain.

Resolves #85

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: scotwells

.github/workflows/publish.yaml

@@ -18,6 +18,7 @@ jobs:
     secrets: inherit
 
   publish-kustomize-bundles:
+    needs: publish-container-image
     permissions:
       id-token: write
       contents: read
@@ -26,4 +27,6 @@ jobs:
     with:
       bundle-name: ghcr.io/datum-cloud/compute-kustomize
       bundle-path: config
+      image-name: ghcr.io/datum-cloud/compute
+      image-overlays: config/base/manager
     secrets: inherit

.gitignore

@@ -25,3 +25,6 @@ go.work.sum
 .env
 
 bin/
+
+# Local e2e environment artefacts (Kind kubeconfigs, etc.)
+tmp/

Taskfile.yaml

@@ -2,8 +2,6 @@ package v1alpha
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha"
 )
 
 // WorkloadDeploymentSpec defines the desired state of WorkloadDeployment
@@ -37,11 +35,6 @@ type WorkloadDeploymentSpec struct {
 
 // WorkloadDeploymentStatus defines the observed state of WorkloadDeployment
 type WorkloadDeploymentStatus struct {
-	// The location which the deployment has been scheduled to
-	//
-	// +kubebuilder:validation:Optional
-	Location *networkingv1alpha.LocationReference `json:"location,omitempty"`
-
 	// Represents the observations of a deployment's current state.
 	// Known condition types are: "Available", "Progressing"
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
@@ -80,8 +73,6 @@ const (
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas`
 // +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas`
 // +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas`
-// +kubebuilder:printcolumn:name="Location Namespace",type=string,JSONPath=`.status.location.namespace`,priority=1
-// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name`,priority=1
 type WorkloadDeployment struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

api/v1alpha/workloaddeployment_types.go

@@ -18,17 +18,22 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager"
 	"sigs.k8s.io/multicluster-runtime/pkg/multicluster"
 	mcsingle "sigs.k8s.io/multicluster-runtime/providers/single"
 
+	karmadaclusterv1alpha1 "github.com/karmada-io/api/cluster/v1alpha1"
+	karmadapolicyv1alpha1 "github.com/karmada-io/api/policy/v1alpha1"
 	computev1alpha "go.datum.net/compute/api/v1alpha"
 	"go.datum.net/compute/internal/config"
 	"go.datum.net/compute/internal/controller"
@@ -51,6 +56,11 @@ var (
 	gitCommit    = "unknown"
 	gitTreeState = "unknown"
 	buildDate    = "unknown"
+
+	// downstreamRestConfig holds the REST config for the downstream control plane.
+	// It is populated from --downstream-kubeconfig when set, and is nil when the
+	// flag is omitted (e.g. in non-federation deployments).
+	downstreamRestConfig *rest.Config
 )
 
 func init() {
@@ -61,6 +71,8 @@ func init() {
 	utilruntime.Must(computev1alpha.AddToScheme(scheme))
 	utilruntime.Must(networkingv1alpha.AddToScheme(scheme))
 	utilruntime.Must(quotav1alpha1.AddToScheme(scheme))
+	utilruntime.Must(karmadapolicyv1alpha1.Install(scheme))
+	utilruntime.Must(karmadaclusterv1alpha1.Install(scheme))
 
 	// +kubebuilder:scaffold:scheme
 }
@@ -71,12 +83,26 @@ func main() {
 	var leaderElectionNamespace string
 	var probeAddr string
 	var serverConfigFile string
+	var downstreamKubeconfig string
+	var downstreamContext string
+	var enableManagementControllers bool
+	var enableCellControllers bool
 
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&leaderElectionNamespace, "leader-elect-namespace", "", "The namespace to use for leader election.")
+	flag.StringVar(&downstreamKubeconfig, "downstream-kubeconfig", "",
+		"Path to the kubeconfig file for the downstream control plane. When omitted, downstream federation features are disabled.")
+	flag.StringVar(&downstreamContext, "downstream-context", "",
+		"Context to use from the downstream kubeconfig. When omitted, the current context is used.")
+	flag.BoolVar(&enableManagementControllers, "enable-management-controllers", true,
+		"Enable management-plane controllers (WorkloadDeploymentFederator, InstanceProjector). "+
+			"Disable when running a cell-only operator instance.")
+	flag.BoolVar(&enableCellControllers, "enable-cell-controllers", true,
+		"Enable cell controllers (WorkloadDeploymentReconciler, InstanceReconciler). "+
+			"Disable when running a management-only operator instance.")
 
 	opts := zap.Options{
 		Development: true,
@@ -89,6 +115,23 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Load the downstream REST config when --downstream-kubeconfig is provided.
+	// When the flag is omitted, downstreamRestConfig remains nil and federation
+	// features will be skipped at controller setup time.
+	if downstreamKubeconfig != "" {
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+			&clientcmd.ClientConfigLoadingRules{ExplicitPath: downstreamKubeconfig},
+			&clientcmd.ConfigOverrides{CurrentContext: downstreamContext},
+		)
+		var err error
+		downstreamRestConfig, err = loader.ClientConfig()
+		if err != nil {
+			setupLog.Error(err, "unable to load downstream kubeconfig", "path", downstreamKubeconfig)
+			os.Exit(1)
+		}
+		setupLog.Info("downstream kubeconfig loaded", "path", downstreamKubeconfig)
+	}
+
 	setupLog.Info("starting compute",
 		"version", version,
 		"gitCommit", gitCommit,
@@ -180,17 +223,62 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "Workload")
 		os.Exit(1)
 	}
-	if err = (&controller.WorkloadDeploymentReconciler{}).SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment")
-		os.Exit(1)
+
+	// Build a single downstream client shared across all controllers that need
+	// to read or write to the downstream control plane. Nil when federation is disabled.
+	var downstreamClient client.Client
+	if downstreamRestConfig != nil {
+		downstreamClient, err = client.New(downstreamRestConfig, client.Options{Scheme: scheme})
+		if err != nil {
+			setupLog.Error(err, "unable to create downstream client")
+			os.Exit(1)
+		}
 	}
-	if err = (&controller.WorkloadDeploymentScheduler{}).SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeploymentScheduler")
-		os.Exit(1)
+
+	if enableCellControllers {
+		if err = (&controller.WorkloadDeploymentReconciler{}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment")
+			os.Exit(1)
+		}
 	}
-	if err = (&controller.InstanceReconciler{}).SetupWithManager(mgr, deploymentCluster); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "Instance")
-		os.Exit(1)
+
+	if enableCellControllers {
+		instanceReconciler := &controller.InstanceReconciler{DownstreamClient: downstreamClient}
+		if err = instanceReconciler.SetupWithManager(mgr, deploymentCluster); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "Instance")
+			os.Exit(1)
+		}
+	}
+
+	// WorkloadDeploymentFederator and InstanceProjector are management-plane
+	// controllers that run on the control-plane cluster. They require a downstream
+	// control plane to be configured (--downstream-kubeconfig provided).
+	if enableManagementControllers && downstreamRestConfig != nil {
+		if err = (&controller.WorkloadDeploymentFederator{DownstreamClient: downstreamClient}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeploymentFederator")
+			os.Exit(1)
+		}
+
+		// InstanceProjector: runs in the Control Plane Cell, watches Instances
+		// written back to the downstream control plane by POP-cell operators, and
+		// projects them into the corresponding project namespaces via the
+		// multicluster manager.
+		downstreamMgr, err := manager.New(downstreamRestConfig, manager.Options{
+			Scheme:  scheme,
+			Metrics: metricsserver.Options{BindAddress: "0"},
+		})
+		if err != nil {
+			setupLog.Error(err, "unable to create downstream manager for InstanceProjector")
+			os.Exit(1)
+		}
+		if err = (&controller.InstanceProjector{
+			DownstreamClient: downstreamClient,
+			MCManager:        mgr,
+		}).SetupWithManager(downstreamMgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "InstanceProjector")
+			os.Exit(1)
+		}
+		runnables = append(runnables, downstreamMgr)
 	}
 
 	if serverConfig.WebhookServer != nil {
@@ -284,6 +372,7 @@ func initializeClusterDiscovery(
 		}
 
 		discoveryManager, err := manager.New(discoveryRestConfig, manager.Options{
+			Metrics: metricsserver.Options{BindAddress: "0"},
 			Client: client.Options{
 				Cache: &client.CacheOptions{
 					Unstructured: true,

api/v1alpha/zz_generated.deepcopy.go

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - rbac.yaml

cmd/main.go

@@ -0,0 +1,32 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: compute-manager
+rules:
+  - apiGroups: ["compute.datumapis.com"]
+    resources: ["workloaddeployments", "workloaddeployments/status", "instances", "instances/status"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["policy.karmada.io"]
+    resources: ["propagationpolicies", "clusterpropagationpolicies"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["cluster.karmada.io"]
+    resources: ["clusters"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["work.karmada.io"]
+    resources: ["resourcebindings", "clusterresourcebindings"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["config.karmada.io"]
+    resources: ["resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: compute-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: compute-manager
+subjects:
+  - kind: User
+    name: system:serviceaccount:compute-system:compute-manager

config/base/downstream-rbac/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../crd/bases/compute.datumapis.com_instances.yaml
+  - ../crd/bases/compute.datumapis.com_workloaddeployments.yaml
+  - ../crd/bases/compute.datumapis.com_workloads.yaml
+
+components:
+  - ../../components/federation

config/base/downstream-rbac/rbac.yaml

@@ -26,14 +26,30 @@ spec:
         seccompProfile:
           type: RuntimeDefault
       containers:
-      - command:
+      - name: manager
+        command:
         - /manager
         args:
-          - --leader-elect
-          - --health-probe-bind-address=:8081
-          - --server-config=/config/config.yaml
+          - --leader-elect=$(LEADER_ELECT)
+          - --health-probe-bind-address=$(HEALTH_PROBE_BIND_ADDRESS)
+          - --server-config=$(SERVER_CONFIG)
+          - --downstream-kubeconfig=$(DOWNSTREAM_KUBECONFIG)
+          - --enable-management-controllers=$(ENABLE_MANAGEMENT_CONTROLLERS)
+          - --enable-cell-controllers=$(ENABLE_CELL_CONTROLLERS)
+        env:
+          - name: LEADER_ELECT
+            value: "true"
+          - name: HEALTH_PROBE_BIND_ADDRESS
+            value: ":8081"
+          - name: SERVER_CONFIG
+            value: /config/config.yaml
+          - name: DOWNSTREAM_KUBECONFIG
+            value: ""
+          - name: ENABLE_MANAGEMENT_CONTROLLERS
+            value: "false"
+          - name: ENABLE_CELL_CONTROLLERS
+            value: "false"
         image: ghcr.io/datum-cloud/compute:latest
-        name: manager
         ports:
         - containerPort: 9443
           name: webhook-server
@@ -69,7 +85,7 @@ spec:
         - name: webhook-cert
           mountPath: /tmp/k8s-webhook-server/serving-certs
           readOnly: true
-      serviceAccountName: compute
+      serviceAccountName: compute-manager
       terminationGracePeriodSeconds: 10
       volumes:
       - name: config