Feat: Add Multi-Tenant Manager to CCM

cmd/cloud-controller-manager/BUILD

@@ -20,27 +20,36 @@ go_library(
     srcs = [
         "gkenetworkparamsetcontroller.go",
         "gkeservicecontroller.go",
+        "gketenantcontrollermanager.go",
         "main.go",
         "nodeipamcontroller.go",
     ],
     importpath = "k8s.io/cloud-provider-gcp/cmd/cloud-controller-manager",
     deps = [
         "//cmd/cloud-controller-manager/options",
         "//pkg/controller/gkenetworkparamset",
+        "//pkg/controller/gketenantcontrollers",
         "//pkg/controller/nodeipam",
         "//pkg/controller/nodeipam/config",
         "//pkg/controller/nodeipam/ipam",
         "//pkg/controller/service",
         "//providers/gce",
+        "//vendor/github.com/GoogleCloudPlatform/gke-enterprise-mt/apis/providerconfig/v1:providerconfig",
+        "//vendor/github.com/GoogleCloudPlatform/gke-enterprise-mt/pkg/framework",
         "//vendor/github.com/GoogleCloudPlatform/gke-networking-api/client/network/clientset/versioned",
         "//vendor/github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions",
         "//vendor/github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/clientset/versioned",
         "//vendor/github.com/spf13/pflag",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema",
         "//vendor/k8s.io/apimachinery/pkg/util/wait",
         "//vendor/k8s.io/apiserver/pkg/util/feature",
+        "//vendor/k8s.io/client-go/dynamic",
+        "//vendor/k8s.io/client-go/dynamic/dynamicinformer",
         "//vendor/k8s.io/cloud-provider",
         "//vendor/k8s.io/cloud-provider/app",
         "//vendor/k8s.io/cloud-provider/app/config",
+        "//vendor/k8s.io/cloud-provider/controllers/node",
+        "//vendor/k8s.io/cloud-provider/controllers/nodelifecycle",
         "//vendor/k8s.io/cloud-provider/names",
         "//vendor/k8s.io/cloud-provider/options",
         "//vendor/k8s.io/component-base/cli/flag",
@@ -51,7 +60,6 @@ go_library(
         "//vendor/k8s.io/controller-manager/controller",
         "//vendor/k8s.io/klog/v2:klog",
         "//vendor/k8s.io/kubernetes/cmd/kube-controller-manager/names",
-        "//vendor/k8s.io/utils/net",
     ],
 )
 
@@ -63,16 +71,22 @@ go_test(
     name = "cloud-controller-manager_test",
     srcs = [
         "gkeservicecontroller_test.go",
+        "gketenantcontrollermanager_test.go",
         "nodeipamcontroller_test.go",
     ],
     embed = [":cloud-controller-manager_lib"],
     deps = [
         "//pkg/controller/nodeipam/config",
         "//pkg/controller/service",
         "//vendor/k8s.io/api/core/v1:core",
+        "//vendor/k8s.io/client-go/informers",
+        "//vendor/k8s.io/client-go/kubernetes/fake",
+        "//vendor/k8s.io/client-go/rest",
         "//vendor/k8s.io/cloud-provider",
+        "//vendor/k8s.io/cloud-provider/app",
         "//vendor/k8s.io/cloud-provider/app/config",
         "//vendor/k8s.io/cloud-provider/config",
         "//vendor/k8s.io/controller-manager/app",
+        "//vendor/k8s.io/controller-manager/pkg/clientbuilder",
     ],
 )

cmd/cloud-controller-manager/gkenetworkparamsetcontroller.go

@@ -10,6 +10,7 @@ import (
 	networkinformers "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
 	cloudprovider "k8s.io/cloud-provider"
 	gkenetworkparamsetcontroller "k8s.io/cloud-provider-gcp/pkg/controller/gkenetworkparamset"
+	nodeipam "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam"
 	"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/ipam"
 	"k8s.io/cloud-provider-gcp/providers/gce"
 	"k8s.io/cloud-provider/app"
@@ -73,7 +74,7 @@ func startGkeNetworkParamsController(ccmConfig *cloudcontrollerconfig.CompletedC
 // with stack type and returns a list of typed cidrs and error
 func validClusterCIDR(clusterCIDRFromFlag string) ([]*net.IPNet, error) {
 	// failure: bad cidrs in config
-	clusterCIDRs, dualStack, err := processCIDRs(clusterCIDRFromFlag)
+	clusterCIDRs, dualStack, err := nodeipam.ProcessCIDRs(clusterCIDRFromFlag)
 	if err != nil {
 		return nil, err
 	}

cmd/cloud-controller-manager/gketenantcontrollermanager.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"context"
+
+	providerconfigv1 "github.com/GoogleCloudPlatform/gke-enterprise-mt/apis/providerconfig/v1"
+	"github.com/GoogleCloudPlatform/gke-enterprise-mt/pkg/framework"
+	networkclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/network/clientset/versioned"
+	networkinformers "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
+	topologyclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/clientset/versioned"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	dynamicinformer "k8s.io/client-go/dynamic/dynamicinformer"
+	cloudprovider "k8s.io/cloud-provider"
+	nodeipamcontrolleroptions "k8s.io/cloud-provider-gcp/cmd/cloud-controller-manager/options"
+	"k8s.io/cloud-provider-gcp/pkg/controller/gketenantcontrollers"
+	nodeipamconfig "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/config"
+	"k8s.io/cloud-provider/app"
+	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
+	controllermanagerapp "k8s.io/controller-manager/app"
+	"k8s.io/controller-manager/controller"
+	"k8s.io/klog/v2"
+
+	"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam"
+	"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/ipam"
+	"k8s.io/cloud-provider/controllers/node"
+	"k8s.io/cloud-provider/controllers/nodelifecycle"
+)
+
+// startGKETenantControllerManagerWrapper is used to take cloud config as input and start the GKE TenantControllerManager controller
+func startGKETenantControllerManagerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions) app.InitFunc {
+	return func(ctx context.Context, controllerContext controllermanagerapp.ControllerContext) (controller.Interface, bool, error) {
+		return startGKETenantControllerManager(ctx, initContext, controllerContext, completedConfig, cloud, *nodeIPAMControllerOptions.NodeIPAMControllerConfiguration)
+	}
+}
+
+func startGKETenantControllerManager(ctx context.Context, initContext app.ControllerInitContext, controlexContext controllermanagerapp.ControllerContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration) (controller.Interface, bool, error) {
+	if !enableProviderConfigController {
+		klog.Infof("GKE Tenant Controller Manager is disabled (enable with --enable-provider-config-controller)")
+		return nil, false, nil
+	}
+
+	clientConfig := completedConfig.Kubeconfig
+
+	// Create network clients and informers
+	networkClient, err := networkclientset.NewForConfig(clientConfig)
+	if err != nil {
+		klog.Errorf("Failed to create network client: %v", err)
+		return nil, false, err
+	}
+	networkInformerFactory := networkinformers.NewSharedInformerFactory(networkClient, 0)
+	networkInformer := networkInformerFactory.Networking().V1().Networks()
+	gnpInformer := networkInformerFactory.Networking().V1().GKENetworkParamSets()
+
+	// Create topology client
+	nodeTopologyClient, err := topologyclientset.NewForConfig(clientConfig)
+	if err != nil {
+		klog.Errorf("Failed to create topology client: %v", err)
+		return nil, false, err
+	}
+
+	// Eagerly request the main Node informer so the SharedInformerFactory starts it.
+	// If the tenant controller manager is the only enabled controller, the informer
+	// factory won't start the node cache unless we explicitly ask for it here.
+	_ = controlexContext.InformerFactory.Core().V1().Nodes().Informer()
+
+	// Create dynamic client for framework
+	dynamicClient, err := dynamic.NewForConfig(clientConfig)
+	if err != nil {
+		klog.Errorf("Failed to create dynamic client: %v", err)
+		return nil, false, err
+	}
+
+	// Create dynamic informer factory for ProviderConfig
+	gvr := schema.GroupVersionResource{
+		Group:    providerconfigv1.GroupName,
+		Version:  providerconfigv1.SchemeGroupVersion.Version,
+		Resource: "providerconfigs",
+	}
+	dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0)
+	providerConfigInformer := dynamicInformerFactory.ForResource(gvr).Informer()
+
+	// Define controllers
+	controllers := map[string]gketenantcontrollers.ControllerStartFunc{
+		"node-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
+			klog.Infof("Creating OSS Cloud Node Controller for %s...", cfg.ProviderConfig.Name)
+			nodeController, err := node.NewCloudNodeController(
+				cfg.NodeInformer,
+				cfg.KubeClient,
+				cfg.Cloud,
+				completedConfig.ComponentConfig.NodeStatusUpdateFrequency.Duration,
+				completedConfig.ComponentConfig.NodeController.ConcurrentNodeSyncs,
+			)
+			if err != nil {
+				return err
+			}
+			klog.Infof("Starting OSS Cloud Node Controller for %s (blocking)", cfg.ProviderConfig.Name)
+			nodeController.Run(cfg.Context.Done(), cfg.ControllerContext.ControllerManagerMetrics)
+			return nil
+		},
+		"node-ipam-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
+			klog.Infof("Starting Node IPAM Controller for %s...", cfg.ProviderConfig.Name)
+			clusterCIDR, err := gketenantcontrollers.GetClusterCIDRsFromProviderConfig(cfg.ProviderConfig)
+			if err != nil {
+				klog.Errorf("Failed to get ClusterCIDRs from ProviderConfig: %v. Node IPAM Controller will be disabled.", err)
+				return nil // Don't fail the whole start
+			}
+
+			_, started, err := nodeipam.StartNodeIpamController(
+				cfg.Context,
+				cfg.NodeInformer,
+				cfg.KubeClient,
+				cfg.Cloud,
+				clusterCIDR,
+				completedConfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs,
+				nodeIPAMConfig.ServiceCIDR,
+				nodeIPAMConfig.SecondaryServiceCIDR,
+				nodeIPAMConfig,
+				networkInformer,
+				gnpInformer,
+				nodeTopologyClient,
+				ipam.CIDRAllocatorType(completedConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
+				cfg.ControllerContext.ControllerManagerMetrics,
+			)
+			if err != nil {
+				return err
+			}
+			if !started {
+				klog.Infof("Node IPAM Controller not started (disabled in config) for %s", cfg.ProviderConfig.Name)
+			} else {
+				klog.Infof("Node IPAM Controller started with ClusterCIDR: %s for %s", clusterCIDR, cfg.ProviderConfig.Name)
+			}
+			// Block until context is canceled so starter doesn't exit early
+			<-cfg.Context.Done()
+			return nil
+		},
+		"node-lifecycle-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
+			klog.Infof("Creating Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
+			nodeMonitorPeriod := completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration
+			lifecycleController, err := nodelifecycle.NewCloudNodeLifecycleController(
+				cfg.NodeInformer,
+				cfg.KubeClient,
+				cfg.Cloud,
+				nodeMonitorPeriod,
+			)
+			if err != nil {
+				return err
+			}
+			klog.Infof("Starting Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
+			lifecycleController.Run(cfg.Context, cfg.ControllerContext.ControllerManagerMetrics)
+			return nil
+		},
+	}
+
+	// Create the starter
+	starter := gketenantcontrollers.NewNodeControllerStarter(
+		completedConfig.ClientBuilder,
+		completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName),
+		controlexContext.InformerFactory,
+		completedConfig,
+		controlexContext,
+		controllers,
+	)
+
+	// Create the framework manager
+	mgr := framework.New(
+		dynamicClient,
+		providerConfigInformer,
+		gkeTenantControllerManagerName,
+		starter,
+		ctx.Done(),
+	)
+
+	// Start network informers
+	networkInformerFactory.Start(ctx.Done())
+	// Start dynamic informers
+	dynamicInformerFactory.Start(ctx.Done())
+
+	// Run the manager
+	go mgr.Run()
+
+	return nil, true, nil
+}

cmd/cloud-controller-manager/gketenantcontrollermanager_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/rest"
+	nodeipamconfig "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/config"
+	"k8s.io/cloud-provider/app"
+	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
+	genericcontrollermanager "k8s.io/controller-manager/app"
+	"k8s.io/controller-manager/pkg/clientbuilder"
+)
+
+func TestStartGKETenantControllerManager(t *testing.T) {
+	originalEnableProviderConfigController := enableProviderConfigController
+	defer func() {
+		enableProviderConfigController = originalEnableProviderConfigController
+	}()
+
+	testCases := []struct {
+		desc    string
+		enable  bool
+		wantRun bool
+	}{
+		{
+			desc:    "disabled",
+			enable:  false,
+			wantRun: false,
+		},
+		{
+			desc:    "enabled",
+			enable:  true,
+			wantRun: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			enableProviderConfigController = tc.enable
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			kubeClient := fake.NewSimpleClientset()
+			informerFactory := informers.NewSharedInformerFactory(kubeClient, time.Second)
+
+			initContext := app.ControllerInitContext{
+				ClientName: "test-client",
+			}
+
+			controllerContext := genericcontrollermanager.ControllerContext{
+				InformerFactory: informerFactory,
+			}
+
+			ccmConfig := &cloudcontrollerconfig.Config{
+				Kubeconfig: &rest.Config{
+					Host: "https://example.com",
+				},
+			}
+			ccmConfig.ClientBuilder = clientbuilder.SimpleControllerClientBuilder{
+				ClientConfig: ccmConfig.Kubeconfig,
+			}
+			completedConfig := ccmConfig.Complete()
+
+			cloud := &fakeCloudProvider{}
+			nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{}
+
+			_, started, err := startGKETenantControllerManager(
+				ctx,
+				initContext,
+				controllerContext,
+				completedConfig,
+				cloud,
+				nodeIPAMConfig,
+			)
+
+			if err != nil {
+				t.Fatalf("startGKETenantControllerManager failed: %v", err)
+			}
+
+			if started != tc.wantRun {
+				t.Errorf("startGKETenantControllerManager started = %v, want %v", started, tc.wantRun)
+			}
+		})
+	}
+}