Feat: Add support for GKETenantControllerManager

Author: priyapande

cmd/cloud-controller-manager/gkenetworkparamsetcontroller.go

@@ -10,6 +10,7 @@ import (
 	networkinformers "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
 	cloudprovider "k8s.io/cloud-provider"
 	gkenetworkparamsetcontroller "k8s.io/cloud-provider-gcp/pkg/controller/gkenetworkparamset"
+	nodeipam "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam"
 	"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/ipam"
 	"k8s.io/cloud-provider-gcp/providers/gce"
 	"k8s.io/cloud-provider/app"
@@ -73,7 +74,7 @@ func startGkeNetworkParamsController(ccmConfig *cloudcontrollerconfig.CompletedC
 // with stack type and returns a list of typed cidrs and error
 func validClusterCIDR(clusterCIDRFromFlag string) ([]*net.IPNet, error) {
 	// failure: bad cidrs in config
-	clusterCIDRs, dualStack, err := processCIDRs(clusterCIDRFromFlag)
+	clusterCIDRs, dualStack, err := nodeipam.ProcessCIDRs(clusterCIDRFromFlag)
 	if err != nil {
 		return nil, err
 	}

cmd/cloud-controller-manager/gketenantcontrollermanager.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	providerconfigv1 "github.com/GoogleCloudPlatform/gke-enterprise-mt/apis/providerconfig/v1"
+	"github.com/GoogleCloudPlatform/gke-enterprise-mt/pkg/framework"
+	networkclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/network/clientset/versioned"
+	networkinformers "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
+	topologyclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/clientset/versioned"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	dynamicinformer "k8s.io/client-go/dynamic/dynamicinformer"
+	cloudprovider "k8s.io/cloud-provider"
+	nodeipamcontrolleroptions "k8s.io/cloud-provider-gcp/cmd/cloud-controller-manager/options"
+	"k8s.io/cloud-provider-gcp/pkg/controller/gketenantcontrollers"
+	nodeipamconfig "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/config"
+	"k8s.io/cloud-provider/app"
+	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
+	controllermanagerapp "k8s.io/controller-manager/app"
+	"k8s.io/controller-manager/controller"
+	"k8s.io/klog/v2"
+
+	"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam"
+	"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/ipam"
+	"k8s.io/cloud-provider/controllers/node"
+	"k8s.io/cloud-provider/controllers/nodelifecycle"
+)
+
+// startGKETenantControllerManagerWrapper is used to take cloud config as input and start the GKE TenantControllerManager controller
+func startGKETenantControllerManagerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions) app.InitFunc {
+	return func(ctx context.Context, controllerContext controllermanagerapp.ControllerContext) (controller.Interface, bool, error) {
+		return startGKETenantControllerManager(ctx, initContext, controllerContext, completedConfig, cloud, *nodeIPAMControllerOptions.NodeIPAMControllerConfiguration)
+	}
+}
+
+func startGKETenantControllerManager(ctx context.Context, initContext app.ControllerInitContext, controlexContext controllermanagerapp.ControllerContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration) (controller.Interface, bool, error) {
+	if !enableProviderConfigController {
+		klog.Infof("GKE Tenant Controller Manager is disabled (enable with --enable-provider-config-controller)")
+		return nil, false, nil
+	}
+
+	clientConfig := completedConfig.Kubeconfig
+
+	// Create network clients and informers
+	networkClient, err := networkclientset.NewForConfig(clientConfig)
+	if err != nil {
+		klog.Errorf("Failed to create network client: %v", err)
+		return nil, false, err
+	}
+	networkInformerFactory := networkinformers.NewSharedInformerFactory(networkClient, 12*time.Hour)
+	networkInformer := networkInformerFactory.Networking().V1().Networks()
+	gnpInformer := networkInformerFactory.Networking().V1().GKENetworkParamSets()
+
+	// Create topology client
+	nodeTopologyClient, err := topologyclientset.NewForConfig(clientConfig)
+	if err != nil {
+		klog.Errorf("Failed to create topology client: %v", err)
+		return nil, false, err
+	}
+
+	// Create dynamic client for framework
+	dynamicClient, err := dynamic.NewForConfig(clientConfig)
+	if err != nil {
+		klog.Errorf("Failed to create dynamic client: %v", err)
+		return nil, false, err
+	}
+
+	// Create dynamic informer factory for ProviderConfig
+	gvr := schema.GroupVersionResource{
+		Group:    providerconfigv1.GroupName,
+		Version:  providerconfigv1.SchemeGroupVersion.Version,
+		Resource: "providerconfigs",
+	}
+	dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 12*time.Hour)
+	providerConfigInformer := dynamicInformerFactory.ForResource(gvr).Informer()
+
+	// Define controllers
+	controllers := map[string]gketenantcontrollers.ControllerStartFunc{
+		"node-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
+			klog.Infof("Creating OSS Cloud Node Controller for %s...", cfg.ProviderConfig.Name)
+			nodeController, err := node.NewCloudNodeController(
+				cfg.NodeInformer,
+				cfg.KubeClient,
+				cfg.Cloud,
+				completedConfig.ComponentConfig.NodeStatusUpdateFrequency.Duration,
+				completedConfig.ComponentConfig.NodeController.ConcurrentNodeSyncs,
+			)
+			if err != nil {
+				return err
+			}
+			klog.Infof("Starting OSS Cloud Node Controller for %s (blocking)", cfg.ProviderConfig.Name)
+			nodeController.Run(cfg.Context.Done(), cfg.ControllerContext.ControllerManagerMetrics)
+			return nil
+		},
+		"node-ipam-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
+			klog.Infof("Starting Node IPAM Controller for %s...", cfg.ProviderConfig.Name)
+			clusterCIDR, err := gketenantcontrollers.GetClusterCIDRsFromProviderConfig(cfg.ProviderConfig)
+			if err != nil {
+				klog.Errorf("Failed to get ClusterCIDRs from ProviderConfig: %v. Node IPAM Controller will be disabled.", err)
+				return nil // Don't fail the whole start
+			}
+
+			_, started, err := nodeipam.StartNodeIpamController(
+				cfg.Context,
+				cfg.NodeInformer,
+				cfg.KubeClient,
+				cfg.Cloud,
+				clusterCIDR,
+				completedConfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs,
+				nodeIPAMConfig.ServiceCIDR,
+				nodeIPAMConfig.SecondaryServiceCIDR,
+				nodeIPAMConfig,
+				networkInformer,
+				gnpInformer,
+				nodeTopologyClient,
+				ipam.CIDRAllocatorType(completedConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
+				cfg.ControllerContext.ControllerManagerMetrics,
+			)
+			if err != nil {
+				return err
+			}
+			if !started {
+				klog.Infof("Node IPAM Controller not started (disabled in config) for %s", cfg.ProviderConfig.Name)
+			} else {
+				klog.Infof("Node IPAM Controller started with ClusterCIDR: %s for %s", clusterCIDR, cfg.ProviderConfig.Name)
+			}
+			// Block until context is canceled so starter doesn't exit early
+			<-cfg.Context.Done()
+			return nil
+		},
+		"node-lifecycle-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
+			klog.Infof("Creating Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
+			nodeMonitorPeriod := completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration
+			lifecycleController, err := nodelifecycle.NewCloudNodeLifecycleController(
+				cfg.NodeInformer,
+				cfg.KubeClient,
+				cfg.Cloud,
+				nodeMonitorPeriod,
+			)
+			if err != nil {
+				return err
+			}
+			klog.Infof("Starting Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
+			lifecycleController.Run(cfg.Context, cfg.ControllerContext.ControllerManagerMetrics)
+			return nil
+		},
+	}
+
+	// Create the starter
+	starter := gketenantcontrollers.NewNodeControllerStarter(
+		completedConfig.ClientBuilder,
+		completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName),
+		controlexContext.InformerFactory,
+		completedConfig,
+		controlexContext,
+		controllers,
+	)
+
+	// Create the framework manager
+	mgr := framework.New(
+		dynamicClient,
+		providerConfigInformer,
+		gkeTenantControllerManagerName,
+		starter,
+		ctx.Done(),
+	)
+
+	// Start network informers
+	networkInformerFactory.Start(ctx.Done())
+	// Start dynamic informers
+	dynamicInformerFactory.Start(ctx.Done())
+
+	// Run the manager
+	go mgr.Run()
+
+	return nil, true, nil
+}

cmd/cloud-controller-manager/main.go

@@ -28,9 +28,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/cloud-provider-gcp/providers/gce"
-	_ "k8s.io/cloud-provider-gcp/providers/gce"
+
 	"k8s.io/cloud-provider/app"
-	"k8s.io/cloud-provider/app/config"
+	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	"k8s.io/cloud-provider/names"
 	"k8s.io/cloud-provider/options"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -45,15 +45,11 @@ const (
 	gkeServiceLBControllerName     = "gke-service-lb-controller"
 	gkeServiceControllerClientName = "gke-service-controller"
 	gkeServiceAlias                = "gke-service"
+	gkeTenantControllerManagerName = "gke-tenant-controller-manager"
+	gkeTenantControllerClientName  = "gke-tenant-controller-manager"
+	gkeTenantControllerManagerAlias = "gke-tenant-controller-manager"
 )
 
-// enableMultiProject is bound to a command-line flag. When true, it enables the
-// projectFromNodeProviderID option of the GCE cloud provider, instructing it to
-// use the project specified in the Node's providerID for GCE API calls.
-//
-// This flag should only be enabled when the Node's providerID can be fully
-// trusted.
-//
 // Flag binding occurs in main()
 var enableMultiProject bool
 
@@ -72,6 +68,9 @@ var enableRBSDefaultForL4NetLB bool
 // for L4 Load Balancers services
 var enableL4LBAnnotations bool
 
+// enableProviderConfigController enables the gke-tenant-controller-manager.
+var enableProviderConfigController bool
+
 func main() {
 	rand.Seed(time.Now().UnixNano())
 
@@ -91,6 +90,7 @@ func main() {
 	cloudProviderFS.BoolVar(&enableDiscretePortForwarding, "enable-discrete-port-forwarding", false, "Enables forwarding of individual ports instead of port ranges for GCE external load balancers.")
 	cloudProviderFS.BoolVar(&enableRBSDefaultForL4NetLB, "enable-rbs-default-l4-netlb", false, "Enables RBS defaulting for GCE L4 NetLB")
 	cloudProviderFS.BoolVar(&enableL4LBAnnotations, "enable-l4-lb-annotations", false, "Enables Annotations for GCE L4 LB Services")
+	cloudProviderFS.BoolVar(&enableProviderConfigController, "enable-provider-config-controller", false, "Enables the GKE Tenant Controller Manager for Multi-Tenancy.")
 
 	// add new controllers and initializers
 	nodeIpamController := nodeIPAMController{}
@@ -111,12 +111,26 @@ func main() {
 		Constructor: startGkeServiceControllerWrapper,
 	}
 
+	controllerInitializers[gkeTenantControllerManagerName] = app.ControllerInitFuncConstructor{
+		InitContext: app.ControllerInitContext{
+			ClientName: gkeTenantControllerClientName,
+		},
+		Constructor: func(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
+			return startGKETenantControllerManagerWrapper(initContext, completedConfig, cloud, nodeIpamController.nodeIPAMControllerOptions)
+		},
+	}
+
 	// add controllers disabled by default
 	app.ControllersDisabledByDefault.Insert("gkenetworkparamset")
 	app.ControllersDisabledByDefault.Insert(gkeServiceLBControllerName)
+	app.ControllersDisabledByDefault.Insert(gkeTenantControllerManagerName)
+
+
 	aliasMap := names.CCMControllerAliases()
 	aliasMap["nodeipam"] = kcmnames.NodeIpamController
 	aliasMap[gkeServiceAlias] = gkeServiceLBControllerName
+	aliasMap[gkeTenantControllerManagerAlias] = gkeTenantControllerManagerName
+
 	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, aliasMap, fss, wait.NeverStop)
 
 	logs.InitLogs()
@@ -127,7 +141,7 @@ func main() {
 	}
 }
 
-func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
+func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
 	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
 
 	// initialize cloud provider with the cloud provider name and config file provided
@@ -147,46 +161,28 @@ func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
 		}
 	}
 
-	if enableMultiProject {
+	if enableMultiProject || enableDiscretePortForwarding || enableRBSDefaultForL4NetLB || enableL4LBAnnotations {
 		gceCloud, ok := (cloud).(*gce.Cloud)
 		if !ok {
-			// Fail-fast: If enableMultiProject is set, the cloud provider MUST
-			// be GCE. A non-GCE provider indicates a misconfiguration. Ideally,
-			// we never expect this to be executed.
-			klog.Fatalf("multi-project mode requires GCE cloud provider, but got %T", cloud)
+			klog.Fatalf("One or more GCE-specific flags (enable-multi-project, enable-discrete-port-forwarding, enable-rbs-default-l4-netlb, enable-l4-lb-annotations) were set, but cloud provider is %T, not GCE.", cloud)
 		}
-		gceCloud.SetProjectFromNodeProviderID(true)
-	}
 
-	if enableDiscretePortForwarding {
-		gceCloud, ok := (cloud).(*gce.Cloud)
-		if !ok {
-			// Fail-fast: If enableDiscretePortForwarding is set, the cloud
-			// provider MUST be GCE.
-			klog.Fatalf("enable-discrete-port-forwarding requires GCE cloud provider, but got %T", cloud)
+		if !enableProviderConfigController && enableMultiProject {
+			gceCloud.SetProjectFromNodeProviderID(true)
 		}
-		gceCloud.SetEnableDiscretePortForwarding(true)
-	}
 
-	if enableRBSDefaultForL4NetLB {
-		gceCloud, ok := (cloud).(*gce.Cloud)
-		if !ok {
-			// Fail-fast: If enableRBSDefaultForL4NetLB is set, the cloud
-			// provider MUST be GCE.
-			klog.Fatalf("enable-rbs-default-l4-netlb requires GCE cloud provider, but got %T", cloud)
+		if enableDiscretePortForwarding {
+			gceCloud.SetEnableDiscretePortForwarding(true)
 		}
-		gceCloud.SetEnableRBSDefaultForL4NetLB(true)
-	}
 
-	if enableL4LBAnnotations {
-		gceCloud, ok := (cloud).(*gce.Cloud)
-		if !ok {
-			// Fail-fast: If enableL4LBAnnotations is set, the cloud
-			// provider MUST be GCE.
-			klog.Fatalf("enable-l4-lb-annotations requires GCE cloud provider, but got %T", cloud)
+		if enableRBSDefaultForL4NetLB {
+			gceCloud.SetEnableRBSDefaultForL4NetLB(true)
 		}
-		gceCloud.SetEnableL4LBAnnotations(true)
-	}
 
+		if enableL4LBAnnotations {
+			gceCloud.SetEnableL4LBAnnotations(true)
+		}
+	}
+	
 	return cloud
 }