Skip to content

Commit fced45e

Browse files
priyapandepriyapande
committed
Feat: Add support for GKETenantControllerManager
1 parent 205348b commit fced45e

File tree

20 files changed

+1531
-258
lines changed

20 files changed

+1531
-258
lines changed

‎cmd/cloud-controller-manager/gkenetworkparamsetcontroller.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
networkinformers "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
1111
cloudprovider "k8s.io/cloud-provider"
1212
gkenetworkparamsetcontroller "k8s.io/cloud-provider-gcp/pkg/controller/gkenetworkparamset"
13+
nodeipam "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam"
1314
"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/ipam"
1415
"k8s.io/cloud-provider-gcp/providers/gce"
1516
"k8s.io/cloud-provider/app"
@@ -73,7 +74,7 @@ func startGkeNetworkParamsController(ccmConfig *cloudcontrollerconfig.CompletedC
7374
// with stack type and returns a list of typed cidrs and error
7475
func validClusterCIDR(clusterCIDRFromFlag string) ([]*net.IPNet, error) {
7576
// failure: bad cidrs in config
76-
clusterCIDRs, dualStack, err := processCIDRs(clusterCIDRFromFlag)
77+
clusterCIDRs, dualStack, err := nodeipam.ProcessCIDRs(clusterCIDRFromFlag)
7778
if err != nil {
7879
return nil, err
7980
}

‎cmd/cloud-controller-manager/gketenantcontrollermanager.go‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
providerconfigv1 "github.com/GoogleCloudPlatform/gke-enterprise-mt/apis/providerconfig/v1"
8+
"github.com/GoogleCloudPlatform/gke-enterprise-mt/pkg/framework"
9+
networkclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/network/clientset/versioned"
10+
networkinformers "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
11+
topologyclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/clientset/versioned"
12+
"k8s.io/apimachinery/pkg/runtime/schema"
13+
"k8s.io/client-go/dynamic"
14+
dynamicinformer "k8s.io/client-go/dynamic/dynamicinformer"
15+
cloudprovider "k8s.io/cloud-provider"
16+
nodeipamcontrolleroptions "k8s.io/cloud-provider-gcp/cmd/cloud-controller-manager/options"
17+
"k8s.io/cloud-provider-gcp/pkg/controller/gketenantcontrollers"
18+
nodeipamconfig "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/config"
19+
"k8s.io/cloud-provider/app"
20+
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
21+
controllermanagerapp "k8s.io/controller-manager/app"
22+
"k8s.io/controller-manager/controller"
23+
"k8s.io/klog/v2"
24+
25+
"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam"
26+
"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/ipam"
27+
"k8s.io/cloud-provider/controllers/node"
28+
"k8s.io/cloud-provider/controllers/nodelifecycle"
29+
)
30+
31+
// startGKETenantControllerManagerWrapper is used to take cloud config as input and start the GKE TenantControllerManager controller
32+
func startGKETenantControllerManagerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions) app.InitFunc {
33+
return func(ctx context.Context, controllerContext controllermanagerapp.ControllerContext) (controller.Interface, bool, error) {
34+
return startGKETenantControllerManager(ctx, initContext, controllerContext, completedConfig, cloud, *nodeIPAMControllerOptions.NodeIPAMControllerConfiguration)
35+
}
36+
}
37+
38+
func startGKETenantControllerManager(ctx context.Context, initContext app.ControllerInitContext, controlexContext controllermanagerapp.ControllerContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration) (controller.Interface, bool, error) {
39+
if !enableProviderConfigController {
40+
klog.Infof("GKE Tenant Controller Manager is disabled (enable with --enable-provider-config-controller)")
41+
return nil, false, nil
42+
}
43+
44+
clientConfig := completedConfig.Kubeconfig
45+
46+
// Create network clients and informers
47+
networkClient, err := networkclientset.NewForConfig(clientConfig)
48+
if err != nil {
49+
klog.Errorf("Failed to create network client: %v", err)
50+
return nil, false, err
51+
}
52+
networkInformerFactory := networkinformers.NewSharedInformerFactory(networkClient, 12*time.Hour)
53+
networkInformer := networkInformerFactory.Networking().V1().Networks()
54+
gnpInformer := networkInformerFactory.Networking().V1().GKENetworkParamSets()
55+
56+
// Create topology client
57+
nodeTopologyClient, err := topologyclientset.NewForConfig(clientConfig)
58+
if err != nil {
59+
klog.Errorf("Failed to create topology client: %v", err)
60+
return nil, false, err
61+
}
62+
63+
// Create dynamic client for framework
64+
dynamicClient, err := dynamic.NewForConfig(clientConfig)
65+
if err != nil {
66+
klog.Errorf("Failed to create dynamic client: %v", err)
67+
return nil, false, err
68+
}
69+
70+
// Create dynamic informer factory for ProviderConfig
71+
gvr := schema.GroupVersionResource{
72+
Group: providerconfigv1.GroupName,
73+
Version: providerconfigv1.SchemeGroupVersion.Version,
74+
Resource: "providerconfigs",
75+
}
76+
dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 12*time.Hour)
77+
providerConfigInformer := dynamicInformerFactory.ForResource(gvr).Informer()
78+
79+
// Define controllers
80+
controllers := map[string]gketenantcontrollers.ControllerStartFunc{
81+
"node-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
82+
klog.Infof("Creating OSS Cloud Node Controller for %s...", cfg.ProviderConfig.Name)
83+
nodeController, err := node.NewCloudNodeController(
84+
cfg.NodeInformer,
85+
cfg.KubeClient,
86+
cfg.Cloud,
87+
completedConfig.ComponentConfig.NodeStatusUpdateFrequency.Duration,
88+
completedConfig.ComponentConfig.NodeController.ConcurrentNodeSyncs,
89+
)
90+
if err != nil {
91+
return err
92+
}
93+
klog.Infof("Starting OSS Cloud Node Controller for %s (blocking)", cfg.ProviderConfig.Name)
94+
nodeController.Run(cfg.Context.Done(), cfg.ControllerContext.ControllerManagerMetrics)
95+
return nil
96+
},
97+
"node-ipam-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
98+
klog.Infof("Starting Node IPAM Controller for %s...", cfg.ProviderConfig.Name)
99+
clusterCIDR, err := gketenantcontrollers.GetClusterCIDRsFromProviderConfig(cfg.ProviderConfig)
100+
if err != nil {
101+
klog.Errorf("Failed to get ClusterCIDRs from ProviderConfig: %v. Node IPAM Controller will be disabled.", err)
102+
return nil // Don't fail the whole start
103+
}
104+
105+
_, started, err := nodeipam.StartNodeIpamController(
106+
cfg.Context,
107+
cfg.NodeInformer,
108+
cfg.KubeClient,
109+
cfg.Cloud,
110+
clusterCIDR,
111+
completedConfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs,
112+
nodeIPAMConfig.ServiceCIDR,
113+
nodeIPAMConfig.SecondaryServiceCIDR,
114+
nodeIPAMConfig,
115+
networkInformer,
116+
gnpInformer,
117+
nodeTopologyClient,
118+
ipam.CIDRAllocatorType(completedConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
119+
cfg.ControllerContext.ControllerManagerMetrics,
120+
)
121+
if err != nil {
122+
return err
123+
}
124+
if !started {
125+
klog.Infof("Node IPAM Controller not started (disabled in config) for %s", cfg.ProviderConfig.Name)
126+
} else {
127+
klog.Infof("Node IPAM Controller started with ClusterCIDR: %s for %s", clusterCIDR, cfg.ProviderConfig.Name)
128+
}
129+
// Block until context is canceled so starter doesn't exit early
130+
<-cfg.Context.Done()
131+
return nil
132+
},
133+
"node-lifecycle-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
134+
klog.Infof("Creating Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
135+
nodeMonitorPeriod := completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration
136+
lifecycleController, err := nodelifecycle.NewCloudNodeLifecycleController(
137+
cfg.NodeInformer,
138+
cfg.KubeClient,
139+
cfg.Cloud,
140+
nodeMonitorPeriod,
141+
)
142+
if err != nil {
143+
return err
144+
}
145+
klog.Infof("Starting Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
146+
lifecycleController.Run(cfg.Context, cfg.ControllerContext.ControllerManagerMetrics)
147+
return nil
148+
},
149+
}
150+
151+
// Create the starter
152+
starter := gketenantcontrollers.NewNodeControllerStarter(
153+
completedConfig.ClientBuilder,
154+
completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName),
155+
controlexContext.InformerFactory,
156+
completedConfig,
157+
controlexContext,
158+
controllers,
159+
)
160+
161+
// Create the framework manager
162+
mgr := framework.New(
163+
dynamicClient,
164+
providerConfigInformer,
165+
gkeTenantControllerManagerName,
166+
starter,
167+
ctx.Done(),
168+
)
169+
170+
// Start network informers
171+
networkInformerFactory.Start(ctx.Done())
172+
// Start dynamic informers
173+
dynamicInformerFactory.Start(ctx.Done())
174+
175+
// Run the manager
176+
go mgr.Run()
177+
178+
return nil, true, nil
179+
}

‎cmd/cloud-controller-manager/main.go‎

Lines changed: 39 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
"k8s.io/apimachinery/pkg/util/wait"
2929
cloudprovider "k8s.io/cloud-provider"
3030
"k8s.io/cloud-provider/app"
31-
"k8s.io/cloud-provider/app/config"
31+
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
3232
"k8s.io/cloud-provider/names"
3333
"k8s.io/cloud-provider/options"
3434
cliflag "k8s.io/component-base/cli/flag"
@@ -43,9 +43,12 @@ import (
4343
)
4444

4545
const (
46-
gkeServiceLBControllerName = "gke-service-lb-controller"
47-
gkeServiceControllerClientName = "gke-service-controller"
48-
gkeServiceAlias = "gke-service"
46+
gkeServiceLBControllerName = "gke-service-lb-controller"
47+
gkeServiceControllerClientName = "gke-service-controller"
48+
gkeServiceAlias = "gke-service"
49+
gkeTenantControllerManagerName = "gke-tenant-controller-manager"
50+
gkeTenantControllerClientName = "gke-tenant-controller-manager"
51+
gkeTenantControllerManagerAlias = "gke-tenant-controller-manager"
4952
)
5053

5154
var (
@@ -77,6 +80,9 @@ var (
7780
// The reason for it not being enabled by default is the additional GCE API calls that are made
7881
// for checking if the deny firewalls exist/deletion which will eat up the quota unnecessarily.
7982
enableL4DenyFirewallRollbackCleanup bool
83+
84+
// enableProviderConfigController enables the gke-tenant-controller-manager.
85+
enableProviderConfigController bool
8086
)
8187

8288
func main() {
@@ -99,6 +105,7 @@ func main() {
99105
cloudProviderFS.BoolVar(&enableL4LBAnnotations, "enable-l4-lb-annotations", false, "Enables Annotations for GCE L4 LB Services")
100106
cloudProviderFS.BoolVar(&enableL4DenyFirewall, "enable-l4-deny-firewall", false, "Enable creation and updates of Deny VPC Firewall Rules for L4 external load balancers. Requires --enable-pinhole and --enable-l4-deny-firewall-rollback-cleanup to be true.")
101107
cloudProviderFS.BoolVar(&enableL4DenyFirewallRollbackCleanup, "enable-l4-deny-firewall-rollback-cleanup", false, "Enable cleanup codepath of the deny firewalls for rollback. The reason for it not being enabled by default is the additional GCE API calls that are made for checking if the deny firewalls exist/deletion which will eat up the quota unnecessarily.")
108+
cloudProviderFS.BoolVar(&enableProviderConfigController, "enable-provider-config-controller", false, "Enables the GKE Tenant Controller Manager for Multi-Tenancy.")
102109

103110
// add new controllers and initializers
104111
nodeIpamController := nodeIPAMController{}
@@ -119,12 +126,25 @@ func main() {
119126
Constructor: startGkeServiceControllerWrapper,
120127
}
121128

129+
controllerInitializers[gkeTenantControllerManagerName] = app.ControllerInitFuncConstructor{
130+
InitContext: app.ControllerInitContext{
131+
ClientName: gkeTenantControllerClientName,
132+
},
133+
Constructor: func(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
134+
return startGKETenantControllerManagerWrapper(initContext, completedConfig, cloud, nodeIpamController.nodeIPAMControllerOptions)
135+
},
136+
}
137+
122138
// add controllers disabled by default
123139
app.ControllersDisabledByDefault.Insert("gkenetworkparamset")
124140
app.ControllersDisabledByDefault.Insert(gkeServiceLBControllerName)
141+
app.ControllersDisabledByDefault.Insert(gkeTenantControllerManagerName)
142+
125143
aliasMap := names.CCMControllerAliases()
126144
aliasMap["nodeipam"] = kcmnames.NodeIpamController
127145
aliasMap[gkeServiceAlias] = gkeServiceLBControllerName
146+
aliasMap[gkeTenantControllerManagerAlias] = gkeTenantControllerManagerName
147+
128148
command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, aliasMap, fss, wait.NeverStop)
129149

130150
logs.InitLogs()
@@ -135,7 +155,7 @@ func main() {
135155
}
136156
}
137157

138-
func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
158+
func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
139159
cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
140160

141161
// initialize cloud provider with the cloud provider name and config file provided
@@ -155,35 +175,27 @@ func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
155175
}
156176
}
157177

158-
if enableMultiProject {
178+
if enableMultiProject || enableDiscretePortForwarding || enableRBSDefaultForL4NetLB || enableL4LBAnnotations {
159179
gceCloud, ok := (cloud).(*gce.Cloud)
160180
if !ok {
161-
// Fail-fast: If enableMultiProject is set, the cloud provider MUST
162-
// be GCE. A non-GCE provider indicates a misconfiguration. Ideally,
163-
// we never expect this to be executed.
164-
klog.Fatalf("multi-project mode requires GCE cloud provider, but got %T", cloud)
181+
klog.Fatalf("One or more GCE-specific flags (enable-multi-project, enable-discrete-port-forwarding, enable-rbs-default-l4-netlb, enable-l4-lb-annotations) were set, but cloud provider is %T, not GCE.", cloud)
165182
}
166-
gceCloud.SetProjectFromNodeProviderID(true)
167-
}
168183

169-
if enableRBSDefaultForL4NetLB {
170-
gceCloud, ok := (cloud).(*gce.Cloud)
171-
if !ok {
172-
// Fail-fast: If enableRBSDefaultForL4NetLB is set, the cloud
173-
// provider MUST be GCE.
174-
klog.Fatalf("enable-rbs-default-l4-netlb requires GCE cloud provider, but got %T", cloud)
184+
if !enableProviderConfigController && enableMultiProject {
185+
gceCloud.SetProjectFromNodeProviderID(true)
175186
}
176-
gceCloud.SetEnableRBSDefaultForL4NetLB(true)
177-
}
178187

179-
if enableL4LBAnnotations {
180-
gceCloud, ok := (cloud).(*gce.Cloud)
181-
if !ok {
182-
// Fail-fast: If enableL4LBAnnotations is set, the cloud
183-
// provider MUST be GCE.
184-
klog.Fatalf("enable-l4-lb-annotations requires GCE cloud provider, but got %T", cloud)
188+
if enableDiscretePortForwarding {
189+
gceCloud.SetEnableDiscretePortForwarding(true)
190+
}
191+
192+
if enableRBSDefaultForL4NetLB {
193+
gceCloud.SetEnableRBSDefaultForL4NetLB(true)
194+
}
195+
196+
if enableL4LBAnnotations {
197+
gceCloud.SetEnableL4LBAnnotations(true)
185198
}
186-
gceCloud.SetEnableL4LBAnnotations(true)
187199
}
188200

189201
if enableL4DenyFirewall || enableL4DenyFirewallRollbackCleanup {

0 commit comments

Comments
 (0)