Skip to content

Commit 088172d

Browse files
priyapandepriyapande
committed
Feat: Add Multi-Tenant Manager to CCM
1 parent ddb20a0 commit 088172d

File tree

14 files changed

+1342
-10
lines changed

14 files changed

+1342
-10
lines changed

‎cmd/cloud-controller-manager/BUILD‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,27 +20,36 @@ go_library(
2020
srcs = [
2121
"gkenetworkparamsetcontroller.go",
2222
"gkeservicecontroller.go",
23+
"gketenantcontrollermanager.go",
2324
"main.go",
2425
"nodeipamcontroller.go",
2526
],
2627
importpath = "k8s.io/cloud-provider-gcp/cmd/cloud-controller-manager",
2728
deps = [
2829
"//cmd/cloud-controller-manager/options",
2930
"//pkg/controller/gkenetworkparamset",
31+
"//pkg/controller/gketenantcontrollers",
3032
"//pkg/controller/nodeipam",
3133
"//pkg/controller/nodeipam/config",
3234
"//pkg/controller/nodeipam/ipam",
3335
"//pkg/controller/service",
3436
"//providers/gce",
37+
"//vendor/github.com/GoogleCloudPlatform/gke-enterprise-mt/apis/providerconfig/v1:providerconfig",
38+
"//vendor/github.com/GoogleCloudPlatform/gke-enterprise-mt/pkg/framework",
3539
"//vendor/github.com/GoogleCloudPlatform/gke-networking-api/client/network/clientset/versioned",
3640
"//vendor/github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions",
3741
"//vendor/github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/clientset/versioned",
3842
"//vendor/github.com/spf13/pflag",
43+
"//vendor/k8s.io/apimachinery/pkg/runtime/schema",
3944
"//vendor/k8s.io/apimachinery/pkg/util/wait",
4045
"//vendor/k8s.io/apiserver/pkg/util/feature",
46+
"//vendor/k8s.io/client-go/dynamic",
47+
"//vendor/k8s.io/client-go/dynamic/dynamicinformer",
4148
"//vendor/k8s.io/cloud-provider",
4249
"//vendor/k8s.io/cloud-provider/app",
4350
"//vendor/k8s.io/cloud-provider/app/config",
51+
"//vendor/k8s.io/cloud-provider/controllers/node",
52+
"//vendor/k8s.io/cloud-provider/controllers/nodelifecycle",
4453
"//vendor/k8s.io/cloud-provider/names",
4554
"//vendor/k8s.io/cloud-provider/options",
4655
"//vendor/k8s.io/component-base/cli/flag",
@@ -51,7 +60,6 @@ go_library(
5160
"//vendor/k8s.io/controller-manager/controller",
5261
"//vendor/k8s.io/klog/v2:klog",
5362
"//vendor/k8s.io/kubernetes/cmd/kube-controller-manager/names",
54-
"//vendor/k8s.io/utils/net",
5563
],
5664
)
5765

@@ -63,16 +71,22 @@ go_test(
6371
name = "cloud-controller-manager_test",
6472
srcs = [
6573
"gkeservicecontroller_test.go",
74+
"gketenantcontrollermanager_test.go",
6675
"nodeipamcontroller_test.go",
6776
],
6877
embed = [":cloud-controller-manager_lib"],
6978
deps = [
7079
"//pkg/controller/nodeipam/config",
7180
"//pkg/controller/service",
7281
"//vendor/k8s.io/api/core/v1:core",
82+
"//vendor/k8s.io/client-go/informers",
83+
"//vendor/k8s.io/client-go/kubernetes/fake",
84+
"//vendor/k8s.io/client-go/rest",
7385
"//vendor/k8s.io/cloud-provider",
86+
"//vendor/k8s.io/cloud-provider/app",
7487
"//vendor/k8s.io/cloud-provider/app/config",
7588
"//vendor/k8s.io/cloud-provider/config",
7689
"//vendor/k8s.io/controller-manager/app",
90+
"//vendor/k8s.io/controller-manager/pkg/clientbuilder",
7791
],
7892
)

‎cmd/cloud-controller-manager/gketenantcontrollermanager.go‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package main
2+
3+
import (
4+
"context"
5+
6+
providerconfigv1 "github.com/GoogleCloudPlatform/gke-enterprise-mt/apis/providerconfig/v1"
7+
"github.com/GoogleCloudPlatform/gke-enterprise-mt/pkg/framework"
8+
networkclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/network/clientset/versioned"
9+
networkinformers "github.com/GoogleCloudPlatform/gke-networking-api/client/network/informers/externalversions"
10+
topologyclientset "github.com/GoogleCloudPlatform/gke-networking-api/client/nodetopology/clientset/versioned"
11+
"k8s.io/apimachinery/pkg/runtime/schema"
12+
"k8s.io/client-go/dynamic"
13+
dynamicinformer "k8s.io/client-go/dynamic/dynamicinformer"
14+
cloudprovider "k8s.io/cloud-provider"
15+
nodeipamcontrolleroptions "k8s.io/cloud-provider-gcp/cmd/cloud-controller-manager/options"
16+
"k8s.io/cloud-provider-gcp/pkg/controller/gketenantcontrollers"
17+
nodeipamconfig "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/config"
18+
"k8s.io/cloud-provider/app"
19+
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
20+
controllermanagerapp "k8s.io/controller-manager/app"
21+
"k8s.io/controller-manager/controller"
22+
"k8s.io/klog/v2"
23+
24+
"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam"
25+
"k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/ipam"
26+
"k8s.io/cloud-provider/controllers/node"
27+
"k8s.io/cloud-provider/controllers/nodelifecycle"
28+
)
29+
30+
// startGKETenantControllerManagerWrapper is used to take cloud config as input and start the GKE TenantControllerManager controller
31+
func startGKETenantControllerManagerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions) app.InitFunc {
32+
return func(ctx context.Context, controllerContext controllermanagerapp.ControllerContext) (controller.Interface, bool, error) {
33+
return startGKETenantControllerManager(ctx, initContext, controllerContext, completedConfig, cloud, *nodeIPAMControllerOptions.NodeIPAMControllerConfiguration)
34+
}
35+
}
36+
37+
func startGKETenantControllerManager(ctx context.Context, initContext app.ControllerInitContext, controlexContext controllermanagerapp.ControllerContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration) (controller.Interface, bool, error) {
38+
if !enableProviderConfigController {
39+
klog.Infof("GKE Tenant Controller Manager is disabled (enable with --enable-provider-config-controller)")
40+
return nil, false, nil
41+
}
42+
43+
clientConfig := completedConfig.Kubeconfig
44+
45+
// Create network clients and informers
46+
networkClient, err := networkclientset.NewForConfig(clientConfig)
47+
if err != nil {
48+
klog.Errorf("Failed to create network client: %v", err)
49+
return nil, false, err
50+
}
51+
networkInformerFactory := networkinformers.NewSharedInformerFactory(networkClient, 0)
52+
networkInformer := networkInformerFactory.Networking().V1().Networks()
53+
gnpInformer := networkInformerFactory.Networking().V1().GKENetworkParamSets()
54+
55+
// Create topology client
56+
nodeTopologyClient, err := topologyclientset.NewForConfig(clientConfig)
57+
if err != nil {
58+
klog.Errorf("Failed to create topology client: %v", err)
59+
return nil, false, err
60+
}
61+
62+
// Eagerly request the main Node informer so the SharedInformerFactory starts it.
63+
// If the tenant controller manager is the only enabled controller, the informer
64+
// factory won't start the node cache unless we explicitly ask for it here.
65+
_ = controlexContext.InformerFactory.Core().V1().Nodes().Informer()
66+
67+
// Create dynamic client for framework
68+
dynamicClient, err := dynamic.NewForConfig(clientConfig)
69+
if err != nil {
70+
klog.Errorf("Failed to create dynamic client: %v", err)
71+
return nil, false, err
72+
}
73+
74+
// Create dynamic informer factory for ProviderConfig
75+
gvr := schema.GroupVersionResource{
76+
Group: providerconfigv1.GroupName,
77+
Version: providerconfigv1.SchemeGroupVersion.Version,
78+
Resource: "providerconfigs",
79+
}
80+
dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0)
81+
providerConfigInformer := dynamicInformerFactory.ForResource(gvr).Informer()
82+
83+
// Define controllers
84+
controllers := map[string]gketenantcontrollers.ControllerStartFunc{
85+
"node-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
86+
klog.Infof("Creating OSS Cloud Node Controller for %s...", cfg.ProviderConfig.Name)
87+
nodeController, err := node.NewCloudNodeController(
88+
cfg.NodeInformer,
89+
cfg.KubeClient,
90+
cfg.Cloud,
91+
completedConfig.ComponentConfig.NodeStatusUpdateFrequency.Duration,
92+
completedConfig.ComponentConfig.NodeController.ConcurrentNodeSyncs,
93+
)
94+
if err != nil {
95+
return err
96+
}
97+
klog.Infof("Starting OSS Cloud Node Controller for %s (blocking)", cfg.ProviderConfig.Name)
98+
nodeController.Run(cfg.Context.Done(), cfg.ControllerContext.ControllerManagerMetrics)
99+
return nil
100+
},
101+
"node-ipam-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
102+
klog.Infof("Starting Node IPAM Controller for %s...", cfg.ProviderConfig.Name)
103+
clusterCIDR, err := gketenantcontrollers.GetClusterCIDRsFromProviderConfig(cfg.ProviderConfig)
104+
if err != nil {
105+
klog.Errorf("Failed to get ClusterCIDRs from ProviderConfig: %v. Node IPAM Controller will be disabled.", err)
106+
return nil // Don't fail the whole start
107+
}
108+
109+
_, started, err := nodeipam.StartNodeIpamController(
110+
cfg.Context,
111+
cfg.NodeInformer,
112+
cfg.KubeClient,
113+
cfg.Cloud,
114+
clusterCIDR,
115+
completedConfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs,
116+
nodeIPAMConfig.ServiceCIDR,
117+
nodeIPAMConfig.SecondaryServiceCIDR,
118+
nodeIPAMConfig,
119+
networkInformer,
120+
gnpInformer,
121+
nodeTopologyClient,
122+
ipam.CIDRAllocatorType(completedConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
123+
cfg.ControllerContext.ControllerManagerMetrics,
124+
)
125+
if err != nil {
126+
return err
127+
}
128+
if !started {
129+
klog.Infof("Node IPAM Controller not started (disabled in config) for %s", cfg.ProviderConfig.Name)
130+
} else {
131+
klog.Infof("Node IPAM Controller started with ClusterCIDR: %s for %s", clusterCIDR, cfg.ProviderConfig.Name)
132+
}
133+
// Block until context is canceled so starter doesn't exit early
134+
<-cfg.Context.Done()
135+
return nil
136+
},
137+
"node-lifecycle-controller": func(cfg *gketenantcontrollers.ControllerConfig) error {
138+
klog.Infof("Creating Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
139+
nodeMonitorPeriod := completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration
140+
lifecycleController, err := nodelifecycle.NewCloudNodeLifecycleController(
141+
cfg.NodeInformer,
142+
cfg.KubeClient,
143+
cfg.Cloud,
144+
nodeMonitorPeriod,
145+
)
146+
if err != nil {
147+
return err
148+
}
149+
klog.Infof("Starting Node Lifecycle Controller for %s...", cfg.ProviderConfig.Name)
150+
lifecycleController.Run(cfg.Context, cfg.ControllerContext.ControllerManagerMetrics)
151+
return nil
152+
},
153+
}
154+
155+
// Create the starter
156+
starter := gketenantcontrollers.NewNodeControllerStarter(
157+
completedConfig.ClientBuilder,
158+
completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName),
159+
controlexContext.InformerFactory,
160+
completedConfig,
161+
controlexContext,
162+
controllers,
163+
)
164+
165+
// Create the framework manager
166+
mgr := framework.New(
167+
dynamicClient,
168+
providerConfigInformer,
169+
gkeTenantControllerManagerName,
170+
starter,
171+
ctx.Done(),
172+
)
173+
174+
// Start network informers
175+
networkInformerFactory.Start(ctx.Done())
176+
// Start dynamic informers
177+
dynamicInformerFactory.Start(ctx.Done())
178+
179+
// Run the manager
180+
go mgr.Run()
181+
182+
return nil, true, nil
183+
}

‎cmd/cloud-controller-manager/gketenantcontrollermanager_test.go‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"testing"
6+
"time"
7+
8+
"k8s.io/client-go/informers"
9+
"k8s.io/client-go/kubernetes/fake"
10+
"k8s.io/client-go/rest"
11+
nodeipamconfig "k8s.io/cloud-provider-gcp/pkg/controller/nodeipam/config"
12+
"k8s.io/cloud-provider/app"
13+
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
14+
genericcontrollermanager "k8s.io/controller-manager/app"
15+
"k8s.io/controller-manager/pkg/clientbuilder"
16+
)
17+
18+
func TestStartGKETenantControllerManager(t *testing.T) {
19+
originalEnableProviderConfigController := enableProviderConfigController
20+
defer func() {
21+
enableProviderConfigController = originalEnableProviderConfigController
22+
}()
23+
24+
testCases := []struct {
25+
desc string
26+
enable bool
27+
wantRun bool
28+
}{
29+
{
30+
desc: "disabled",
31+
enable: false,
32+
wantRun: false,
33+
},
34+
{
35+
desc: "enabled",
36+
enable: true,
37+
wantRun: true,
38+
},
39+
}
40+
41+
for _, tc := range testCases {
42+
t.Run(tc.desc, func(t *testing.T) {
43+
enableProviderConfigController = tc.enable
44+
45+
ctx, cancel := context.WithCancel(context.Background())
46+
defer cancel()
47+
48+
kubeClient := fake.NewSimpleClientset()
49+
informerFactory := informers.NewSharedInformerFactory(kubeClient, time.Second)
50+
51+
initContext := app.ControllerInitContext{
52+
ClientName: "test-client",
53+
}
54+
55+
controllerContext := genericcontrollermanager.ControllerContext{
56+
InformerFactory: informerFactory,
57+
}
58+
59+
ccmConfig := &cloudcontrollerconfig.Config{
60+
Kubeconfig: &rest.Config{
61+
Host: "https://example.com",
62+
},
63+
}
64+
ccmConfig.ClientBuilder = clientbuilder.SimpleControllerClientBuilder{
65+
ClientConfig: ccmConfig.Kubeconfig,
66+
}
67+
completedConfig := ccmConfig.Complete()
68+
69+
cloud := &fakeCloudProvider{}
70+
nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{}
71+
72+
_, started, err := startGKETenantControllerManager(
73+
ctx,
74+
initContext,
75+
controllerContext,
76+
completedConfig,
77+
cloud,
78+
nodeIPAMConfig,
79+
)
80+
81+
if err != nil {
82+
t.Fatalf("startGKETenantControllerManager failed: %v", err)
83+
}
84+
85+
if started != tc.wantRun {
86+
t.Errorf("startGKETenantControllerManager started = %v, want %v", started, tc.wantRun)
87+
}
88+
})
89+
}
90+
}

0 commit comments

Comments
 (0)