Skip to content

Commit fa2bd50

Browse files
sergelogvinovsergelogvinov
committed
support multi region cluster
1 parent 2f186d6 commit fa2bd50

File tree

4 files changed

+148
-57
lines changed

4 files changed

+148
-57
lines changed

‎docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ The options in `Global` section are used for openstack-cloud-controller-manager
114114
Keystone user password. If you are using [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html), this option is not required.
115115
* `region`
116116
Required. Keystone region name.
117+
* `regions`
118+
Optional. Keystone region name, witch is used to specify regions for the cloud provider where the instance is running. Region is default region name. Can be specified multiple times.
117119
* `domain-id`
118120
Keystone user domain ID. If you are using [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html), this option is not required.
119121
* `domain-name`
@@ -207,7 +209,7 @@ Although the openstack-cloud-controller-manager was initially implemented with N
207209
* `ROUND_ROBIN` (default)
208210
* `LEAST_CONNECTIONS`
209211
* `SOURCE_IP`
210-
212+
211213
If `lb-provider` is set to "ovn" the value must be set to `SOURCE_IP_PORT`.
212214
213215
* `lb-provider`
@@ -300,7 +302,7 @@ Although the openstack-cloud-controller-manager was initially implemented with N
300302
call](https://docs.openstack.org/api-ref/load-balancer/v2/?expanded=create-a-load-balancer-detail#creating-a-fully-populated-load-balancer).
301303
Setting this option to true will create loadbalancers using serial API calls which first create an unpopulated
302304
loadbalancer, then populate its listeners, pools and members. This is a compatibility option at the expense of
303-
increased load on the OpenStack API. Default: false
305+
increased load on the OpenStack API. Default: false
304306
305307
NOTE:
306308

‎pkg/client/client.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ type AuthOpts struct {
5454
UserDomainID string `gcfg:"user-domain-id" mapstructure:"user-domain-id" name:"os-userDomainID" value:"optional"`
5555
UserDomainName string `gcfg:"user-domain-name" mapstructure:"user-domain-name" name:"os-userDomainName" value:"optional"`
5656
Region string `name:"os-region"`
57+
Regions []string `name:"os-regions" value:"optional"`
5758
EndpointType gophercloud.Availability `gcfg:"os-endpoint-type" mapstructure:"os-endpoint-type" name:"os-endpointType" value:"optional"`
5859
CAFile string `gcfg:"ca-file" mapstructure:"ca-file" name:"os-certAuthorityPath" value:"optional"`
5960
TLSInsecure string `gcfg:"tls-insecure" mapstructure:"tls-insecure" name:"os-TLSInsecure" value:"optional" matches:"^true|false$"`
@@ -88,6 +89,7 @@ func LogCfg(authOpts AuthOpts) {
8889
klog.V(5).Infof("UserDomainID: %s", authOpts.UserDomainID)
8990
klog.V(5).Infof("UserDomainName: %s", authOpts.UserDomainName)
9091
klog.V(5).Infof("Region: %s", authOpts.Region)
92+
klog.V(5).Infof("Regions: %s", authOpts.Regions)
9193
klog.V(5).Infof("EndpointType: %s", authOpts.EndpointType)
9294
klog.V(5).Infof("CAFile: %s", authOpts.CAFile)
9395
klog.V(5).Infof("CertFile: %s", authOpts.CertFile)
@@ -233,6 +235,20 @@ func ReadClouds(authOpts *AuthOpts) error {
233235
authOpts.ApplicationCredentialName = replaceEmpty(authOpts.ApplicationCredentialName, cloud.AuthInfo.ApplicationCredentialName)
234236
authOpts.ApplicationCredentialSecret = replaceEmpty(authOpts.ApplicationCredentialSecret, cloud.AuthInfo.ApplicationCredentialSecret)
235237

238+
regions := strings.Split(authOpts.Region, ",")
239+
if len(regions) > 1 {
240+
authOpts.Region = regions[0]
241+
}
242+
243+
for _, r := range cloud.Regions {
244+
// Support only single auth section in clouds.yaml
245+
if r.Values.AuthInfo == nil && r.Name != authOpts.Region {
246+
regions = append(regions, r.Name)
247+
}
248+
}
249+
250+
authOpts.Regions = regions
251+
236252
return nil
237253
}
238254

‎pkg/openstack/instancesv2.go

Lines changed: 120 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ import (
2020
"context"
2121
"fmt"
2222
sysos "os"
23+
"slices"
24+
"strings"
2325

2426
"github.com/gophercloud/gophercloud"
2527
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
@@ -33,9 +35,9 @@ import (
3335

3436
// InstancesV2 encapsulates an implementation of InstancesV2 for OpenStack.
3537
type InstancesV2 struct {
36-
compute *gophercloud.ServiceClient
37-
network *gophercloud.ServiceClient
38-
region string
38+
compute map[string]*gophercloud.ServiceClient
39+
network map[string]*gophercloud.ServiceClient
40+
regions []string
3941
regionProviderID bool
4042
networkingOpts NetworkingOpts
4143
}
@@ -51,16 +53,25 @@ func (os *OpenStack) InstancesV2() (cloudprovider.InstancesV2, bool) {
5153
func (os *OpenStack) instancesv2() (*InstancesV2, bool) {
5254
klog.V(4).Info("openstack.Instancesv2() called")
5355

54-
compute, err := client.NewComputeV2(os.provider, os.epOpts)
55-
if err != nil {
56-
klog.Errorf("unable to access compute v2 API : %v", err)
57-
return nil, false
58-
}
56+
var err error
57+
compute := make(map[string]*gophercloud.ServiceClient, len(os.regions))
58+
network := make(map[string]*gophercloud.ServiceClient, len(os.regions))
5959

60-
network, err := client.NewNetworkV2(os.provider, os.epOpts)
61-
if err != nil {
62-
klog.Errorf("unable to access network v2 API : %v", err)
63-
return nil, false
60+
for _, region := range os.regions {
61+
opt := os.epOpts
62+
opt.Region = region
63+
64+
compute[region], err = client.NewComputeV2(os.provider, opt)
65+
if err != nil {
66+
klog.Errorf("unable to access compute v2 API : %v", err)
67+
return nil, false
68+
}
69+
70+
network[region], err = client.NewNetworkV2(os.provider, opt)
71+
if err != nil {
72+
klog.Errorf("unable to access network v2 API : %v", err)
73+
return nil, false
74+
}
6475
}
6576

6677
regionalProviderID := false
@@ -71,17 +82,23 @@ func (os *OpenStack) instancesv2() (*InstancesV2, bool) {
7182
return &InstancesV2{
7283
compute: compute,
7384
network: network,
74-
region: os.epOpts.Region,
85+
regions: os.regions,
7586
regionProviderID: regionalProviderID,
7687
networkingOpts: os.networkingOpts,
7788
}, true
7889
}
7990

8091
// InstanceExists indicates whether a given node exists according to the cloud provider
8192
func (i *InstancesV2) InstanceExists(ctx context.Context, node *v1.Node) (bool, error) {
82-
_, err := i.getInstance(ctx, node)
93+
klog.V(4).InfoS("openstack.InstanceExists() called", "node", klog.KObj(node),
94+
"providerID", node.Spec.ProviderID,
95+
"region", node.Labels[v1.LabelTopologyRegion])
96+
97+
_, _, err := i.getInstance(ctx, node)
8398
if err == cloudprovider.InstanceNotFound {
84-
klog.V(6).Infof("instance not found for node: %s", node.Name)
99+
klog.V(6).InfoS("Node is not found in cloud provider", "node", klog.KObj(node),
100+
"providerID", node.Spec.ProviderID,
101+
"region", node.Labels[v1.LabelTopologyRegion])
85102
return false, nil
86103
}
87104

@@ -94,7 +111,11 @@ func (i *InstancesV2) InstanceExists(ctx context.Context, node *v1.Node) (bool,
94111

95112
// InstanceShutdown returns true if the instance is shutdown according to the cloud provider.
96113
func (i *InstancesV2) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) {
97-
server, err := i.getInstance(ctx, node)
114+
klog.V(4).InfoS("openstack.InstanceShutdown() called", "node", klog.KObj(node),
115+
"providerID", node.Spec.ProviderID,
116+
"region", node.Labels[v1.LabelTopologyRegion])
117+
118+
server, _, err := i.getInstance(ctx, node)
98119
if err != nil {
99120
return false, err
100121
}
@@ -109,7 +130,7 @@ func (i *InstancesV2) InstanceShutdown(ctx context.Context, node *v1.Node) (bool
109130

110131
// InstanceMetadata returns the instance's metadata.
111132
func (i *InstancesV2) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) {
112-
srv, err := i.getInstance(ctx, node)
133+
srv, region, err := i.getInstance(ctx, node)
113134
if err != nil {
114135
return nil, err
115136
}
@@ -118,79 +139,124 @@ func (i *InstancesV2) InstanceMetadata(ctx context.Context, node *v1.Node) (*clo
118139
server = *srv
119140
}
120141

121-
instanceType, err := srvInstanceType(i.compute, &server.Server)
142+
instanceType, err := srvInstanceType(i.compute[region], &server.Server)
122143
if err != nil {
123144
return nil, err
124145
}
125146

126-
ports, err := getAttachedPorts(i.network, server.ID)
147+
ports, err := getAttachedPorts(i.network[region], server.ID)
127148
if err != nil {
128149
return nil, err
129150
}
130151

131-
addresses, err := nodeAddresses(&server.Server, ports, i.network, i.networkingOpts)
152+
addresses, err := nodeAddresses(&server.Server, ports, i.network[region], i.networkingOpts)
132153
if err != nil {
133154
return nil, err
134155
}
135156

136157
return &cloudprovider.InstanceMetadata{
137-
ProviderID: i.makeInstanceID(&server.Server),
158+
ProviderID: i.makeInstanceID(&server.Server, region),
138159
InstanceType: instanceType,
139160
NodeAddresses: addresses,
140161
Zone: server.AvailabilityZone,
141-
Region: i.region,
162+
Region: region,
142163
}, nil
143164
}
144165

145-
func (i *InstancesV2) makeInstanceID(srv *servers.Server) string {
166+
func (i *InstancesV2) makeInstanceID(srv *servers.Server, region string) string {
146167
if i.regionProviderID {
147-
return fmt.Sprintf("%s://%s/%s", ProviderName, i.region, srv.ID)
168+
return fmt.Sprintf("%s://%s/%s", ProviderName, region, srv.ID)
148169
}
149170
return fmt.Sprintf("%s:///%s", ProviderName, srv.ID)
150171
}
151172

152-
func (i *InstancesV2) getInstance(ctx context.Context, node *v1.Node) (*ServerAttributesExt, error) {
153-
if node.Spec.ProviderID == "" {
154-
opt := servers.ListOpts{
155-
Name: fmt.Sprintf("^%s$", node.Name),
173+
func (i *InstancesV2) getInstance(ctx context.Context, node *v1.Node) (*ServerAttributesExt, string, error) {
174+
klog.V(4).InfoS("openstack.getInstance() called", "node", klog.KObj(node),
175+
"providerID", node.Spec.ProviderID,
176+
"region", node.Labels[v1.LabelTopologyRegion])
177+
178+
instanceID, instanceRegion, err := instanceIDFromProviderID(node.Spec.ProviderID)
179+
if err == nil && instanceRegion != "" {
180+
if slices.Contains(i.regions, instanceRegion) {
181+
return i.getInstanceByID(instanceID, []string{instanceRegion})
182+
}
183+
184+
return nil, "", fmt.Errorf("getInstance: ProviderID \"%s\" didn't match supported regions \"%s\"", node.Spec.ProviderID, strings.Join(i.regions, ","))
185+
}
186+
187+
// At this point we know that ProviderID is not properly set or it doesn't contain region information
188+
// We need to search for the instance in all regions
189+
var searchRegions []string
190+
191+
// We cannot trust the region label, so we need to check the region
192+
instanceRegion = node.Labels[v1.LabelTopologyRegion]
193+
if slices.Contains(i.regions, instanceRegion) {
194+
searchRegions = []string{instanceRegion}
195+
}
196+
197+
for _, r := range i.regions {
198+
if r != instanceRegion {
199+
searchRegions = append(searchRegions, r)
156200
}
157-
mc := metrics.NewMetricContext("server", "list")
158-
allPages, err := servers.List(i.compute, opt).AllPages()
201+
}
202+
203+
klog.V(6).InfoS("openstack.getInstance() trying to find the instance in regions", "node", klog.KObj(node),
204+
"instanceID", instanceID,
205+
"regions", strings.Join(searchRegions, ","))
206+
207+
if instanceID == "" {
208+
return i.getInstanceByName(node, searchRegions)
209+
}
210+
211+
return i.getInstanceByID(instanceID, searchRegions)
212+
}
213+
214+
func (i *InstancesV2) getInstanceByID(instanceID string, searchRegions []string) (*ServerAttributesExt, string, error) {
215+
server := ServerAttributesExt{}
216+
217+
mc := metrics.NewMetricContext("server", "get")
218+
for _, r := range searchRegions {
219+
err := servers.Get(i.compute[r], instanceID).ExtractInto(&server)
159220
if mc.ObserveRequest(err) != nil {
160-
return nil, fmt.Errorf("error listing servers %v: %v", opt, err)
221+
if errors.IsNotFound(err) {
222+
continue
223+
}
224+
225+
return nil, "", err
226+
}
227+
228+
return &server, r, nil
229+
}
230+
231+
return nil, "", cloudprovider.InstanceNotFound
232+
}
233+
234+
func (i *InstancesV2) getInstanceByName(node *v1.Node, searchRegions []string) (*ServerAttributesExt, string, error) {
235+
opt := servers.ListOpts{
236+
Name: fmt.Sprintf("^%s$", node.Name),
237+
}
238+
mc := metrics.NewMetricContext("server", "list")
239+
serverList := []ServerAttributesExt{}
240+
241+
for _, r := range searchRegions {
242+
allPages, err := servers.List(i.compute[r], opt).AllPages()
243+
if mc.ObserveRequest(err) != nil {
244+
return nil, "", fmt.Errorf("error listing servers %v: %v", opt, err)
161245
}
162246

163-
serverList := []ServerAttributesExt{}
164247
err = servers.ExtractServersInto(allPages, &serverList)
165248
if err != nil {
166-
return nil, fmt.Errorf("error extracting servers from pages: %v", err)
249+
return nil, "", fmt.Errorf("error extracting servers from pages: %v", err)
167250
}
168251
if len(serverList) == 0 {
169-
return nil, cloudprovider.InstanceNotFound
252+
continue
170253
}
171254
if len(serverList) > 1 {
172-
return nil, fmt.Errorf("getInstance: multiple instances found")
255+
return nil, "", fmt.Errorf("getInstanceByName: multiple instances found")
173256
}
174-
return &serverList[0], nil
175-
}
176-
177-
instanceID, instanceRegion, err := instanceIDFromProviderID(node.Spec.ProviderID)
178-
if err != nil {
179-
return nil, err
180-
}
181257

182-
if instanceRegion != "" && instanceRegion != i.region {
183-
return nil, fmt.Errorf("ProviderID \"%s\" didn't match supported region \"%s\"", node.Spec.ProviderID, i.region)
258+
return &serverList[0], r, nil
184259
}
185260

186-
server := ServerAttributesExt{}
187-
mc := metrics.NewMetricContext("server", "get")
188-
err = servers.Get(i.compute, instanceID).ExtractInto(&server)
189-
if mc.ObserveRequest(err) != nil {
190-
if errors.IsNotFound(err) {
191-
return nil, cloudprovider.InstanceNotFound
192-
}
193-
return nil, err
194-
}
195-
return &server, nil
261+
return nil, "", cloudprovider.InstanceNotFound
196262
}

‎pkg/openstack/openstack.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ import (
3737
cloudprovider "k8s.io/cloud-provider"
3838
"k8s.io/klog/v2"
3939

40-
"k8s.io/api/core/v1"
40+
v1 "k8s.io/api/core/v1"
4141
"k8s.io/client-go/informers"
4242
coreinformers "k8s.io/client-go/informers/core/v1"
4343
"k8s.io/client-go/kubernetes/scheme"
@@ -159,6 +159,7 @@ type ServerAttributesExt struct {
159159

160160
// OpenStack is an implementation of cloud provider Interface for OpenStack.
161161
type OpenStack struct {
162+
regions []string
162163
provider *gophercloud.ProviderClient
163164
epOpts *gophercloud.EndpointOpts
164165
lbOpts LoadBalancerOpts
@@ -257,6 +258,11 @@ func ReadConfig(config io.Reader) (Config, error) {
257258
klog.V(5).Infof("Config, loaded from the %s:", cfg.Global.CloudsFile)
258259
client.LogCfg(cfg.Global)
259260
}
261+
262+
if len(cfg.Global.Regions) == 0 {
263+
cfg.Global.Regions = []string{cfg.Global.Region}
264+
}
265+
260266
// Set the default values for search order if not set
261267
if cfg.Metadata.SearchOrder == "" {
262268
cfg.Metadata.SearchOrder = fmt.Sprintf("%s,%s", metadata.ConfigDriveID, metadata.MetadataID)
@@ -309,6 +315,7 @@ func NewOpenStack(cfg Config) (*OpenStack, error) {
309315
}
310316

311317
os := OpenStack{
318+
regions: cfg.Global.Regions,
312319
provider: provider,
313320
epOpts: &gophercloud.EndpointOpts{
314321
Region: cfg.Global.Region,

0 commit comments

Comments
 (0)