Assign multiple hyperscaler accounts for global accounts with a large…

cmd/broker/broker_suite_test.go

@@ -84,6 +84,7 @@ type BrokerSuiteTest struct {
 	eventBroker              *event.PubSub
 	metrics                  *metrics.RegisterContainer
 	k8sDeletionObjectTracker Deleter
+	gardenerClient           *dynamicFake.FakeDynamicClient
 }
 
 func (s *BrokerSuiteTest) AddNotCompletedStep(suspensionOpID string) {
@@ -226,6 +227,7 @@ func NewBrokerSuiteTestWithConfig(t *testing.T, cfg *Config, version ...string)
 		eventBroker:    eventBroker,
 
 		k8sDeletionObjectTracker: ot,
+		gardenerClient:           gardenerClient,
 	}
 	ts.poller = &broker.TimerPoller{PollInterval: 3 * time.Millisecond, PollTimeout: 800 * time.Millisecond, Log: ts.t.Log}
 
@@ -248,7 +250,7 @@ func (s *BrokerSuiteTest) GetKcpClient() client.Client {
 
 func createSubscriptions(t *testing.T, gardenerClient *dynamicFake.FakeDynamicClient, bindingResource string) {
 	resource := gardener.SecretBindingResource
-	if strings.ToLower(bindingResource) == "credentialsbinding" {
+	if strings.ToLower(bindingResource) == credentialsBinding {
 		resource = gardener.CredentialsBindingResource
 	}
 
@@ -1025,3 +1027,41 @@ func fixDiscoveredZones() map[string][]string {
 		"c7i.large": {"zone-l", "zone-m"},
 	}
 }
+
+func (s *BrokerSuiteTest) CreateTestShoot(shootName, credentialsBindingName string) {
+	shoot := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "core.gardener.cloud/v1beta1",
+			"kind":       "Shoot",
+			"metadata": map[string]interface{}{
+				"name":      shootName,
+				"namespace": gardenerKymaNamespace,
+			},
+			"spec": map[string]interface{}{
+				"credentialsBindingName": credentialsBindingName,
+			},
+		},
+	}
+
+	_, err := s.gardenerClient.Resource(schema.GroupVersionResource{
+		Group:    "core.gardener.cloud",
+		Version:  "v1beta1",
+		Resource: "shoots",
+	}).Namespace(gardenerKymaNamespace).Create(context.Background(), shoot, metav1.CreateOptions{})
+
+	require.NoError(s.t, err)
+}
+
+func (s *BrokerSuiteTest) CreateAdditionalCredentialsBinding(name, hyperscalerType string) {
+	cb := &gardener.CredentialsBinding{}
+	cb.SetName(name)
+	cb.SetNamespace(gardenerKymaNamespace)
+	cb.SetLabels(map[string]string{
+		"hyperscalerType": hyperscalerType,
+	})
+	cb.SetSecretRefName(name)
+	cb.SetSecretRefNamespace(gardenerKymaNamespace)
+
+	_, err := s.gardenerClient.Resource(gardener.CredentialsBindingResource).Namespace(gardenerKymaNamespace).Create(context.Background(), &cb.Unstructured, metav1.CreateOptions{})
+	require.NoError(s.t, err)
+}

cmd/broker/main.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/kyma-project/kyma-environment-broker/common/gardener"
+	"github.com/kyma-project/kyma-environment-broker/common/hyperscaler/multiaccount"
 	"github.com/kyma-project/kyma-environment-broker/common/hyperscaler/rules"
 	"github.com/kyma-project/kyma-environment-broker/internal"
 	"github.com/kyma-project/kyma-environment-broker/internal/additionalproperties"
@@ -125,6 +126,8 @@ type Config struct {
 
 	HapRuleFilePath string
 
+	HapMultiHyperscalerAccount multiaccount.MultiAccountConfig `envconfig:"optional"`
+
 	ProvidersConfigurationFilePath string
 
 	PlansConfigurationFilePath string

cmd/broker/provisioning.go

@@ -57,7 +57,7 @@ func NewProvisioningProcessingQueue(ctx context.Context, provisionManager *proce
 		},
 		{
 			step: steps.NewHolderStep(cfg.HoldHapSteps,
-				provisioning.NewResolveCredentialsBindingStep(db, gardenerClient, rulesService, internal.RetryTuple{Timeout: resolveSubscriptionSecretTimeout, Interval: resolveSubscriptionSecretRetryInterval})),
+				provisioning.NewResolveCredentialsBindingStep(db, gardenerClient, rulesService, internal.RetryTuple{Timeout: resolveSubscriptionSecretTimeout, Interval: resolveSubscriptionSecretRetryInterval}, &cfg.HapMultiHyperscalerAccount)),
 			disabled: !useCredentialsBinding,
 		},
 		{

cmd/broker/provisioning_test.go

@@ -3440,3 +3440,178 @@ kyma-template: |-
 		assert.Equal(t, "fast", channelValue, "User selection should override plan default")
 	})
 }
+
+func TestProvisioning_MultiHyperscalerAccounts(t *testing.T) {
+	t.Run("disabled feature uses single-account behavior", func(t *testing.T) {
+		// given
+		cfg := fixConfig()
+		cfg.SubscriptionGardenerResource = credentialsBinding
+		cfg.HapMultiHyperscalerAccount.AllowedGlobalAccounts = []string{}
+		cfg.HapMultiHyperscalerAccount.Limits.Default = 100
+		cfg.HapMultiHyperscalerAccount.Limits.AWS = 2
+		cfg.HapMultiHyperscalerAccount.Limits.Azure = 2
+		cfg.HapMultiHyperscalerAccount.Limits.GCP = 2
+		suite := NewBrokerSuiteTestWithConfig(t, cfg)
+		defer suite.TearDown()
+
+		// Create 3 instances for same GA to verify they all use the same binding
+		instanceIDs := []string{
+			uuid.New().String(),
+			uuid.New().String(),
+			uuid.New().String(),
+		}
+
+		// when - provision 3 instances for the same global account
+		suite.provisionMultipleInstances(t, instanceIDs, "test-ga-001")
+
+		// then - verify all instances use the SAME credentials binding (single-account behavior)
+		suite.verifySingleAccountBehavior(t, instanceIDs)
+	})
+
+	t.Run("GA not in allowed global accounts uses single-account behavior", func(t *testing.T) {
+		// given
+		cfg := fixConfig()
+		cfg.SubscriptionGardenerResource = credentialsBinding
+		cfg.HapMultiHyperscalerAccount.AllowedGlobalAccounts = []string{"whitelisted-ga-001"}
+		cfg.HapMultiHyperscalerAccount.Limits.Default = 100
+		cfg.HapMultiHyperscalerAccount.Limits.AWS = 2
+		cfg.HapMultiHyperscalerAccount.Limits.Azure = 2
+		cfg.HapMultiHyperscalerAccount.Limits.GCP = 2
+		suite := NewBrokerSuiteTestWithConfig(t, cfg)
+		defer suite.TearDown()
+
+		// Create 3 instances for same GA (not in allowed global accounts) to verify they all use the same binding
+		instanceIDs := []string{
+			uuid.New().String(),
+			uuid.New().String(),
+			uuid.New().String(),
+		}
+
+		// when - provision 3 instances for GA that's NOT in allowed global accounts
+		suite.provisionMultipleInstances(t, instanceIDs, "not-whitelisted-ga")
+
+		// then - verify all instances use the SAME credentials binding (single-account behavior even with multi-account enabled)
+		suite.verifySingleAccountBehavior(t, instanceIDs)
+	})
+
+	t.Run("enabled for specific GA selects most populated account below limit", func(t *testing.T) {
+		// given
+		cfg := fixConfig()
+		cfg.SubscriptionGardenerResource = credentialsBinding
+		cfg.HapMultiHyperscalerAccount.AllowedGlobalAccounts = []string{"multi-account-ga-001"}
+		// Set low limit to force rotation to multiple bindings
+		cfg.HapMultiHyperscalerAccount.Limits.Default = 100
+		cfg.HapMultiHyperscalerAccount.Limits.AWS = 2
+		cfg.HapMultiHyperscalerAccount.Limits.Azure = 2
+		cfg.HapMultiHyperscalerAccount.Limits.GCP = 2
+		suite := NewBrokerSuiteTestWithConfig(t, cfg)
+		defer suite.TearDown()
+
+		// Create additional AWS bindings for multi-account test
+		// We need at least 3 bindings to accommodate 5 instances with limit of 2
+		suite.CreateAdditionalCredentialsBinding("sb-aws-2", "aws")
+		suite.CreateAdditionalCredentialsBinding("sb-aws-3", "aws")
+
+		// Create 5 instances for same GA - should use at least 3 different bindings (2+2+1)
+		instanceIDs := []string{
+			uuid.New().String(),
+			uuid.New().String(),
+			uuid.New().String(),
+			uuid.New().String(),
+			uuid.New().String(),
+		}
+
+		// when - provision 5 instances to trigger multi-account rotation
+		suite.provisionMultipleInstances(t, instanceIDs, "multi-account-ga-001")
+
+		// then - verify instances are distributed across multiple bindings
+		suite.verifyMultiAccountBehavior(t, instanceIDs, 3,
+			"Should use at least 3 bindings for 5 instances with limit=2")
+	})
+
+	t.Run("wildcard allowed global accounts enables feature for all GAs", func(t *testing.T) {
+		// given
+		cfg := fixConfig()
+		cfg.SubscriptionGardenerResource = credentialsBinding
+		cfg.HapMultiHyperscalerAccount.AllowedGlobalAccounts = []string{"*"}
+		cfg.HapMultiHyperscalerAccount.Limits.Default = 100
+		cfg.HapMultiHyperscalerAccount.Limits.AWS = 2
+		suite := NewBrokerSuiteTestWithConfig(t, cfg)
+		defer suite.TearDown()
+
+		// Create additional AWS bindings
+		suite.CreateAdditionalCredentialsBinding("sb-aws-wildcard-2", "aws")
+
+		// Provision 3 instances with random GA ID to verify wildcard enables multi-account
+		instanceIDs := []string{
+			uuid.New().String(),
+			uuid.New().String(),
+			uuid.New().String(),
+		}
+
+		// when - provision with random GA ID (wildcard should enable multi-account for any GA)
+		suite.provisionMultipleInstances(t, instanceIDs, "any-random-ga-id")
+
+		// then - verify multi-account logic was applied (should use at least 2 different bindings with limit=2)
+		suite.verifyMultiAccountBehavior(t, instanceIDs, 2,
+			"Wildcard allowed global accounts should enable multi-account for any GA - expected at least 2 bindings with limit=2")
+	})
+}
+
+func (s *BrokerSuiteTest) provisionMultipleInstances(t *testing.T, instanceIDs []string, globalAccountID string) {
+	const (
+		serviceID         = "47c9dcbf-ff30-448e-ab36-d3bad66ba281"
+		planID            = "361c511f-f939-4621-b228-d0fb79a1fe15"
+		clusterNamePrefix = "test-cluster"
+		region            = "eu-central-1"
+	)
+
+	for idx, iid := range instanceIDs {
+		resp := s.CallAPI("PUT", fmt.Sprintf("oauth/v2/service_instances/%s?accepts_incomplete=true", iid),
+			fmt.Sprintf(`{
+				"service_id": "%s",
+				"plan_id": "%s",
+				"context": {
+					"globalaccount_id": "%s",
+					"subaccount_id": "sub-id-%d",
+					"user_id": "john.smith@email.com"
+				},
+				"parameters": {
+					"name": "%s-%d",
+					"region": "%s"
+				}
+		}`, serviceID, planID, globalAccountID, idx, clusterNamePrefix, idx, region))
+		defer func() { _ = resp.Body.Close() }()
+
+		opID := s.DecodeOperationID(resp)
+		s.processKIMProvisioningByOperationID(opID)
+		s.WaitForOperationState(opID, domain.Succeeded)
+
+		runtime := s.GetRuntimeResourceByInstanceID(iid)
+		s.CreateTestShoot(fmt.Sprintf("%s-%d", clusterNamePrefix, idx), runtime.Spec.Shoot.SecretBindingName)
+	}
+}
+
+func (s *BrokerSuiteTest) verifySingleAccountBehavior(t *testing.T, instanceIDs []string) {
+	var bindingName string
+	for idx, iid := range instanceIDs {
+		runtime := s.GetRuntimeResourceByInstanceID(iid)
+		if idx == 0 {
+			bindingName = runtime.Spec.Shoot.SecretBindingName
+			assert.NotEmpty(t, bindingName, "First instance should have a binding")
+		} else {
+			assert.Equal(t, bindingName, runtime.Spec.Shoot.SecretBindingName,
+				"Instance %d should use the same binding as instance 0 (single-account behavior)", idx)
+		}
+	}
+}
+
+func (s *BrokerSuiteTest) verifyMultiAccountBehavior(t *testing.T, instanceIDs []string, minExpectedBindings int, testDescription string) {
+	uniqueBindings := make(map[string]bool)
+	for _, iid := range instanceIDs {
+		runtime := s.GetRuntimeResourceByInstanceID(iid)
+		assert.NotEmpty(t, runtime.Spec.Shoot.SecretBindingName)
+		uniqueBindings[runtime.Spec.Shoot.SecretBindingName] = true
+	}
+	assert.GreaterOrEqual(t, len(uniqueBindings), minExpectedBindings, testDescription)
+}

common/gardener/client.go

@@ -327,3 +327,58 @@ func NewGardenerClusterConfig(kubeconfigPath string) (*restclient.Config, error)
 func RESTConfig(kubeconfig []byte) (*restclient.Config, error) {
 	return clientcmd.RESTConfigFromKubeConfig(kubeconfig)
 }
+
+func (c *Client) GetShootCountPerCredentialsBinding() (map[string]int, error) {
+	shoots, err := c.GetShoots()
+	if err != nil {
+		return nil, fmt.Errorf("while listing shoots: %w", err)
+	}
+
+	shootCount := make(map[string]int)
+	if shoots == nil || len(shoots.Items) == 0 {
+		return shootCount, nil
+	}
+
+	for _, shoot := range shoots.Items {
+		s := Shoot{Unstructured: shoot}
+		bindingName := s.GetSpecCredentialsBindingName()
+		if bindingName != "" {
+			shootCount[bindingName]++
+		}
+	}
+
+	return shootCount, nil
+}
+
+func (c *Client) GetMostPopulatedCredentialsBindingBelowLimit(credentialsBindings []unstructured.Unstructured, hyperscalerAccountLimit int) (*CredentialsBinding, error) {
+	if len(credentialsBindings) == 0 {
+		return nil, fmt.Errorf("no credentials bindings provided")
+	}
+
+	shootCount, err := c.GetShootCountPerCredentialsBinding()
+	if err != nil {
+		return nil, fmt.Errorf("while getting shoot count: %w", err)
+	}
+
+	var bestBinding *unstructured.Unstructured
+	maxCount := -1
+
+	for i := range credentialsBindings {
+		cb := &credentialsBindings[i]
+		count := shootCount[cb.GetName()]
+
+		if count < hyperscalerAccountLimit {
+			// Select the one with most clusters (fill-most-populated strategy)
+			if count > maxCount {
+				maxCount = count
+				bestBinding = cb
+			}
+		}
+	}
+
+	if bestBinding == nil {
+		return nil, nil
+	}
+
+	return NewCredentialsBinding(*bestBinding), nil
+}

common/hyperscaler/multiaccount/config.go

@@ -0,0 +1,16 @@
+package multiaccount
+
+type MultiAccountConfig struct {
+	AllowedGlobalAccounts []string
+	Limits                HyperscalerAccountLimits
+}
+
+type HyperscalerAccountLimits struct {
+	Default int `envconfig:"default=100"`
+
+	AWS       int `envconfig:"optional"`
+	GCP       int `envconfig:"optional"`
+	Azure     int `envconfig:"optional"`
+	OpenStack int `envconfig:"optional"`
+	AliCloud  int `envconfig:"optional"`
+}

docs/contributor/02-30-keb-configuration.md

@@ -52,6 +52,13 @@ Kyma Environment Broker (KEB) binary allows you to override some configuration p
 | **APP_GARDENER_&#x200b;KUBECONFIG_PATH** | <code>/gardener/kubeconfig/kubeconfig</code> | Path to the kubeconfig file for accessing the Gardener cluster. |
 | **APP_GARDENER_PROJECT** | <code>kyma-dev</code> | Gardener project connected to SA for HAP credentials lookup. |
 | **APP_GARDENER_SHOOT_&#x200b;DOMAIN** | <code>kyma-dev.shoot.canary.k8s-hana.ondemand.com</code> | Default domain for shoots (clusters) created by Gardener. |
+| **APP_HAP_MULTI_&#x200b;HYPERSCALER_ACCOUNT_&#x200b;ALLOWED_GLOBAL_&#x200b;ACCOUNTS** | <code>[]</code> | assigning multiple hyperscaler accounts per Global Account when capacity limits are reached - Empty array [] = feature disabled - Specific GAs = enabled only for listed Global Accounts - ["*"] = enabled for all Global Accounts |
+| **APP_HAP_MULTI_&#x200b;HYPERSCALER_ACCOUNT_&#x200b;LIMITS_ALICLOUD** | <code>100</code> | - |
+| **APP_HAP_MULTI_&#x200b;HYPERSCALER_ACCOUNT_&#x200b;LIMITS_AWS** | <code>180</code> | - |
+| **APP_HAP_MULTI_&#x200b;HYPERSCALER_ACCOUNT_&#x200b;LIMITS_AZURE** | <code>135</code> | - |
+| **APP_HAP_MULTI_&#x200b;HYPERSCALER_ACCOUNT_&#x200b;LIMITS_DEFAULT** | <code>1000</code> | - |
+| **APP_HAP_MULTI_&#x200b;HYPERSCALER_ACCOUNT_&#x200b;LIMITS_GCP** | <code>135</code> | - |
+| **APP_HAP_MULTI_&#x200b;HYPERSCALER_ACCOUNT_&#x200b;LIMITS_OPENSTACK** | <code>100</code> | - |
 | **APP_HAP_RULE_FILE_&#x200b;PATH** | <code>/config/hapRule.yaml</code> | Path to the rules for mapping plans and regions to hyperscaler account pools. |
 | **APP_HOLD_HAP_STEPS** | <code>false</code> | If true, the broker holds any operation with HAP assignments. It is designed for migration (SecretBinding to CredentialBinding). |
 | **APP_INFRASTRUCTURE_&#x200b;MANAGER_CONTROL_&#x200b;PLANE_FAILURE_&#x200b;TOLERANCE** | None | Sets the failure tolerance level for the Kubernetes control plane in Gardener clusters. Possible values: empty (default), "node", or "zone". |

docs/contributor/02-70-chart-config.md

@@ -114,6 +114,13 @@
 | gardener.secretName | Name of the Kubernetes Secret containing Gardener credentials. | `gardener-credentials` |
 | gardener.shootDomain | Default domain for shoots (clusters) created by Gardener. | `kyma-dev.shoot.canary.k8s-hana.ondemand.com` |
 | hap.rule | Rules for mapping plans and regions to hyperscaler account pools. | `- aws  - aws(PR=cf-eu11) -> EU  - azure  - azure(PR=cf-ch20) -> EU  - gcp  - gcp(PR=cf-sa30) -> PR  - trial -> S  - sap-converged-cloud(HR=*) -> S  - azure_lite  - preview  - free` |
+| hap.multiHyperscalerAccount.<br>allowedGlobalAccounts | assigning multiple hyperscaler accounts per Global Account when capacity limits are reached - Empty array [] = feature disabled - Specific GAs = enabled only for listed Global Accounts - ["*"] = enabled for all Global Accounts | `[]` |
+| hap.multiHyperscalerAccount.<br>limits.default | - | `1000` |
+| hap.multiHyperscalerAccount.<br>limits.aws | - | `180` |
+| hap.multiHyperscalerAccount.<br>limits.gcp | - | `135` |
+| hap.multiHyperscalerAccount.<br>limits.azure | - | `135` |
+| hap.multiHyperscalerAccount.<br>limits.openstack | - | `100` |
+| hap.multiHyperscalerAccount.<br>limits.alicloud | - | `100` |
 | infrastructureManager.<br>controlPlaneFailureTolerance | Sets the failure tolerance level for the Kubernetes control plane in Gardener clusters. Possible values: empty (default), "node", or "zone". | `` |
 | infrastructureManager.<br>defaultShootPurpose | Sets the default purpose for Gardener shoots (clusters) created by the broker. Possible values: development, evaluation, production, testing. | `development` |
 | infrastructureManager.<br>defaultTrialProvider | Sets the default cloud provider for trial Kyma runtimes, for example, Azure, AWS. | `Azure` |