Register cross-account TargetGroupBinding IP targets with the pod's availability zone (opt-in) (#4877)

Author: cqi1217

apis/elbv2/v1alpha1/targetgroupbinding_types.go

@@ -136,6 +136,13 @@ type TargetGroupBindingSpec struct {
 	// IAM Role ARN to assume when calling AWS APIs. Needed to assume a role in another account and prevent the confused deputy problem. https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html
 	// +optional
 	AssumeRoleExternalId string `json:"assumeRoleExternalId,omitempty"`
+
+	// registerTargetsWithPodAvailabilityZone specifies whether cross-account targets are registered with the
+	// availability zone of the pod's node instead of "all". Requires iamRoleArnToAssume. Enables in-AZ routing
+	// when cross-zone load balancing is disabled. Availability zone names are translated across accounts using
+	// availability zone IDs, so the assumed role requires ec2:DescribeAvailabilityZones.
+	// +optional
+	RegisterTargetsWithPodAvailabilityZone *bool `json:"registerTargetsWithPodAvailabilityZone,omitempty"`
 }
 
 // TargetGroupBindingStatus defines the observed state of TargetGroupBinding

apis/elbv2/v1alpha1/zz_generated.deepcopy.go

@@ -171,6 +171,13 @@ type TargetGroupBindingSpec struct {
 	// IAM Role ARN to assume when calling AWS APIs. Needed to assume a role in another account and prevent the confused deputy problem. https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html
 	// +optional
 	AssumeRoleExternalId string `json:"assumeRoleExternalId,omitempty"`
+
+	// registerTargetsWithPodAvailabilityZone specifies whether cross-account targets are registered with the
+	// availability zone of the pod's node instead of "all". Requires iamRoleArnToAssume. Enables in-AZ routing
+	// when cross-zone load balancing is disabled. Availability zone names are translated across accounts using
+	// availability zone IDs, so the assumed role requires ec2:DescribeAvailabilityZones.
+	// +optional
+	RegisterTargetsWithPodAvailabilityZone *bool `json:"registerTargetsWithPodAvailabilityZone,omitempty"`
 }
 
 // TargetGroupBindingStatus defines the observed state of TargetGroupBinding

apis/elbv2/v1beta1/targetgroupbinding_types.go

@@ -153,6 +153,13 @@ spec:
                       type: object
                     type: array
                 type: object
+              registerTargetsWithPodAvailabilityZone:
+                description: |-
+                  registerTargetsWithPodAvailabilityZone specifies whether cross-account targets are registered with the
+                  availability zone of the pod's node instead of "all". Requires iamRoleArnToAssume. Enables in-AZ routing
+                  when cross-zone load balancing is disabled. Availability zone names are translated across accounts using
+                  availability zone IDs, so the assumed role requires ec2:DescribeAvailabilityZones.
+                type: boolean
               serviceRef:
                 description: serviceRef is a reference to a Kubernetes Service and
                   ServicePort.
@@ -397,6 +404,13 @@ spec:
                     type: object
                 type: object
                 x-kubernetes-map-type: atomic
+              registerTargetsWithPodAvailabilityZone:
+                description: |-
+                  registerTargetsWithPodAvailabilityZone specifies whether cross-account targets are registered with the
+                  availability zone of the pod's node instead of "all". Requires iamRoleArnToAssume. Enables in-AZ routing
+                  when cross-zone load balancing is disabled. Availability zone names are translated across accounts using
+                  availability zone IDs, so the assumed role requires ec2:DescribeAvailabilityZones.
+                type: boolean
               serviceRef:
                 description: serviceRef is a reference to a Kubernetes Service and
                   ServicePort.

apis/elbv2/v1beta1/zz_generated.deepcopy.go

@@ -143,6 +143,7 @@ Use this feature when you need to manage TargetGroups in a different AWS account
 **Spec fields:**
 * `iamRoleArnToAssume`: The ARN of the role in the TGO account that the controller will assume
 * `assumeRoleExternalId`: External ID for the assume role operation (optional but recommended to prevent the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html))
+* `registerTargetsWithPodAvailabilityZone`: Register cross-account IP targets with the availability zone of the pod's node instead of `all`. See [Availability zones for cross-account targets](#availability-zones-for-cross-account-targets). Requires `iamRoleArnToAssume`.
 
 
 ### Sample YAML
@@ -244,6 +245,47 @@ Add the following permission to the AWS Load Balancer Controller's IAM role. Thi
 }
 ```
 
+### Availability zones for cross-account targets
+
+By default, cross-account IP targets are registered with `AvailabilityZone: all`. Every zonal node of
+the load balancer can then forward to every target, so turning off cross-zone load balancing has no
+effect and inter-AZ data transfer charges are unavoidable.
+
+Set `registerTargetsWithPodAvailabilityZone: true` to register each target with the availability zone
+of the node its pod runs on. With cross-zone load balancing disabled, a zonal load balancer node then
+only forwards to targets in its own zone.
+
+Availability zone *names* are randomized per AWS account — `us-west-2a` in the CO account is usually a
+different physical zone than `us-west-2a` in the TGO account. Availability zone *IDs* (for example
+`usw2-az1`) are stable, so the controller resolves the pod's zone name to a zone ID in the CO account
+and then back to the matching zone name in the TGO account. Both lookups use
+`ec2:DescribeAvailabilityZones`, so the TGO role from Step 2 needs it:
+
+```json
+{
+  "Effect": "Allow",
+  "Action": [
+    "ec2:DescribeAvailabilityZones"
+  ],
+  "Resource": "*"
+}
+```
+
+!!!note ""
+    The zone must be one the load balancer has a subnet in. If a cluster runs nodes in a zone the load
+    balancer does not use, registration is rejected and the TargetGroupBinding reports an error.
+
+!!!note ""
+    If the zone cannot be translated — for instance when the TGO role is missing
+    `ec2:DescribeAvailabilityZones` — the controller logs the failure and falls back to
+    `AvailabilityZone: all` rather than failing registration.
+
+!!!note ""
+    The setting only affects targets registered after it is set. Targets that are already registered
+    keep their current availability zone (`all` or a zone name) until they are deregistered and
+    registered again, for example when the pod is replaced. This avoids disrupting live traffic when
+    the setting is toggled on an existing TargetGroupBinding.
+
 
 ## MultiCluster TargetGroup
 TargetGroupBinding CR supports sharing the same TargetGroup ARN among multiple TargetGroupBindings. Setting this flag allows

config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml

@@ -1041,6 +1041,13 @@ spec:
                       type: object
                     type: array
                 type: object
+              registerTargetsWithPodAvailabilityZone:
+                description: |-
+                  registerTargetsWithPodAvailabilityZone specifies whether cross-account targets are registered with the
+                  availability zone of the pod's node instead of "all". Requires iamRoleArnToAssume. Enables in-AZ routing
+                  when cross-zone load balancing is disabled. Availability zone names are translated across accounts using
+                  availability zone IDs, so the assumed role requires ec2:DescribeAvailabilityZones.
+                type: boolean
               serviceRef:
                 description: serviceRef is a reference to a Kubernetes Service and
                   ServicePort.
@@ -1285,6 +1292,13 @@ spec:
                     type: object
                 type: object
                 x-kubernetes-map-type: atomic
+              registerTargetsWithPodAvailabilityZone:
+                description: |-
+                  registerTargetsWithPodAvailabilityZone specifies whether cross-account targets are registered with the
+                  availability zone of the pod's node instead of "all". Requires iamRoleArnToAssume. Enables in-AZ routing
+                  when cross-zone load balancing is disabled. Availability zone names are translated across accounts using
+                  availability zone IDs, so the assumed role requires ec2:DescribeAvailabilityZones.
+                type: boolean
               serviceRef:
                 description: serviceRef is a reference to a Kubernetes Service and
                   ServicePort.

docs/guide/targetgroupbinding/targetgroupbinding.md

@@ -206,8 +206,9 @@ func main() {
 
 	tgArnMapper := shared_utils.NewTargetGroupNameToArnMapper(cloud.ELBV2())
 
+	azIDTranslator := networking.NewDefaultAZIDTranslator(cloud.EC2(), ctrl.Log.WithName("az-id-translator"))
 	tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(),
-		podInfoRepo, networkingManager, vpcInfoProvider, multiClusterManager, lbcMetricsCollector,
+		podInfoRepo, networkingManager, vpcInfoProvider, azIDTranslator, multiClusterManager, lbcMetricsCollector,
 		cloud.VpcID(), controllerCFG.FeatureGates.Enabled(config.EndpointsFailOpen), controllerCFG.EnableEndpointSlices,
 		mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log, controllerCFG.MaxTargetsPerTargetGroup, controllerCFG.TargetGroupBindingRequeueDuration)
 	backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup,

helm/aws-load-balancer-controller/crds/crds.yaml

@@ -89,7 +89,9 @@ func NewCloud(cfg CloudConfig, clusterName string, metricsCollector *aws_metrics
 			return nil, errors.Wrap(err, "failed to create aws clients provider")
 		}
 	}
-	ec2Service := services.NewEC2(awsClientsProvider)
+	// Constructed without a Cloud back-reference, which does not exist yet. Only used for the VPC ID
+	// lookup below, which never assumes a role. thisObj.ec2 below is the instance the controller uses.
+	ec2Service := services.NewEC2(awsClientsProvider, nil)
 
 	vpcID, err := getVpcID(cfg, ec2Service, ec2Metadata, logger)
 	if err != nil {
@@ -101,7 +103,6 @@ func NewCloud(cfg CloudConfig, clusterName string, metricsCollector *aws_metrics
 	thisObj := &defaultCloud{
 		cfg:               cfg,
 		clusterName:       clusterName,
-		ec2:               ec2Service,
 		route53:           services.NewRoute53(awsClientsProvider),
 		acm:               services.NewACM(awsClientsProvider),
 		wafv2:             services.NewWAFv2(awsClientsProvider),
@@ -113,12 +114,14 @@ func NewCloud(cfg CloudConfig, clusterName string, metricsCollector *aws_metrics
 		awsConfigGenerator: awsConfigGenerator,
 
 		assumeRoleElbV2Cache: cache.NewExpiring(),
+		assumeRoleEc2Cache:   cache.NewExpiring(),
 
 		awsClientsProvider: awsClientsProvider,
 		logger:             logger,
 	}
 
 	thisObj.elbv2 = services.NewELBV2(awsClientsProvider, thisObj, lbStabilizationTime)
+	thisObj.ec2 = services.NewEC2(awsClientsProvider, thisObj)
 
 	return thisObj, nil
 }
@@ -224,6 +227,11 @@ type defaultCloud struct {
 	// assumeRoleElbV2CacheMutex protects assumeRoleElbV2Cache
 	assumeRoleElbV2CacheMutex sync.RWMutex
 
+	// A cache holding ec2 clients that are assuming a role.
+	assumeRoleEc2Cache *cache.Expiring
+	// assumeRoleEc2CacheMutex protects assumeRoleEc2Cache
+	assumeRoleEc2CacheMutex sync.RWMutex
+
 	awsClientsProvider provider.AWSClientsProvider
 	logger             logr.Logger
 }
@@ -255,10 +263,56 @@ func (c *defaultCloud) GetAssumedRoleELBV2(ctx context.Context, assumeRoleArn st
 	}
 	c.logger.Info("Constructing new elbv2 client", "AssumeRoleArn", assumeRoleArn, "externalId", externalId)
 
+	newAwsConfig, cacheTTL, err := c.generateAssumedRoleConfig(ctx, assumeRoleArn, externalId)
+	if err != nil {
+		return nil, err
+	}
+
+	elbv2WithAssumedRole := services.NewELBV2FromStaticClient(c.awsClientsProvider.GenerateNewELBv2Client(newAwsConfig), c, DefaultLbStabilizationTime)
+
+	c.assumeRoleElbV2CacheMutex.Lock()
+	defer c.assumeRoleElbV2CacheMutex.Unlock()
+	c.assumeRoleElbV2Cache.Set(cacheKey, elbv2WithAssumedRole, cacheTTL-cacheTTLBufferTime)
+	return elbv2WithAssumedRole, nil
+}
+
+// GetAssumedRoleEC2 returns EC2 client for the given assumeRoleArn, or the default EC2 client if assumeRoleArn is empty
+func (c *defaultCloud) GetAssumedRoleEC2(ctx context.Context, assumeRoleArn string, externalId string) (services.EC2, error) {
+	if assumeRoleArn == "" {
+		return c.ec2, nil
+	}
+
+	cacheKey := assumedRoleCacheKey{roleArn: assumeRoleArn, externalId: externalId}
+
+	c.assumeRoleEc2CacheMutex.RLock()
+	assumedRoleEC2, exists := c.assumeRoleEc2Cache.Get(cacheKey)
+	c.assumeRoleEc2CacheMutex.RUnlock()
+
+	if exists {
+		return assumedRoleEC2.(services.EC2), nil
+	}
+	c.logger.Info("Constructing new ec2 client", "AssumeRoleArn", assumeRoleArn, "externalId", externalId)
+
+	newAwsConfig, cacheTTL, err := c.generateAssumedRoleConfig(ctx, assumeRoleArn, externalId)
+	if err != nil {
+		return nil, err
+	}
+
+	ec2WithAssumedRole := services.NewEC2FromStaticClient(c.awsClientsProvider.GenerateNewEC2Client(newAwsConfig), c)
+
+	c.assumeRoleEc2CacheMutex.Lock()
+	defer c.assumeRoleEc2CacheMutex.Unlock()
+	c.assumeRoleEc2Cache.Set(cacheKey, ec2WithAssumedRole, cacheTTL-cacheTTLBufferTime)
+	return ec2WithAssumedRole, nil
+}
+
+// generateAssumedRoleConfig assumes the given role and returns an AWS config using the resulting
+// credentials, along with the remaining lifetime of those credentials.
+func (c *defaultCloud) generateAssumedRoleConfig(ctx context.Context, assumeRoleArn string, externalId string) (aws.Config, time.Duration, error) {
 	stsClient, err := c.awsClientsProvider.GetSTSClient(ctx, "AssumeRole")
 	if err != nil {
 		// This should never happen, but let's be forward-looking.
-		return nil, err
+		return aws.Config{}, 0, err
 	}
 
 	assumeRoleInput := &sts.AssumeRoleInput{
@@ -271,23 +325,17 @@ func (c *defaultCloud) GetAssumedRoleELBV2(ctx context.Context, assumeRoleArn st
 	response, err := stsClient.AssumeRole(ctx, assumeRoleInput)
 	if err != nil {
 		c.logger.Error(err, "Unable to assume target role", "roleArn", assumeRoleArn)
-		return nil, err
+		return aws.Config{}, 0, err
 	}
 	assumedRoleCreds := response.Credentials
 	newCreds := credentials.NewStaticCredentialsProvider(*assumedRoleCreds.AccessKeyId, *assumedRoleCreds.SecretAccessKey, *assumedRoleCreds.SessionToken)
 	newAwsConfig, err := c.awsConfigGenerator.GenerateAWSConfig(config.WithCredentialsProvider(newCreds))
 	if err != nil {
 		c.logger.Error(err, "Create new service client config service client config", "roleArn", assumeRoleArn)
-		return nil, err
+		return aws.Config{}, 0, err
 	}
 
-	cacheTTL := assumedRoleCreds.Expiration.Sub(time.Now())
-	elbv2WithAssumedRole := services.NewELBV2FromStaticClient(c.awsClientsProvider.GenerateNewELBv2Client(newAwsConfig), c, DefaultLbStabilizationTime)
-
-	c.assumeRoleElbV2CacheMutex.Lock()
-	defer c.assumeRoleElbV2CacheMutex.Unlock()
-	c.assumeRoleElbV2Cache.Set(cacheKey, elbv2WithAssumedRole, cacheTTL-cacheTTLBufferTime)
-	return elbv2WithAssumedRole, nil
+	return newAwsConfig, assumedRoleCreds.Expiration.Sub(time.Now()), nil
 }
 
 func (c *defaultCloud) EC2() services.EC2 {