WIP: Add support for dual stack load balancers #1313
+1,602
−55
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -234,6 +234,16 @@ const ServiceAnnotationLoadBalancerEIPAllocations = "service.beta.kubernetes.io/ | |||||
| // static IP addresses for the NLB. Only supported on elbv2 (NLB) | ||||||
| const ServiceAnnotationLoadBalancerPrivateIPv4Addresses = "service.beta.kubernetes.io/aws-load-balancer-private-ipv4-addresses" | ||||||
|
|
||||||
| // ServiceAnnotationLoadBalancerIPAddressType is the annotation used on the service | ||||||
| // to specify the IP address type for the load balancer. Supported values are "ipv4" and "dualstack". | ||||||
| // Defaults to "ipv4". Only supported on NLB. | ||||||
| const ServiceAnnotationLoadBalancerIPAddressType = "service.beta.kubernetes.io/aws-load-balancer-ip-address-type" | ||||||
|
|
||||||
| // ServiceAnnotationLoadBalancerTargetGroupIPAddressType is the annotation used on the service | ||||||
| // to specify the IP address type for the target groups. Supported values are "ipv4" and "ipv6". | ||||||
| // Defaults to "ipv4". Only supported on NLB. | ||||||
| const ServiceAnnotationLoadBalancerTargetGroupIPAddressType = "service.beta.kubernetes.io/aws-load-balancer-target-group-ip-address-type" | ||||||
|
|
||||||
|
Comment on lines
+237
to
+246
Contributor
There was a problem hiding this comment. Please also update the service controller documentation: https://github.com/kubernetes/cloud-provider-aws/blob/master/docs/service_controller.md |
||||||
| // ServiceAnnotationLoadBalancerTargetNodeLabels is the annotation used on the service | ||||||
| // to specify a comma-separated list of key-value pairs which will be used to select | ||||||
| // the target nodes for the load balancer | ||||||
|
|
@@ -2116,7 +2126,16 @@ func (c *Cloud) getSubnetCidrs(ctx context.Context, subnetIDs []string) ([]strin | |||||
|
|
||||||
| cidrs := make([]string, 0, len(subnets)) | ||||||
| for _, subnet := range subnets { | ||||||
| // Add IPv4 CIDR | ||||||
| cidrs = append(cidrs, aws.ToString(subnet.CidrBlock)) | ||||||
|
|
||||||
| // Add IPv6 CIDRs if present | ||||||
| for _, ipv6Association := range subnet.Ipv6CidrBlockAssociationSet { | ||||||
| if ipv6Association.Ipv6CidrBlockState != nil && | ||||||
| ipv6Association.Ipv6CidrBlockState.State == ec2types.SubnetCidrBlockStateCodeAssociated { | ||||||
| cidrs = append(cidrs, aws.ToString(ipv6Association.Ipv6CidrBlock)) | ||||||
| } | ||||||
| } | ||||||
| } | ||||||
| return cidrs, nil | ||||||
| } | ||||||
|
|
@@ -2430,11 +2449,6 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS | |||||
| loadBalancerName := c.GetLoadBalancerName(ctx, clusterName, apiService) | ||||||
| serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name} | ||||||
|
|
||||||
| securityGroups, err := c.ensureNLBSecurityGroup(ctx, | ||||||
| loadBalancerName, | ||||||
| clusterName, | ||||||
|
|
@@ -2447,7 +2461,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS | |||||
| serviceName, | ||||||
| loadBalancerName, | ||||||
| v2Mappings, | ||||||
| instances, | ||||||
| discoveredSubnetIDs, | ||||||
| internalELB, | ||||||
| annotations, | ||||||
|
|
@@ -2483,6 +2497,12 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS | |||||
| } | ||||||
| if len(sourceRangeCidrs) == 0 { | ||||||
| sourceRangeCidrs = append(sourceRangeCidrs, "0.0.0.0/0") | ||||||
|
|
||||||
| // For dual-stack or IPv6 load balancers, also add IPv6 default route | ||||||
|
Contributor
There was a problem hiding this comment. route or rule?
Suggested change
There was a problem hiding this comment. Route. "0.0.0.0/0" and "::/0" act as a gateway of last resort, directing unmatched traffic to a specific next-hop or upstream router. |
||||||
| lbIPAddressType := annotations[ServiceAnnotationLoadBalancerIPAddressType] | ||||||
| if lbIPAddressType == string(elbv2types.IpAddressTypeDualstack) { | ||||||
| sourceRangeCidrs = append(sourceRangeCidrs, "::/0") | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| err = c.updateInstanceSecurityGroupsForNLB(ctx, loadBalancerName, instances, subnetCidrs, sourceRangeCidrs, v2Mappings) | ||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,6 +22,7 @@ import ( | |
| "encoding/hex" | ||
| "errors" | ||
| "fmt" | ||
| "net/netip" | ||
| "reflect" | ||
| "regexp" | ||
| "strconv" | ||
|
|
@@ -145,7 +146,7 @@ func getKeyValuePropertiesFromAnnotation(annotations map[string]string, annotati | |
| } | ||
|
|
||
| // ensureLoadBalancerv2 ensures a v2 load balancer is created | ||
| func (c *Cloud) ensureLoadBalancerv2(ctx context.Context, namespacedName types.NamespacedName, loadBalancerName string, mappings []nlbPortMapping, instances map[InstanceID]*ec2types.Instance, discoveredSubnetIDs []string, internalELB bool, annotations map[string]string, securityGroups []string) (*elbv2types.LoadBalancer, error) { | ||
| loadBalancer, err := c.describeLoadBalancerv2(ctx, loadBalancerName) | ||
| if err != nil { | ||
| return nil, err | ||
|
|
@@ -169,6 +170,16 @@ func (c *Cloud) ensureLoadBalancerv2(ctx context.Context, namespacedName types.N | |
| createRequest.Scheme = elbv2types.LoadBalancerSchemeEnumInternal | ||
| } | ||
|
|
||
| // Set IP address type based on annotation | ||
| if ipAddressType, ok := annotations[ServiceAnnotationLoadBalancerIPAddressType]; ok { | ||
| if ipAddressType == string(elbv2types.IpAddressTypeDualstack) || ipAddressType == string(elbv2types.IpAddressTypeIpv4) { | ||
| createRequest.IpAddressType = elbv2types.IpAddressType(ipAddressType) | ||
| } else { | ||
| klog.Warningf("Invalid ip-address-type annotation value: %s, defaulting to ipv4", ipAddressType) | ||
| createRequest.IpAddressType = elbv2types.IpAddressTypeIpv4 | ||
| } | ||
| } | ||
|
|
||
| var allocationIDs []string | ||
| if eipList, present := annotations[ServiceAnnotationLoadBalancerEIPAllocations]; present { | ||
| allocationIDs = strings.Split(eipList, ",") | ||
|
|
@@ -208,7 +219,7 @@ func (c *Cloud) ensureLoadBalancerv2(ctx context.Context, namespacedName types.N | |
| for i := range mappings { | ||
| // It is easier to keep track of updates by having possibly | ||
| // duplicate target groups where the backend port is the same | ||
| _, err := c.createListenerV2(ctx, createResponse.LoadBalancers[0].LoadBalancerArn, mappings[i], namespacedName, instances, *createResponse.LoadBalancers[0].VpcId, tags, annotations) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error creating listener: %q", err) | ||
| } | ||
|
|
@@ -302,9 +313,10 @@ func (c *Cloud) ensureLoadBalancerv2(ctx context.Context, namespacedName types.N | |
| nil, | ||
| namespacedName, | ||
| mapping, | ||
| instances, | ||
| *loadBalancer.VpcId, | ||
| tags, | ||
| annotations, | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
|
|
@@ -351,9 +363,10 @@ func (c *Cloud) ensureLoadBalancerv2(ctx context.Context, namespacedName types.N | |
| targetGroup, | ||
| namespacedName, | ||
| mapping, | ||
| instances, | ||
| *loadBalancer.VpcId, | ||
| tags, | ||
| annotations, | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
|
|
@@ -364,7 +377,7 @@ func (c *Cloud) ensureLoadBalancerv2(ctx context.Context, namespacedName types.N | |
| } | ||
|
|
||
| // Additions | ||
| _, err := c.createListenerV2(ctx, loadBalancer.LoadBalancerArn, mapping, namespacedName, instances, *loadBalancer.VpcId, tags, annotations) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
@@ -685,14 +698,15 @@ func (c *Cloud) buildTargetGroupName(serviceName types.NamespacedName, servicePo | |
| return fmt.Sprintf("k8s-%.8s-%.8s-%.10s", sanitizedNamespace, sanitizedServiceName, tgUUID) | ||
| } | ||
|
|
||
| func (c *Cloud) createListenerV2(ctx context.Context, loadBalancerArn *string, mapping nlbPortMapping, namespacedName types.NamespacedName, instances map[InstanceID]*ec2types.Instance, vpcID string, tags map[string]string, annotations map[string]string) (listener *elbv2types.Listener, err error) { | ||
| target, err := c.ensureTargetGroup(ctx, | ||
| nil, | ||
| namespacedName, | ||
| mapping, | ||
| instances, | ||
| vpcID, | ||
| tags, | ||
| annotations, | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
|
|
@@ -749,9 +763,18 @@ func (c *Cloud) deleteListenerV2(ctx context.Context, listener *elbv2types.Liste | |
| } | ||
|
|
||
| // ensureTargetGroup creates a target group with a set of instances. | ||
| func (c *Cloud) ensureTargetGroup(ctx context.Context, targetGroup *elbv2types.TargetGroup, serviceName types.NamespacedName, mapping nlbPortMapping, instances map[InstanceID]*ec2types.Instance, vpcID string, tags map[string]string, annotations map[string]string) (*elbv2types.TargetGroup, error) { | ||
| dirty := false | ||
|
|
||
| // Determine target group IP address type | ||
| var tgIPAddressType elbv2types.TargetGroupIpAddressTypeEnum | ||
| if ipType, ok := annotations[ServiceAnnotationLoadBalancerTargetGroupIPAddressType]; ok { | ||
| tgIPAddressType = elbv2types.TargetGroupIpAddressTypeEnum(ipType) | ||
| } else { | ||
| tgIPAddressType = elbv2types.TargetGroupIpAddressTypeEnumIpv4 | ||
| } | ||
|
|
||
| expectedTargets := c.computeTargetGroupExpectedTargets(instances, mapping.TrafficPort, tgIPAddressType) | ||
| if targetGroup == nil { | ||
| targetType := elbv2types.TargetTypeEnumInstance | ||
| name := c.buildTargetGroupName(serviceName, mapping.FrontendPort, mapping.TrafficPort, mapping.TrafficProtocol, targetType, mapping) | ||
|
|
@@ -774,6 +797,16 @@ func (c *Cloud) ensureTargetGroup(ctx context.Context, targetGroup *elbv2types.T | |
| input.HealthCheckPath = aws.String(mapping.HealthCheckConfig.Path) | ||
| } | ||
|
|
||
| // Set IP address type based on annotation | ||
| if tgIPAddressType, ok := annotations[ServiceAnnotationLoadBalancerTargetGroupIPAddressType]; ok { | ||
| if tgIPAddressType == string(elbv2types.TargetGroupIpAddressTypeEnumIpv6) || tgIPAddressType == string(elbv2types.TargetGroupIpAddressTypeEnumIpv4) { | ||
|
||
| input.IpAddressType = elbv2types.TargetGroupIpAddressTypeEnum(tgIPAddressType) | ||
| } else { | ||
| klog.Warningf("Invalid target-group-ip-address-type annotation value: %s, defaulting to ipv4", tgIPAddressType) | ||
| input.IpAddressType = elbv2types.TargetGroupIpAddressTypeEnumIpv4 | ||
| } | ||
| } | ||
|
|
||
| if len(tags) != 0 { | ||
| targetGroupTags := make([]elbv2types.Tag, 0, len(tags)) | ||
| for k, v := range tags { | ||
|
|
@@ -886,13 +919,48 @@ func (c *Cloud) ensureTargetGroupTargets(ctx context.Context, tgARN string, expe | |
| return nil | ||
| } | ||
|
|
||
| // extractInstanceIPv6Address extracts the first IPv6 address from an EC2 instance. | ||
| // Returns empty string if no IPv6 address is found. | ||
| func extractInstanceIPv6Address(instance *ec2types.Instance) string { | ||
| if instance == nil { | ||
| return "" | ||
| } | ||
|
|
||
| // Check network interfaces for IPv6 addresses | ||
| for _, networkInterface := range instance.NetworkInterfaces { | ||
| if networkInterface.Status != ec2types.NetworkInterfaceStatusInUse { | ||
| continue | ||
| } | ||
| if len(networkInterface.Ipv6Addresses) > 0 { | ||
| // Return the first IPv6 address | ||
| return aws.ToString(networkInterface.Ipv6Addresses[0].Ipv6Address) | ||
|
Contributor
There was a problem hiding this comment. In order to satisfy the requirement of the target ip family ipv6, I believe you need to verify if the Ipv6Address is primary ( https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_NetworkInterfaceIpv6Address.html
There was a problem hiding this comment. Hey @mtulio That's right, but the primary IPv6 requirement actually depends on the target type of the Target Group. In our case, the following type is applicable:
My understanding is that we should prefer @nrb IIUC, we should make the CCM prefer registering by |
||
| } | ||
| } | ||
| return "" | ||
| } | ||
|
|
||
| func (c *Cloud) computeTargetGroupExpectedTargets(instances map[InstanceID]*ec2types.Instance, port int32, ipAddressType elbv2types.TargetGroupIpAddressTypeEnum) []*elbv2types.TargetDescription { | ||
| expectedTargets := make([]*elbv2types.TargetDescription, 0, len(instances)) | ||
|
|
||
| for instanceID, instance := range instances { | ||
| if ipAddressType == elbv2types.TargetGroupIpAddressTypeEnumIpv6 { | ||
| // For IPv6 target groups, register using the instance's IPv6 address | ||
| ipv6Address := extractInstanceIPv6Address(instance) | ||
| if ipv6Address != "" { | ||
| expectedTargets = append(expectedTargets, &elbv2types.TargetDescription{ | ||
| Id: aws.String(ipv6Address), | ||
| Port: aws.Int32(port), | ||
| }) | ||
| } else { | ||
| klog.Warningf("Instance %s has no IPv6 address, skipping registration to IPv6 target group", instanceID) | ||
| } | ||
| } else { | ||
| // For IPv4 target groups, register using the instance ID | ||
| expectedTargets = append(expectedTargets, &elbv2types.TargetDescription{ | ||
| Id: aws.String(string(instanceID)), | ||
| Port: aws.Int32(port), | ||
| }) | ||
| } | ||
| } | ||
| return expectedTargets | ||
| } | ||
|
|
@@ -1044,23 +1112,49 @@ func (c *Cloud) updateInstanceSecurityGroupsForNLB(ctx context.Context, lbName s | |
| return nil | ||
| } | ||
|
|
||
| // isIPv6CIDR returns true if the given CIDR is an IPv6 CIDR. | ||
| // It uses netip.ParsePrefix to properly parse and validate the CIDR notation. | ||
| func isIPv6CIDR(cidr string) bool { | ||
| prefix, err := netip.ParsePrefix(cidr) | ||
| if err != nil { | ||
| // If parsing fails, fall back to simple string check for backward compatibility | ||
| // This shouldn't happen with valid AWS CIDR blocks, but we handle it gracefully | ||
| klog.Warningf("Failed to parse CIDR %s: %v, falling back to string-based detection", cidr, err) | ||
| return strings.Contains(cidr, ":") | ||
| } | ||
| return prefix.Addr().Is6() | ||
| } | ||
|
|
||
| // updateInstanceSecurityGroupForNLBTraffic will manage permissions set(identified by ruleDesc) on securityGroup to match desired set(allow protocol traffic from ports/cidr). | ||
| // Note: sgPerms will be updated to reflect the current permission set on SG after update. | ||
| func (c *Cloud) updateInstanceSecurityGroupForNLBTraffic(ctx context.Context, sgID string, sgPerms IPPermissionSet, ruleDesc string, protocol string, ports sets.Set[int32], cidrs []string) error { | ||
| desiredPerms := NewIPPermissionSet() | ||
| for port := range ports { | ||
| for _, cidr := range cidrs { | ||
| perm := ec2types.IpPermission{ | ||
| IpProtocol: aws.String(protocol), | ||
|
There was a problem hiding this comment. Should IpProtocol be set below based on result of isIPv6CIDR(cidr)? |
||
| FromPort: aws.Int32(int32(port)), | ||
| ToPort: aws.Int32(int32(port)), | ||
| } | ||
|
|
||
| // Determine if this is an IPv4 or IPv6 CIDR | ||
| if isIPv6CIDR(cidr) { | ||
| perm.Ipv6Ranges = []ec2types.Ipv6Range{ | ||
| { | ||
| CidrIpv6: aws.String(cidr), | ||
| Description: aws.String(ruleDesc), | ||
| }, | ||
| } | ||
| } else { | ||
| perm.IpRanges = []ec2types.IpRange{ | ||
| { | ||
| CidrIp: aws.String(cidr), | ||
| Description: aws.String(ruleDesc), | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| desiredPerms.Insert(perm) | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -1606,9 +1700,9 @@ func (c *Cloud) ensureLoadBalancerHealthCheck(ctx context.Context, loadBalancer | |
| } | ||
|
|
||
| // Makes sure that exactly the specified hosts are registered as instances with the load balancer | ||
| func (c *Cloud) ensureLoadBalancerInstances(ctx context.Context, loadBalancerName string, lbInstances []elbtypes.Instance, instances map[InstanceID]*ec2types.Instance) error { | ||
| expected := sets.NewString() | ||
| for id := range instances { | ||
| expected.Insert(string(id)) | ||
| } | ||
|
|
||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.