Release IPPrefixes: support AWS ENI IP Prefix unassignment

Cilium currently does not support releasing IPPrefixes and secondary
IPs when IPPrefix delegation enabled due to which IP starvation is
happening in our Kubernetes clusters. We have to manually delete a
node in order to free unused IPPrefixes.

With this fix, the operator will release those unused IPPrefixes and
secondary IPs to reassign them again in AWS ENI.

Added a no-op stub implementation in Alibaba and Azure Clouds to satisfy the
NodeOperations interface.

Fixes: cilium#32209, cilium#39904

Signed-off-by: Harish Salluri <hsalluri259@gmail.com>

Author: hsalluri259

pkg/alibabacloud/eni/node.go

@@ -377,6 +377,13 @@ func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *ipam.Rel
 	return r
 }
 
+// ReleaseIPPrefixes is a no-op on AlibabaCloud since Alibaba ENIs don't
+// support prefix delegation.
+func (n *Node) ReleaseIPPrefixes(ctx context.Context, r *ipam.ReleaseAction) error {
+	// nothing to do
+	return nil
+}
+
 // ReleaseIPs performs the ENI IP release operation
 func (n *Node) ReleaseIPs(ctx context.Context, r *ipam.ReleaseAction) error {
 	return n.manager.api.UnassignPrivateIPAddresses(ctx, r.InterfaceID, r.IPsToRelease)
@@ -417,13 +424,6 @@ func (n *Node) IsPrefixDelegated() bool {
 	return false
 }
 
-func (n *Node) GetUsedIPWithPrefixes() int {
-	if n.k8sObj == nil {
-		return 0
-	}
-	return len(n.k8sObj.Status.IPAM.Used)
-}
-
 // getLimits returns the interface and IP limits of this node
 func (n *Node) getLimits() (ipamTypes.Limits, bool) {
 	n.mutex.RLock()

pkg/aws/eni/node.go

@@ -154,8 +154,84 @@ func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *ipam.Rel
 	for _, eniId := range slices.Sorted(maps.Keys(n.enis)) {
 		e := n.enis[eniId]
 
-		// IP release for prefixes is not currently supported. Will skip releasing from this ENI
-		if len(e.Prefixes) > 0 {
+		// Ignore if the ENI is not managed by Cilium
+		if e.IsExcludedBySpec(n.k8sObj.Spec.ENI) {
+			continue
+		}
+		ipPrefixes := e.Prefixes
+		ipsOnENI := e.Addresses
+		usedIPs := n.k8sObj.Status.IPAM.Used
+
+		matchedIPs := []string{}
+		// Returns the first ENI with either IPPrefixes/secondary IPs to release instead of
+		// looking for an ENI with max IPPrefixes/secondary IPs to release for faster and
+		// lower latency when early ENIs are eligible.
+		if len(ipPrefixes) > 0 {
+			scopedLog.Debug(
+				"Considering ENI for IPPrefix release",
+				fieldEniID, e.ID,
+				logfields.NeedIndex, *n.k8sObj.Spec.ENI.FirstInterfaceIndex,
+				logfields.Index, e.Number,
+				logfields.NumAddresses, len(e.Addresses),
+				logfields.LenPrefixes, len(ipPrefixes),
+				logfields.ExcessIPs, excessIPs,
+			)
+
+			unusedIPPrefixes := []string{}
+			if excessIPs >= option.ENIPDBlockSizeIPv4 {
+				// Identify unused IP prefixes to release
+				for _, prefix := range ipPrefixes {
+					prefixAddr, err := netip.ParsePrefix(prefix)
+					if err != nil {
+						continue
+					}
+
+					found := false
+					for ip := range usedIPs {
+						if prefixAddr.Contains(netip.MustParseAddr(ip)) {
+							found = true
+							break
+						}
+					}
+					if !found {
+						unusedIPPrefixes = append(unusedIPPrefixes, prefix)
+						for _, ipStr := range e.Addresses {
+							ip := netip.MustParseAddr(ipStr)
+							if prefixAddr.Contains(ip) {
+								matchedIPs = append(matchedIPs, ipStr)
+							}
+						}
+						// Reduce excessIPs with option.ENIPDBlockSizeIPv4 number of IPs after adding a prefix
+						excessIPs = excessIPs - option.ENIPDBlockSizeIPv4
+					}
+
+					if excessIPs < option.ENIPDBlockSizeIPv4 {
+						break
+					}
+				}
+			}
+
+			secondaryIPs := getIndividualIPs(ipPrefixes, e.Addresses)
+			if len(unusedIPPrefixes) > 0 || len(secondaryIPs) > 0 {
+				r.InterfaceID = eniId
+				r.PoolID = ipamTypes.PoolID(e.Subnet.ID)
+				r.IPPrefixesToRelease = unusedIPPrefixes
+				if len(secondaryIPs) > 0 {
+					secondaryIPs = getUnusedIPs(usedIPs, secondaryIPs, e.IP)
+					maxReleaseOnENI := min(excessIPs, len(secondaryIPs))
+					matchedIPs = append(matchedIPs, secondaryIPs[:maxReleaseOnENI]...)
+				}
+				scopedLog.Debug(
+					"ENI has unused secondary IPs and/or IPPrefixes that can be released",
+					fieldEniID, e.ID,
+					logfields.Prefix, unusedIPPrefixes,
+					logfields.IPAddrs, matchedIPs,
+				)
+				r.IPsToRelease = matchedIPs
+				// Return since we have either IPPrefixes/secondary IPs to release
+				return r
+			}
+			// Look for next ENI if we do not have an ENI with either a Prefix/secondary IP to release
 			continue
 		}
 		scopedLog.Debug(
@@ -166,22 +242,9 @@ func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *ipam.Rel
 			logfields.NumAddresses, len(e.Addresses),
 		)
 
-		if e.IsExcludedBySpec(n.k8sObj.Spec.ENI) {
-			continue
-		}
-
 		// Count free IP addresses on this ENI
-		ipsOnENI := n.k8sObj.Status.ENI.ENIs[e.ID].Addresses
-		freeIpsOnENI := []string{}
-		for _, ip := range ipsOnENI {
-			_, ipUsed := n.k8sObj.Status.IPAM.Used[ip]
-			// exclude primary IPs
-			if !ipUsed && ip != e.IP {
-				freeIpsOnENI = append(freeIpsOnENI, ip)
-			}
-		}
+		freeIpsOnENI := getUnusedIPs(usedIPs, ipsOnENI, e.IP)
 		freeOnENICount := len(freeIpsOnENI)
-
 		if freeOnENICount <= 0 {
 			continue
 		}
@@ -193,7 +256,6 @@ func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *ipam.Rel
 			logfields.FreeOnENICount, freeOnENICount,
 		)
 		maxReleaseOnENI := min(freeOnENICount, excessIPs)
-
 		firstENIWithFreeIPFound := r.IPsToRelease == nil
 		eniWithMoreFreeIPsFound := maxReleaseOnENI > len(r.IPsToRelease)
 		// Select the ENI with the most addresses available for release
@@ -203,12 +265,62 @@ func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *ipam.Rel
 			r.IPsToRelease = freeIpsOnENI[:maxReleaseOnENI]
 		}
 	}
-
 	return r
 }
 
+// ReleaseIPPrefixes performs the ENI IPPrefixes release operation
+func (n *Node) ReleaseIPPrefixes(ctx context.Context, r *ipam.ReleaseAction) error {
+	if err := n.manager.api.UnassignENIPrefixes(ctx, r.InterfaceID, r.IPPrefixesToRelease); err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+// Get Unused Individual IPs
+func getUnusedIPs(usedIPs ipamTypes.AllocationMap, ipAddresses []string, primaryIP string) (unusedIPs []string) {
+	for _, ipStr := range ipAddresses {
+		_, usedIP := usedIPs[ipStr]
+		if !usedIP && ipStr != primaryIP {
+			unusedIPs = append(unusedIPs, ipStr)
+		}
+	}
+	return unusedIPs
+}
+
+// Get Individual IPs that do not belong to any IPPrefix
+func getIndividualIPs(ipPrefixes, ipAddresses []string) (individualIPs []string) {
+	for _, ipStr := range ipAddresses {
+		matched := false
+		ip := netip.MustParseAddr(ipStr)
+		for _, prefix := range ipPrefixes {
+			prefixAddr, err := netip.ParsePrefix(prefix)
+			if err != nil {
+				continue
+			}
+			if prefixAddr.Contains(ip) {
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			individualIPs = append(individualIPs, ipStr)
+		}
+	}
+	return individualIPs
+}
+
 // ReleaseIPs performs the ENI IP release operation
 func (n *Node) ReleaseIPs(ctx context.Context, r *ipam.ReleaseAction) error {
+	// Filter IPs that do not belong to any IPPrefix
+	if len(r.IPPrefixesToRelease) > 0 {
+		r.IPsToRelease = getIndividualIPs(r.IPPrefixesToRelease, r.IPsToRelease)
+	}
+
+	if len(r.IPsToRelease) <= 0 {
+		return nil
+	}
 	if err := n.manager.api.UnassignPrivateIpAddresses(ctx, r.InterfaceID, r.IPsToRelease); err != nil {
 		return err
 	}
@@ -822,58 +934,6 @@ func (n *Node) IsPrefixDelegated() bool {
 	return true
 }
 
-// GetUsedIPWithPrefixes returns the total number of used IPs on the node including the prefixes allocated.
-// A prefix is considered as used if there is at least one allocated IP from that prefix. All IPs from a used prefix
-// are included in the count returned.
-func (n *Node) GetUsedIPWithPrefixes() int {
-	var usedIps int
-	eniPrefixes := make(map[string][]netip.Prefix)
-
-	// Populate ENI -> Prefix mapping
-	for eniName, eni := range n.k8sObj.Status.ENI.ENIs {
-		var prefixes []netip.Prefix
-		for _, pfx := range eni.Prefixes {
-			ipNet, err := netip.ParsePrefix(pfx)
-			if err != nil {
-				continue
-			}
-			prefixes = append(prefixes, ipNet)
-		}
-		eniPrefixes[eniName] = prefixes
-	}
-	usedPfx := make(map[netip.Prefix]bool)
-	for ip, resource := range n.k8sObj.Status.IPAM.Used {
-		// Fetch prefixes on this IP's ENI
-		prefixNetworks, exists := eniPrefixes[resource.Resource]
-		if !exists {
-			continue
-		}
-		var prefixBased bool
-		var pfx netip.Prefix
-		addr, err := netip.ParseAddr(ip)
-		if err != nil {
-			continue
-		}
-		// Check if the IP is from any of the prefixes attached to this ENI
-		for _, ipNet := range prefixNetworks {
-			if ipNet.Contains(addr) {
-				prefixBased = true
-				pfx = ipNet
-				break
-			}
-		}
-		if prefixBased {
-			if !usedPfx[pfx] {
-				usedIps = usedIps + option.ENIPDBlockSizeIPv4
-				usedPfx[pfx] = true
-			}
-		} else {
-			usedIps++
-		}
-	}
-	return usedIps
-}
-
 // getEffectiveIPLimits computing the effective number of available addresses on the ENI
 // based on limits (which includes any left over prefix delegation capacity), as well as
 // just the left over prefix delegation capacity.

pkg/aws/eni/node_test.go

@@ -45,23 +45,6 @@ func TestGetMaximumAllocatableIPv4(t *testing.T) {
 	require.Equal(t, 0, n.GetMaximumAllocatableIPv4())
 }
 
-// TestGetUsedIPWithPrefixes tests the logic computing used IPs on a node when prefix delegation is enabled.
-func TestGetUsedIPWithPrefixes(t *testing.T) {
-	cn := newCiliumNode("node1", withInstanceType("m5a.large"))
-	n := &Node{k8sObj: cn}
-	eniName := "eni-1"
-	prefixes := []string{"10.10.128.0/28", "10.10.128.16/28"}
-	eniMap := make(map[string]types.ENI)
-	eniMap[eniName] = types.ENI{Prefixes: prefixes}
-	cn.Status.ENI.ENIs = eniMap
-
-	allocationMap := make(ipamTypes.AllocationMap)
-	allocationMap["10.10.128.2"] = ipamTypes.AllocationIP{Resource: eniName}
-	allocationMap["10.10.128.18"] = ipamTypes.AllocationIP{Resource: eniName}
-	n.k8sObj.Status.IPAM.Used = allocationMap
-	require.Equal(t, 32, n.GetUsedIPWithPrefixes())
-}
-
 func Test_findSubnetInSameRouteTableWithNodeSubnet(t *testing.T) {
 	routeTableMap := ipamTypes.RouteTableMap{
 		"rt-1": &ipamTypes.RouteTable{

pkg/azure/ipam/node.go

@@ -59,6 +59,13 @@ func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *ipam.Rel
 	return &ipam.ReleaseAction{}
 }
 
+// ReleaseIPPrefixes is a no-op on Azure since Azure ENIs don't
+// support prefix delegation.
+func (n *Node) ReleaseIPPrefixes(ctx context.Context, r *ipam.ReleaseAction) error {
+	// nothing to do
+	return nil
+}
+
 // ReleaseIPs performs the IP release operation
 func (n *Node) ReleaseIPs(ctx context.Context, r *ipam.ReleaseAction) error {
 	return fmt.Errorf("not implemented")
@@ -225,13 +232,6 @@ func (n *Node) IsPrefixDelegated() bool {
 	return false
 }
 
-func (n *Node) GetUsedIPWithPrefixes() int {
-	if n.k8sObj == nil {
-		return 0
-	}
-	return len(n.k8sObj.Status.IPAM.Used)
-}
-
 // isAvailableInterface returns whether interface is available and the number of available IPs to allocate in interface
 func isAvailableInterface(requiredIfaceName string, iface *types.AzureInterface, scopedLog *slog.Logger) (availableOnInterface int, available bool) {
 	if requiredIfaceName != "" {