[azure] implement excess IP release for IPAM nodes

Signed-off-by: Jared Ledvina <jared.ledvina@datadoghq.com>

Author: jaredledvina

pkg/azure/api/mock/mock.go

@@ -276,6 +276,23 @@ func (a *API) AssignPrivateIpAddressesVMSS(ctx context.Context, vmName, vmssName
 	return nil
 }
 
+// AllocateSubnetIP marks ip as allocated in the named subnet's allocator. It is
+// a test helper so callers can observe the IP being freed back when it is later
+// released via Unassign*.
+func (a *API) AllocateSubnetIP(subnetID, ip string) error {
+	a.mutex.Lock()
+	defer a.mutex.Unlock()
+	s, ok := a.subnets[subnetID]
+	if !ok {
+		return fmt.Errorf("subnet %s does not exist", subnetID)
+	}
+	addr, err := netip.ParseAddr(ip)
+	if err != nil {
+		return err
+	}
+	return s.allocator.Allocate(addr)
+}
+
 // SetPrimaryIPs marks the given identifiers (IPs for the VM path or IPConfig
 // names for the VMSS path) as primary on the named interface. Calls to
 // UnassignPrivateIpAddresses* that intersect this set return

pkg/azure/ipam/instances.go

@@ -9,6 +9,7 @@ import (
 	"log/slog"
 	"maps"
 	"net/netip"
+	"slices"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -225,6 +226,44 @@ func (m *InstancesManager) InstanceSync(ctx context.Context, instanceID string)
 	return m.resyncInstance(ctx, instanceID)
 }
 
+// RemoveIPsFromInterface removes the given IPs from the cached interface so the
+// IP pool reflects a release before the next full resync. It mirrors the AWS
+// RemoveIPsFromENI helper. Unknown instances/interfaces are logged and ignored;
+// the next resync reconciles the cache with Azure regardless.
+func (m *InstancesManager) RemoveIPsFromInterface(instanceID, interfaceID string, ips []string) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	iface, ok := m.instances.GetInterface(instanceID, interfaceID)
+	if !ok {
+		// A concurrent resync may legitimately have dropped the interface
+		// between the successful release and this cache update; the next
+		// resync reconciles the cache regardless.
+		m.logger.Warn("Interface not found while removing released IPs from cache",
+			logfields.InstanceID, instanceID,
+			logfields.Interface, interfaceID,
+		)
+		return
+	}
+
+	azIface, ok := iface.DeepCopyInterface().(*types.AzureInterface)
+	if !ok {
+		// A non-Azure interface on an Azure node is an invariant violation, not
+		// a benign race — surface it at error level.
+		m.logger.Error("Unexpected interface type while removing released IPs from cache",
+			logfields.InstanceID, instanceID,
+			logfields.Interface, interfaceID,
+		)
+		return
+	}
+
+	release := sets.New[string](ips...)
+	azIface.Addresses = slices.DeleteFunc(azIface.Addresses, func(a types.AzureAddress) bool {
+		return release.Has(a.IP.String())
+	})
+	m.instances.Update(instanceID, azIface)
+}
+
 // DeleteInstance delete instance from m.instances
 func (m *InstancesManager) DeleteInstance(instanceID string) {
 	m.mutex.Lock()

pkg/azure/ipam/node.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"log/slog"
 	"net/netip"
+	"slices"
+	"strings"
 
 	"github.com/cilium/cilium/operator/pkg/ipam/nodemanager"
 	"github.com/cilium/cilium/operator/pkg/ipam/stats"
@@ -58,9 +60,73 @@ func (n *Node) PopulateStatusFields(k8sObj *v2.CiliumNode) {
 	})
 }
 
-// PrepareIPRelease prepares the release of IPs
+// PrepareIPRelease selects up to excessIPs free IP addresses on a single
+// interface attached to this node and returns them as a ReleaseAction. An IP is
+// considered "free" if it is in Succeeded state, IPv4, not the NIC's primary
+// IPConfiguration, and not present in CiliumNode.Status.IPAM.Used.
+//
+// Interfaces are iterated in sorted ID order and the one with the most
+// releasable IPs is chosen (mirroring the AWS path). Deterministic selection
+// matters: if the chosen interface flapped between runs, the node manager would
+// repeatedly reset the per-IP release-delay timer and never release anything.
+//
+// Primary IPConfigurations are excluded because Azure ARM rejects an update
+// that drops the primary, failing the whole batch atomically — so a primary in
+// the release set would jam every secondary selected alongside it. The
+// SDK-level Unassign* methods keep a pre-flight guard as defense-in-depth.
 func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *nodemanager.ReleaseAction {
-	return &nodemanager.ReleaseAction{}
+	r := &nodemanager.ReleaseAction{}
+	requiredIfaceName := n.k8sObj.Spec.Azure.InterfaceName
+	usedIPs := n.k8sObj.Status.IPAM.Used
+
+	n.manager.mutex.RLock()
+	defer n.manager.mutex.RUnlock()
+
+	var ifaces []*types.AzureInterface
+	err := n.manager.instances.ForeachInterface(n.node.InstanceID(),
+		func(_, _ string, ifaceObj ipamTypes.Interface) error {
+			iface, ok := ifaceObj.(*types.AzureInterface)
+			if !ok {
+				return fmt.Errorf("invalid interface object")
+			}
+			if requiredIfaceName != "" && iface.Name != requiredIfaceName {
+				return nil
+			}
+			ifaces = append(ifaces, iface)
+			return nil
+		})
+	if err != nil {
+		scopedLog.Warn(
+			"Unable to enumerate interfaces while preparing IP release",
+			logfields.InstanceID, n.node.InstanceID(),
+			logfields.Error, err,
+		)
+		return r
+	}
+	slices.SortFunc(ifaces, func(a, b *types.AzureInterface) int {
+		return strings.Compare(a.ID, b.ID)
+	})
+
+	for _, iface := range ifaces {
+		free := freeIPsOnInterface(iface, usedIPs)
+		if len(free) == 0 {
+			continue
+		}
+		maxRelease := min(len(free), excessIPs)
+		// Select the interface with the most addresses available for release.
+		if r.IPsToRelease == nil || maxRelease > len(r.IPsToRelease) {
+			r.InterfaceID = iface.ID
+			r.PoolID = ipamTypes.PoolID(iface.Subnet.ID)
+			r.IPsToRelease = free[:maxRelease]
+			scopedLog.Debug(
+				"Interface has unused IPs that can be released",
+				logfields.ID, iface.ID,
+				logfields.ExcessIPs, excessIPs,
+				logfields.IPAddrs, r.IPsToRelease,
+			)
+		}
+	}
+	return r
 }
 
 // ReleaseIPPrefixes is a no-op on Azure since Azure ENIs don't
@@ -70,9 +136,102 @@ func (n *Node) ReleaseIPPrefixes(ctx context.Context, r *nodemanager.ReleaseActi
 	return nil
 }
 
-// ReleaseIPs performs the IP release operation
+// ReleaseIPs performs the IP release operation. For VM (non-VMSS) interfaces the
+// network model carries the IP on each IPConfiguration, so IPs are passed to
+// the SDK directly. For VMSS, the compute model only exposes IPConfiguration
+// names, so IPs are translated to names via the in-memory mapping populated by
+// parseInterface. On success the released IPs are removed from the cached
+// interface so the pool reflects the release before the next full resync.
 func (n *Node) ReleaseIPs(ctx context.Context, r *nodemanager.ReleaseAction) error {
-	return fmt.Errorf("not implemented")
+	if len(r.IPsToRelease) == 0 {
+		return nil
+	}
+
+	iface, err := n.findInterface(r.InterfaceID)
+	if err != nil {
+		return err
+	}
+
+	if iface.GetVMScaleSetName() == "" {
+		if err := n.manager.api.UnassignPrivateIpAddressesVM(ctx, iface.Name, r.IPsToRelease); err != nil {
+			return err
+		}
+	} else {
+		names, missing := ipsToConfigNames(iface, r.IPsToRelease)
+		if len(missing) > 0 {
+			return fmt.Errorf("interface %s: missing IPConfiguration name mapping for IPs %v (cache out of sync, will retry after next resync)", iface.Name, missing)
+		}
+		if err := n.manager.api.UnassignPrivateIpAddressesVMSS(ctx, iface.GetVMID(), iface.GetVMScaleSetName(), iface.Name, names); err != nil {
+			return err
+		}
+	}
+
+	n.manager.RemoveIPsFromInterface(n.node.InstanceID(), r.InterfaceID, r.IPsToRelease)
+	return nil
+}
+
+// freeIPsOnInterface returns the releasable IPv4 addresses on iface as sorted
+// strings: Succeeded state, not the NIC's primary IPConfiguration, and absent
+// from used. The result is sorted so truncation to excessIPs is deterministic.
+func freeIPsOnInterface(iface *types.AzureInterface, used ipamTypes.AllocationMap) []string {
+	free := make([]string, 0, len(iface.Addresses))
+	for _, a := range iface.Addresses {
+		if a.State != types.StateSucceeded {
+			continue
+		}
+		// The release handshake is driven by IPv4 excess; never release an
+		// IPv6 address through this path.
+		if !a.IP.Addr.Is4() {
+			continue
+		}
+		// Never release the primary IPConfiguration (see PrepareIPRelease).
+		if a.IP == iface.IP {
+			continue
+		}
+		ip := a.IP.String()
+		if _, inUse := used[ip]; inUse {
+			continue
+		}
+		free = append(free, ip)
+	}
+	slices.Sort(free)
+	return free
+}
+
+// findInterface returns a copy of the AzureInterface with the given ID on this
+// node, or an error if not found. A copy is returned so callers can read it
+// without holding the manager mutex.
+func (n *Node) findInterface(interfaceID string) (*types.AzureInterface, error) {
+	n.manager.mutex.RLock()
+	defer n.manager.mutex.RUnlock()
+	iface, ok := n.manager.instances.GetInterface(n.node.InstanceID(), interfaceID)
+	if !ok {
+		return nil, fmt.Errorf("interface %s not found on instance %s", interfaceID, n.node.InstanceID())
+	}
+	azIface, ok := iface.DeepCopyInterface().(*types.AzureInterface)
+	if !ok {
+		return nil, fmt.Errorf("interface %s on instance %s has unexpected type", interfaceID, n.node.InstanceID())
+	}
+	return azIface, nil
+}
+
+// ipsToConfigNames maps each IP in ips to its IPConfiguration name on iface.
+// IPs without a known mapping are returned in missing.
+func ipsToConfigNames(iface *types.AzureInterface, ips []string) (names, missing []string) {
+	byIP := make(map[string]string, len(iface.Addresses))
+	for _, a := range iface.Addresses {
+		if name := a.IPConfigName(); name != "" {
+			byIP[a.IP.String()] = name
+		}
+	}
+	for _, ip := range ips {
+		if name, ok := byIP[ip]; ok {
+			names = append(names, name)
+		} else {
+			missing = append(missing, ip)
+		}
+	}
+	return
 }
 
 // PrepareIPAllocation returns the number of IPs that can be allocated/created.