[azure] Implement excess IP release support

operator/pkg/ipam/alibabacloud.go

@@ -46,6 +46,7 @@ var defaultAlibabaCloudConfig = AlibabaCloudConfig{
 func (cfg AlibabaCloudConfig) Flags(flags *pflag.FlagSet) {
 	flags.String(operatorOption.AlibabaCloudVPCID, defaultAlibabaCloudConfig.AlibabaCloudVPCID, "Specific VPC ID for AlibabaCloud ENI. If not set use same VPC as operator")
 	flags.Bool(operatorOption.AlibabaCloudReleaseExcessIPs, defaultAlibabaCloudConfig.AlibabaCloudReleaseExcessIPs, "Enable releasing excess free IP addresses from Alibaba Cloud ENI.")
+	flags.MarkDeprecated(operatorOption.AlibabaCloudReleaseExcessIPs, "use --ipam-release-excess-ips instead")
 }
 
 type alibabaParams struct {
@@ -67,7 +68,8 @@ type alibabaParams struct {
 func startAlibabaAllocator(p alibabaParams) {
 	alloc := &alibabacloud.AllocatorAlibabaCloud{
 		AlibabaCloudVPCID:            p.AlibabaCfg.AlibabaCloudVPCID,
-		AlibabaCloudReleaseExcessIPs: p.AlibabaCfg.AlibabaCloudReleaseExcessIPs,
+		AlibabaCloudReleaseExcessIPs: p.AlibabaCfg.AlibabaCloudReleaseExcessIPs || p.Cfg.IPAMReleaseExcessIPs,
+		ExcessIPReleaseDelay:         p.Cfg.ExcessIPReleaseDelay,
 		ParallelAllocWorkers:         p.Cfg.ParallelAllocWorkers,
 		LimitIPAMAPIBurst:            p.Cfg.LimitIPAMAPIBurst,
 		LimitIPAMAPIQPS:              p.Cfg.LimitIPAMAPIQPS,

operator/pkg/ipam/allocator/alibabacloud/alibabacloud.go

@@ -31,6 +31,7 @@ var subsysLogAttr = []any{logfields.LogSubsys, "ipam-allocator-alibaba-cloud"}
 type AllocatorAlibabaCloud struct {
 	AlibabaCloudVPCID            string
 	AlibabaCloudReleaseExcessIPs bool
+	ExcessIPReleaseDelay         int
 	ParallelAllocWorkers         int64
 	LimitIPAMAPIBurst            int
 	LimitIPAMAPIQPS              float64
@@ -95,7 +96,7 @@ func (a *AllocatorAlibabaCloud) Start(ctx context.Context, getterUpdater allocat
 
 	instances := ipam.NewInstancesManager(a.rootLogger, a.client)
 	nodeManager, err := nodemanager.NewNodeManager(a.logger, instances, getterUpdater, iMetrics,
-		a.ParallelAllocWorkers, a.AlibabaCloudReleaseExcessIPs, 0, false)
+		a.ParallelAllocWorkers, a.AlibabaCloudReleaseExcessIPs, a.ExcessIPReleaseDelay, false)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize AlibabaCloud node manager: %w", err)
 	}

operator/pkg/ipam/allocator/azure/azure.go

@@ -22,6 +22,8 @@ type AllocatorAzure struct {
 	AzureResourceGroup          string
 	AzureUserAssignedIdentityID string
 	AzureUsePrimaryAddress      bool
+	AzureReleaseExcessIPs       bool
+	ExcessIPReleaseDelay        int
 	ParallelAllocWorkers        int64
 	LimitIPAMAPIBurst           int
 	LimitIPAMAPIQPS             float64
@@ -75,7 +77,7 @@ func (a *AllocatorAzure) Start(ctx context.Context, getterUpdater allocator.Cili
 		return nil, fmt.Errorf("unable to create Azure client: %w", err)
 	}
 	instances := ipam.NewInstancesManager(a.rootLogger, azureClient, a.AzureUsePrimaryAddress)
-	nodeManager, err := nodemanager.NewNodeManager(a.logger, instances, getterUpdater, iMetrics, a.ParallelAllocWorkers, false, 0, false)
+	nodeManager, err := nodemanager.NewNodeManager(a.logger, instances, getterUpdater, iMetrics, a.ParallelAllocWorkers, a.AzureReleaseExcessIPs, a.ExcessIPReleaseDelay, false)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize Azure node manager: %w", err)
 	}

operator/pkg/ipam/aws.go

@@ -35,7 +35,6 @@ func init() {
 
 type AWSConfig struct {
 	AWSReleaseExcessIPs          bool
-	ExcessIPReleaseDelay         int
 	AWSEnablePrefixDelegation    bool
 	ENITags                      map[string]string
 	ENIGarbageCollectionTags     map[string]string `mapstructure:"eni-gc-tags"`
@@ -49,7 +48,6 @@ type AWSConfig struct {
 
 var awsDefaultConfig = AWSConfig{
 	AWSReleaseExcessIPs:          false,
-	ExcessIPReleaseDelay:         180,
 	AWSEnablePrefixDelegation:    false,
 	ENITags:                      nil,
 	ENIGarbageCollectionTags:     nil,
@@ -63,7 +61,7 @@ var awsDefaultConfig = AWSConfig{
 
 func (cfg AWSConfig) Flags(flags *pflag.FlagSet) {
 	flags.Bool("aws-release-excess-ips", awsDefaultConfig.AWSReleaseExcessIPs, "Enable releasing excess free IP addresses from AWS ENI.")
-	flags.Int("excess-ip-release-delay", awsDefaultConfig.ExcessIPReleaseDelay, "Number of seconds operator would wait before it releases an IP previously marked as excess")
+	flags.MarkDeprecated("aws-release-excess-ips", "use --ipam-release-excess-ips instead")
 	flags.Bool("aws-enable-prefix-delegation", awsDefaultConfig.AWSEnablePrefixDelegation, "Allows operator to allocate prefixes to ENIs instead of individual IP addresses")
 	flags.StringToString("eni-tags", awsDefaultConfig.ENITags,
 		"ENI tags in the form of k1=v1 (multiple k/v pairs can be passed by repeating the CLI flag)")
@@ -97,8 +95,8 @@ type awsParams struct {
 
 func startAWSAllocator(p awsParams) {
 	alloc := &aws.AllocatorAWS{
-		AWSReleaseExcessIPs:          p.AwsCfg.AWSReleaseExcessIPs,
-		ExcessIPReleaseDelay:         p.AwsCfg.ExcessIPReleaseDelay,
+		AWSReleaseExcessIPs:          p.AwsCfg.AWSReleaseExcessIPs || p.Cfg.IPAMReleaseExcessIPs,
+		ExcessIPReleaseDelay:         p.Cfg.ExcessIPReleaseDelay,
 		AWSEnablePrefixDelegation:    p.AwsCfg.AWSEnablePrefixDelegation,
 		ENITags:                      p.AwsCfg.ENITags,
 		ENIGarbageCollectionTags:     p.AwsCfg.ENIGarbageCollectionTags,

operator/pkg/ipam/azure.go

@@ -76,6 +76,8 @@ func startAzureAllocator(p azureParams) {
 		AzureResourceGroup:          p.AzureCfg.AzureResourceGroup,
 		AzureUserAssignedIdentityID: p.AzureCfg.AzureUserAssignedIdentityID,
 		AzureUsePrimaryAddress:      p.AzureCfg.AzureUsePrimaryAddress,
+		AzureReleaseExcessIPs:       p.Cfg.IPAMReleaseExcessIPs,
+		ExcessIPReleaseDelay:        p.Cfg.ExcessIPReleaseDelay,
 		ParallelAllocWorkers:        p.Cfg.ParallelAllocWorkers,
 		LimitIPAMAPIBurst:           p.Cfg.LimitIPAMAPIBurst,
 		LimitIPAMAPIQPS:             p.Cfg.LimitIPAMAPIQPS,

operator/pkg/ipam/cell.go

@@ -31,16 +31,22 @@ type Config struct {
 	ParallelAllocWorkers int64
 	LimitIPAMAPIBurst    int
 	LimitIPAMAPIQPS      float64
+	IPAMReleaseExcessIPs bool
+	ExcessIPReleaseDelay int
 }
 
 var defaultConfig = Config{
 	ParallelAllocWorkers: 50,
 	LimitIPAMAPIBurst:    20,
 	LimitIPAMAPIQPS:      4.0,
+	IPAMReleaseExcessIPs: false,
+	ExcessIPReleaseDelay: 180,
 }
 
 func (cfg Config) Flags(flags *pflag.FlagSet) {
 	flags.Int64(option.ParallelAllocWorkers, defaultConfig.ParallelAllocWorkers, "Maximum number of parallel IPAM workers")
 	flags.Int("limit-ipam-api-burst", defaultConfig.LimitIPAMAPIBurst, "Upper burst limit when accessing external APIs")
 	flags.Float64("limit-ipam-api-qps", defaultConfig.LimitIPAMAPIQPS, "Queries per second limit when accessing external IPAM APIs")
+	flags.Bool("ipam-release-excess-ips", defaultConfig.IPAMReleaseExcessIPs, "Enable releasing excess free IP addresses from the cloud provider, regardless of the provider in use.")
+	flags.Int("excess-ip-release-delay", defaultConfig.ExcessIPReleaseDelay, "Number of seconds operator would wait before it releases an IP previously marked as excess")
 }

pkg/azure/api/api.go

@@ -380,6 +380,9 @@ func parseInterface(logger *slog.Logger, iface *armnetwork.Interface, subnets ip
 				IP:    iputil.AddrFrom(parsedIP),
 				State: strings.ToLower(string(*ip.Properties.ProvisioningState)),
 			}
+			if ip.Name != nil {
+				addr.SetIPConfigName(*ip.Name)
+			}
 			if ip.Properties.Subnet != nil {
 				addr.Subnet = *ip.Properties.Subnet.ID //nolint:staticcheck // transitional, see https://github.com/cilium/cilium/issues/46074
 			}
@@ -839,6 +842,215 @@ func (c *Client) getVMPublicIP(ctx context.Context, publicIPRef *armnetwork.Publ
 	return addr, nil
 }
 
+// PrimaryReleaseError is returned by the Unassign* methods when a release set
+// would drop a primary IPConfiguration. Azure rejects such an update and fails
+// the whole batch, so the methods refuse pre-flight without mutating the NIC.
+type PrimaryReleaseError struct {
+	// InterfaceName is the NIC the primary IPConfiguration belongs to.
+	InterfaceName string
+	// Items holds the offending IPs (VM path) or IPConfiguration names (VMSS path).
+	Items []string
+}
+
+func (e *PrimaryReleaseError) Error() string {
+	return fmt.Sprintf("interface %s: refusing to release primary IPConfiguration(s) %v", e.InterfaceName, e.Items)
+}
+
+// dropMatchingIPConfigsVM splits ipConfigs by whether their IP is in releaseSet.
+// Requested primaries are kept and reported via primaryBlocked, never dropped.
+func dropMatchingIPConfigsVM(
+	ipConfigs []*armnetwork.InterfaceIPConfiguration,
+	releaseSet map[string]struct{},
+) (kept []*armnetwork.InterfaceIPConfiguration, dropped int, primaryBlocked []string) {
+	kept = make([]*armnetwork.InterfaceIPConfiguration, 0, len(ipConfigs))
+	for _, c := range ipConfigs {
+		if c == nil || c.Properties == nil || c.Properties.PrivateIPAddress == nil {
+			kept = append(kept, c)
+			continue
+		}
+		ip := *c.Properties.PrivateIPAddress
+		_, requested := releaseSet[ip]
+		isPrimary := c.Properties.Primary != nil && *c.Properties.Primary
+		switch {
+		case requested && isPrimary:
+			primaryBlocked = append(primaryBlocked, ip)
+			kept = append(kept, c)
+		case requested:
+			dropped++
+		default:
+			kept = append(kept, c)
+		}
+	}
+	return
+}
+
+// dropMatchingIPConfigsVMSS is like dropMatchingIPConfigsVM but matches by
+// IPConfiguration name, as the VMSS compute model does not carry the IP.
+func dropMatchingIPConfigsVMSS(
+	ipConfigs []*armcompute.VirtualMachineScaleSetIPConfiguration,
+	releaseNames map[string]struct{},
+) (kept []*armcompute.VirtualMachineScaleSetIPConfiguration, dropped int, primaryBlocked []string) {
+	kept = make([]*armcompute.VirtualMachineScaleSetIPConfiguration, 0, len(ipConfigs))
+	for _, c := range ipConfigs {
+		if c == nil || c.Name == nil {
+			kept = append(kept, c)
+			continue
+		}
+		name := *c.Name
+		_, requested := releaseNames[name]
+		isPrimary := c.Properties != nil && c.Properties.Primary != nil && *c.Properties.Primary
+		switch {
+		case requested && isPrimary:
+			primaryBlocked = append(primaryBlocked, name)
+			kept = append(kept, c)
+		case requested:
+			dropped++
+		default:
+			kept = append(kept, c)
+		}
+	}
+	return
+}
+
+// UnassignPrivateIpAddressesVM releases the given IPs from the named NIC of a
+// standalone VM, returning *PrimaryReleaseError if any IP backs the primary.
+func (c *Client) UnassignPrivateIpAddressesVM(ctx context.Context, interfaceName string, addresses []string) error {
+	if len(addresses) == 0 {
+		return nil
+	}
+
+	c.limiter.Limit(ctx, interfacesGet)
+	sinceStart := spanstat.Start()
+
+	iface, err := c.interfaces.Get(ctx, c.resourceGroup, interfaceName, nil)
+
+	c.metricsAPI.ObserveAPICall(interfacesGet, deriveStatus(err), sinceStart.Seconds())
+	if err != nil {
+		return fmt.Errorf("failed to get standalone instance's interface %s: %w", interfaceName, err)
+	}
+
+	releaseSet := make(map[string]struct{}, len(addresses))
+	for _, ip := range addresses {
+		releaseSet[ip] = struct{}{}
+	}
+
+	kept, dropped, primaryBlocked := dropMatchingIPConfigsVM(iface.Properties.IPConfigurations, releaseSet)
+	if len(primaryBlocked) > 0 {
+		return &PrimaryReleaseError{InterfaceName: interfaceName, Items: primaryBlocked}
+	}
+	if dropped < len(addresses) {
+		// Requested IPs no longer on the NIC: likely a stale cache, harmless.
+		c.logger.Debug("Some requested IPs were not present on the interface during release",
+			logfields.Interface, interfaceName,
+			logfields.IPAddrs, addresses,
+		)
+	}
+	if dropped == 0 {
+		return nil
+	}
+	iface.Properties.IPConfigurations = kept
+
+	c.limiter.Limit(ctx, interfacesCreateOrUpdate)
+	sinceStart = spanstat.Start()
+
+	poller, err := c.interfaces.BeginCreateOrUpdate(ctx, c.resourceGroup, interfaceName, iface.Interface, nil)
+
+	defer func() {
+		c.metricsAPI.ObserveAPICall(interfacesCreateOrUpdate, deriveStatus(err), sinceStart.Seconds())
+	}()
+	if err != nil {
+		return fmt.Errorf("unable to update interface %s: %w", interfaceName, err)
+	}
+
+	// Assign to the outer err so the deferred metric records poll failures.
+	if _, err = poller.PollUntilDone(ctx, nil); err != nil {
+		return fmt.Errorf("error while waiting for interface CreateOrUpdate to complete for %s: %w", interfaceName, err)
+	}
+
+	return nil
+}
+
+// UnassignPrivateIpAddressesVMSS releases the named IPConfigurations from the
+// NIC of a VMSS instance, returning *PrimaryReleaseError if any is the primary.
+func (c *Client) UnassignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, interfaceName string, ipConfigNames []string) error {
+	if len(ipConfigNames) == 0 {
+		return nil
+	}
+
+	vmssGetOptions := &armcompute.VirtualMachineScaleSetVMsClientGetOptions{
+		Expand: new(armcompute.InstanceViewTypesInstanceView),
+	}
+
+	c.limiter.Limit(ctx, virtualMachineScaleSetVMsGet)
+	sinceStart := spanstat.Start()
+
+	result, err := c.virtualMachineScaleSetVMs.Get(ctx, c.resourceGroup, vmssName, instanceID, vmssGetOptions)
+
+	c.metricsAPI.ObserveAPICall(virtualMachineScaleSetVMsGet, deriveStatus(err), sinceStart.Seconds())
+	if err != nil {
+		return fmt.Errorf("failed to get VM %s from VMSS %s: %w", instanceID, vmssName, err)
+	}
+
+	var netIfConfig *armcompute.VirtualMachineScaleSetNetworkConfiguration
+	if result.Properties.NetworkProfileConfiguration != nil {
+		for _, nic := range result.Properties.NetworkProfileConfiguration.NetworkInterfaceConfigurations {
+			if nic.Name != nil && *nic.Name == interfaceName {
+				netIfConfig = nic
+				break
+			}
+		}
+	}
+	if netIfConfig == nil {
+		return fmt.Errorf("interface %s does not exist in VM %s", interfaceName, instanceID)
+	}
+
+	releaseNames := make(map[string]struct{}, len(ipConfigNames))
+	for _, name := range ipConfigNames {
+		releaseNames[name] = struct{}{}
+	}
+
+	kept, dropped, primaryBlocked := dropMatchingIPConfigsVMSS(netIfConfig.Properties.IPConfigurations, releaseNames)
+	if len(primaryBlocked) > 0 {
+		return &PrimaryReleaseError{InterfaceName: interfaceName, Items: primaryBlocked}
+	}
+	if dropped < len(ipConfigNames) {
+		// Requested IPConfigs no longer on the NIC: likely a stale cache, harmless.
+		c.logger.Debug("Some requested IPConfigurations were not present on the interface during release",
+			logfields.Interface, interfaceName,
+			logfields.IPAddrs, ipConfigNames,
+		)
+	}
+	if dropped == 0 {
+		return nil
+	}
+	netIfConfig.Properties.IPConfigurations = kept
+
+	// Unset imageReference to avoid a permissions error on update, as in
+	// AssignPrivateIpAddressesVMSS. See https://github.com/Azure/AKS/issues/1819.
+	if result.Properties.StorageProfile != nil {
+		result.Properties.StorageProfile.ImageReference = nil
+	}
+
+	c.limiter.Limit(ctx, virtualMachineScaleSetVMsUpdate)
+	sinceStart = spanstat.Start()
+
+	poller, err := c.virtualMachineScaleSetVMs.BeginUpdate(ctx, c.resourceGroup, vmssName, instanceID, result.VirtualMachineScaleSetVM, nil)
+
+	defer func() {
+		c.metricsAPI.ObserveAPICall(virtualMachineScaleSetVMsUpdate, deriveStatus(err), sinceStart.Seconds())
+	}()
+	if err != nil {
+		return fmt.Errorf("unable to update virtualMachineScaleSetVMs: %w", err)
+	}
+
+	// Assign to the outer err so the deferred metric records poll failures.
+	if _, err = poller.PollUntilDone(ctx, nil); err != nil {
+		return fmt.Errorf("error while waiting for virtualMachineScaleSetVMs Update to complete: %w", err)
+	}
+
+	return nil
+}
+
 // AssignPublicIPAddressesVMSS assigns a public IP to a VMSS instance.
 // The public IP is allocated from a Public IP Prefix matching publicIpTags
 func (c *Client) AssignPublicIPAddressesVMSS(ctx context.Context, instanceID, vmssName string, publicIpTags ipamTypes.Tags) (netip.Addr, error) {