[1.19 backport] [azure] Add support for ip-release

operator/cmd/provider_azure_flags.go

@@ -34,5 +34,11 @@ func (hook *azureFlagsHooks) RegisterProviderFlag(cmd *cobra.Command, vp *viper.
 	flags.Bool(operatorOption.AzureUsePrimaryAddress, false, "Use Azure IP address from interface's primary IPConfigurations")
 	option.BindEnvWithLegacyEnvFallback(vp, operatorOption.AzureUsePrimaryAddress, "AZURE_USE_PRIMARY_ADDRESS")
 
+	flags.Bool(operatorOption.AzureReleaseExcessIPs, false, "Enable releasing excess free IP addresses from Azure NICs.")
+	option.BindEnvWithLegacyEnvFallback(vp, operatorOption.AzureReleaseExcessIPs, "AZURE_RELEASE_EXCESS_IPS")
+
+	flags.Int(operatorOption.ExcessIPReleaseDelay, 180, "Number of seconds operator would wait before it releases an IP previously marked as excess")
+	option.BindEnv(vp, operatorOption.ExcessIPReleaseDelay)
+
 	vp.BindPFlags(flags)
 }

operator/option/config.go

@@ -148,6 +148,11 @@ const (
 	// primary IPConfiguration
 	AzureUsePrimaryAddress = "azure-use-primary-address"
 
+	// AzureReleaseExcessIPs allows releasing excess free IP addresses from Azure NICs.
+	// Enabling this option reduces waste of IP addresses but may increase
+	// the number of API calls to Azure.
+	AzureReleaseExcessIPs = "azure-release-excess-ips"
+
 	// LeaderElectionLeaseDuration is the duration that non-leader candidates will wait to
 	// force acquire leadership
 	LeaderElectionLeaseDuration = "leader-election-lease-duration"
@@ -349,6 +354,9 @@ type OperatorConfig struct {
 	// primary IPConfiguration
 	AzureUsePrimaryAddress bool
 
+	// AzureReleaseExcessIPs allows releasing excess free IP addresses from Azure NICs
+	AzureReleaseExcessIPs bool
+
 	// AlibabaCloud options
 
 	// AlibabaCloudVPCID allow user to specific vpc
@@ -466,6 +474,7 @@ func (c *OperatorConfig) Populate(logger *slog.Logger, vp *viper.Viper) {
 	c.AzureResourceGroup = vp.GetString(AzureResourceGroup)
 	c.AzureUsePrimaryAddress = vp.GetBool(AzureUsePrimaryAddress)
 	c.AzureUserAssignedIdentityID = vp.GetString(AzureUserAssignedIdentityID)
+	c.AzureReleaseExcessIPs = vp.GetBool(AzureReleaseExcessIPs)
 
 	// AlibabaCloud options
 

pkg/azure/api/api.go

@@ -41,6 +41,7 @@ const (
 
 	interfacesListVirtualMachineScaleSetNetworkInterfaces   = "Interfaces.ListVirtualMachineScaleSetNetworkInterfaces"
 	interfacesListVirtualMachineScaleSetVMNetworkInterfaces = "Interfaces.ListVirtualMachineScaleSetVMNetworkInterfaces"
+	interfacesGetVirtualMachineScaleSetNetworkInterface     = "Interfaces.GetVirtualMachineScaleSetNetworkInterface"
 )
 
 // Client represents an Azure API client
@@ -653,6 +654,180 @@ func (c *Client) AssignPrivateIpAddressesVM(ctx context.Context, subnetID, inter
 	return nil
 }
 
+// UnassignPrivateIpAddressesVM removes the given private IPs from an interface attached to a standalone instance
+func (c *Client) UnassignPrivateIpAddressesVM(ctx context.Context, interfaceName string, addresses []string) error {
+	if len(addresses) == 0 {
+		return nil
+	}
+
+	c.limiter.Limit(ctx, interfacesGet)
+	sinceStart := spanstat.Start()
+
+	iface, err := c.interfaces.Get(ctx, c.resourceGroup, interfaceName, nil)
+
+	c.metricsAPI.ObserveAPICall(interfacesGet, deriveStatus(err), sinceStart.Seconds())
+	if err != nil {
+		return fmt.Errorf("failed to get standalone instance's interface %s: %w", interfaceName, err)
+	}
+
+	releaseSet := make(map[string]struct{}, len(addresses))
+	for _, ip := range addresses {
+		releaseSet[ip] = struct{}{}
+	}
+
+	kept := iface.Properties.IPConfigurations[:0]
+	for _, ipConfig := range iface.Properties.IPConfigurations {
+		if ipConfig.Properties != nil && ipConfig.Properties.Primary != nil && *ipConfig.Properties.Primary {
+			kept = append(kept, ipConfig)
+			continue
+		}
+		if ipConfig.Properties == nil || ipConfig.Properties.PrivateIPAddress == nil {
+			kept = append(kept, ipConfig)
+			continue
+		}
+		if _, drop := releaseSet[*ipConfig.Properties.PrivateIPAddress]; drop {
+			continue
+		}
+		kept = append(kept, ipConfig)
+	}
+	iface.Properties.IPConfigurations = kept
+
+	c.limiter.Limit(ctx, interfacesCreateOrUpdate)
+	sinceStart = spanstat.Start()
+
+	poller, err := c.interfaces.BeginCreateOrUpdate(ctx, c.resourceGroup, interfaceName, iface.Interface, nil)
+
+	defer func() {
+		c.metricsAPI.ObserveAPICall(interfacesCreateOrUpdate, deriveStatus(err), sinceStart.Seconds())
+	}()
+	if err != nil {
+		return fmt.Errorf("unable to update interface %s: %w", interfaceName, err)
+	}
+
+	if _, err := poller.PollUntilDone(ctx, nil); err != nil {
+		return fmt.Errorf("error while waiting for interface CreateOrUpdate to complete for %s: %w", interfaceName, err)
+	}
+
+	return nil
+}
+
+// UnassignPrivateIpAddressesVMSS removes the given private IPs from an interface attached to a VMSS instance.
+// Azure VMSS IP configurations don't carry the assigned IP on the desired-state model, so the underlying
+// network interface is fetched first to map IPs to IP configuration names; configs with matching names
+// (excluding primaries) are then dropped from the VMSS VM model and a VMSS update is issued.
+func (c *Client) UnassignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, interfaceName string, addresses []string) error {
+	if len(addresses) == 0 {
+		return nil
+	}
+
+	c.limiter.Limit(ctx, interfacesGetVirtualMachineScaleSetNetworkInterface)
+	sinceStart := spanstat.Start()
+
+	nicResp, err := c.interfaces.GetVirtualMachineScaleSetNetworkInterface(ctx, c.resourceGroup, vmssName, instanceID, interfaceName, nil)
+
+	c.metricsAPI.ObserveAPICall(interfacesGetVirtualMachineScaleSetNetworkInterface, deriveStatus(err), sinceStart.Seconds())
+	if err != nil {
+		return fmt.Errorf("failed to get VMSS %s instance %s interface %s: %w", vmssName, instanceID, interfaceName, err)
+	}
+
+	releaseSet := make(map[string]struct{}, len(addresses))
+	for _, ip := range addresses {
+		releaseSet[ip] = struct{}{}
+	}
+
+	dropNames := make(map[string]struct{})
+	if nicResp.Properties != nil {
+		for _, ipConfig := range nicResp.Properties.IPConfigurations {
+			if ipConfig.Properties == nil || ipConfig.Name == nil || ipConfig.Properties.PrivateIPAddress == nil {
+				continue
+			}
+			if ipConfig.Properties.Primary != nil && *ipConfig.Properties.Primary {
+				continue
+			}
+			if _, drop := releaseSet[*ipConfig.Properties.PrivateIPAddress]; drop {
+				dropNames[*ipConfig.Name] = struct{}{}
+			}
+		}
+	}
+
+	if len(dropNames) == 0 {
+		return nil
+	}
+
+	vmssGetOptions := &armcompute.VirtualMachineScaleSetVMsClientGetOptions{
+		Expand: to.Ptr(armcompute.InstanceViewTypesInstanceView),
+	}
+
+	c.limiter.Limit(ctx, virtualMachineScaleSetVMsGet)
+	sinceStart = spanstat.Start()
+
+	result, err := c.virtualMachineScaleSetVMs.Get(ctx, c.resourceGroup, vmssName, instanceID, vmssGetOptions)
+
+	c.metricsAPI.ObserveAPICall(virtualMachineScaleSetVMsGet, deriveStatus(err), sinceStart.Seconds())
+	if err != nil {
+		return fmt.Errorf("failed to get VM %s from VMSS %s: %w", instanceID, vmssName, err)
+	}
+
+	var netIfConfig *armcompute.VirtualMachineScaleSetNetworkConfiguration
+	if result.Properties.NetworkProfileConfiguration != nil {
+		for _, networkInterfaceConfiguration := range result.Properties.NetworkProfileConfiguration.NetworkInterfaceConfigurations {
+			if networkInterfaceConfiguration.Name != nil && *networkInterfaceConfiguration.Name == interfaceName {
+				netIfConfig = networkInterfaceConfiguration
+				break
+			}
+		}
+	}
+
+	if netIfConfig == nil {
+		return fmt.Errorf("interface %s does not exist in VM %s", interfaceName, instanceID)
+	}
+
+	kept := netIfConfig.Properties.IPConfigurations[:0]
+	for _, ipConfig := range netIfConfig.Properties.IPConfigurations {
+		if ipConfig.Properties != nil && ipConfig.Properties.Primary != nil && *ipConfig.Properties.Primary {
+			kept = append(kept, ipConfig)
+			continue
+		}
+		if ipConfig.Name == nil {
+			kept = append(kept, ipConfig)
+			continue
+		}
+		if _, drop := dropNames[*ipConfig.Name]; drop {
+			continue
+		}
+		kept = append(kept, ipConfig)
+	}
+	netIfConfig.Properties.IPConfigurations = kept
+
+	// Unset imageReference, because if this contains a reference to an image from the
+	// Azure Compute Gallery, including this reference in an update to the VMSS instance
+	// will cause a permissions error, because the reference includes an Azure-managed
+	// subscription ID.
+	// Removing the image reference indicates to the API that we don't want to change it.
+	// See https://github.com/Azure/AKS/issues/1819.
+	if result.Properties.StorageProfile != nil {
+		result.Properties.StorageProfile.ImageReference = nil
+	}
+
+	c.limiter.Limit(ctx, virtualMachineScaleSetVMsUpdate)
+	sinceStart = spanstat.Start()
+
+	poller, err := c.virtualMachineScaleSetVMs.BeginUpdate(ctx, c.resourceGroup, vmssName, instanceID, result.VirtualMachineScaleSetVM, nil)
+
+	defer func() {
+		c.metricsAPI.ObserveAPICall(virtualMachineScaleSetVMsUpdate, deriveStatus(err), sinceStart.Seconds())
+	}()
+	if err != nil {
+		return fmt.Errorf("unable to update virtualMachineScaleSetVMs: %w", err)
+	}
+
+	if _, err := poller.PollUntilDone(ctx, nil); err != nil {
+		return fmt.Errorf("error while waiting for virtualMachineScaleSetVMs Update to complete: %w", err)
+	}
+
+	return nil
+}
+
 // AssignPublicIPAddressesVMSS assigns a public IP to a VMSS instance.
 // The public IP is allocated from a Public IP Prefix matching publicIpTags
 func (c *Client) AssignPublicIPAddressesVMSS(ctx context.Context, instanceID, vmssName string, publicIpTags ipamTypes.Tags) (string, error) {

pkg/azure/api/mock/mock.go

@@ -27,6 +27,7 @@ const (
 	GetInstances
 	GetVpcsAndSubnets
 	AssignPrivateIpAddressesVMSS
+	UnassignPrivateIpAddressesVMSS
 	MaxOperation
 )
 
@@ -253,6 +254,73 @@ func (a *API) AssignPrivateIpAddressesVMSS(ctx context.Context, vmName, vmssName
 	return nil
 }
 
+func (a *API) UnassignPrivateIpAddressesVM(ctx context.Context, interfaceName string, addresses []string) error {
+	return nil
+}
+
+func (a *API) UnassignPrivateIpAddressesVMSS(ctx context.Context, vmName, vmssName, interfaceName string, addresses []string) error {
+	a.rateLimit()
+	a.delaySim.Delay(UnassignPrivateIpAddressesVMSS)
+
+	a.mutex.Lock()
+	defer a.mutex.Unlock()
+
+	if err, ok := a.errors[UnassignPrivateIpAddressesVMSS]; ok {
+		return err
+	}
+
+	if len(addresses) == 0 {
+		return nil
+	}
+
+	releaseSet := make(map[string]struct{}, len(addresses))
+	for _, ip := range addresses {
+		releaseSet[ip] = struct{}{}
+	}
+
+	foundInterface := false
+	instances := a.instances.DeepCopy()
+	err := instances.ForeachInterface("", func(id, _ string, iface ipamTypes.InterfaceRevision) error {
+		intf, ok := iface.Resource.(*types.AzureInterface)
+		if !ok {
+			return fmt.Errorf("invalid interface object")
+		}
+
+		if intf.Name != interfaceName || intf.GetVMID() != vmName {
+			return nil
+		}
+
+		kept := intf.Addresses[:0]
+		for _, addr := range intf.Addresses {
+			if _, drop := releaseSet[addr.IP]; drop {
+				if s, ok := a.subnets[addr.Subnet]; ok {
+					_, ipNet, err := net.ParseCIDR(addr.IP + "/32")
+					if err == nil {
+						s.allocator.Release(ipNet.IP)
+					}
+				}
+				continue
+			}
+			kept = append(kept, addr)
+		}
+		intf.Addresses = kept
+
+		foundInterface = true
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	a.updateInstancesLocked(instances)
+
+	if !foundInterface {
+		return fmt.Errorf("interface %s not found", interfaceName)
+	}
+
+	return nil
+}
+
 func (a *API) AssignPublicIPAddressesVMSS(ctx context.Context, instanceID, vmssName string, publicIpTags ipamTypes.Tags) (string, error) {
 	a.rateLimit()
 	return "mock-public-ip-prefix-id", nil

pkg/azure/ipam/instances.go

@@ -22,6 +22,8 @@ type AzureAPI interface {
 	GetVpcsAndSubnets(ctx context.Context) (ipamTypes.VirtualNetworkMap, ipamTypes.SubnetMap, error)
 	AssignPrivateIpAddressesVM(ctx context.Context, subnetID, interfaceName string, addresses int) error
 	AssignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, subnetID, interfaceName string, addresses int) error
+	UnassignPrivateIpAddressesVM(ctx context.Context, interfaceName string, addresses []string) error
+	UnassignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, interfaceName string, addresses []string) error
 	AssignPublicIPAddressesVM(ctx context.Context, instanceID string, publicIpTags ipamTypes.Tags) (string, error)
 	AssignPublicIPAddressesVMSS(ctx context.Context, instanceID, vmssName string, publicIpTags ipamTypes.Tags) (string, error)
 }