From 21828e763d471c3a65f0d6622bacd7cf0521219d Mon Sep 17 00:00:00 2001 From: Jared Ledvina Date: Thu, 25 Jun 2026 18:40:17 -0400 Subject: [PATCH 1/3] [azure] add SDK support to release private IP addresses Signed-off-by: Jared Ledvina --- pkg/azure/api/api.go | 212 +++++++++++++++++++++++++++ pkg/azure/api/api_test.go | 212 +++++++++++++++++++++++++++ pkg/azure/api/mock/mock.go | 252 +++++++++++++++++++++++++++++++- pkg/azure/api/mock/mock_test.go | 131 +++++++++++++++++ pkg/azure/ipam/instances.go | 2 + pkg/azure/types/types.go | 15 ++ 6 files changed, 820 insertions(+), 4 deletions(-) diff --git a/pkg/azure/api/api.go b/pkg/azure/api/api.go index 95e697cd13ad4..a44b2da945c54 100644 --- a/pkg/azure/api/api.go +++ b/pkg/azure/api/api.go @@ -380,6 +380,9 @@ func parseInterface(logger *slog.Logger, iface *armnetwork.Interface, subnets ip IP: iputil.AddrFrom(parsedIP), State: strings.ToLower(string(*ip.Properties.ProvisioningState)), } + if ip.Name != nil { + addr.SetIPConfigName(*ip.Name) + } if ip.Properties.Subnet != nil { addr.Subnet = *ip.Properties.Subnet.ID //nolint:staticcheck // transitional, see https://github.com/cilium/cilium/issues/46074 } @@ -839,6 +842,215 @@ func (c *Client) getVMPublicIP(ctx context.Context, publicIPRef *armnetwork.Publ return addr, nil } +// PrimaryReleaseError is returned by the Unassign* methods when a release set +// would drop a primary IPConfiguration. Azure rejects such an update and fails +// the whole batch, so the methods refuse pre-flight without mutating the NIC. +type PrimaryReleaseError struct { + // InterfaceName is the NIC the primary IPConfiguration belongs to. + InterfaceName string + // Items holds the offending IPs (VM path) or IPConfiguration names (VMSS path). + Items []string +} + +func (e *PrimaryReleaseError) Error() string { + return fmt.Sprintf("interface %s: refusing to release primary IPConfiguration(s) %v", e.InterfaceName, e.Items) +} + +// dropMatchingIPConfigsVM splits ipConfigs by whether their IP is in releaseSet. +// Requested primaries are kept and reported via primaryBlocked, never dropped. +func dropMatchingIPConfigsVM( + ipConfigs []*armnetwork.InterfaceIPConfiguration, + releaseSet map[string]struct{}, +) (kept []*armnetwork.InterfaceIPConfiguration, dropped int, primaryBlocked []string) { + kept = make([]*armnetwork.InterfaceIPConfiguration, 0, len(ipConfigs)) + for _, c := range ipConfigs { + if c == nil || c.Properties == nil || c.Properties.PrivateIPAddress == nil { + kept = append(kept, c) + continue + } + ip := *c.Properties.PrivateIPAddress + _, requested := releaseSet[ip] + isPrimary := c.Properties.Primary != nil && *c.Properties.Primary + switch { + case requested && isPrimary: + primaryBlocked = append(primaryBlocked, ip) + kept = append(kept, c) + case requested: + dropped++ + default: + kept = append(kept, c) + } + } + return +} + +// dropMatchingIPConfigsVMSS is like dropMatchingIPConfigsVM but matches by +// IPConfiguration name, as the VMSS compute model does not carry the IP. +func dropMatchingIPConfigsVMSS( + ipConfigs []*armcompute.VirtualMachineScaleSetIPConfiguration, + releaseNames map[string]struct{}, +) (kept []*armcompute.VirtualMachineScaleSetIPConfiguration, dropped int, primaryBlocked []string) { + kept = make([]*armcompute.VirtualMachineScaleSetIPConfiguration, 0, len(ipConfigs)) + for _, c := range ipConfigs { + if c == nil || c.Name == nil { + kept = append(kept, c) + continue + } + name := *c.Name + _, requested := releaseNames[name] + isPrimary := c.Properties != nil && c.Properties.Primary != nil && *c.Properties.Primary + switch { + case requested && isPrimary: + primaryBlocked = append(primaryBlocked, name) + kept = append(kept, c) + case requested: + dropped++ + default: + kept = append(kept, c) + } + } + return +} + +// UnassignPrivateIpAddressesVM releases the given IPs from the named NIC of a +// standalone VM, returning *PrimaryReleaseError if any IP backs the primary. +func (c *Client) UnassignPrivateIpAddressesVM(ctx context.Context, interfaceName string, addresses []string) error { + if len(addresses) == 0 { + return nil + } + + c.limiter.Limit(ctx, interfacesGet) + sinceStart := spanstat.Start() + + iface, err := c.interfaces.Get(ctx, c.resourceGroup, interfaceName, nil) + + c.metricsAPI.ObserveAPICall(interfacesGet, deriveStatus(err), sinceStart.Seconds()) + if err != nil { + return fmt.Errorf("failed to get standalone instance's interface %s: %w", interfaceName, err) + } + + releaseSet := make(map[string]struct{}, len(addresses)) + for _, ip := range addresses { + releaseSet[ip] = struct{}{} + } + + kept, dropped, primaryBlocked := dropMatchingIPConfigsVM(iface.Properties.IPConfigurations, releaseSet) + if len(primaryBlocked) > 0 { + return &PrimaryReleaseError{InterfaceName: interfaceName, Items: primaryBlocked} + } + if dropped < len(addresses) { + // Requested IPs no longer on the NIC: likely a stale cache, harmless. + c.logger.Debug("Some requested IPs were not present on the interface during release", + logfields.Interface, interfaceName, + logfields.IPAddrs, addresses, + ) + } + if dropped == 0 { + return nil + } + iface.Properties.IPConfigurations = kept + + c.limiter.Limit(ctx, interfacesCreateOrUpdate) + sinceStart = spanstat.Start() + + poller, err := c.interfaces.BeginCreateOrUpdate(ctx, c.resourceGroup, interfaceName, iface.Interface, nil) + + defer func() { + c.metricsAPI.ObserveAPICall(interfacesCreateOrUpdate, deriveStatus(err), sinceStart.Seconds()) + }() + if err != nil { + return fmt.Errorf("unable to update interface %s: %w", interfaceName, err) + } + + // Assign to the outer err so the deferred metric records poll failures. + if _, err = poller.PollUntilDone(ctx, nil); err != nil { + return fmt.Errorf("error while waiting for interface CreateOrUpdate to complete for %s: %w", interfaceName, err) + } + + return nil +} + +// UnassignPrivateIpAddressesVMSS releases the named IPConfigurations from the +// NIC of a VMSS instance, returning *PrimaryReleaseError if any is the primary. +func (c *Client) UnassignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, interfaceName string, ipConfigNames []string) error { + if len(ipConfigNames) == 0 { + return nil + } + + vmssGetOptions := &armcompute.VirtualMachineScaleSetVMsClientGetOptions{ + Expand: new(armcompute.InstanceViewTypesInstanceView), + } + + c.limiter.Limit(ctx, virtualMachineScaleSetVMsGet) + sinceStart := spanstat.Start() + + result, err := c.virtualMachineScaleSetVMs.Get(ctx, c.resourceGroup, vmssName, instanceID, vmssGetOptions) + + c.metricsAPI.ObserveAPICall(virtualMachineScaleSetVMsGet, deriveStatus(err), sinceStart.Seconds()) + if err != nil { + return fmt.Errorf("failed to get VM %s from VMSS %s: %w", instanceID, vmssName, err) + } + + var netIfConfig *armcompute.VirtualMachineScaleSetNetworkConfiguration + if result.Properties.NetworkProfileConfiguration != nil { + for _, nic := range result.Properties.NetworkProfileConfiguration.NetworkInterfaceConfigurations { + if nic.Name != nil && *nic.Name == interfaceName { + netIfConfig = nic + break + } + } + } + if netIfConfig == nil { + return fmt.Errorf("interface %s does not exist in VM %s", interfaceName, instanceID) + } + + releaseNames := make(map[string]struct{}, len(ipConfigNames)) + for _, name := range ipConfigNames { + releaseNames[name] = struct{}{} + } + + kept, dropped, primaryBlocked := dropMatchingIPConfigsVMSS(netIfConfig.Properties.IPConfigurations, releaseNames) + if len(primaryBlocked) > 0 { + return &PrimaryReleaseError{InterfaceName: interfaceName, Items: primaryBlocked} + } + if dropped < len(ipConfigNames) { + // Requested IPConfigs no longer on the NIC: likely a stale cache, harmless. + c.logger.Debug("Some requested IPConfigurations were not present on the interface during release", + logfields.Interface, interfaceName, + logfields.IPAddrs, ipConfigNames, + ) + } + if dropped == 0 { + return nil + } + netIfConfig.Properties.IPConfigurations = kept + + // Unset imageReference to avoid a permissions error on update, as in + // AssignPrivateIpAddressesVMSS. See https://github.com/Azure/AKS/issues/1819. + if result.Properties.StorageProfile != nil { + result.Properties.StorageProfile.ImageReference = nil + } + + c.limiter.Limit(ctx, virtualMachineScaleSetVMsUpdate) + sinceStart = spanstat.Start() + + poller, err := c.virtualMachineScaleSetVMs.BeginUpdate(ctx, c.resourceGroup, vmssName, instanceID, result.VirtualMachineScaleSetVM, nil) + + defer func() { + c.metricsAPI.ObserveAPICall(virtualMachineScaleSetVMsUpdate, deriveStatus(err), sinceStart.Seconds()) + }() + if err != nil { + return fmt.Errorf("unable to update virtualMachineScaleSetVMs: %w", err) + } + + // Assign to the outer err so the deferred metric records poll failures. + if _, err = poller.PollUntilDone(ctx, nil); err != nil { + return fmt.Errorf("error while waiting for virtualMachineScaleSetVMs Update to complete: %w", err) + } + + return nil +} + // AssignPublicIPAddressesVMSS assigns a public IP to a VMSS instance. // The public IP is allocated from a Public IP Prefix matching publicIpTags func (c *Client) AssignPublicIPAddressesVMSS(ctx context.Context, instanceID, vmssName string, publicIpTags ipamTypes.Tags) (netip.Addr, error) { diff --git a/pkg/azure/api/api_test.go b/pkg/azure/api/api_test.go index b011a63301ef3..247c638a0e521 100644 --- a/pkg/azure/api/api_test.go +++ b/pkg/azure/api/api_test.go @@ -176,6 +176,76 @@ func TestParseInterface(t *testing.T) { } } +// TestParseInterfaceIPConfigName verifies parseInterface records the +// IPConfiguration name on each address. +func TestParseInterfaceIPConfigName(t *testing.T) { + ifaceID := "/subscriptions/xxx/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic1" + subnetID := "/subscriptions/xxx/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet/subnets/subnet1" + + cfg := func(name, ip string, primary bool) *armnetwork.InterfaceIPConfiguration { + return &armnetwork.InterfaceIPConfiguration{ + Name: new(name), + Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: new(ip), + Primary: new(primary), + ProvisioningState: new(armnetwork.ProvisioningStateSucceeded), + Subnet: &armnetwork.Subnet{ID: new(subnetID)}, + }, + } + } + iface := &armnetwork.Interface{ + ID: new(ifaceID), + Properties: &armnetwork.InterfacePropertiesFormat{ + IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ + cfg("pods", "10.0.0.4", true), + cfg("pod-01", "10.0.0.5", false), + cfg("pod-02", "10.0.0.6", false), + }, + }, + } + + // usePrimary=true so the primary address is also exposed and carries its name. + _, got := parseInterface(hivetest.Logger(t), iface, ipamTypes.SubnetMap{}, true) + require.NotNil(t, got) + + byIP := make(map[string]string, len(got.Addresses)) + for _, a := range got.Addresses { + byIP[a.IP.String()] = a.IPConfigName() + } + require.Equal(t, "pods", byIP["10.0.0.4"]) + require.Equal(t, "pod-01", byIP["10.0.0.5"]) + require.Equal(t, "pod-02", byIP["10.0.0.6"]) +} + +// TestParseInterfaceDeepEqualIgnoresIPConfigName verifies ipConfigName is +// excluded from DeepEqual, so a changed name never triggers a status update. +func TestParseInterfaceDeepEqualIgnoresIPConfigName(t *testing.T) { + ifaceID := "/subscriptions/xxx/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic1" + mk := func(ipConfigName string) *armnetwork.Interface { + return &armnetwork.Interface{ + ID: new(ifaceID), + Properties: &armnetwork.InterfacePropertiesFormat{ + IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ + { + Name: new(ipConfigName), + Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: new("10.0.0.5"), + Primary: new(false), + ProvisioningState: new(armnetwork.ProvisioningStateSucceeded), + }, + }, + }, + }, + } + } + + _, a := parseInterface(hivetest.Logger(t), mk("pod-01"), ipamTypes.SubnetMap{}, false) + _, b := parseInterface(hivetest.Logger(t), mk("pod-renamed"), ipamTypes.SubnetMap{}, false) + require.Equal(t, "pod-01", a.Addresses[0].IPConfigName()) + require.Equal(t, "pod-renamed", b.Addresses[0].IPConfigName()) + require.True(t, a.DeepEqual(b), "ipConfigName must be excluded from DeepEqual to avoid status churn") +} + func TestAvailableIPs(t *testing.T) { cidr := netip.MustParsePrefix("10.0.0.0/8") require.Equal(t, 16777216, availableIPs(cidr)) @@ -391,3 +461,145 @@ func TestParseSubnetID(t *testing.T) { }) } } + +func TestDropMatchingIPConfigsVM(t *testing.T) { + primary := func(ip string) *armnetwork.InterfaceIPConfiguration { + return &armnetwork.InterfaceIPConfiguration{ + Name: new("primary-cfg"), + Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: new(ip), + Primary: new(true), + }, + } + } + secondary := func(name, ip string) *armnetwork.InterfaceIPConfiguration { + return &armnetwork.InterfaceIPConfiguration{ + Name: new(name), + Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: new(ip), + Primary: new(false), + }, + } + } + + t.Run("drops requested non-primary IPs only", func(t *testing.T) { + input := []*armnetwork.InterfaceIPConfiguration{ + primary("10.0.0.1"), + secondary("a", "10.0.0.2"), + secondary("b", "10.0.0.3"), + secondary("c", "10.0.0.4"), + } + releaseSet := map[string]struct{}{"10.0.0.2": {}, "10.0.0.4": {}} + kept, dropped, blocked := dropMatchingIPConfigsVM(input, releaseSet) + require.Equal(t, 2, dropped) + require.Empty(t, blocked) + require.Len(t, kept, 2) + require.Equal(t, "10.0.0.1", *kept[0].Properties.PrivateIPAddress) + require.Equal(t, "10.0.0.3", *kept[1].Properties.PrivateIPAddress) + }) + + t.Run("primary in release set is reported and retained", func(t *testing.T) { + input := []*armnetwork.InterfaceIPConfiguration{ + primary("10.0.0.1"), + secondary("a", "10.0.0.2"), + } + releaseSet := map[string]struct{}{"10.0.0.1": {}, "10.0.0.2": {}} + kept, dropped, blocked := dropMatchingIPConfigsVM(input, releaseSet) + require.Equal(t, []string{"10.0.0.1"}, blocked) + require.Equal(t, 1, dropped) + require.Len(t, kept, 1) + require.Equal(t, "10.0.0.1", *kept[0].Properties.PrivateIPAddress) + }) + + t.Run("no overlap drops nothing", func(t *testing.T) { + input := []*armnetwork.InterfaceIPConfiguration{ + primary("10.0.0.1"), + secondary("a", "10.0.0.2"), + } + releaseSet := map[string]struct{}{"10.0.0.99": {}} + kept, dropped, blocked := dropMatchingIPConfigsVM(input, releaseSet) + require.Equal(t, 0, dropped) + require.Empty(t, blocked) + require.Len(t, kept, 2) + }) + + t.Run("nil properties retained", func(t *testing.T) { + input := []*armnetwork.InterfaceIPConfiguration{ + {Name: new("nilprops")}, + secondary("a", "10.0.0.2"), + } + releaseSet := map[string]struct{}{"10.0.0.2": {}} + kept, dropped, blocked := dropMatchingIPConfigsVM(input, releaseSet) + require.Equal(t, 1, dropped) + require.Empty(t, blocked) + require.Len(t, kept, 1) + require.Equal(t, "nilprops", *kept[0].Name) + }) +} + +func TestDropMatchingIPConfigsVMSS(t *testing.T) { + primary := func(name string) *armcompute.VirtualMachineScaleSetIPConfiguration { + return &armcompute.VirtualMachineScaleSetIPConfiguration{ + Name: new(name), + Properties: &armcompute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: new(true), + }, + } + } + secondary := func(name string) *armcompute.VirtualMachineScaleSetIPConfiguration { + return &armcompute.VirtualMachineScaleSetIPConfiguration{ + Name: new(name), + Properties: &armcompute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: new(false), + }, + } + } + + t.Run("drops requested non-primary names only", func(t *testing.T) { + input := []*armcompute.VirtualMachineScaleSetIPConfiguration{ + primary("pods"), + secondary("pod-01"), + secondary("pod-02"), + secondary("pod-03"), + } + releaseNames := map[string]struct{}{"pod-01": {}, "pod-03": {}} + kept, dropped, blocked := dropMatchingIPConfigsVMSS(input, releaseNames) + require.Equal(t, 2, dropped) + require.Empty(t, blocked) + require.Len(t, kept, 2) + require.Equal(t, "pods", *kept[0].Name) + require.Equal(t, "pod-02", *kept[1].Name) + }) + + t.Run("primary name in release set is reported and retained", func(t *testing.T) { + input := []*armcompute.VirtualMachineScaleSetIPConfiguration{ + primary("pods"), + secondary("pod-01"), + } + releaseNames := map[string]struct{}{"pods": {}, "pod-01": {}} + kept, dropped, blocked := dropMatchingIPConfigsVMSS(input, releaseNames) + require.Equal(t, []string{"pods"}, blocked) + require.Equal(t, 1, dropped) + require.Len(t, kept, 1) + require.Equal(t, "pods", *kept[0].Name) + }) + + t.Run("nil name retained", func(t *testing.T) { + input := []*armcompute.VirtualMachineScaleSetIPConfiguration{ + {}, + secondary("pod-01"), + } + releaseNames := map[string]struct{}{"pod-01": {}} + kept, dropped, blocked := dropMatchingIPConfigsVMSS(input, releaseNames) + require.Equal(t, 1, dropped) + require.Empty(t, blocked) + require.Len(t, kept, 1) + require.Nil(t, kept[0].Name) + }) +} + +func TestPrimaryReleaseError(t *testing.T) { + err := &PrimaryReleaseError{InterfaceName: "pods", Items: []string{"pods", "pod-99"}} + require.Contains(t, err.Error(), "pods") + require.Contains(t, err.Error(), "pod-99") +} diff --git a/pkg/azure/api/mock/mock.go b/pkg/azure/api/mock/mock.go index 66482c159cdbd..7cb503700bd7c 100644 --- a/pkg/azure/api/mock/mock.go +++ b/pkg/azure/api/mock/mock.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "github.com/cilium/cilium/pkg/api/helpers" + azureAPI "github.com/cilium/cilium/pkg/azure/api" "github.com/cilium/cilium/pkg/azure/types" iputil "github.com/cilium/cilium/pkg/ip" "github.com/cilium/cilium/pkg/ipam/service/ipallocator" @@ -30,6 +31,8 @@ const ( ListAllNetworkInterfaces GetSubnetsByIDs AssignPrivateIpAddressesVMSS + UnassignPrivateIpAddressesVM + UnassignPrivateIpAddressesVMSS MaxOperation ) @@ -45,14 +48,17 @@ type API struct { errors map[Operation]error delaySim *helpers.DelaySimulator limiter *rate.Limiter + // primaryIPs records the identifiers treated as primary, keyed by interface ID. + primaryIPs map[string]map[string]struct{} } func NewAPI(subnets []*ipamTypes.Subnet) *API { api := &API{ - instances: ipamTypes.NewInstanceMap(), - subnets: map[string]*subnet{}, - errors: map[Operation]error{}, - delaySim: helpers.NewDelaySimulator(), + instances: ipamTypes.NewInstanceMap(), + subnets: map[string]*subnet{}, + errors: map[Operation]error{}, + delaySim: helpers.NewDelaySimulator(), + primaryIPs: map[string]map[string]struct{}{}, } api.UpdateSubnets(subnets) @@ -151,6 +157,58 @@ func (a *API) GetSubnetsByIDs(ctx context.Context, nodeSubnetIDs []string) (ipam } func (a *API) AssignPrivateIpAddressesVM(ctx context.Context, subnetID, interfaceName string, addresses int) error { + a.rateLimit() + + a.mutex.Lock() + defer a.mutex.Unlock() + + foundInterface := false + instances := a.instances.DeepCopy() + err := instances.ForeachInterface("", func(id, _ string, iface ipamTypes.Interface) error { + intf, ok := iface.(*types.AzureInterface) + if !ok { + return fmt.Errorf("invalid interface object") + } + + // Standalone-VM interfaces are not part of a scale set. + if intf.Name != interfaceName || intf.GetVMScaleSetName() != "" { + return nil + } + + if len(intf.Addresses)+addresses > types.InterfaceAddressLimit { + return fmt.Errorf("exceeded interface limit") + } + + s, ok := a.subnets[subnetID] + if !ok { + return fmt.Errorf("subnet %s does not exist", subnetID) + } + + for range addresses { + ip, err := s.allocator.AllocateNext() + if err != nil { + panic("Unable to allocate IP from allocator") + } + intf.Addresses = append(intf.Addresses, types.AzureAddress{ + IP: iputil.AddrFrom(ip), + Subnet: subnetID, //nolint:staticcheck // deprecated mirror; matches parseInterface, see https://github.com/cilium/cilium/issues/46074 + State: types.StateSucceeded, + }) + } + + foundInterface = true + return nil + }) + if err != nil { + return err + } + + a.updateInstancesLocked(instances) + + if !foundInterface { + return fmt.Errorf("interface %s not found", interfaceName) + } + return nil } @@ -214,6 +272,192 @@ func (a *API) AssignPrivateIpAddressesVMSS(ctx context.Context, vmName, vmssName return nil } +// SetPrimaryIPs marks the given identifiers as primary on the named interface, +// so Unassign* returns *api.PrimaryReleaseError when asked to release them. +func (a *API) SetPrimaryIPs(interfaceID string, items ...string) { + a.mutex.Lock() + defer a.mutex.Unlock() + if a.primaryIPs == nil { + a.primaryIPs = map[string]map[string]struct{}{} + } + set, ok := a.primaryIPs[interfaceID] + if !ok { + set = map[string]struct{}{} + a.primaryIPs[interfaceID] = set + } + for _, item := range items { + set[item] = struct{}{} + } +} + +// findPrimaryBlocked returns the subset of items recorded as primary on +// interfaceID via SetPrimaryIPs. Caller must hold a.mutex. +func (a *API) findPrimaryBlocked(interfaceID string, items []string) []string { + set, ok := a.primaryIPs[interfaceID] + if !ok { + return nil + } + var blocked []string + for _, item := range items { + if _, isPrimary := set[item]; isPrimary { + blocked = append(blocked, item) + } + } + return blocked +} + +func (a *API) UnassignPrivateIpAddressesVM(ctx context.Context, interfaceName string, addresses []string) error { + a.rateLimit() + a.delaySim.Delay(UnassignPrivateIpAddressesVM) + + a.mutex.Lock() + defer a.mutex.Unlock() + + if err, ok := a.errors[UnassignPrivateIpAddressesVM]; ok { + return err + } + if len(addresses) == 0 { + return nil + } + + if blocked := a.findPrimaryBlocked(interfaceName, addresses); len(blocked) > 0 { + return &azureAPI.PrimaryReleaseError{InterfaceName: interfaceName, Items: blocked} + } + + releaseSet := make(map[string]struct{}, len(addresses)) + for _, ip := range addresses { + releaseSet[ip] = struct{}{} + } + + instances := a.instances.DeepCopy() + foundInterface := false + err := instances.ForeachInterface("", func(_, _ string, iface ipamTypes.Interface) error { + intf, ok := iface.(*types.AzureInterface) + if !ok { + return fmt.Errorf("invalid interface object") + } + if intf.Name != interfaceName || intf.GetVMScaleSetName() != "" { + return nil + } + foundInterface = true + intf.Addresses = a.dropAddressesByIP(intf, releaseSet) + return nil + }) + if err != nil { + return err + } + if !foundInterface { + return fmt.Errorf("interface %s not found", interfaceName) + } + + a.updateInstancesLocked(instances) + return nil +} + +func (a *API) UnassignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, interfaceName string, ipConfigNames []string) error { + a.rateLimit() + a.delaySim.Delay(UnassignPrivateIpAddressesVMSS) + + a.mutex.Lock() + defer a.mutex.Unlock() + + if err, ok := a.errors[UnassignPrivateIpAddressesVMSS]; ok { + return err + } + if len(ipConfigNames) == 0 { + return nil + } + + releaseNames := make(map[string]struct{}, len(ipConfigNames)) + for _, name := range ipConfigNames { + releaseNames[name] = struct{}{} + } + + // Run the primary guard before mutating: dropAddressesByIPConfigName frees + // IPs to the shared allocator, which a fail-closed return would not undo. + var ifaceID string + foundInterface := false + a.instances.ForeachInterface("", func(_, _ string, iface ipamTypes.Interface) error { + intf, ok := iface.(*types.AzureInterface) + if !ok { + return nil + } + if intf.Name != interfaceName || intf.GetVMID() != instanceID || intf.GetVMScaleSetName() != vmssName { + return nil + } + foundInterface = true + ifaceID = intf.ID + return nil + }) + if !foundInterface { + return fmt.Errorf("interface %s not found on VM %s in VMSS %s", interfaceName, instanceID, vmssName) + } + + if blocked := a.findPrimaryBlocked(ifaceID, ipConfigNames); len(blocked) > 0 { + return &azureAPI.PrimaryReleaseError{InterfaceName: interfaceName, Items: blocked} + } + + instances := a.instances.DeepCopy() + err := instances.ForeachInterface("", func(_, _ string, iface ipamTypes.Interface) error { + intf, ok := iface.(*types.AzureInterface) + if !ok { + return fmt.Errorf("invalid interface object") + } + if intf.Name != interfaceName || intf.GetVMID() != instanceID || intf.GetVMScaleSetName() != vmssName { + return nil + } + intf.Addresses = a.dropAddressesByIPConfigName(intf, releaseNames) + return nil + }) + if err != nil { + return err + } + + a.updateInstancesLocked(instances) + return nil +} + +// dropAddressesByIP removes addresses in releaseSet, freeing their IPs. +// Caller must hold a.mutex. +func (a *API) dropAddressesByIP(intf *types.AzureInterface, releaseSet map[string]struct{}) []types.AzureAddress { + kept := make([]types.AzureAddress, 0, len(intf.Addresses)) + for _, addr := range intf.Addresses { + if _, drop := releaseSet[addr.IP.String()]; drop { + a.releaseToSubnetAllocator(addr) + continue + } + kept = append(kept, addr) + } + return kept +} + +// dropAddressesByIPConfigName removes addresses whose IPConfigName is in +// releaseNames, freeing their IPs. Caller must hold a.mutex. +func (a *API) dropAddressesByIPConfigName(intf *types.AzureInterface, releaseNames map[string]struct{}) []types.AzureAddress { + kept := make([]types.AzureAddress, 0, len(intf.Addresses)) + for _, addr := range intf.Addresses { + if _, drop := releaseNames[addr.IPConfigName()]; drop { + a.releaseToSubnetAllocator(addr) + continue + } + kept = append(kept, addr) + } + return kept +} + +// releaseToSubnetAllocator returns addr.IP to the subnet allocator if known. +// Caller must hold a.mutex. +func (a *API) releaseToSubnetAllocator(addr types.AzureAddress) { + s, ok := a.subnets[addr.Subnet] + if !ok { + return + } + if !addr.IP.Addr.IsValid() { + return + } + s.allocator.Release(addr.IP.Addr) +} + func (a *API) AssignPublicIPAddressesVMSS(ctx context.Context, instanceID, vmssName string, publicIpTags ipamTypes.Tags) (netip.Addr, error) { a.rateLimit() return netip.MustParseAddr("192.0.2.1"), nil diff --git a/pkg/azure/api/mock/mock_test.go b/pkg/azure/api/mock/mock_test.go index 62ed5dcab1cc2..e8f299857225d 100644 --- a/pkg/azure/api/mock/mock_test.go +++ b/pkg/azure/api/mock/mock_test.go @@ -10,10 +10,13 @@ import ( "github.com/stretchr/testify/require" + azureAPI "github.com/cilium/cilium/pkg/azure/api" "github.com/cilium/cilium/pkg/azure/types" + // Register the Azure resource-ID parser so AzureInterface.SetID() can // populate VMSS/VM/RG fields used by AssignPrivateIpAddressesVMSS lookup. _ "github.com/cilium/cilium/pkg/azure/types/azureid" + iputil "github.com/cilium/cilium/pkg/ip" ipamTypes "github.com/cilium/cilium/pkg/ipam/types" ) @@ -109,3 +112,131 @@ func TestSetLimiter(t *testing.T) { _, err := api.ListAllNetworkInterfaces(t.Context()) require.NoError(t, err) } + +// addrWithName builds an AzureAddress with both IP and IPConfig name set. +func addrWithName(ip, name, subnet string) types.AzureAddress { + addr := types.AzureAddress{IP: iputil.AddrFrom(netip.MustParseAddr(ip)), Subnet: subnet, State: types.StateSucceeded} + addr.SetIPConfigName(name) + return addr +} + +func TestUnassignPrivateIpAddressesVMSS(t *testing.T) { + cidr := netip.MustParsePrefix("10.0.0.0/16") + subnet := &ipamTypes.Subnet{ID: "s-1", CIDR: cidr, AvailableAddresses: 65534} + api := NewAPI([]*ipamTypes.Subnet{subnet}) + + const vmFullID = "/subscriptions/xxx/resourceGroups/g1/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines/0" + const ifaceID = vmFullID + "/networkInterfaces/pods" + // Per AzureInterface.extractIDs, GetVMID returns just the instance index. + const vmIndex = "0" + + resource := &types.AzureInterface{ + Name: "pods", + State: types.StateSucceeded, + Addresses: []types.AzureAddress{ + addrWithName("10.0.0.1", "pods", "s-1"), + addrWithName("10.0.0.2", "pod-01", "s-1"), + addrWithName("10.0.0.3", "pod-02", "s-1"), + }, + } + resource.SetID(ifaceID) + + instances := ipamTypes.NewInstanceMap() + instances.Update(vmFullID, resource.DeepCopy()) + api.UpdateInstances(instances) + + // Pre-allocate the IPs so a release frees them back observably. + for _, ip := range []string{"10.0.0.1", "10.0.0.2", "10.0.0.3"} { + require.NoError(t, api.subnets["s-1"].allocator.Allocate(netip.MustParseAddr(ip))) + } + + t.Run("removes requested IPConfigurations and frees them to the subnet", func(t *testing.T) { + before, err := api.GetSubnetsByIDs(t.Context(), []string{"s-1"}) + require.NoError(t, err) + + err = api.UnassignPrivateIpAddressesVMSS(t.Context(), vmIndex, "vmss1", "pods", []string{"pod-02"}) + require.NoError(t, err) + + api.instances.ForeachInterface("", func(_, _ string, rev ipamTypes.Interface) error { + intf := rev.(*types.AzureInterface) + require.Len(t, intf.Addresses, 2) + for _, a := range intf.Addresses { + require.NotEqual(t, "pod-02", a.IPConfigName()) + } + return nil + }) + + after, err := api.GetSubnetsByIDs(t.Context(), []string{"s-1"}) + require.NoError(t, err) + require.Equal(t, before["s-1"].AvailableAddresses+1, after["s-1"].AvailableAddresses, + "released IP must be freed back to the subnet") + }) + + t.Run("primary block returns PrimaryReleaseError without mutating", func(t *testing.T) { + before, err := api.GetSubnetsByIDs(t.Context(), []string{"s-1"}) + require.NoError(t, err) + + api.SetPrimaryIPs(ifaceID, "pods") + err = api.UnassignPrivateIpAddressesVMSS(t.Context(), vmIndex, "vmss1", "pods", []string{"pods"}) + var pErr *azureAPI.PrimaryReleaseError + require.ErrorAs(t, err, &pErr) + require.Equal(t, "pods", pErr.InterfaceName) + require.Contains(t, pErr.Items, "pods") + + api.instances.ForeachInterface("", func(_, _ string, rev ipamTypes.Interface) error { + intf := rev.(*types.AzureInterface) + names := make([]string, 0, len(intf.Addresses)) + for _, a := range intf.Addresses { + names = append(names, a.IPConfigName()) + } + require.Contains(t, names, "pods", "primary IPConfig must remain on the NIC") + return nil + }) + + // Fail-closed must not leak the primary's IP into the subnet allocator. + after, err := api.GetSubnetsByIDs(t.Context(), []string{"s-1"}) + require.NoError(t, err) + require.Equal(t, before["s-1"].AvailableAddresses, after["s-1"].AvailableAddresses, + "primary block must not change subnet availability") + }) +} + +func TestUnassignPrivateIpAddressesVM(t *testing.T) { + cidr := netip.MustParsePrefix("10.0.0.0/16") + subnet := &ipamTypes.Subnet{ID: "s-1", CIDR: cidr, AvailableAddresses: 65534} + api := NewAPI([]*ipamTypes.Subnet{subnet}) + + const vmIfaceID = "/subscriptions/xxx/resourceGroups/g1/providers/Microsoft.Network/networkInterfaces/vm-if" + + resource := &types.AzureInterface{ + Name: "vm-if", + State: types.StateSucceeded, + Addresses: []types.AzureAddress{ + addrWithName("10.0.0.5", "primary", "s-1"), + addrWithName("10.0.0.6", "secondary", "s-1"), + }, + } + resource.SetID(vmIfaceID) + + instances := ipamTypes.NewInstanceMap() + instances.Update("vm-instance", resource.DeepCopy()) + api.UpdateInstances(instances) + + t.Run("removes requested IP", func(t *testing.T) { + err := api.UnassignPrivateIpAddressesVM(t.Context(), "vm-if", []string{"10.0.0.6"}) + require.NoError(t, err) + api.instances.ForeachInterface("", func(_, _ string, rev ipamTypes.Interface) error { + intf := rev.(*types.AzureInterface) + require.Len(t, intf.Addresses, 1) + require.Equal(t, "10.0.0.5", intf.Addresses[0].IP.String()) + return nil + }) + }) + + t.Run("primary block returns error without mutation", func(t *testing.T) { + api.SetPrimaryIPs("vm-if", "10.0.0.5") + err := api.UnassignPrivateIpAddressesVM(t.Context(), "vm-if", []string{"10.0.0.5"}) + var pErr *azureAPI.PrimaryReleaseError + require.ErrorAs(t, err, &pErr) + }) +} diff --git a/pkg/azure/ipam/instances.go b/pkg/azure/ipam/instances.go index f6e04ecfdac44..5d1ce3e1045c5 100644 --- a/pkg/azure/ipam/instances.go +++ b/pkg/azure/ipam/instances.go @@ -32,6 +32,8 @@ type AzureAPI interface { GetSubnetsByIDs(ctx context.Context, nodeSubnetIDs []string) (ipamTypes.SubnetMap, error) AssignPrivateIpAddressesVM(ctx context.Context, subnetID, interfaceName string, addresses int) error AssignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, subnetID, interfaceName string, addresses int) error + UnassignPrivateIpAddressesVM(ctx context.Context, interfaceName string, addresses []string) error + UnassignPrivateIpAddressesVMSS(ctx context.Context, instanceID, vmssName, interfaceName string, ipConfigNames []string) error AssignPublicIPAddressesVM(ctx context.Context, instanceID string, publicIpTags ipamTypes.Tags) (netip.Addr, error) AssignPublicIPAddressesVMSS(ctx context.Context, instanceID, vmssName string, publicIpTags ipamTypes.Tags) (netip.Addr, error) ListAllNetworkInterfaces(ctx context.Context) ([]*armnetwork.Interface, error) diff --git a/pkg/azure/types/types.go b/pkg/azure/types/types.go index 4d0ada6021ad5..a1b5f5204064e 100644 --- a/pkg/azure/types/types.go +++ b/pkg/azure/types/types.go @@ -64,6 +64,21 @@ type AzureAddress struct { // State is the provisioning state of the address State string `json:"state,omitempty"` + + // ipConfigName is the Azure IPConfiguration name, used to release IPs from + // VMSS instances whose compute model exposes the name but not the IP. + // +deepequal-gen=false + ipConfigName string `json:"-"` +} + +// SetIPConfigName sets the Azure IPConfiguration name backing this address. +func (a *AzureAddress) SetIPConfigName(name string) { + a.ipConfigName = name +} + +// IPConfigName returns the Azure IPConfiguration name backing this address. +func (a AzureAddress) IPConfigName() string { + return a.ipConfigName } // AzureSubnet describes the subnet an AzureInterface is attached to. Azure From f6e0b3a1159975f51cf14cb3164f916fc361394c Mon Sep 17 00:00:00 2001 From: Jared Ledvina Date: Thu, 25 Jun 2026 18:45:59 -0400 Subject: [PATCH 2/3] [azure] implement excess IP release for IPAM nodes Signed-off-by: Jared Ledvina --- pkg/azure/api/mock/mock.go | 15 ++ pkg/azure/ipam/instances.go | 33 ++++ pkg/azure/ipam/node.go | 150 ++++++++++++++++- pkg/azure/ipam/node_test.go | 324 ++++++++++++++++++++++++++++++++++++ 4 files changed, 518 insertions(+), 4 deletions(-) diff --git a/pkg/azure/api/mock/mock.go b/pkg/azure/api/mock/mock.go index 7cb503700bd7c..260e1095b5b0b 100644 --- a/pkg/azure/api/mock/mock.go +++ b/pkg/azure/api/mock/mock.go @@ -272,6 +272,21 @@ func (a *API) AssignPrivateIpAddressesVMSS(ctx context.Context, vmName, vmssName return nil } +// AllocateSubnetIP marks ip as allocated in the named subnet's allocator. +func (a *API) AllocateSubnetIP(subnetID, ip string) error { + a.mutex.Lock() + defer a.mutex.Unlock() + s, ok := a.subnets[subnetID] + if !ok { + return fmt.Errorf("subnet %s does not exist", subnetID) + } + addr, err := netip.ParseAddr(ip) + if err != nil { + return err + } + return s.allocator.Allocate(addr) +} + // SetPrimaryIPs marks the given identifiers as primary on the named interface, // so Unassign* returns *api.PrimaryReleaseError when asked to release them. func (a *API) SetPrimaryIPs(interfaceID string, items ...string) { diff --git a/pkg/azure/ipam/instances.go b/pkg/azure/ipam/instances.go index 5d1ce3e1045c5..e47ba73a30b3a 100644 --- a/pkg/azure/ipam/instances.go +++ b/pkg/azure/ipam/instances.go @@ -9,6 +9,7 @@ import ( "log/slog" "maps" "net/netip" + "slices" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v9" "k8s.io/apimachinery/pkg/util/sets" @@ -225,6 +226,38 @@ func (m *InstancesManager) InstanceSync(ctx context.Context, instanceID string) return m.resyncInstance(ctx, instanceID) } +// RemoveIPsFromInterface removes the given IPs from the cached interface so the +// pool reflects the release before the next resync, mirroring AWS RemoveIPsFromENI. +func (m *InstancesManager) RemoveIPsFromInterface(instanceID, interfaceID string, ips []string) { + m.mutex.Lock() + defer m.mutex.Unlock() + + iface, ok := m.instances.GetInterface(instanceID, interfaceID) + if !ok { + // A concurrent resync may have dropped the interface; the next reconciles. + m.logger.Warn("Interface not found while removing released IPs from cache", + logfields.InstanceID, instanceID, + logfields.Interface, interfaceID, + ) + return + } + + azIface, ok := iface.DeepCopyInterface().(*types.AzureInterface) + if !ok { + m.logger.Error("Unexpected interface type while removing released IPs from cache", + logfields.InstanceID, instanceID, + logfields.Interface, interfaceID, + ) + return + } + + release := sets.New[string](ips...) + azIface.Addresses = slices.DeleteFunc(azIface.Addresses, func(a types.AzureAddress) bool { + return release.Has(a.IP.String()) + }) + m.instances.Update(instanceID, azIface) +} + // DeleteInstance delete instance from m.instances func (m *InstancesManager) DeleteInstance(instanceID string) { m.mutex.Lock() diff --git a/pkg/azure/ipam/node.go b/pkg/azure/ipam/node.go index 5542b55925456..0d7408b4571e2 100644 --- a/pkg/azure/ipam/node.go +++ b/pkg/azure/ipam/node.go @@ -8,6 +8,8 @@ import ( "fmt" "log/slog" "net/netip" + "slices" + "strings" "github.com/cilium/cilium/operator/pkg/ipam/nodemanager" "github.com/cilium/cilium/operator/pkg/ipam/stats" @@ -58,9 +60,62 @@ func (n *Node) PopulateStatusFields(k8sObj *v2.CiliumNode) { }) } -// PrepareIPRelease prepares the release of IPs +// PrepareIPRelease selects up to excessIPs free IPv4 addresses from the +// interface with the most releasable IPs. Interfaces are sorted by ID so the +// selection is deterministic across runs (matching the AWS path). func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *slog.Logger) *nodemanager.ReleaseAction { - return &nodemanager.ReleaseAction{} + r := &nodemanager.ReleaseAction{} + requiredIfaceName := n.k8sObj.Spec.Azure.InterfaceName + usedIPs := n.k8sObj.Status.IPAM.Used + + n.manager.mutex.RLock() + defer n.manager.mutex.RUnlock() + + var ifaces []*types.AzureInterface + err := n.manager.instances.ForeachInterface(n.node.InstanceID(), + func(_, _ string, ifaceObj ipamTypes.Interface) error { + iface, ok := ifaceObj.(*types.AzureInterface) + if !ok { + return fmt.Errorf("invalid interface object") + } + if requiredIfaceName != "" && iface.Name != requiredIfaceName { + return nil + } + ifaces = append(ifaces, iface) + return nil + }) + if err != nil { + scopedLog.Warn( + "Unable to enumerate interfaces while preparing IP release", + logfields.InstanceID, n.node.InstanceID(), + logfields.Error, err, + ) + return r + } + slices.SortFunc(ifaces, func(a, b *types.AzureInterface) int { + return strings.Compare(a.ID, b.ID) + }) + + for _, iface := range ifaces { + free := freeIPsOnInterface(iface, usedIPs) + if len(free) == 0 { + continue + } + maxRelease := min(len(free), excessIPs) + // Select the interface with the most addresses available for release. + if r.IPsToRelease == nil || maxRelease > len(r.IPsToRelease) { + r.InterfaceID = iface.ID + r.PoolID = ipamTypes.PoolID(iface.Subnet.ID) + r.IPsToRelease = free[:maxRelease] + scopedLog.Debug( + "Interface has unused IPs that can be released", + logfields.ID, iface.ID, + logfields.ExcessIPs, excessIPs, + logfields.IPAddrs, r.IPsToRelease, + ) + } + } + return r } // ReleaseIPPrefixes is a no-op on Azure since Azure ENIs don't @@ -70,9 +125,96 @@ func (n *Node) ReleaseIPPrefixes(ctx context.Context, r *nodemanager.ReleaseActi return nil } -// ReleaseIPs performs the IP release operation +// ReleaseIPs releases r.IPsToRelease: VM NICs take IPs directly, VMSS NICs need +// them translated to IPConfiguration names. On success the IPs are dropped from +// the cached interface so the pool reflects the release before the next resync. func (n *Node) ReleaseIPs(ctx context.Context, r *nodemanager.ReleaseAction) error { - return fmt.Errorf("not implemented") + if len(r.IPsToRelease) == 0 { + return nil + } + + iface, err := n.findInterface(r.InterfaceID) + if err != nil { + return err + } + + if iface.GetVMScaleSetName() == "" { + if err := n.manager.api.UnassignPrivateIpAddressesVM(ctx, iface.Name, r.IPsToRelease); err != nil { + return err + } + } else { + names, missing := ipsToConfigNames(iface, r.IPsToRelease) + if len(missing) > 0 { + return fmt.Errorf("interface %s: missing IPConfiguration name mapping for IPs %v (cache out of sync, will retry after next resync)", iface.Name, missing) + } + if err := n.manager.api.UnassignPrivateIpAddressesVMSS(ctx, iface.GetVMID(), iface.GetVMScaleSetName(), iface.Name, names); err != nil { + return err + } + } + + n.manager.RemoveIPsFromInterface(n.node.InstanceID(), r.InterfaceID, r.IPsToRelease) + return nil +} + +// freeIPsOnInterface returns the releasable IPv4 addresses on iface (Succeeded, +// non-primary, unused), sorted so truncation to excessIPs is deterministic. +func freeIPsOnInterface(iface *types.AzureInterface, used ipamTypes.AllocationMap) []string { + free := make([]string, 0, len(iface.Addresses)) + for _, a := range iface.Addresses { + if a.State != types.StateSucceeded { + continue + } + // Release is IPv4-only. + if !a.IP.Addr.Is4() { + continue + } + // Never release the primary IPConfiguration. + if a.IP == iface.IP { + continue + } + ip := a.IP.String() + if _, inUse := used[ip]; inUse { + continue + } + free = append(free, ip) + } + slices.Sort(free) + return free +} + +// findInterface returns a copy of the AzureInterface with the given ID, safe to +// read without holding the manager mutex. +func (n *Node) findInterface(interfaceID string) (*types.AzureInterface, error) { + n.manager.mutex.RLock() + defer n.manager.mutex.RUnlock() + iface, ok := n.manager.instances.GetInterface(n.node.InstanceID(), interfaceID) + if !ok { + return nil, fmt.Errorf("interface %s not found on instance %s", interfaceID, n.node.InstanceID()) + } + azIface, ok := iface.DeepCopyInterface().(*types.AzureInterface) + if !ok { + return nil, fmt.Errorf("interface %s on instance %s has unexpected type", interfaceID, n.node.InstanceID()) + } + return azIface, nil +} + +// ipsToConfigNames maps each IP in ips to its IPConfiguration name on iface. +// IPs without a known mapping are returned in missing. +func ipsToConfigNames(iface *types.AzureInterface, ips []string) (names, missing []string) { + byIP := make(map[string]string, len(iface.Addresses)) + for _, a := range iface.Addresses { + if name := a.IPConfigName(); name != "" { + byIP[a.IP.String()] = name + } + } + for _, ip := range ips { + if name, ok := byIP[ip]; ok { + names = append(names, name) + } else { + missing = append(missing, ip) + } + } + return } // PrepareIPAllocation returns the number of IPs that can be allocated/created. diff --git a/pkg/azure/ipam/node_test.go b/pkg/azure/ipam/node_test.go index da0318d7bd793..a72add4d9531e 100644 --- a/pkg/azure/ipam/node_test.go +++ b/pkg/azure/ipam/node_test.go @@ -4,14 +4,338 @@ package ipam import ( + "errors" + "net/netip" "testing" + "github.com/cilium/hive/hivetest" "github.com/stretchr/testify/require" + "github.com/cilium/cilium/operator/pkg/ipam/nodemanager" + "github.com/cilium/cilium/pkg/azure/api/mock" "github.com/cilium/cilium/pkg/azure/types" + iputil "github.com/cilium/cilium/pkg/ip" + ipamTypes "github.com/cilium/cilium/pkg/ipam/types" + v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" + + // Register the Azure resource-ID parser so AzureInterface.SetID() can + // populate the VMSS/VM/RG fields used by the release path. + _ "github.com/cilium/cilium/pkg/azure/types/azureid" ) func TestGetMaximumAllocatableIPv4(t *testing.T) { n := &Node{} require.Equal(t, types.InterfaceAddressLimit, n.GetMaximumAllocatableIPv4()) } + +const testInstanceID = "/subscriptions/sub/resourceGroups/g/providers/Microsoft.Compute/virtualMachineScaleSets/vmss/virtualMachines/0" + +func addr(ip string) iputil.Addr { + return iputil.AddrFrom(netip.MustParseAddr(ip)) +} + +// addrWithName builds a Succeeded AzureAddress with both IP and IPConfig name set. +func addrWithName(ip, name string) types.AzureAddress { + a := types.AzureAddress{IP: addr(ip), Subnet: "subnet-a", State: types.StateSucceeded} + a.SetIPConfigName(name) + return a +} + +func mkIface(t *testing.T, name string, primary iputil.Addr, addresses ...types.AzureAddress) *types.AzureInterface { + t.Helper() + iface := &types.AzureInterface{Name: name, State: types.StateSucceeded, IP: primary, Addresses: addresses} + iface.Subnet.ID = "subnet-a" + iface.SetID(testInstanceID + "/networkInterfaces/" + name) + return iface +} + +func mkNode(ifaceFilter string, used ipamTypes.AllocationMap, ifaces ...*types.AzureInterface) *Node { + mgr := &InstancesManager{instances: ipamTypes.NewInstanceMap()} + for _, iface := range ifaces { + mgr.instances.Update(testInstanceID, iface) + } + k8sObj := &v2.CiliumNode{ + Spec: v2.NodeSpec{Azure: types.AzureSpec{InterfaceName: ifaceFilter}}, + Status: v2.NodeStatus{IPAM: ipamTypes.IPAMStatus{Used: used}}, + } + return &Node{k8sObj: k8sObj, manager: mgr, node: &fakeIpamNode{id: testInstanceID}} +} + +// fakeIpamNode is a minimal ipamNodeActions stub. +type fakeIpamNode struct{ id string } + +func (f *fakeIpamNode) InstanceID() string { return f.id } + +func TestPrepareIPRelease(t *testing.T) { + log := hivetest.Logger(t) + + t.Run("includes free IPs and excludes used", func(t *testing.T) { + used := ipamTypes.AllocationMap{"10.0.0.3": ipamTypes.AllocationIP{}} + iface := mkIface(t, "pods", iputil.Addr{}, + addrWithName("10.0.0.1", "pod-00"), + addrWithName("10.0.0.2", "pod-01"), + addrWithName("10.0.0.3", "pod-02"), + addrWithName("10.0.0.4", "pod-03"), + ) + n := mkNode("", used, iface) + r := n.PrepareIPRelease(2, log) + require.Equal(t, iface.ID, r.InterfaceID) + require.Equal(t, ipamTypes.PoolID("subnet-a"), r.PoolID) + require.Equal(t, []string{"10.0.0.1", "10.0.0.2"}, r.IPsToRelease) + }) + + t.Run("respects excessIPs cap", func(t *testing.T) { + iface := mkIface(t, "pods", iputil.Addr{}, + addrWithName("10.0.0.1", "pod-00"), + addrWithName("10.0.0.2", "pod-01"), + addrWithName("10.0.0.3", "pod-02"), + ) + n := mkNode("", ipamTypes.AllocationMap{}, iface) + r := n.PrepareIPRelease(1, log) + require.Len(t, r.IPsToRelease, 1) + }) + + t.Run("excludes the primary IPConfiguration", func(t *testing.T) { + // usePrimary=true layout: the primary is also present in Addresses and + // equals iface.IP. It must never be selected. + iface := mkIface(t, "pods", addr("10.0.0.1"), + addrWithName("10.0.0.1", "pods"), + addrWithName("10.0.0.2", "pod-01"), + addrWithName("10.0.0.3", "pod-02"), + ) + n := mkNode("", ipamTypes.AllocationMap{}, iface) + r := n.PrepareIPRelease(5, log) + require.NotContains(t, r.IPsToRelease, "10.0.0.1") + require.Equal(t, []string{"10.0.0.2", "10.0.0.3"}, r.IPsToRelease) + }) + + t.Run("Spec.Azure.InterfaceName filters interfaces", func(t *testing.T) { + host := mkIface(t, "primary", iputil.Addr{}, addrWithName("10.0.0.1", "host-00")) + pods := mkIface(t, "pods", iputil.Addr{}, + addrWithName("10.1.0.1", "pod-00"), + addrWithName("10.1.0.2", "pod-01"), + ) + n := mkNode("pods", ipamTypes.AllocationMap{}, host, pods) + r := n.PrepareIPRelease(5, log) + require.Equal(t, pods.ID, r.InterfaceID) + require.NotContains(t, r.IPsToRelease, "10.0.0.1") + }) + + t.Run("picks the interface with the most freeable IPs", func(t *testing.T) { + few := mkIface(t, "few", iputil.Addr{}, addrWithName("10.0.0.1", "few-00")) + many := mkIface(t, "many", iputil.Addr{}, + addrWithName("10.1.0.1", "many-00"), + addrWithName("10.1.0.2", "many-01"), + addrWithName("10.1.0.3", "many-02"), + ) + n := mkNode("", ipamTypes.AllocationMap{}, few, many) + r := n.PrepareIPRelease(5, log) + require.Equal(t, many.ID, r.InterfaceID) + require.Len(t, r.IPsToRelease, 3) + }) + + t.Run("excludes addresses not in Succeeded state", func(t *testing.T) { + bad := types.AzureAddress{IP: addr("10.0.0.99"), Subnet: "subnet-a", State: "failed"} + bad.SetIPConfigName("broken") + iface := mkIface(t, "pods", iputil.Addr{}, addrWithName("10.0.0.1", "pod-00"), bad) + n := mkNode("", ipamTypes.AllocationMap{}, iface) + r := n.PrepareIPRelease(5, log) + require.NotContains(t, r.IPsToRelease, "10.0.0.99") + }) + + t.Run("excludes IPv6 addresses", func(t *testing.T) { + v6 := addrWithName("fd00::2", "v6") + iface := mkIface(t, "pods", iputil.Addr{}, addrWithName("10.0.0.1", "pod-00"), v6) + n := mkNode("", ipamTypes.AllocationMap{}, iface) + r := n.PrepareIPRelease(5, log) + require.Equal(t, []string{"10.0.0.1"}, r.IPsToRelease) + }) + + t.Run("empty subnet yields empty PoolID", func(t *testing.T) { + iface := &types.AzureInterface{Name: "pods", State: types.StateSucceeded, + Addresses: []types.AzureAddress{addrWithName("10.0.0.1", "pod-00")}} + iface.SetID(testInstanceID + "/networkInterfaces/pods") + n := mkNode("", ipamTypes.AllocationMap{}, iface) + r := n.PrepareIPRelease(5, log) + require.Equal(t, ipamTypes.PoolID(""), r.PoolID) + require.Equal(t, []string{"10.0.0.1"}, r.IPsToRelease) + }) + + t.Run("no free IPs yields empty action", func(t *testing.T) { + used := ipamTypes.AllocationMap{"10.0.0.1": ipamTypes.AllocationIP{}} + iface := mkIface(t, "pods", iputil.Addr{}, addrWithName("10.0.0.1", "pod-00")) + n := mkNode("", used, iface) + r := n.PrepareIPRelease(5, log) + require.Empty(t, r.IPsToRelease) + require.Empty(t, r.InterfaceID) + }) +} + +func TestIPsToConfigNames(t *testing.T) { + iface := &types.AzureInterface{ + Addresses: []types.AzureAddress{ + addrWithName("10.0.0.1", "pods"), + addrWithName("10.0.0.2", "pod-01"), + }, + } + names, missing := ipsToConfigNames(iface, []string{"10.0.0.1", "10.0.0.99"}) + require.Equal(t, []string{"pods"}, names) + require.Equal(t, []string{"10.0.0.99"}, missing) +} + +func TestReleaseIPs(t *testing.T) { + ctx := t.Context() + + // newManagerWithMock wires a manager and a mock API that both hold iface, + // keyed by instanceID. The given IPs are pre-allocated in the mock subnet so + // a successful release frees them back and availability changes are + // observable (which proves the SDK actually dropped the right IPConfig). + newManagerWithMock := func(t *testing.T, instanceID string, iface *types.AzureInterface, allocated ...string) (*InstancesManager, *mock.API) { + t.Helper() + subnet := &ipamTypes.Subnet{ID: "subnet-a", CIDR: netip.MustParsePrefix("10.0.0.0/16")} + api := mock.NewAPI([]*ipamTypes.Subnet{subnet}) + mockInstances := ipamTypes.NewInstanceMap() + mockInstances.Update(instanceID, iface.DeepCopy()) + api.UpdateInstances(mockInstances) + for _, ip := range allocated { + require.NoError(t, api.AllocateSubnetIP("subnet-a", ip)) + } + + mgr := &InstancesManager{instances: ipamTypes.NewInstanceMap(), api: api} + mgr.instances.Update(instanceID, iface.DeepCopy()) + return mgr, api + } + + node := func(mgr *InstancesManager, instanceID string) *Node { + return &Node{ + k8sObj: &v2.CiliumNode{}, + manager: mgr, + node: &fakeIpamNode{id: instanceID}, + } + } + + remainingIPs := func(t *testing.T, mgr *InstancesManager, instanceID string) []string { + t.Helper() + var ips []string + mgr.instances.ForeachInterface(instanceID, func(_, _ string, o ipamTypes.Interface) error { + ips = append(ips, ipsOf(o.(*types.AzureInterface))...) + return nil + }) + return ips + } + + subnetAvail := func(t *testing.T, api *mock.API) int { + t.Helper() + subnets, err := api.GetSubnetsByIDs(ctx, []string{"subnet-a"}) + require.NoError(t, err) + return subnets["subnet-a"].AvailableAddresses + } + + t.Run("VMSS path translates IPs to names, releases on Azure, and updates the cache", func(t *testing.T) { + iface := mkIface(t, "pods", addr("10.0.0.1"), + addrWithName("10.0.0.1", "pods"), + addrWithName("10.0.0.2", "pod-01"), + addrWithName("10.0.0.3", "pod-02"), + ) + mgr, api := newManagerWithMock(t, testInstanceID, iface, "10.0.0.2", "10.0.0.3") + n := node(mgr, testInstanceID) + + before := subnetAvail(t, api) + err := n.ReleaseIPs(ctx, releaseAction(iface.ID, "10.0.0.2")) + require.NoError(t, err) + require.NotContains(t, remainingIPs(t, mgr, testInstanceID), "10.0.0.2") + require.Contains(t, remainingIPs(t, mgr, testInstanceID), "10.0.0.3") + // If the IP->name translation were wrong, the mock would drop nothing and + // availability would not move. +1 proves "pod-01" (10.0.0.2) was released. + require.Equal(t, before+1, subnetAvail(t, api), "released IP must be freed on Azure") + }) + + t.Run("VM path passes IPs directly and updates the cache", func(t *testing.T) { + const vmInstanceID = "/subscriptions/sub/resourceGroups/g/providers/Microsoft.Compute/virtualMachines/vm0" + iface := &types.AzureInterface{Name: "vm-if", State: types.StateSucceeded, Addresses: []types.AzureAddress{ + addrWithName("10.0.0.5", "primary"), + addrWithName("10.0.0.6", "secondary"), + }} + iface.Subnet.ID = "subnet-a" + iface.SetID(vmInstanceID + "/networkInterfaces/vm-if") + require.Empty(t, iface.GetVMScaleSetName(), "fixture must route through the VM path") + + mgr, api := newManagerWithMock(t, vmInstanceID, iface, "10.0.0.6") + n := node(mgr, vmInstanceID) + + before := subnetAvail(t, api) + err := n.ReleaseIPs(ctx, releaseAction(iface.ID, "10.0.0.6")) + require.NoError(t, err) + require.NotContains(t, remainingIPs(t, mgr, vmInstanceID), "10.0.0.6") + require.Equal(t, before+1, subnetAvail(t, api)) + }) + + t.Run("VMSS path errors when an IP has no IPConfig name mapping", func(t *testing.T) { + // Address with no IPConfig name simulates a stale/partial cache. + noName := types.AzureAddress{IP: addr("10.0.0.2"), Subnet: "subnet-a", State: types.StateSucceeded} + iface := mkIface(t, "pods", addr("10.0.0.1"), addrWithName("10.0.0.1", "pods"), noName) + mgr, _ := newManagerWithMock(t, testInstanceID, iface) + n := node(mgr, testInstanceID) + + err := n.ReleaseIPs(ctx, releaseAction(iface.ID, "10.0.0.2")) + require.Error(t, err) + require.Contains(t, err.Error(), "missing IPConfiguration name mapping") + // Cache must be untouched on error. + require.Contains(t, remainingIPs(t, mgr, testInstanceID), "10.0.0.2") + }) + + t.Run("cache is untouched when the SDK release fails", func(t *testing.T) { + iface := mkIface(t, "pods", addr("10.0.0.1"), + addrWithName("10.0.0.1", "pods"), + addrWithName("10.0.0.2", "pod-01"), + ) + mgr, api := newManagerWithMock(t, testInstanceID, iface) + api.SetMockError(mock.UnassignPrivateIpAddressesVMSS, errors.New("azure boom")) + n := node(mgr, testInstanceID) + + err := n.ReleaseIPs(ctx, releaseAction(iface.ID, "10.0.0.2")) + require.Error(t, err) + require.Contains(t, remainingIPs(t, mgr, testInstanceID), "10.0.0.2", + "cache must not be mutated when the release fails") + }) + + t.Run("empty action is a no-op", func(t *testing.T) { + iface := mkIface(t, "pods", iputil.Addr{}, addrWithName("10.0.0.2", "pod-01")) + mgr, _ := newManagerWithMock(t, testInstanceID, iface) + n := node(mgr, testInstanceID) + require.NoError(t, n.ReleaseIPs(ctx, releaseAction(iface.ID))) + }) +} + +func TestRemoveIPsFromInterface(t *testing.T) { + iface := mkIface(t, "pods", iputil.Addr{}, + addrWithName("10.0.0.1", "pod-00"), + addrWithName("10.0.0.2", "pod-01"), + ) + mgr := &InstancesManager{instances: ipamTypes.NewInstanceMap(), logger: hivetest.Logger(t)} + mgr.instances.Update(testInstanceID, iface) + + mgr.RemoveIPsFromInterface(testInstanceID, iface.ID, []string{"10.0.0.1"}) + + var ips []string + mgr.instances.ForeachInterface(testInstanceID, func(_, _ string, o ipamTypes.Interface) error { + ips = ipsOf(o.(*types.AzureInterface)) + return nil + }) + require.Equal(t, []string{"10.0.0.2"}, ips) + + // Unknown interface is a safe no-op. + mgr.RemoveIPsFromInterface(testInstanceID, "does-not-exist", []string{"10.0.0.2"}) +} + +func releaseAction(interfaceID string, ips ...string) *nodemanager.ReleaseAction { + return &nodemanager.ReleaseAction{InterfaceID: interfaceID, IPsToRelease: ips} +} + +func ipsOf(iface *types.AzureInterface) []string { + ips := make([]string, 0, len(iface.Addresses)) + for _, a := range iface.Addresses { + ips = append(ips, a.IP.String()) + } + return ips +} From ddc2297c25ab4847ebafac774d4abb9e78c700e3 Mon Sep 17 00:00:00 2001 From: Jared Ledvina Date: Thu, 25 Jun 2026 19:00:57 -0400 Subject: [PATCH 3/3] [ipam] add generic ipam-release-excess-ips flag, deprecate per-cloud flags Signed-off-by: Jared Ledvina --- operator/pkg/ipam/alibabacloud.go | 4 +++- operator/pkg/ipam/allocator/alibabacloud/alibabacloud.go | 3 ++- operator/pkg/ipam/allocator/azure/azure.go | 4 +++- operator/pkg/ipam/aws.go | 8 +++----- operator/pkg/ipam/azure.go | 2 ++ operator/pkg/ipam/cell.go | 6 ++++++ 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/operator/pkg/ipam/alibabacloud.go b/operator/pkg/ipam/alibabacloud.go index c7054b8de2f19..d5032d12395fb 100644 --- a/operator/pkg/ipam/alibabacloud.go +++ b/operator/pkg/ipam/alibabacloud.go @@ -46,6 +46,7 @@ var defaultAlibabaCloudConfig = AlibabaCloudConfig{ func (cfg AlibabaCloudConfig) Flags(flags *pflag.FlagSet) { flags.String(operatorOption.AlibabaCloudVPCID, defaultAlibabaCloudConfig.AlibabaCloudVPCID, "Specific VPC ID for AlibabaCloud ENI. If not set use same VPC as operator") flags.Bool(operatorOption.AlibabaCloudReleaseExcessIPs, defaultAlibabaCloudConfig.AlibabaCloudReleaseExcessIPs, "Enable releasing excess free IP addresses from Alibaba Cloud ENI.") + flags.MarkDeprecated(operatorOption.AlibabaCloudReleaseExcessIPs, "use --ipam-release-excess-ips instead") } type alibabaParams struct { @@ -67,7 +68,8 @@ type alibabaParams struct { func startAlibabaAllocator(p alibabaParams) { alloc := &alibabacloud.AllocatorAlibabaCloud{ AlibabaCloudVPCID: p.AlibabaCfg.AlibabaCloudVPCID, - AlibabaCloudReleaseExcessIPs: p.AlibabaCfg.AlibabaCloudReleaseExcessIPs, + AlibabaCloudReleaseExcessIPs: p.AlibabaCfg.AlibabaCloudReleaseExcessIPs || p.Cfg.IPAMReleaseExcessIPs, + ExcessIPReleaseDelay: p.Cfg.ExcessIPReleaseDelay, ParallelAllocWorkers: p.Cfg.ParallelAllocWorkers, LimitIPAMAPIBurst: p.Cfg.LimitIPAMAPIBurst, LimitIPAMAPIQPS: p.Cfg.LimitIPAMAPIQPS, diff --git a/operator/pkg/ipam/allocator/alibabacloud/alibabacloud.go b/operator/pkg/ipam/allocator/alibabacloud/alibabacloud.go index 1ba57b554a44c..037e3d69f5f75 100644 --- a/operator/pkg/ipam/allocator/alibabacloud/alibabacloud.go +++ b/operator/pkg/ipam/allocator/alibabacloud/alibabacloud.go @@ -31,6 +31,7 @@ var subsysLogAttr = []any{logfields.LogSubsys, "ipam-allocator-alibaba-cloud"} type AllocatorAlibabaCloud struct { AlibabaCloudVPCID string AlibabaCloudReleaseExcessIPs bool + ExcessIPReleaseDelay int ParallelAllocWorkers int64 LimitIPAMAPIBurst int LimitIPAMAPIQPS float64 @@ -95,7 +96,7 @@ func (a *AllocatorAlibabaCloud) Start(ctx context.Context, getterUpdater allocat instances := ipam.NewInstancesManager(a.rootLogger, a.client) nodeManager, err := nodemanager.NewNodeManager(a.logger, instances, getterUpdater, iMetrics, - a.ParallelAllocWorkers, a.AlibabaCloudReleaseExcessIPs, 0, false) + a.ParallelAllocWorkers, a.AlibabaCloudReleaseExcessIPs, a.ExcessIPReleaseDelay, false) if err != nil { return nil, fmt.Errorf("unable to initialize AlibabaCloud node manager: %w", err) } diff --git a/operator/pkg/ipam/allocator/azure/azure.go b/operator/pkg/ipam/allocator/azure/azure.go index 56f107f8cb9f7..48271dc8afc4f 100644 --- a/operator/pkg/ipam/allocator/azure/azure.go +++ b/operator/pkg/ipam/allocator/azure/azure.go @@ -22,6 +22,8 @@ type AllocatorAzure struct { AzureResourceGroup string AzureUserAssignedIdentityID string AzureUsePrimaryAddress bool + AzureReleaseExcessIPs bool + ExcessIPReleaseDelay int ParallelAllocWorkers int64 LimitIPAMAPIBurst int LimitIPAMAPIQPS float64 @@ -75,7 +77,7 @@ func (a *AllocatorAzure) Start(ctx context.Context, getterUpdater allocator.Cili return nil, fmt.Errorf("unable to create Azure client: %w", err) } instances := ipam.NewInstancesManager(a.rootLogger, azureClient, a.AzureUsePrimaryAddress) - nodeManager, err := nodemanager.NewNodeManager(a.logger, instances, getterUpdater, iMetrics, a.ParallelAllocWorkers, false, 0, false) + nodeManager, err := nodemanager.NewNodeManager(a.logger, instances, getterUpdater, iMetrics, a.ParallelAllocWorkers, a.AzureReleaseExcessIPs, a.ExcessIPReleaseDelay, false) if err != nil { return nil, fmt.Errorf("unable to initialize Azure node manager: %w", err) } diff --git a/operator/pkg/ipam/aws.go b/operator/pkg/ipam/aws.go index 87e4c950fa7b4..af8e78f83b154 100644 --- a/operator/pkg/ipam/aws.go +++ b/operator/pkg/ipam/aws.go @@ -35,7 +35,6 @@ func init() { type AWSConfig struct { AWSReleaseExcessIPs bool - ExcessIPReleaseDelay int AWSEnablePrefixDelegation bool ENITags map[string]string ENIGarbageCollectionTags map[string]string `mapstructure:"eni-gc-tags"` @@ -49,7 +48,6 @@ type AWSConfig struct { var awsDefaultConfig = AWSConfig{ AWSReleaseExcessIPs: false, - ExcessIPReleaseDelay: 180, AWSEnablePrefixDelegation: false, ENITags: nil, ENIGarbageCollectionTags: nil, @@ -63,7 +61,7 @@ var awsDefaultConfig = AWSConfig{ func (cfg AWSConfig) Flags(flags *pflag.FlagSet) { flags.Bool("aws-release-excess-ips", awsDefaultConfig.AWSReleaseExcessIPs, "Enable releasing excess free IP addresses from AWS ENI.") - flags.Int("excess-ip-release-delay", awsDefaultConfig.ExcessIPReleaseDelay, "Number of seconds operator would wait before it releases an IP previously marked as excess") + flags.MarkDeprecated("aws-release-excess-ips", "use --ipam-release-excess-ips instead") flags.Bool("aws-enable-prefix-delegation", awsDefaultConfig.AWSEnablePrefixDelegation, "Allows operator to allocate prefixes to ENIs instead of individual IP addresses") flags.StringToString("eni-tags", awsDefaultConfig.ENITags, "ENI tags in the form of k1=v1 (multiple k/v pairs can be passed by repeating the CLI flag)") @@ -97,8 +95,8 @@ type awsParams struct { func startAWSAllocator(p awsParams) { alloc := &aws.AllocatorAWS{ - AWSReleaseExcessIPs: p.AwsCfg.AWSReleaseExcessIPs, - ExcessIPReleaseDelay: p.AwsCfg.ExcessIPReleaseDelay, + AWSReleaseExcessIPs: p.AwsCfg.AWSReleaseExcessIPs || p.Cfg.IPAMReleaseExcessIPs, + ExcessIPReleaseDelay: p.Cfg.ExcessIPReleaseDelay, AWSEnablePrefixDelegation: p.AwsCfg.AWSEnablePrefixDelegation, ENITags: p.AwsCfg.ENITags, ENIGarbageCollectionTags: p.AwsCfg.ENIGarbageCollectionTags, diff --git a/operator/pkg/ipam/azure.go b/operator/pkg/ipam/azure.go index 3cb625aba4ea0..e34481ac43a67 100644 --- a/operator/pkg/ipam/azure.go +++ b/operator/pkg/ipam/azure.go @@ -76,6 +76,8 @@ func startAzureAllocator(p azureParams) { AzureResourceGroup: p.AzureCfg.AzureResourceGroup, AzureUserAssignedIdentityID: p.AzureCfg.AzureUserAssignedIdentityID, AzureUsePrimaryAddress: p.AzureCfg.AzureUsePrimaryAddress, + AzureReleaseExcessIPs: p.Cfg.IPAMReleaseExcessIPs, + ExcessIPReleaseDelay: p.Cfg.ExcessIPReleaseDelay, ParallelAllocWorkers: p.Cfg.ParallelAllocWorkers, LimitIPAMAPIBurst: p.Cfg.LimitIPAMAPIBurst, LimitIPAMAPIQPS: p.Cfg.LimitIPAMAPIQPS, diff --git a/operator/pkg/ipam/cell.go b/operator/pkg/ipam/cell.go index 52a8813d47f5a..ef27741d45146 100644 --- a/operator/pkg/ipam/cell.go +++ b/operator/pkg/ipam/cell.go @@ -31,16 +31,22 @@ type Config struct { ParallelAllocWorkers int64 LimitIPAMAPIBurst int LimitIPAMAPIQPS float64 + IPAMReleaseExcessIPs bool + ExcessIPReleaseDelay int } var defaultConfig = Config{ ParallelAllocWorkers: 50, LimitIPAMAPIBurst: 20, LimitIPAMAPIQPS: 4.0, + IPAMReleaseExcessIPs: false, + ExcessIPReleaseDelay: 180, } func (cfg Config) Flags(flags *pflag.FlagSet) { flags.Int64(option.ParallelAllocWorkers, defaultConfig.ParallelAllocWorkers, "Maximum number of parallel IPAM workers") flags.Int("limit-ipam-api-burst", defaultConfig.LimitIPAMAPIBurst, "Upper burst limit when accessing external APIs") flags.Float64("limit-ipam-api-qps", defaultConfig.LimitIPAMAPIQPS, "Queries per second limit when accessing external IPAM APIs") + flags.Bool("ipam-release-excess-ips", defaultConfig.IPAMReleaseExcessIPs, "Enable releasing excess free IP addresses from the cloud provider, regardless of the provider in use.") + flags.Int("excess-ip-release-delay", defaultConfig.ExcessIPReleaseDelay, "Number of seconds operator would wait before it releases an IP previously marked as excess") }