[IPAM] Grace Period

cmd/ipam/main.go

@@ -72,8 +72,10 @@ func main() {
 
 	// Server options.
 	cmd.Flags().IntVar(&options.ServerOpts.Port, "port", consts.IpamPort, "The port on which to listen for incoming gRPC requests.")
-	cmd.Flags().DurationVar(&options.ServerOpts.SyncFrequency, "sync-interval", consts.SyncInterval,
+	cmd.Flags().DurationVar(&options.ServerOpts.SyncInterval, "sync-interval", consts.SyncInterval,
 		"The interval at which the IPAM will synchronize the IPAM storage.")
+	cmd.Flags().DurationVar(&options.ServerOpts.SyncGracePeriod, "sync-graceperiod", consts.SyncGracePeriod,
+		"The grace period the sync routine wait before releasing an ip or a network.")
 	cmd.Flags().BoolVar(&options.ServerOpts.GraphvizEnabled, "enable-graphviz", false, "Enable the graphviz output for the IPAM.")
 	cmd.Flags().StringSliceVar(&options.ServerOpts.Pools, "pools",
 		[]string{"10.0.0.0/8", "192.168.0.0/16", "172.16.0.0/12"},

deployments/liqo/README.md

@@ -59,6 +59,7 @@
 | ipam.internal.pod.priorityClassName | string | `""` | PriorityClassName (https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority) for the IPAM pod. |
 | ipam.internal.pod.resources | object | `{"limits":{},"requests":{}}` | Resource requests and limits (https://kubernetes.io/docs/user-guide/compute-resources/) for the IPAM pod. |
 | ipam.internal.replicas | int | `1` | The number of IPAM instances to run, which can be increased for active/passive high availability. |
+| ipam.internal.syncGracePeriod | string | `"30s"` |  |
 | ipam.internal.syncInterval | string | `"2m"` | Set the interval at which the IPAM pod will synchronize it's in-memory status with the local cluster. If you want to disable the synchronization, set the interval to 0. |
 | ipam.internalCIDR | string | `"10.80.0.0/16"` | The subnet used for the internal CIDR. These IPs are assigned to the Liqo internal-network interfaces. |
 | ipam.podCIDR | string | `""` | The subnet used by the pods in your cluster, in CIDR notation (e.g., 10.0.0.0/16). |

deployments/liqo/templates/liqo-ipam-deployment.yaml

@@ -52,6 +52,7 @@ spec:
             - --pod-name=$(POD_NAME)
             - --port=6000
             - --sync-interval={{ .Values.ipam.internal.syncInterval }}
+            - --sync-graceperiod={{ .Values.ipam.internal.syncGracePeriod }}
             {{- if $ha }}
             - --leader-election
             - --leader-election-namespace=$(POD_NAMESPACE)

deployments/liqo/values.yaml

@@ -449,6 +449,8 @@ ipam:
     # -- Set the interval at which the IPAM pod will synchronize it's in-memory status with the local cluster.
     # If you want to disable the synchronization, set the interval to 0.
     syncInterval: 2m
+    ## -- Set the grace period the sync routine will wait before deleting an ip or a network.
+    syncGracePeriod: 30s
   # -- The subnet used by the pods in your cluster, in CIDR notation (e.g., 10.0.0.0/16).
   podCIDR: ""
   # -- The subnet used by the services in you cluster, in CIDR notation (e.g., 172.16.0.0/16).

pkg/consts/ipam.go

@@ -24,7 +24,8 @@ const (
 	IpamPort = 6000
 	// SyncInterval is the frequency at which the IPAM should periodically sync its status.
 	SyncInterval = 2 * time.Minute
-
+	// SyncGracePeriod is the time the IPAM sync routine should wait before performing a deletion.
+	SyncGracePeriod = 30 * time.Second
 	// NetworkNotRemappedLabelKey is the label key used to mark a Network that does not need CIDR remapping.
 	NetworkNotRemappedLabelKey = "ipam.liqo.io/network-not-remapped"
 	// NetworkNotRemappedLabelValue is the label value used to mark a Network that does not need CIDR remapping.

pkg/ipam/core/ipam.go

@@ -17,7 +17,7 @@ package ipamcore
 import (
 	"fmt"
 	"net/netip"
-	"slices"
+	"time"
 )
 
 // Ipam represents the IPAM core structure.
@@ -74,10 +74,10 @@ func (ipam *Ipam) NetworkAcquireWithPrefix(prefix netip.Prefix) *netip.Prefix {
 
 // NetworkRelease frees the network with the given prefix.
 // It returns the freed network or nil if the network is not found.
-func (ipam *Ipam) NetworkRelease(prefix netip.Prefix) *netip.Prefix {
+func (ipam *Ipam) NetworkRelease(prefix netip.Prefix, gracePeriod time.Duration) *netip.Prefix {
 	for i := range ipam.roots {
 		if isPrefixChildOf(ipam.roots[i].prefix, prefix) {
-			if result := networkRelease(prefix, &ipam.roots[i]); result != nil {
+			if result := networkRelease(prefix, &ipam.roots[i], gracePeriod); result != nil {
 				return result
 			}
 		}
@@ -137,13 +137,13 @@ func (ipam *Ipam) IPAcquireWithAddr(prefix netip.Prefix, addr netip.Addr) (*neti
 
 // IPRelease frees the IP address from the given prefix.
 // It returns the freed IP address or nil if the IP address is not found.
-func (ipam *Ipam) IPRelease(prefix netip.Prefix, addr netip.Addr) (*netip.Addr, error) {
+func (ipam *Ipam) IPRelease(prefix netip.Prefix, addr netip.Addr, gracePeriod time.Duration) (*netip.Addr, error) {
 	node, err := ipam.search(prefix)
 	if err != nil {
 		return nil, err
 	}
 	if node != nil {
-		return node.ipRelease(addr), nil
+		return node.ipRelease(addr, gracePeriod), nil
 	}
 	return nil, nil
 }
@@ -155,14 +155,18 @@ func (ipam *Ipam) ListIPs(prefix netip.Prefix) ([]netip.Addr, error) {
 		return nil, err
 	}
 	if node != nil {
-		return slices.Clone(node.ips), nil
+		addrs := make([]netip.Addr, len(node.ips))
+		for i := range node.ips {
+			addrs[i] = node.ips[i].addr
+		}
+		return addrs, nil
 	}
 	return nil, nil
 }
 
-// IsAllocatedIP checks if the IP address is allocated from the given prefix.
+// IPIsAllocated checks if the IP address is allocated from the given prefix.
 // It returns true if the IP address is allocated, false otherwise.
-func (ipam *Ipam) IsAllocatedIP(prefix netip.Prefix, addr netip.Addr) (bool, error) {
+func (ipam *Ipam) IPIsAllocated(prefix netip.Prefix, addr netip.Addr) (bool, error) {
 	node, err := ipam.search(prefix)
 	if err != nil {
 		return false, err
@@ -216,3 +220,37 @@ func checkRoots(roots []netip.Prefix) error {
 	}
 	return nil
 }
+
+// NetworkSetLastUpdateTimestamp sets the last update time of the network with the given prefix.
+// This function is for testing purposes only.
+func (ipam *Ipam) NetworkSetLastUpdateTimestamp(prefix netip.Prefix, lastUpdateTimestamp time.Time) error {
+	node, err := ipam.search(prefix)
+	if err != nil {
+		return err
+	}
+	if node == nil {
+		return fmt.Errorf("prefix %s not found", prefix)
+	}
+	node.lastUpdateTimestamp = lastUpdateTimestamp
+	return nil
+}
+
+// IPSetCreationTimestamp sets the creation timestamp of the IP address with the given address.
+// This function is for testing purposes only.
+func (ipam *Ipam) IPSetCreationTimestamp(addr netip.Addr, prefix netip.Prefix, creationTimestamp time.Time) error {
+	node, err := ipam.search(prefix)
+	if err != nil {
+		return err
+	}
+	if node == nil {
+		return fmt.Errorf("prefix %s not found", prefix)
+	}
+
+	for i := range node.ips {
+		if node.ips[i].addr.Compare(addr) == 0 {
+			node.ips[i].creationTimestamp = creationTimestamp
+			return nil
+		}
+	}
+	return nil
+}

pkg/ipam/core/node.go

@@ -21,16 +21,25 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
+// nodeIP represents an IP address acquired by a node.
+type nodeIP struct {
+	addr              netip.Addr
+	creationTimestamp time.Time
+}
+
 // node represents a node in the binary tree.
 type node struct {
+	lastUpdateTimestamp time.Time
+
 	prefix   netip.Prefix
 	acquired bool
 	left     *node
 	right    *node
 
-	ips    []netip.Addr
+	ips    []nodeIP
 	lastip netip.Addr
 }
 
@@ -39,10 +48,12 @@ type nodeDirection string
 const (
 	leftDirection  nodeDirection = "left"
 	rightDirection nodeDirection = "right"
+
+	graphvizFolder = "./graphviz"
 )
 
 func newNode(prefix netip.Prefix) node {
-	return node{prefix: prefix}
+	return node{prefix: prefix, lastUpdateTimestamp: time.Now()}
 }
 
 func allocateNetwork(size int, node *node) *netip.Prefix {
@@ -52,6 +63,7 @@ func allocateNetwork(size int, node *node) *netip.Prefix {
 	if node.prefix.Bits() == size {
 		if !node.isSplitted() {
 			node.acquired = true
+			node.lastUpdateTimestamp = time.Now()
 			return &node.prefix
 		}
 		return nil
@@ -76,6 +88,7 @@ func allocateNetworkWithPrefix(prefix netip.Prefix, node *node) *netip.Prefix {
 	if node.prefix.Addr().Compare(prefix.Addr()) == 0 && node.prefix.Bits() == prefix.Bits() {
 		if !node.acquired && node.left == nil && node.right == nil {
 			node.acquired = true
+			node.lastUpdateTimestamp = time.Now()
 			return &node.prefix
 		}
 		return nil
@@ -95,29 +108,31 @@ func allocateNetworkWithPrefix(prefix netip.Prefix, node *node) *netip.Prefix {
 	return nil
 }
 
-func networkRelease(prefix netip.Prefix, node *node) *netip.Prefix {
+func networkRelease(prefix netip.Prefix, node *node, gracePeriod time.Duration) *netip.Prefix {
 	var result *netip.Prefix
 
 	if node == nil {
 		return nil
 	}
 
-	if node.prefix.Addr().Compare(prefix.Addr()) == 0 && node.prefix.Bits() == prefix.Bits() {
+	if node.prefix.Addr().Compare(prefix.Addr()) == 0 && node.prefix.Bits() == prefix.Bits() &&
+		node.lastUpdateTimestamp.Add(gracePeriod).Before(time.Now()) {
 		if node.acquired {
 			node.acquired = false
+			node.lastUpdateTimestamp = time.Now()
 			return &node.prefix
 		}
 		return nil
 	}
 
 	if node.left != nil && node.left.prefix.Overlaps(prefix) {
-		result = networkRelease(prefix, node.left)
+		result = networkRelease(prefix, node.left, gracePeriod)
 	}
 	if node.right != nil && node.right.prefix.Overlaps(prefix) {
-		result = networkRelease(prefix, node.right)
+		result = networkRelease(prefix, node.right, gracePeriod)
 	}
 
-	node.merge()
+	node.merge(gracePeriod)
 	return result
 }
 
@@ -170,7 +185,7 @@ func listNetworks(node *node) []netip.Prefix {
 
 func (n *node) isAllocatedIP(ip netip.Addr) bool {
 	for i := range n.ips {
-		if n.ips[i].Compare(ip) == 0 {
+		if n.ips[i].addr.Compare(ip) == 0 {
 			return true
 		}
 	}
@@ -202,8 +217,9 @@ func (n *node) ipAcquire() *netip.Addr {
 			addr = n.prefix.Addr()
 		}
 		if !n.isAllocatedIP(addr) {
-			n.ips = append(n.ips, addr)
+			n.ips = append(n.ips, nodeIP{addr: addr, creationTimestamp: time.Now()})
 			n.lastip = addr
+			n.lastUpdateTimestamp = time.Now()
 			return &addr
 		}
 		addr = addr.Next()
@@ -221,25 +237,30 @@ func (n *node) allocateIPWithAddr(addr netip.Addr) *netip.Addr {
 	}
 
 	for i := range n.ips {
-		if n.ips[i].Compare(addr) == 0 {
+		if n.ips[i].addr.Compare(addr) == 0 {
 			return nil
 		}
 	}
 
-	n.ips = append(n.ips, addr)
+	n.ips = append(n.ips, nodeIP{addr: addr, creationTimestamp: time.Now()})
+	n.lastUpdateTimestamp = time.Now()
 
-	return &n.ips[len(n.ips)-1]
+	return &n.ips[len(n.ips)-1].addr
 }
 
-func (n *node) ipRelease(ip netip.Addr) *netip.Addr {
+func (n *node) ipRelease(ip netip.Addr, gracePeriod time.Duration) *netip.Addr {
 	if !n.acquired {
 		return nil
 	}
 
-	for i, addr := range n.ips {
-		if addr.Compare(ip) == 0 {
+	for i, nodeIP := range n.ips {
+		if !nodeIP.creationTimestamp.Add(gracePeriod).Before(time.Now()) {
+			continue
+		}
+		if nodeIP.addr.Compare(ip) == 0 {
 			n.ips = append(n.ips[:i], n.ips[i+1:]...)
-			return &addr
+			n.lastUpdateTimestamp = time.Now()
+			return &nodeIP.addr
 		}
 	}
 	return nil
@@ -273,21 +294,33 @@ func (n *node) split() {
 	n.insert(rightDirection, right)
 }
 
-func (n *node) merge() {
-	if n.left.isLeaf() && n.right.isLeaf() && !n.left.acquired && !n.right.acquired {
-		n.left = nil
-		n.right = nil
+func (n *node) merge(gracePeriod time.Duration) {
+	if n.left == nil || n.right == nil {
+		return
+	}
+	if !n.left.lastUpdateTimestamp.Add(gracePeriod).Before(time.Now()) || !n.right.lastUpdateTimestamp.Add(gracePeriod).Before(time.Now()) {
+		return // grace period not expired
+	}
+	if !n.left.isLeaf() || !n.right.isLeaf() {
+		return
 	}
+	if n.left.acquired || n.right.acquired {
+		return
+	}
+
+	n.left = nil
+	n.right = nil
+	n.lastUpdateTimestamp = time.Now()
 }
 
 func (n *node) insert(nd nodeDirection, prefix netip.Prefix) {
-	newNode := &node{prefix: prefix}
+	newNode := newNode(prefix)
 	switch nd {
 	case leftDirection:
-		n.left = newNode
+		n.left = &newNode
 		return
 	case rightDirection:
-		n.right = newNode
+		n.right = &newNode
 		return
 	default:
 		return
@@ -335,13 +368,13 @@ func (n *node) toGraphviz() error {
 	n.toGraphvizRecursive(&sb)
 	sb.WriteString("}\n")
 
-	if _, err := os.Stat("./graphviz"); os.IsNotExist(err) {
-		if err := os.Mkdir("./graphviz", 0o700); err != nil {
+	if _, err := os.Stat(graphvizFolder + ""); os.IsNotExist(err) {
+		if err := os.Mkdir(graphvizFolder+"", 0o700); err != nil {
 			return err
 		}
 	}
 
-	filePath := filepath.Clean("./graphviz/" + strings.NewReplacer("/", "_", ".", "_").Replace(n.prefix.String()) + ".dot")
+	filePath := filepath.Clean(graphvizFolder + "/" + strings.NewReplacer("/", "_", ".", "_").Replace(n.prefix.String()) + ".dot")
 	file, err := os.Create(filePath)
 	if err != nil {
 		return err
@@ -360,7 +393,7 @@ func (n *node) toGraphvizRecursive(sb *strings.Builder) {
 	if len(n.ips) > 0 {
 		ipsString := []string{}
 		for i := range n.ips {
-			ipsString = append(ipsString, n.ips[i].String())
+			ipsString = append(ipsString, n.ips[i].addr.String())
 		}
 		label += "\\n" + strings.Join(ipsString, "\\n")
 	}

pkg/ipam/initialize.go

@@ -57,7 +57,7 @@ func (lipam *LiqoIPAM) initializeNetworks(ctx context.Context) error {
 		}
 		for i := 0; i < int(netdetails.preallocated); i++ {
 			if _, err := lipam.ipAcquire(net); err != nil {
-				return errors.Join(err, lipam.networkRelease(net))
+				return errors.Join(err, lipam.networkRelease(net, 0))
 			}
 		}
 	}