Merge pull request #658 from DataDog/hadrien/1.19/oracle

cilium-cni: Add support for route/rules creation on Kubernetes IPAM

Author: HadrienPatte

daemon/infraendpoints/infra_ip_allocation.go

@@ -282,9 +282,13 @@ func (r *infraIPAllocator) reallocateRouterIPs(ctx context.Context, family datap
 		}
 	}
 
-	if (r.daemonConfig.IPAM == ipamOption.IPAMENI ||
-		r.daemonConfig.IPAM == ipamOption.IPAMAlibabaCloud ||
-		r.daemonConfig.IPAM == ipamOption.IPAMAzure) && result != nil {
+	// Configure routing if we have the necessary routing information,
+	// regardless of IPAM mode. This allows any IPAM mode (including kubernetes)
+	// to work with multi-VNIC setups by providing routing information.
+	if result != nil &&
+		result.GatewayIP != "" &&
+		result.PrimaryMAC != "" &&
+		len(result.CIDRs) > 0 {
 		var routingInfo *linuxrouting.RoutingInfo
 		routingInfo, err = linuxrouting.NewRoutingInfo(r.logger, result.GatewayIP, result.CIDRs,
 			result.PrimaryMAC, result.InterfaceNumber, r.daemonConfig.IPAM,
@@ -388,12 +392,12 @@ func (r *infraIPAllocator) allocateHealthIPs(oldV4HealthIP net.IP, oldV6HealthIP
 
 		r.logger.Debug("Allocated IPv4 health endpoint address", logfields.IPAddr, result.IP)
 
-		// In ENI and AlibabaCloud ENI mode, we require the gateway, CIDRs, and the ENI MAC addr
-		// in order to set up rules and routes on the local node to direct
-		// endpoint traffic out of the ENIs.
-		if r.daemonConfig.IPAM == ipamOption.IPAMENI || r.daemonConfig.IPAM == ipamOption.IPAMAlibabaCloud {
+		// If routing information is available (gateway, CIDRs, MAC address),
+		// parse and store it for setting up health endpoint routing rules.
+		// This works with any IPAM mode that provides routing information.
+		if result.GatewayIP != "" && result.PrimaryMAC != "" && len(result.CIDRs) > 0 {
 			if r.healthEndpointRouting, err = r.parseRoutingInfo(result); err != nil {
-				r.logger.Warn("Unable to allocate health information for ENI", logfields.Error, err)
+				r.logger.Warn("Unable to parse health endpoint routing information", logfields.Error, err)
 			}
 		}
 	}
@@ -475,12 +479,12 @@ func (r *infraIPAllocator) allocateIngressIPs(oldV4IngressIP net.IP, oldV6Ingres
 		r.localNodeStore.Update(func(n *node.LocalNode) { n.IPv4IngressIP = result.IP })
 		r.logger.Debug("Allocated IPv4 Ingress address", logfields.IPAddr, result.IP)
 
-		// In ENI and AlibabaCloud ENI mode, we require the gateway, CIDRs, and the
-		// ENI MAC addr in order to set up rules and routes on the local node to
-		// direct ingress traffic out of the ENIs.
-		if r.daemonConfig.IPAM == ipamOption.IPAMENI || r.daemonConfig.IPAM == ipamOption.IPAMAlibabaCloud {
+		// If routing information is available (gateway, CIDRs, MAC address),
+		// configure ingress routing rules. This works with any IPAM mode that
+		// provides routing information.
+		if result.GatewayIP != "" && result.PrimaryMAC != "" && len(result.CIDRs) > 0 {
 			if ingressRouting, err := r.parseRoutingInfo(result); err != nil {
-				r.logger.Warn("Unable to allocate ingress information for ENI", logfields.Error, err)
+				r.logger.Warn("Unable to parse ingress routing information", logfields.Error, err)
 			} else {
 				if err := ingressRouting.Configure(
 					result.IP,

pkg/ipam/hostscope.go

@@ -7,37 +7,59 @@ import (
 	"fmt"
 	"math/big"
 	"net"
+	"strconv"
+	"strings"
 
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/cilium/cilium/pkg/datapath/linux/safenetlink"
 	"github.com/cilium/cilium/pkg/ip"
 	"github.com/cilium/cilium/pkg/ipam/service/ipallocator"
 )
 
+// routingInfo caches the auto-detected routing information for the allocation
+// CIDR. nil when no matching interface was found.
+type routingInfo struct {
+	primaryMAC      string
+	interfaceNumber string
+	cidrs           []string
+	gatewayIP       string
+}
+
 type hostScopeAllocator struct {
-	allocCIDR *net.IPNet
-	allocator *ipallocator.Range
+	allocCIDR   *net.IPNet
+	allocator   *ipallocator.Range
+	routingInfo *routingInfo
 }
 
 func newHostScopeAllocator(n *net.IPNet) Allocator {
-	return &hostScopeAllocator{
+	h := &hostScopeAllocator{
 		allocCIDR: n,
 		allocator: ipallocator.NewCIDRRange(n),
 	}
+	h.routingInfo = detectRoutingInfo(n)
+	return h
 }
 
 func (h *hostScopeAllocator) Allocate(ip net.IP, owner string, pool Pool) (*AllocationResult, error) {
 	if err := h.allocator.Allocate(ip); err != nil {
 		return nil, err
 	}
 
-	return &AllocationResult{IP: ip}, nil
+	result := &AllocationResult{IP: ip}
+	h.applyRoutingInfo(result)
+	return result, nil
 }
 
 func (h *hostScopeAllocator) AllocateWithoutSyncUpstream(ip net.IP, owner string, pool Pool) (*AllocationResult, error) {
 	if err := h.allocator.Allocate(ip); err != nil {
 		return nil, err
 	}
 
-	return &AllocationResult{IP: ip}, nil
+	result := &AllocationResult{IP: ip}
+	h.applyRoutingInfo(result)
+	return result, nil
 }
 
 func (h *hostScopeAllocator) Release(ip net.IP, pool Pool) error {
@@ -51,7 +73,9 @@ func (h *hostScopeAllocator) AllocateNext(owner string, pool Pool) (*AllocationR
 		return nil, err
 	}
 
-	return &AllocationResult{IP: ip}, nil
+	result := &AllocationResult{IP: ip}
+	h.applyRoutingInfo(result)
+	return result, nil
 }
 
 func (h *hostScopeAllocator) AllocateNextWithoutSyncUpstream(owner string, pool Pool) (*AllocationResult, error) {
@@ -60,7 +84,9 @@ func (h *hostScopeAllocator) AllocateNextWithoutSyncUpstream(owner string, pool
 		return nil, err
 	}
 
-	return &AllocationResult{IP: ip}, nil
+	result := &AllocationResult{IP: ip}
+	h.applyRoutingInfo(result)
+	return result, nil
 }
 
 func (h *hostScopeAllocator) Dump() (map[Pool]map[string]string, string) {
@@ -95,3 +121,115 @@ func (h *hostScopeAllocator) Capacity() uint64 {
 
 // RestoreFinished marks the status of restoration as done
 func (h *hostScopeAllocator) RestoreFinished() {}
+
+// applyRoutingInfo stamps the cached routing information onto an AllocationResult.
+func (h *hostScopeAllocator) applyRoutingInfo(result *AllocationResult) {
+	if h.routingInfo == nil || result == nil {
+		return
+	}
+	result.PrimaryMAC = h.routingInfo.primaryMAC
+	result.InterfaceNumber = h.routingInfo.interfaceNumber
+	result.CIDRs = h.routingInfo.cidrs
+	result.GatewayIP = h.routingInfo.gatewayIP
+}
+
+// detectRoutingInfo attempts to auto-detect routing information for the
+// allocation CIDR by finding a network interface that has an IP address within
+// the same subnet. This enables kubernetes IPAM mode to work with multi-VNIC
+// setups (e.g., Oracle Cloud, bare metal) without requiring manual configuration.
+// Returns nil when no matching interface is found.
+func detectRoutingInfo(allocCIDR *net.IPNet) *routingInfo {
+	if allocCIDR == nil {
+		return nil
+	}
+
+	links, err := safenetlink.LinkList()
+	if err != nil {
+		return nil
+	}
+
+	for _, link := range links {
+		// Skip interfaces that are not up and operational
+		if link.Attrs().OperState != netlink.OperUp &&
+			link.Attrs().OperState != netlink.OperUnknown {
+			continue
+		}
+
+		// Skip slave devices (we want the master device)
+		if link.Attrs().RawFlags&unix.IFF_SLAVE != 0 {
+			continue
+		}
+
+		// Skip loopback and other special interfaces
+		if link.Attrs().Flags&net.FlagLoopback != 0 {
+			continue
+		}
+
+		// Skip Cilium-managed interfaces (cilium_host, cilium_net, lxc*)
+		if strings.HasPrefix(link.Attrs().Name, "cilium_") ||
+			strings.HasPrefix(link.Attrs().Name, "lxc") {
+			continue
+		}
+
+		// Get addresses on this interface
+		family := netlink.FAMILY_V4
+		if allocCIDR.IP.To4() == nil {
+			family = netlink.FAMILY_V6
+		}
+
+		addrs, err := safenetlink.AddrList(link, family)
+		if err != nil {
+			continue
+		}
+
+		// Check if any address on this interface is within our allocation CIDR
+		for _, addr := range addrs {
+			// The interface's subnet should CONTAIN our allocation CIDR, not the other way around
+			// This ensures we find the physical interface (e.g., enp1s0 with 100.64.0.0/18)
+			// rather than virtual interfaces like cilium_host
+			if addr.IPNet.Contains(allocCIDR.IP) && addr.IPNet.Contains(lastIPInCIDR(allocCIDR)) {
+				return &routingInfo{
+					primaryMAC:      link.Attrs().HardwareAddr.String(),
+					interfaceNumber: strconv.Itoa(link.Attrs().Index),
+					cidrs:           []string{addr.IPNet.String()},
+					gatewayIP:       deriveGatewayFromSubnet(addr.IPNet),
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// lastIPInCIDR returns the last IP address in a CIDR range
+func lastIPInCIDR(cidr *net.IPNet) net.IP {
+	ip := make(net.IP, len(cidr.IP))
+	copy(ip, cidr.IP)
+	for i := range ip {
+		ip[i] |= ^cidr.Mask[i]
+	}
+	return ip
+}
+
+// deriveGatewayFromSubnet derives the gateway IP from a subnet by using the first
+// usable IP address in the subnet (typically x.x.x.1 for IPv4).
+func deriveGatewayFromSubnet(subnet *net.IPNet) string {
+	if subnet == nil {
+		return ""
+	}
+
+	// Get the network address
+	ip := subnet.IP.Mask(subnet.Mask)
+
+	if ip.To4() != nil {
+		// For IPv4, use the first address in the subnet (x.x.x.1)
+		ip = ip.To4()
+		ip[3] = 1
+		return ip.String()
+	} else {
+		// For IPv6, use the first address in the subnet (ending in ::1)
+		ip = ip.To16()
+		ip[15] = 1
+		return ip.String()
+	}
+}

plugins/cilium-cni/cmd/cmd.go

@@ -775,19 +775,17 @@ func (cmd *Cmd) Add(args *skel.CmdArgs) (err error) {
 			res.Routes = append(res.Routes, routes...)
 		}
 
-		if needsEndpointRoutingOnHost(conf) {
-			if ipam.IPV4 != nil && ipConfig != nil {
-				err = interfaceAdd(scopedLogger, ipConfig, ipam.IPV4, conf)
-				if err != nil {
-					return fmt.Errorf("unable to setup interface datapath: %w", err)
-				}
+		if needsEndpointRoutingOnHost(conf, ipam.IPV4, ipConfig) {
+			err = interfaceAdd(scopedLogger, ipConfig, ipam.IPV4, conf)
+			if err != nil {
+				return fmt.Errorf("unable to setup interface datapath: %w", err)
 			}
+		}
 
-			if ipam.IPV6 != nil && ipv6Config != nil {
-				err = interfaceAdd(scopedLogger, ipv6Config, ipam.IPV6, conf)
-				if err != nil {
-					return fmt.Errorf("unable to setup interface datapath: %w", err)
-				}
+		if needsEndpointRoutingOnHost(conf, ipam.IPV6, ipv6Config) {
+			err = interfaceAdd(scopedLogger, ipv6Config, ipam.IPV6, conf)
+			if err != nil {
+				return fmt.Errorf("unable to setup interface datapath: %w", err)
 			}
 		}
 
@@ -1337,12 +1335,19 @@ func buildLogAttrsWithCNIArgs(logger *slog.Logger, cniArgs *types.ArgsSpec) *slo
 // on host for the Pod. This is needed for following IPAM modes:
 // - Cloud ENI IPAM modes.
 // - DelegatedPlugin mode with InstallUplinkRoutesForDelegatedIPAM set to true.
-func needsEndpointRoutingOnHost(conf *models.DaemonConfigurationStatus) bool {
+// - Some cases where we use Kubernetes IPAM with multiple NICs (eg. Oracle Cloud)
+func needsEndpointRoutingOnHost(conf *models.DaemonConfigurationStatus, ipam *models.IPAMAddressResponse, ipConfig *cniTypesV1.IPConfig) bool {
+	if ipam == nil || ipConfig == nil {
+		return false
+	}
+
 	switch conf.IpamMode {
 	case ipamOption.IPAMENI, ipamOption.IPAMAzure, ipamOption.IPAMAlibabaCloud:
 		return true
 	case ipamOption.IPAMDelegatedPlugin:
 		return conf.InstallUplinkRoutesForDelegatedIPAM
+	case ipamOption.IPAMKubernetes:
+		return ipam.Gateway != "" && ipam.MasterMac != ""
 	}
 	return false
 }