diff --git a/daemon/infraendpoints/infra_ip_allocation.go b/daemon/infraendpoints/infra_ip_allocation.go index 42dc54de181e7..948ce5e5fea93 100644 --- a/daemon/infraendpoints/infra_ip_allocation.go +++ b/daemon/infraendpoints/infra_ip_allocation.go @@ -282,9 +282,13 @@ func (r *infraIPAllocator) reallocateRouterIPs(ctx context.Context, family datap } } - if (r.daemonConfig.IPAM == ipamOption.IPAMENI || - r.daemonConfig.IPAM == ipamOption.IPAMAlibabaCloud || - r.daemonConfig.IPAM == ipamOption.IPAMAzure) && result != nil { + // Configure routing if we have the necessary routing information, + // regardless of IPAM mode. This allows any IPAM mode (including kubernetes) + // to work with multi-VNIC setups by providing routing information. + if result != nil && + result.GatewayIP != "" && + result.PrimaryMAC != "" && + len(result.CIDRs) > 0 { var routingInfo *linuxrouting.RoutingInfo routingInfo, err = linuxrouting.NewRoutingInfo(r.logger, result.GatewayIP, result.CIDRs, result.PrimaryMAC, result.InterfaceNumber, r.daemonConfig.IPAM, @@ -388,12 +392,12 @@ func (r *infraIPAllocator) allocateHealthIPs(oldV4HealthIP net.IP, oldV6HealthIP r.logger.Debug("Allocated IPv4 health endpoint address", logfields.IPAddr, result.IP) - // In ENI and AlibabaCloud ENI mode, we require the gateway, CIDRs, and the ENI MAC addr - // in order to set up rules and routes on the local node to direct - // endpoint traffic out of the ENIs. - if r.daemonConfig.IPAM == ipamOption.IPAMENI || r.daemonConfig.IPAM == ipamOption.IPAMAlibabaCloud { + // If routing information is available (gateway, CIDRs, MAC address), + // parse and store it for setting up health endpoint routing rules. + // This works with any IPAM mode that provides routing information. + if result.GatewayIP != "" && result.PrimaryMAC != "" && len(result.CIDRs) > 0 { if r.healthEndpointRouting, err = r.parseRoutingInfo(result); err != nil { - r.logger.Warn("Unable to allocate health information for ENI", logfields.Error, err) + r.logger.Warn("Unable to parse health endpoint routing information", logfields.Error, err) } } } @@ -475,12 +479,12 @@ func (r *infraIPAllocator) allocateIngressIPs(oldV4IngressIP net.IP, oldV6Ingres r.localNodeStore.Update(func(n *node.LocalNode) { n.IPv4IngressIP = result.IP }) r.logger.Debug("Allocated IPv4 Ingress address", logfields.IPAddr, result.IP) - // In ENI and AlibabaCloud ENI mode, we require the gateway, CIDRs, and the - // ENI MAC addr in order to set up rules and routes on the local node to - // direct ingress traffic out of the ENIs. - if r.daemonConfig.IPAM == ipamOption.IPAMENI || r.daemonConfig.IPAM == ipamOption.IPAMAlibabaCloud { + // If routing information is available (gateway, CIDRs, MAC address), + // configure ingress routing rules. This works with any IPAM mode that + // provides routing information. + if result.GatewayIP != "" && result.PrimaryMAC != "" && len(result.CIDRs) > 0 { if ingressRouting, err := r.parseRoutingInfo(result); err != nil { - r.logger.Warn("Unable to allocate ingress information for ENI", logfields.Error, err) + r.logger.Warn("Unable to parse ingress routing information", logfields.Error, err) } else { if err := ingressRouting.Configure( result.IP, diff --git a/pkg/ipam/hostscope.go b/pkg/ipam/hostscope.go index 52061d96e37d7..1dba2eae26d4f 100644 --- a/pkg/ipam/hostscope.go +++ b/pkg/ipam/hostscope.go @@ -7,21 +7,39 @@ import ( "fmt" "math/big" "net" + "strconv" + "strings" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/cilium/cilium/pkg/datapath/linux/safenetlink" "github.com/cilium/cilium/pkg/ip" "github.com/cilium/cilium/pkg/ipam/service/ipallocator" ) +// routingInfo caches the auto-detected routing information for the allocation +// CIDR. nil when no matching interface was found. +type routingInfo struct { + primaryMAC string + interfaceNumber string + cidrs []string + gatewayIP string +} + type hostScopeAllocator struct { - allocCIDR *net.IPNet - allocator *ipallocator.Range + allocCIDR *net.IPNet + allocator *ipallocator.Range + routingInfo *routingInfo } func newHostScopeAllocator(n *net.IPNet) Allocator { - return &hostScopeAllocator{ + h := &hostScopeAllocator{ allocCIDR: n, allocator: ipallocator.NewCIDRRange(n), } + h.routingInfo = detectRoutingInfo(n) + return h } func (h *hostScopeAllocator) Allocate(ip net.IP, owner string, pool Pool) (*AllocationResult, error) { @@ -29,7 +47,9 @@ func (h *hostScopeAllocator) Allocate(ip net.IP, owner string, pool Pool) (*Allo return nil, err } - return &AllocationResult{IP: ip}, nil + result := &AllocationResult{IP: ip} + h.applyRoutingInfo(result) + return result, nil } func (h *hostScopeAllocator) AllocateWithoutSyncUpstream(ip net.IP, owner string, pool Pool) (*AllocationResult, error) { @@ -37,7 +57,9 @@ func (h *hostScopeAllocator) AllocateWithoutSyncUpstream(ip net.IP, owner string return nil, err } - return &AllocationResult{IP: ip}, nil + result := &AllocationResult{IP: ip} + h.applyRoutingInfo(result) + return result, nil } func (h *hostScopeAllocator) Release(ip net.IP, pool Pool) error { @@ -51,7 +73,9 @@ func (h *hostScopeAllocator) AllocateNext(owner string, pool Pool) (*AllocationR return nil, err } - return &AllocationResult{IP: ip}, nil + result := &AllocationResult{IP: ip} + h.applyRoutingInfo(result) + return result, nil } func (h *hostScopeAllocator) AllocateNextWithoutSyncUpstream(owner string, pool Pool) (*AllocationResult, error) { @@ -60,7 +84,9 @@ func (h *hostScopeAllocator) AllocateNextWithoutSyncUpstream(owner string, pool return nil, err } - return &AllocationResult{IP: ip}, nil + result := &AllocationResult{IP: ip} + h.applyRoutingInfo(result) + return result, nil } func (h *hostScopeAllocator) Dump() (map[Pool]map[string]string, string) { @@ -95,3 +121,115 @@ func (h *hostScopeAllocator) Capacity() uint64 { // RestoreFinished marks the status of restoration as done func (h *hostScopeAllocator) RestoreFinished() {} + +// applyRoutingInfo stamps the cached routing information onto an AllocationResult. +func (h *hostScopeAllocator) applyRoutingInfo(result *AllocationResult) { + if h.routingInfo == nil || result == nil { + return + } + result.PrimaryMAC = h.routingInfo.primaryMAC + result.InterfaceNumber = h.routingInfo.interfaceNumber + result.CIDRs = h.routingInfo.cidrs + result.GatewayIP = h.routingInfo.gatewayIP +} + +// detectRoutingInfo attempts to auto-detect routing information for the +// allocation CIDR by finding a network interface that has an IP address within +// the same subnet. This enables kubernetes IPAM mode to work with multi-VNIC +// setups (e.g., Oracle Cloud, bare metal) without requiring manual configuration. +// Returns nil when no matching interface is found. +func detectRoutingInfo(allocCIDR *net.IPNet) *routingInfo { + if allocCIDR == nil { + return nil + } + + links, err := safenetlink.LinkList() + if err != nil { + return nil + } + + for _, link := range links { + // Skip interfaces that are not up and operational + if link.Attrs().OperState != netlink.OperUp && + link.Attrs().OperState != netlink.OperUnknown { + continue + } + + // Skip slave devices (we want the master device) + if link.Attrs().RawFlags&unix.IFF_SLAVE != 0 { + continue + } + + // Skip loopback and other special interfaces + if link.Attrs().Flags&net.FlagLoopback != 0 { + continue + } + + // Skip Cilium-managed interfaces (cilium_host, cilium_net, lxc*) + if strings.HasPrefix(link.Attrs().Name, "cilium_") || + strings.HasPrefix(link.Attrs().Name, "lxc") { + continue + } + + // Get addresses on this interface + family := netlink.FAMILY_V4 + if allocCIDR.IP.To4() == nil { + family = netlink.FAMILY_V6 + } + + addrs, err := safenetlink.AddrList(link, family) + if err != nil { + continue + } + + // Check if any address on this interface is within our allocation CIDR + for _, addr := range addrs { + // The interface's subnet should CONTAIN our allocation CIDR, not the other way around + // This ensures we find the physical interface (e.g., enp1s0 with 100.64.0.0/18) + // rather than virtual interfaces like cilium_host + if addr.IPNet.Contains(allocCIDR.IP) && addr.IPNet.Contains(lastIPInCIDR(allocCIDR)) { + return &routingInfo{ + primaryMAC: link.Attrs().HardwareAddr.String(), + interfaceNumber: strconv.Itoa(link.Attrs().Index), + cidrs: []string{addr.IPNet.String()}, + gatewayIP: deriveGatewayFromSubnet(addr.IPNet), + } + } + } + } + + return nil +} + +// lastIPInCIDR returns the last IP address in a CIDR range +func lastIPInCIDR(cidr *net.IPNet) net.IP { + ip := make(net.IP, len(cidr.IP)) + copy(ip, cidr.IP) + for i := range ip { + ip[i] |= ^cidr.Mask[i] + } + return ip +} + +// deriveGatewayFromSubnet derives the gateway IP from a subnet by using the first +// usable IP address in the subnet (typically x.x.x.1 for IPv4). +func deriveGatewayFromSubnet(subnet *net.IPNet) string { + if subnet == nil { + return "" + } + + // Get the network address + ip := subnet.IP.Mask(subnet.Mask) + + if ip.To4() != nil { + // For IPv4, use the first address in the subnet (x.x.x.1) + ip = ip.To4() + ip[3] = 1 + return ip.String() + } else { + // For IPv6, use the first address in the subnet (ending in ::1) + ip = ip.To16() + ip[15] = 1 + return ip.String() + } +} diff --git a/plugins/cilium-cni/cmd/cmd.go b/plugins/cilium-cni/cmd/cmd.go index ab2167ff9ae47..a94b25f2b3d88 100644 --- a/plugins/cilium-cni/cmd/cmd.go +++ b/plugins/cilium-cni/cmd/cmd.go @@ -775,19 +775,17 @@ func (cmd *Cmd) Add(args *skel.CmdArgs) (err error) { res.Routes = append(res.Routes, routes...) } - if needsEndpointRoutingOnHost(conf) { - if ipam.IPV4 != nil && ipConfig != nil { - err = interfaceAdd(scopedLogger, ipConfig, ipam.IPV4, conf) - if err != nil { - return fmt.Errorf("unable to setup interface datapath: %w", err) - } + if needsEndpointRoutingOnHost(conf, ipam.IPV4, ipConfig) { + err = interfaceAdd(scopedLogger, ipConfig, ipam.IPV4, conf) + if err != nil { + return fmt.Errorf("unable to setup interface datapath: %w", err) } + } - if ipam.IPV6 != nil && ipv6Config != nil { - err = interfaceAdd(scopedLogger, ipv6Config, ipam.IPV6, conf) - if err != nil { - return fmt.Errorf("unable to setup interface datapath: %w", err) - } + if needsEndpointRoutingOnHost(conf, ipam.IPV6, ipv6Config) { + err = interfaceAdd(scopedLogger, ipv6Config, ipam.IPV6, conf) + if err != nil { + return fmt.Errorf("unable to setup interface datapath: %w", err) } } @@ -1337,12 +1335,19 @@ func buildLogAttrsWithCNIArgs(logger *slog.Logger, cniArgs *types.ArgsSpec) *slo // on host for the Pod. This is needed for following IPAM modes: // - Cloud ENI IPAM modes. // - DelegatedPlugin mode with InstallUplinkRoutesForDelegatedIPAM set to true. -func needsEndpointRoutingOnHost(conf *models.DaemonConfigurationStatus) bool { +// - Some cases where we use Kubernetes IPAM with multiple NICs (eg. Oracle Cloud) +func needsEndpointRoutingOnHost(conf *models.DaemonConfigurationStatus, ipam *models.IPAMAddressResponse, ipConfig *cniTypesV1.IPConfig) bool { + if ipam == nil || ipConfig == nil { + return false + } + switch conf.IpamMode { case ipamOption.IPAMENI, ipamOption.IPAMAzure, ipamOption.IPAMAlibabaCloud: return true case ipamOption.IPAMDelegatedPlugin: return conf.InstallUplinkRoutesForDelegatedIPAM + case ipamOption.IPAMKubernetes: + return ipam.Gateway != "" && ipam.MasterMac != "" } return false }