[1.17] Backport ip-masq-agent changes

daemon/cmd/cells.go

@@ -41,6 +41,7 @@ import (
 	"github.com/cilium/cilium/pkg/identity/identitymanager"
 	ipamcell "github.com/cilium/cilium/pkg/ipam/cell"
 	ipcache "github.com/cilium/cilium/pkg/ipcache/cell"
+	ipmasq "github.com/cilium/cilium/pkg/ipmasq/cell"
 	"github.com/cilium/cilium/pkg/k8s"
 	k8sClient "github.com/cilium/cilium/pkg/k8s/client"
 	k8sSynced "github.com/cilium/cilium/pkg/k8s/synced"
@@ -50,6 +51,7 @@ import (
 	loadbalancer_experimental "github.com/cilium/cilium/pkg/loadbalancer/experimental"
 	"github.com/cilium/cilium/pkg/logging/logfields"
 	"github.com/cilium/cilium/pkg/maglev"
+	ipmasqmaps "github.com/cilium/cilium/pkg/maps/ipmasq"
 	"github.com/cilium/cilium/pkg/maps/metricsmap"
 	natStats "github.com/cilium/cilium/pkg/maps/nat/stats"
 	"github.com/cilium/cilium/pkg/maps/ratelimitmap"
@@ -247,6 +249,12 @@ var (
 		// ServiceCache holds the list of known services correlated with the matching endpoints.
 		k8s.ServiceCacheCell,
 
+		// Provides the BPF ip-masq-agent maps
+		ipmasqmaps.Cell,
+
+		// Provides the BPF ip-masq-agent implementation, which is responsible for managing IP masquerading rules
+		ipmasq.Cell,
+
 		// Provides PolicyRepository (List of policy rules)
 		policy.Cell,
 

daemon/cmd/daemon_main.go

@@ -65,7 +65,6 @@ import (
 	"github.com/cilium/cilium/pkg/ipam"
 	ipamOption "github.com/cilium/cilium/pkg/ipam/option"
 	"github.com/cilium/cilium/pkg/ipcache"
-	"github.com/cilium/cilium/pkg/ipmasq"
 	"github.com/cilium/cilium/pkg/k8s"
 	k8sClient "github.com/cilium/cilium/pkg/k8s/client"
 	k8sSynced "github.com/cilium/cilium/pkg/k8s/synced"
@@ -681,18 +680,12 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
 	flags.Bool(option.EnableMasqueradeRouteSource, false, "Masquerade packets to the source IP provided from the routing layer rather than interface address")
 	option.BindEnv(vp, option.EnableMasqueradeRouteSource)
 
-	flags.Bool(option.EnableIPMasqAgent, false, "Enable BPF ip-masq-agent")
-	option.BindEnv(vp, option.EnableIPMasqAgent)
-
 	flags.Bool(option.EnableIPv4EgressGateway, false, "Enable egress gateway for IPv4")
 	option.BindEnv(vp, option.EnableIPv4EgressGateway)
 
 	flags.Bool(option.EnableEnvoyConfig, false, "Enable Envoy Config CRDs")
 	option.BindEnv(vp, option.EnableEnvoyConfig)
 
-	flags.String(option.IPMasqAgentConfigPath, "/etc/config/ip-masq-agent", "ip-masq-agent configuration file path")
-	option.BindEnv(vp, option.IPMasqAgentConfigPath)
-
 	flags.Bool(option.InstallIptRules, true, "Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading)")
 	flags.MarkHidden(option.InstallIptRules)
 	option.BindEnv(vp, option.InstallIptRules)
@@ -1804,14 +1797,6 @@ func startDaemon(d *Daemon, restoredEndpoints *endpointRestoreState, cleaner *da
 		}
 	}
 
-	if option.Config.EnableIPMasqAgent {
-		ipmasqAgent, err := ipmasq.NewIPMasqAgent(option.Config.IPMasqAgentConfigPath)
-		if err != nil {
-			return fmt.Errorf("failed to create ipmasq agent: %w", err)
-		}
-		ipmasqAgent.Start()
-	}
-
 	go func() {
 		if d.endpointRestoreComplete != nil {
 			select {

daemon/cmd/datapath.go

@@ -26,7 +26,6 @@ import (
 	"github.com/cilium/cilium/pkg/maps/encrypt"
 	"github.com/cilium/cilium/pkg/maps/fragmap"
 	ipcachemap "github.com/cilium/cilium/pkg/maps/ipcache"
-	"github.com/cilium/cilium/pkg/maps/ipmasq"
 	"github.com/cilium/cilium/pkg/maps/lbmap"
 	"github.com/cilium/cilium/pkg/maps/lxcmap"
 	"github.com/cilium/cilium/pkg/maps/metricsmap"
@@ -241,19 +240,6 @@ func (d *Daemon) initMaps() error {
 		}
 	}
 
-	if option.Config.EnableIPMasqAgent {
-		if option.Config.EnableIPv4Masquerade {
-			if err := ipmasq.IPMasq4Map().OpenOrCreate(); err != nil {
-				return fmt.Errorf("initializing IPv4 masquerading map: %w", err)
-			}
-		}
-		if option.Config.EnableIPv6Masquerade {
-			if err := ipmasq.IPMasq6Map().OpenOrCreate(); err != nil {
-				return fmt.Errorf("initializing IPv6 masquerading map: %w", err)
-			}
-		}
-	}
-
 	if option.Config.EnableIPSec {
 		if err := encrypt.MapCreate(); err != nil {
 			return fmt.Errorf("initializing IPsec map: %w", err)

pkg/ipam/allocator_test.go

@@ -59,7 +59,7 @@ var mtuMock = fakeMTU{}
 func TestAllocatedIPDump(t *testing.T) {
 	fakeAddressing := fakeTypes.NewNodeAddressing()
 	localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
-	ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil)
+	ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil)
 	ipam.ConfigureAllocator()
 
 	allocv4, allocv6, status := ipam.Dump()
@@ -80,7 +80,7 @@ func TestExpirationTimer(t *testing.T) {
 
 	fakeAddressing := fakeTypes.NewNodeAddressing()
 	localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
-	ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil)
+	ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil)
 	ipam.ConfigureAllocator()
 
 	err := ipam.AllocateIP(ip, "foo", PoolDefault())
@@ -148,7 +148,7 @@ func TestAllocateNextWithExpiration(t *testing.T) {
 	fakeAddressing := fakeTypes.NewNodeAddressing()
 	localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
 	fakeMetadata := fakeMetadataFunc(func(owner string, family Family) (pool string, err error) { return "some-pool", nil })
-	ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil)
+	ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil, nil)
 	ipam.ConfigureAllocator()
 
 	// Allocate IPs and test expiration timer. 'pool' is empty in order to test

pkg/ipam/cell/cell.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cilium/cilium/pkg/ipam"
 	ipamapi "github.com/cilium/cilium/pkg/ipam/api"
 	ipamMetadata "github.com/cilium/cilium/pkg/ipam/metadata"
+	"github.com/cilium/cilium/pkg/ipmasq"
 	k8sClient "github.com/cilium/cilium/pkg/k8s/client"
 	"github.com/cilium/cilium/pkg/k8s/watchers"
 	"github.com/cilium/cilium/pkg/mtu"
@@ -49,10 +50,11 @@ type ipamParams struct {
 	IPAMMetadataManager ipamMetadata.Manager
 	NodeDiscovery       *nodediscovery.NodeDiscovery
 	Sysctl              sysctl.Sysctl
+	IPMasqAgent         *ipmasq.IPMasqAgent
 }
 
 func newIPAddressManager(params ipamParams) *ipam.IPAM {
-	return ipam.NewIPAM(params.NodeAddressing, params.AgentConfig, params.NodeDiscovery, params.LocalNodeStore, params.K8sEventReporter, params.NodeResource, params.MTU, params.Clientset, params.IPAMMetadataManager, params.Sysctl)
+	return ipam.NewIPAM(params.NodeAddressing, params.AgentConfig, params.NodeDiscovery, params.LocalNodeStore, params.K8sEventReporter, params.NodeResource, params.MTU, params.Clientset, params.IPAMMetadataManager, params.Sysctl, params.IPMasqAgent)
 }
 
 type ipamAPIHandlerParams struct {

pkg/ipam/crd.go

@@ -27,6 +27,7 @@ import (
 	"github.com/cilium/cilium/pkg/ip"
 	ipamOption "github.com/cilium/cilium/pkg/ipam/option"
 	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
+	"github.com/cilium/cilium/pkg/ipmasq"
 	ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	"github.com/cilium/cilium/pkg/k8s/client"
 	"github.com/cilium/cilium/pkg/k8s/informer"
@@ -692,6 +693,8 @@ type crdAllocator struct {
 	family Family
 
 	conf *option.DaemonConfig
+
+	ipMasqAgent *ipmasq.IPMasqAgent
 }
 
 // newCRDAllocator creates a new CRD-backed IP allocator
@@ -704,16 +707,18 @@ func newCRDAllocator(
 	k8sEventReg K8sEventRegister,
 	mtuConfig MtuConfiguration,
 	sysctl sysctl.Sysctl,
+	ipMasqAgent *ipmasq.IPMasqAgent,
 ) Allocator {
 	initNodeStore.Do(func() {
 		sharedNodeStore = newNodeStore(nodeTypes.GetName(), c, owner, localNodeStore, clientset, k8sEventReg, mtuConfig, sysctl)
 	})
 
 	allocator := &crdAllocator{
-		allocated: ipamTypes.AllocationMap{},
-		family:    family,
-		store:     sharedNodeStore,
-		conf:      c,
+		allocated:   ipamTypes.AllocationMap{},
+		family:      family,
+		store:       sharedNodeStore,
+		conf:        c,
+		ipMasqAgent: ipMasqAgent,
 	}
 
 	sharedNodeStore.addAllocator(allocator)
@@ -759,6 +764,19 @@ func (a *crdAllocator) buildAllocationResult(ip net.IP, ipInfo *ipamTypes.Alloca
 				if a.conf.IPv4NativeRoutingCIDR != nil {
 					result.CIDRs = append(result.CIDRs, a.conf.IPv4NativeRoutingCIDR.String())
 				}
+				// If the ip-masq-agent is enabled, get the CIDRs that are not masqueraded.
+				// Note that the resulting ip rules will not be dynamically regenerated if the
+				// ip-masq-agent configuration changes.
+				if a.conf.EnableIPMasqAgent {
+					nonMasqCidrs := a.ipMasqAgent.NonMasqCIDRsFromConfig()
+					for _, prefix := range nonMasqCidrs {
+						if ip.To4() != nil && prefix.Addr().Is4() {
+							result.CIDRs = append(result.CIDRs, prefix.String())
+						} else if ip.To4() == nil && prefix.Addr().Is6() {
+							result.CIDRs = append(result.CIDRs, prefix.String())
+						}
+					}
+				}
 				if eni.Subnet.CIDR != "" {
 					// The gateway for a subnet and VPC is always x.x.x.1
 					// Ref: https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Route_Tables.html

pkg/ipam/crd_test.go

@@ -18,6 +18,7 @@ import (
 	fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types"
 	ipamOption "github.com/cilium/cilium/pkg/ipam/option"
 	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
+	"github.com/cilium/cilium/pkg/ipmasq"
 	ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	"github.com/cilium/cilium/pkg/node"
 	"github.com/cilium/cilium/pkg/option"
@@ -95,12 +96,12 @@ func TestMarkForReleaseNoAllocate(t *testing.T) {
 
 	fakeAddressing := fakeTypes.NewNodeAddressing()
 	conf := testConfigurationCRD
-	initNodeStore.Do(func() {
-		sharedNodeStore = newFakeNodeStore(conf, t)
-		sharedNodeStore.ownNode = cn
-	})
+	initNodeStore.Do(func() {}) // Ensure the real initNodeStore is not called
+	sharedNodeStore = newFakeNodeStore(conf, t)
+	sharedNodeStore.ownNode = cn
+
 	localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
-	ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil)
+	ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil)
 	ipam.ConfigureAllocator()
 	sharedNodeStore.updateLocalNodeResource(cn)
 
@@ -127,6 +128,79 @@ func TestMarkForReleaseNoAllocate(t *testing.T) {
 	require.Equal(t, ipamOption.IPAMDoNotRelease, string(cn.Status.IPAM.ReleaseIPs["1.1.1.3"]))
 }
 
+type ipMasqMapDummy struct{}
+
+func (m ipMasqMapDummy) Update(netip.Prefix) error     { return nil }
+func (m ipMasqMapDummy) Delete(netip.Prefix) error     { return nil }
+func (m ipMasqMapDummy) Dump() ([]netip.Prefix, error) { return []netip.Prefix{}, nil }
+
+func TestIPMasq(t *testing.T) {
+	cn := newCiliumNode("node1", 4, 4, 0)
+	dummyResource := ipamTypes.AllocationIP{Resource: "eni-1"}
+	cn.Spec.IPAM.Pool["10.1.1.226"] = dummyResource
+	cn.Status.ENI.ENIs = map[string]eniTypes.ENI{
+		"eni-1": {
+			ID: "eni-1",
+			Addresses: []string{
+				"10.1.1.226",
+				"10.1.1.229",
+			},
+			VPC: eniTypes.AwsVPC{
+				ID:          "vpc-1",
+				PrimaryCIDR: "10.1.0.0/16",
+				CIDRs: []string{
+					"10.2.0.0/16",
+				},
+			},
+		},
+	}
+
+	fakeAddressing := fakeTypes.NewNodeAddressing()
+	conf := testConfigurationCRD
+	conf.IPAM = ipamOption.IPAMENI
+	conf.EnableIPMasqAgent = true
+	ipMasqAgent := ipmasq.NewIPMasqAgent("", ipMasqMapDummy{})
+	err := ipMasqAgent.Start()
+	require.NoError(t, err)
+
+	initNodeStore.Do(func() {}) // Ensure the real initNodeStore is not called
+	sharedNodeStore = newFakeNodeStore(conf, t)
+	sharedNodeStore.ownNode = cn
+
+	localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
+	ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, ipMasqAgent)
+	ipam.ConfigureAllocator()
+
+	epipv4 := netip.MustParseAddr("10.1.1.226")
+	result, err := ipam.IPv4Allocator.Allocate(epipv4.AsSlice(), "test1", PoolDefault())
+	require.NoError(t, err)
+	// The resulting CIDRs should contain the VPC CIDRs and the default ip-masq-agent CIDRs from pkg/ipmasq/ipmasq.go
+	require.ElementsMatch(
+		t,
+		[]string{
+			// VPC CIDRs
+			"10.1.0.0/16",
+			"10.2.0.0/16",
+			// Default ip-masq-agent CIDRs
+			"10.0.0.0/8",
+			"172.16.0.0/12",
+			"192.168.0.0/16",
+			"100.64.0.0/10",
+			"192.0.0.0/24",
+			"192.0.2.0/24",
+			"192.88.99.0/24",
+			"198.18.0.0/15",
+			"198.51.100.0/24",
+			"203.0.113.0/24",
+			"240.0.0.0/4",
+			"169.254.0.0/16",
+		},
+		result.CIDRs,
+	)
+
+	ipMasqAgent.Stop()
+}
+
 func Test_validateENIConfig(t *testing.T) {
 	type args struct {
 		node *ciliumv2.CiliumNode

pkg/ipam/ipam.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cilium/cilium/pkg/datapath/linux/sysctl"
 	"github.com/cilium/cilium/pkg/datapath/types"
 	ipamOption "github.com/cilium/cilium/pkg/ipam/option"
+	"github.com/cilium/cilium/pkg/ipmasq"
 	"github.com/cilium/cilium/pkg/k8s/client"
 	"github.com/cilium/cilium/pkg/logging"
 	"github.com/cilium/cilium/pkg/logging/logfields"
@@ -79,6 +80,7 @@ func NewIPAM(
 	clientset client.Clientset,
 	metadata Metadata,
 	sysctl sysctl.Sysctl,
+	ipMasqAgent *ipmasq.IPMasqAgent,
 ) *IPAM {
 	return &IPAM{
 		nodeAddressing:   nodeAddressing,
@@ -95,6 +97,7 @@ func NewIPAM(
 		nodeDiscovery:  nodeDiscovery,
 		metadata:       metadata,
 		sysctl:         sysctl,
+		ipMasqAgent:    ipMasqAgent,
 	}
 }
 
@@ -129,11 +132,11 @@ func (ipam *IPAM) ConfigureAllocator() {
 	case ipamOption.IPAMCRD, ipamOption.IPAMENI, ipamOption.IPAMAzure, ipamOption.IPAMAlibabaCloud:
 		log.Info("Initializing CRD-based IPAM")
 		if ipam.config.IPv6Enabled() {
-			ipam.IPv6Allocator = newCRDAllocator(IPv6, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl)
+			ipam.IPv6Allocator = newCRDAllocator(IPv6, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl, ipam.ipMasqAgent)
 		}
 
 		if ipam.config.IPv4Enabled() {
-			ipam.IPv4Allocator = newCRDAllocator(IPv4, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl)
+			ipam.IPv4Allocator = newCRDAllocator(IPv4, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl, ipam.ipMasqAgent)
 		}
 	case ipamOption.IPAMDelegatedPlugin:
 		log.Info("Initializing no-op IPAM since we're using a CNI delegated plugin")