Skip to content

Commit 4a08974

Browse files
antonippantonipp
authored
Merge pull request #616 from DataDog/ai/backport-40141
[1.17] Backport ip-masq-agent changes
2 parents 11ad9a1 + d4c3738 commit 4a08974

19 files changed

Lines changed: 437 additions & 97 deletions

File tree

‎Documentation/cmdref/cilium-agent_hive.md‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Documentation/cmdref/cilium-agent_hive_dot-graph.md‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎daemon/cmd/cells.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import (
4141
"github.com/cilium/cilium/pkg/identity/identitymanager"
4242
ipamcell "github.com/cilium/cilium/pkg/ipam/cell"
4343
ipcache "github.com/cilium/cilium/pkg/ipcache/cell"
44+
ipmasq "github.com/cilium/cilium/pkg/ipmasq/cell"
4445
"github.com/cilium/cilium/pkg/k8s"
4546
k8sClient "github.com/cilium/cilium/pkg/k8s/client"
4647
k8sSynced "github.com/cilium/cilium/pkg/k8s/synced"
@@ -50,6 +51,7 @@ import (
5051
loadbalancer_experimental "github.com/cilium/cilium/pkg/loadbalancer/experimental"
5152
"github.com/cilium/cilium/pkg/logging/logfields"
5253
"github.com/cilium/cilium/pkg/maglev"
54+
ipmasqmaps "github.com/cilium/cilium/pkg/maps/ipmasq"
5355
"github.com/cilium/cilium/pkg/maps/metricsmap"
5456
natStats "github.com/cilium/cilium/pkg/maps/nat/stats"
5557
"github.com/cilium/cilium/pkg/maps/ratelimitmap"
@@ -247,6 +249,12 @@ var (
247249
// ServiceCache holds the list of known services correlated with the matching endpoints.
248250
k8s.ServiceCacheCell,
249251

252+
// Provides the BPF ip-masq-agent maps
253+
ipmasqmaps.Cell,
254+
255+
// Provides the BPF ip-masq-agent implementation, which is responsible for managing IP masquerading rules
256+
ipmasq.Cell,
257+
250258
// Provides PolicyRepository (List of policy rules)
251259
policy.Cell,
252260

‎daemon/cmd/daemon_main.go‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@ import (
6565
"github.com/cilium/cilium/pkg/ipam"
6666
ipamOption "github.com/cilium/cilium/pkg/ipam/option"
6767
"github.com/cilium/cilium/pkg/ipcache"
68-
"github.com/cilium/cilium/pkg/ipmasq"
6968
"github.com/cilium/cilium/pkg/k8s"
7069
k8sClient "github.com/cilium/cilium/pkg/k8s/client"
7170
k8sSynced "github.com/cilium/cilium/pkg/k8s/synced"
@@ -681,18 +680,12 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
681680
flags.Bool(option.EnableMasqueradeRouteSource, false, "Masquerade packets to the source IP provided from the routing layer rather than interface address")
682681
option.BindEnv(vp, option.EnableMasqueradeRouteSource)
683682

684-
flags.Bool(option.EnableIPMasqAgent, false, "Enable BPF ip-masq-agent")
685-
option.BindEnv(vp, option.EnableIPMasqAgent)
686-
687683
flags.Bool(option.EnableIPv4EgressGateway, false, "Enable egress gateway for IPv4")
688684
option.BindEnv(vp, option.EnableIPv4EgressGateway)
689685

690686
flags.Bool(option.EnableEnvoyConfig, false, "Enable Envoy Config CRDs")
691687
option.BindEnv(vp, option.EnableEnvoyConfig)
692688

693-
flags.String(option.IPMasqAgentConfigPath, "/etc/config/ip-masq-agent", "ip-masq-agent configuration file path")
694-
option.BindEnv(vp, option.IPMasqAgentConfigPath)
695-
696689
flags.Bool(option.InstallIptRules, true, "Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading)")
697690
flags.MarkHidden(option.InstallIptRules)
698691
option.BindEnv(vp, option.InstallIptRules)
@@ -1804,14 +1797,6 @@ func startDaemon(d *Daemon, restoredEndpoints *endpointRestoreState, cleaner *da
18041797
}
18051798
}
18061799

1807-
if option.Config.EnableIPMasqAgent {
1808-
ipmasqAgent, err := ipmasq.NewIPMasqAgent(option.Config.IPMasqAgentConfigPath)
1809-
if err != nil {
1810-
return fmt.Errorf("failed to create ipmasq agent: %w", err)
1811-
}
1812-
ipmasqAgent.Start()
1813-
}
1814-
18151800
go func() {
18161801
if d.endpointRestoreComplete != nil {
18171802
select {

‎daemon/cmd/datapath.go‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"github.com/cilium/cilium/pkg/maps/encrypt"
2727
"github.com/cilium/cilium/pkg/maps/fragmap"
2828
ipcachemap "github.com/cilium/cilium/pkg/maps/ipcache"
29-
"github.com/cilium/cilium/pkg/maps/ipmasq"
3029
"github.com/cilium/cilium/pkg/maps/lbmap"
3130
"github.com/cilium/cilium/pkg/maps/lxcmap"
3231
"github.com/cilium/cilium/pkg/maps/metricsmap"
@@ -241,19 +240,6 @@ func (d *Daemon) initMaps() error {
241240
}
242241
}
243242

244-
if option.Config.EnableIPMasqAgent {
245-
if option.Config.EnableIPv4Masquerade {
246-
if err := ipmasq.IPMasq4Map().OpenOrCreate(); err != nil {
247-
return fmt.Errorf("initializing IPv4 masquerading map: %w", err)
248-
}
249-
}
250-
if option.Config.EnableIPv6Masquerade {
251-
if err := ipmasq.IPMasq6Map().OpenOrCreate(); err != nil {
252-
return fmt.Errorf("initializing IPv6 masquerading map: %w", err)
253-
}
254-
}
255-
}
256-
257243
if option.Config.EnableIPSec {
258244
if err := encrypt.MapCreate(); err != nil {
259245
return fmt.Errorf("initializing IPsec map: %w", err)

‎pkg/ipam/allocator_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ var mtuMock = fakeMTU{}
5959
func TestAllocatedIPDump(t *testing.T) {
6060
fakeAddressing := fakeTypes.NewNodeAddressing()
6161
localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
62-
ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil)
62+
ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil)
6363
ipam.ConfigureAllocator()
6464

6565
allocv4, allocv6, status := ipam.Dump()
@@ -80,7 +80,7 @@ func TestExpirationTimer(t *testing.T) {
8080

8181
fakeAddressing := fakeTypes.NewNodeAddressing()
8282
localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
83-
ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil)
83+
ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil)
8484
ipam.ConfigureAllocator()
8585

8686
err := ipam.AllocateIP(ip, "foo", PoolDefault())
@@ -148,7 +148,7 @@ func TestAllocateNextWithExpiration(t *testing.T) {
148148
fakeAddressing := fakeTypes.NewNodeAddressing()
149149
localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
150150
fakeMetadata := fakeMetadataFunc(func(owner string, family Family) (pool string, err error) { return "some-pool", nil })
151-
ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil)
151+
ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil, nil)
152152
ipam.ConfigureAllocator()
153153

154154
// Allocate IPs and test expiration timer. 'pool' is empty in order to test

‎pkg/ipam/cell/cell.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/cilium/cilium/pkg/ipam"
1616
ipamapi "github.com/cilium/cilium/pkg/ipam/api"
1717
ipamMetadata "github.com/cilium/cilium/pkg/ipam/metadata"
18+
"github.com/cilium/cilium/pkg/ipmasq"
1819
k8sClient "github.com/cilium/cilium/pkg/k8s/client"
1920
"github.com/cilium/cilium/pkg/k8s/watchers"
2021
"github.com/cilium/cilium/pkg/mtu"
@@ -49,10 +50,11 @@ type ipamParams struct {
4950
IPAMMetadataManager ipamMetadata.Manager
5051
NodeDiscovery *nodediscovery.NodeDiscovery
5152
Sysctl sysctl.Sysctl
53+
IPMasqAgent *ipmasq.IPMasqAgent
5254
}
5355

5456
func newIPAddressManager(params ipamParams) *ipam.IPAM {
55-
return ipam.NewIPAM(params.NodeAddressing, params.AgentConfig, params.NodeDiscovery, params.LocalNodeStore, params.K8sEventReporter, params.NodeResource, params.MTU, params.Clientset, params.IPAMMetadataManager, params.Sysctl)
57+
return ipam.NewIPAM(params.NodeAddressing, params.AgentConfig, params.NodeDiscovery, params.LocalNodeStore, params.K8sEventReporter, params.NodeResource, params.MTU, params.Clientset, params.IPAMMetadataManager, params.Sysctl, params.IPMasqAgent)
5658
}
5759

5860
type ipamAPIHandlerParams struct {

‎pkg/ipam/crd.go‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/cilium/cilium/pkg/ip"
2828
ipamOption "github.com/cilium/cilium/pkg/ipam/option"
2929
ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
30+
"github.com/cilium/cilium/pkg/ipmasq"
3031
ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
3132
"github.com/cilium/cilium/pkg/k8s/client"
3233
"github.com/cilium/cilium/pkg/k8s/informer"
@@ -692,6 +693,8 @@ type crdAllocator struct {
692693
family Family
693694

694695
conf *option.DaemonConfig
696+
697+
ipMasqAgent *ipmasq.IPMasqAgent
695698
}
696699

697700
// newCRDAllocator creates a new CRD-backed IP allocator
@@ -704,16 +707,18 @@ func newCRDAllocator(
704707
k8sEventReg K8sEventRegister,
705708
mtuConfig MtuConfiguration,
706709
sysctl sysctl.Sysctl,
710+
ipMasqAgent *ipmasq.IPMasqAgent,
707711
) Allocator {
708712
initNodeStore.Do(func() {
709713
sharedNodeStore = newNodeStore(nodeTypes.GetName(), c, owner, localNodeStore, clientset, k8sEventReg, mtuConfig, sysctl)
710714
})
711715

712716
allocator := &crdAllocator{
713-
allocated: ipamTypes.AllocationMap{},
714-
family: family,
715-
store: sharedNodeStore,
716-
conf: c,
717+
allocated: ipamTypes.AllocationMap{},
718+
family: family,
719+
store: sharedNodeStore,
720+
conf: c,
721+
ipMasqAgent: ipMasqAgent,
717722
}
718723

719724
sharedNodeStore.addAllocator(allocator)
@@ -759,6 +764,19 @@ func (a *crdAllocator) buildAllocationResult(ip net.IP, ipInfo *ipamTypes.Alloca
759764
if a.conf.IPv4NativeRoutingCIDR != nil {
760765
result.CIDRs = append(result.CIDRs, a.conf.IPv4NativeRoutingCIDR.String())
761766
}
767+
// If the ip-masq-agent is enabled, get the CIDRs that are not masqueraded.
768+
// Note that the resulting ip rules will not be dynamically regenerated if the
769+
// ip-masq-agent configuration changes.
770+
if a.conf.EnableIPMasqAgent {
771+
nonMasqCidrs := a.ipMasqAgent.NonMasqCIDRsFromConfig()
772+
for _, prefix := range nonMasqCidrs {
773+
if ip.To4() != nil && prefix.Addr().Is4() {
774+
result.CIDRs = append(result.CIDRs, prefix.String())
775+
} else if ip.To4() == nil && prefix.Addr().Is6() {
776+
result.CIDRs = append(result.CIDRs, prefix.String())
777+
}
778+
}
779+
}
762780
if eni.Subnet.CIDR != "" {
763781
// The gateway for a subnet and VPC is always x.x.x.1
764782
// Ref: https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Route_Tables.html

‎pkg/ipam/crd_test.go‎

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types"
1919
ipamOption "github.com/cilium/cilium/pkg/ipam/option"
2020
ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
21+
"github.com/cilium/cilium/pkg/ipmasq"
2122
ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
2223
"github.com/cilium/cilium/pkg/node"
2324
"github.com/cilium/cilium/pkg/option"
@@ -95,12 +96,12 @@ func TestMarkForReleaseNoAllocate(t *testing.T) {
9596

9697
fakeAddressing := fakeTypes.NewNodeAddressing()
9798
conf := testConfigurationCRD
98-
initNodeStore.Do(func() {
99-
sharedNodeStore = newFakeNodeStore(conf, t)
100-
sharedNodeStore.ownNode = cn
101-
})
99+
initNodeStore.Do(func() {}) // Ensure the real initNodeStore is not called
100+
sharedNodeStore = newFakeNodeStore(conf, t)
101+
sharedNodeStore.ownNode = cn
102+
102103
localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
103-
ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil)
104+
ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil)
104105
ipam.ConfigureAllocator()
105106
sharedNodeStore.updateLocalNodeResource(cn)
106107

@@ -127,6 +128,79 @@ func TestMarkForReleaseNoAllocate(t *testing.T) {
127128
require.Equal(t, ipamOption.IPAMDoNotRelease, string(cn.Status.IPAM.ReleaseIPs["1.1.1.3"]))
128129
}
129130

131+
type ipMasqMapDummy struct{}
132+
133+
func (m ipMasqMapDummy) Update(netip.Prefix) error { return nil }
134+
func (m ipMasqMapDummy) Delete(netip.Prefix) error { return nil }
135+
func (m ipMasqMapDummy) Dump() ([]netip.Prefix, error) { return []netip.Prefix{}, nil }
136+
137+
func TestIPMasq(t *testing.T) {
138+
cn := newCiliumNode("node1", 4, 4, 0)
139+
dummyResource := ipamTypes.AllocationIP{Resource: "eni-1"}
140+
cn.Spec.IPAM.Pool["10.1.1.226"] = dummyResource
141+
cn.Status.ENI.ENIs = map[string]eniTypes.ENI{
142+
"eni-1": {
143+
ID: "eni-1",
144+
Addresses: []string{
145+
"10.1.1.226",
146+
"10.1.1.229",
147+
},
148+
VPC: eniTypes.AwsVPC{
149+
ID: "vpc-1",
150+
PrimaryCIDR: "10.1.0.0/16",
151+
CIDRs: []string{
152+
"10.2.0.0/16",
153+
},
154+
},
155+
},
156+
}
157+
158+
fakeAddressing := fakeTypes.NewNodeAddressing()
159+
conf := testConfigurationCRD
160+
conf.IPAM = ipamOption.IPAMENI
161+
conf.EnableIPMasqAgent = true
162+
ipMasqAgent := ipmasq.NewIPMasqAgent("", ipMasqMapDummy{})
163+
err := ipMasqAgent.Start()
164+
require.NoError(t, err)
165+
166+
initNodeStore.Do(func() {}) // Ensure the real initNodeStore is not called
167+
sharedNodeStore = newFakeNodeStore(conf, t)
168+
sharedNodeStore.ownNode = cn
169+
170+
localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{})
171+
ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, ipMasqAgent)
172+
ipam.ConfigureAllocator()
173+
174+
epipv4 := netip.MustParseAddr("10.1.1.226")
175+
result, err := ipam.IPv4Allocator.Allocate(epipv4.AsSlice(), "test1", PoolDefault())
176+
require.NoError(t, err)
177+
// The resulting CIDRs should contain the VPC CIDRs and the default ip-masq-agent CIDRs from pkg/ipmasq/ipmasq.go
178+
require.ElementsMatch(
179+
t,
180+
[]string{
181+
// VPC CIDRs
182+
"10.1.0.0/16",
183+
"10.2.0.0/16",
184+
// Default ip-masq-agent CIDRs
185+
"10.0.0.0/8",
186+
"172.16.0.0/12",
187+
"192.168.0.0/16",
188+
"100.64.0.0/10",
189+
"192.0.0.0/24",
190+
"192.0.2.0/24",
191+
"192.88.99.0/24",
192+
"198.18.0.0/15",
193+
"198.51.100.0/24",
194+
"203.0.113.0/24",
195+
"240.0.0.0/4",
196+
"169.254.0.0/16",
197+
},
198+
result.CIDRs,
199+
)
200+
201+
ipMasqAgent.Stop()
202+
}
203+
130204
func Test_validateENIConfig(t *testing.T) {
131205
type args struct {
132206
node *ciliumv2.CiliumNode

‎pkg/ipam/ipam.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/cilium/cilium/pkg/datapath/linux/sysctl"
1313
"github.com/cilium/cilium/pkg/datapath/types"
1414
ipamOption "github.com/cilium/cilium/pkg/ipam/option"
15+
"github.com/cilium/cilium/pkg/ipmasq"
1516
"github.com/cilium/cilium/pkg/k8s/client"
1617
"github.com/cilium/cilium/pkg/logging"
1718
"github.com/cilium/cilium/pkg/logging/logfields"
@@ -79,6 +80,7 @@ func NewIPAM(
7980
clientset client.Clientset,
8081
metadata Metadata,
8182
sysctl sysctl.Sysctl,
83+
ipMasqAgent *ipmasq.IPMasqAgent,
8284
) *IPAM {
8385
return &IPAM{
8486
nodeAddressing: nodeAddressing,
@@ -95,6 +97,7 @@ func NewIPAM(
9597
nodeDiscovery: nodeDiscovery,
9698
metadata: metadata,
9799
sysctl: sysctl,
100+
ipMasqAgent: ipMasqAgent,
98101
}
99102
}
100103

@@ -129,11 +132,11 @@ func (ipam *IPAM) ConfigureAllocator() {
129132
case ipamOption.IPAMCRD, ipamOption.IPAMENI, ipamOption.IPAMAzure, ipamOption.IPAMAlibabaCloud:
130133
log.Info("Initializing CRD-based IPAM")
131134
if ipam.config.IPv6Enabled() {
132-
ipam.IPv6Allocator = newCRDAllocator(IPv6, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl)
135+
ipam.IPv6Allocator = newCRDAllocator(IPv6, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl, ipam.ipMasqAgent)
133136
}
134137

135138
if ipam.config.IPv4Enabled() {
136-
ipam.IPv4Allocator = newCRDAllocator(IPv4, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl)
139+
ipam.IPv4Allocator = newCRDAllocator(IPv4, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl, ipam.ipMasqAgent)
137140
}
138141
case ipamOption.IPAMDelegatedPlugin:
139142
log.Info("Initializing no-op IPAM since we're using a CNI delegated plugin")

0 commit comments

Comments
 (0)