From b0003140b8befbdf1d07f976d2611a944cb4c8b7 Mon Sep 17 00:00:00 2001 From: Anton Ippolitov Date: Mon, 23 Jun 2025 14:35:31 +0200 Subject: [PATCH 1/2] ip-masq-agent: refactor into a Hive Cell Signed-off-by: Anton Ippolitov --- Documentation/cmdref/cilium-agent_hive.md | 2 + .../cmdref/cilium-agent_hive_dot-graph.md | 2 + daemon/cmd/cells.go | 8 ++ daemon/cmd/daemon_main.go | 17 +--- daemon/cmd/datapath.go | 14 --- pkg/ipmasq/cell/cell.go | 56 +++++++++++ pkg/ipmasq/cell/cell_test.go | 96 +++++++++++++++++++ pkg/ipmasq/cell/config.go | 25 +++++ pkg/ipmasq/ipmasq.go | 41 ++++---- pkg/ipmasq/ipmasq_test.go | 53 +++++----- pkg/maps/ipmasq/cell.go | 56 +++++++++++ pkg/option/config.go | 2 - 12 files changed, 294 insertions(+), 78 deletions(-) create mode 100644 pkg/ipmasq/cell/cell.go create mode 100644 pkg/ipmasq/cell/cell_test.go create mode 100644 pkg/ipmasq/cell/config.go create mode 100644 pkg/maps/ipmasq/cell.go diff --git a/Documentation/cmdref/cilium-agent_hive.md b/Documentation/cmdref/cilium-agent_hive.md index 29254dade7983..628dbf3cab758 100644 --- a/Documentation/cmdref/cilium-agent_hive.md +++ b/Documentation/cmdref/cilium-agent_hive.md @@ -47,6 +47,7 @@ cilium-agent hive [flags] --enable-hubble-open-metrics Enable exporting hubble metrics in OpenMetrics format --enable-hubble-recorder-api Enable the Hubble recorder API (default true) --enable-ingress-controller Enables Envoy secret sync for Ingress controller related TLS secrets + --enable-ip-masq-agent Enable BPF ip-masq-agent --enable-ipv4-big-tcp Enable IPv4 BIG TCP option which increases device's maximum GRO/GSO limits for IPv4 --enable-ipv6-big-tcp Enable IPv6 BIG TCP option which increases device's maximum GRO/GSO limits for IPv6 --enable-k8s Enable the k8s clientset (default true) @@ -118,6 +119,7 @@ cilium-agent hive [flags] --hubble-tls-key-file string Path to the private key file for the Hubble server. The file must contain PEM encoded data. --ignore-flags-drift-checker strings Ignores specified flags during drift checking --ingress-secrets-namespace string IngressSecretsNamespace is the namespace having tls secrets used by CEC, originating from Ingress controller + --ip-masq-agent-config-path string ip-masq-agent configuration file path (default "/etc/config/ip-masq-agent") --iptables-lock-timeout duration Time to pass to each iptables invocation to wait for xtables lock acquisition (default 5s) --iptables-random-fully Set iptables flag random-fully on masquerading rules --k8s-api-server string Kubernetes API server URL diff --git a/Documentation/cmdref/cilium-agent_hive_dot-graph.md b/Documentation/cmdref/cilium-agent_hive_dot-graph.md index 3eef124e73e2a..384a18406840d 100644 --- a/Documentation/cmdref/cilium-agent_hive_dot-graph.md +++ b/Documentation/cmdref/cilium-agent_hive_dot-graph.md @@ -53,6 +53,7 @@ cilium-agent hive dot-graph [flags] --enable-hubble-open-metrics Enable exporting hubble metrics in OpenMetrics format --enable-hubble-recorder-api Enable the Hubble recorder API (default true) --enable-ingress-controller Enables Envoy secret sync for Ingress controller related TLS secrets + --enable-ip-masq-agent Enable BPF ip-masq-agent --enable-ipv4-big-tcp Enable IPv4 BIG TCP option which increases device's maximum GRO/GSO limits for IPv4 --enable-ipv6-big-tcp Enable IPv6 BIG TCP option which increases device's maximum GRO/GSO limits for IPv6 --enable-k8s Enable the k8s clientset (default true) @@ -123,6 +124,7 @@ cilium-agent hive dot-graph [flags] --hubble-tls-key-file string Path to the private key file for the Hubble server. The file must contain PEM encoded data. --ignore-flags-drift-checker strings Ignores specified flags during drift checking --ingress-secrets-namespace string IngressSecretsNamespace is the namespace having tls secrets used by CEC, originating from Ingress controller + --ip-masq-agent-config-path string ip-masq-agent configuration file path (default "/etc/config/ip-masq-agent") --iptables-lock-timeout duration Time to pass to each iptables invocation to wait for xtables lock acquisition (default 5s) --iptables-random-fully Set iptables flag random-fully on masquerading rules --k8s-api-server string Kubernetes API server URL diff --git a/daemon/cmd/cells.go b/daemon/cmd/cells.go index 98de7f7e9d5dc..d4038a491f543 100644 --- a/daemon/cmd/cells.go +++ b/daemon/cmd/cells.go @@ -41,6 +41,7 @@ import ( "github.com/cilium/cilium/pkg/identity/identitymanager" ipamcell "github.com/cilium/cilium/pkg/ipam/cell" ipcache "github.com/cilium/cilium/pkg/ipcache/cell" + ipmasq "github.com/cilium/cilium/pkg/ipmasq/cell" "github.com/cilium/cilium/pkg/k8s" k8sClient "github.com/cilium/cilium/pkg/k8s/client" k8sSynced "github.com/cilium/cilium/pkg/k8s/synced" @@ -50,6 +51,7 @@ import ( loadbalancer_experimental "github.com/cilium/cilium/pkg/loadbalancer/experimental" "github.com/cilium/cilium/pkg/logging/logfields" "github.com/cilium/cilium/pkg/maglev" + ipmasqmaps "github.com/cilium/cilium/pkg/maps/ipmasq" "github.com/cilium/cilium/pkg/maps/metricsmap" natStats "github.com/cilium/cilium/pkg/maps/nat/stats" "github.com/cilium/cilium/pkg/maps/ratelimitmap" @@ -247,6 +249,12 @@ var ( // ServiceCache holds the list of known services correlated with the matching endpoints. k8s.ServiceCacheCell, + // Provides the BPF ip-masq-agent maps + ipmasqmaps.Cell, + + // Provides the BPF ip-masq-agent implementation, which is responsible for managing IP masquerading rules + ipmasq.Cell, + // Provides PolicyRepository (List of policy rules) policy.Cell, diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go index 3e7dc7a2c7178..db224e0867a4d 100644 --- a/daemon/cmd/daemon_main.go +++ b/daemon/cmd/daemon_main.go @@ -681,18 +681,12 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) { flags.Bool(option.EnableMasqueradeRouteSource, false, "Masquerade packets to the source IP provided from the routing layer rather than interface address") option.BindEnv(vp, option.EnableMasqueradeRouteSource) - flags.Bool(option.EnableIPMasqAgent, false, "Enable BPF ip-masq-agent") - option.BindEnv(vp, option.EnableIPMasqAgent) - flags.Bool(option.EnableIPv4EgressGateway, false, "Enable egress gateway for IPv4") option.BindEnv(vp, option.EnableIPv4EgressGateway) flags.Bool(option.EnableEnvoyConfig, false, "Enable Envoy Config CRDs") option.BindEnv(vp, option.EnableEnvoyConfig) - flags.String(option.IPMasqAgentConfigPath, "/etc/config/ip-masq-agent", "ip-masq-agent configuration file path") - option.BindEnv(vp, option.IPMasqAgentConfigPath) - flags.Bool(option.InstallIptRules, true, "Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading)") flags.MarkHidden(option.InstallIptRules) option.BindEnv(vp, option.InstallIptRules) @@ -1631,6 +1625,7 @@ type daemonParams struct { Hubble hubblecell.HubbleIntegration LRPManager *redirectpolicy.Manager MaglevConfig maglev.Config + IPMasqAgent *ipmasq.IPMasqAgent } func newDaemonPromise(params daemonParams) promise.Promise[*Daemon] { @@ -1803,15 +1798,7 @@ func startDaemon(d *Daemon, restoredEndpoints *endpointRestoreState, cleaner *da } } } - - if option.Config.EnableIPMasqAgent { - ipmasqAgent, err := ipmasq.NewIPMasqAgent(option.Config.IPMasqAgentConfigPath) - if err != nil { - return fmt.Errorf("failed to create ipmasq agent: %w", err) - } - ipmasqAgent.Start() - } - + go func() { if d.endpointRestoreComplete != nil { select { diff --git a/daemon/cmd/datapath.go b/daemon/cmd/datapath.go index 09d7c26df53bf..bb418244251e6 100644 --- a/daemon/cmd/datapath.go +++ b/daemon/cmd/datapath.go @@ -26,7 +26,6 @@ import ( "github.com/cilium/cilium/pkg/maps/encrypt" "github.com/cilium/cilium/pkg/maps/fragmap" ipcachemap "github.com/cilium/cilium/pkg/maps/ipcache" - "github.com/cilium/cilium/pkg/maps/ipmasq" "github.com/cilium/cilium/pkg/maps/lbmap" "github.com/cilium/cilium/pkg/maps/lxcmap" "github.com/cilium/cilium/pkg/maps/metricsmap" @@ -241,19 +240,6 @@ func (d *Daemon) initMaps() error { } } - if option.Config.EnableIPMasqAgent { - if option.Config.EnableIPv4Masquerade { - if err := ipmasq.IPMasq4Map().OpenOrCreate(); err != nil { - return fmt.Errorf("initializing IPv4 masquerading map: %w", err) - } - } - if option.Config.EnableIPv6Masquerade { - if err := ipmasq.IPMasq6Map().OpenOrCreate(); err != nil { - return fmt.Errorf("initializing IPv6 masquerading map: %w", err) - } - } - } - if option.Config.EnableIPSec { if err := encrypt.MapCreate(); err != nil { return fmt.Errorf("initializing IPsec map: %w", err) diff --git a/pkg/ipmasq/cell/cell.go b/pkg/ipmasq/cell/cell.go new file mode 100644 index 0000000000000..e2e0e27b09f35 --- /dev/null +++ b/pkg/ipmasq/cell/cell.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package cell + +import ( + "fmt" + "log/slog" + + "github.com/cilium/cilium/pkg/ipmasq" + ipmasqmaps "github.com/cilium/cilium/pkg/maps/ipmasq" + + "github.com/cilium/hive/cell" +) + +var Cell = cell.Module( + "ip-masq-agent", + "BPF ip-masq-agent implementation", + + cell.Provide(newIPMasqAgentCell), + cell.Config(defaultConfig), +) + +type ipMasqAgentParams struct { + cell.In + + Logger *slog.Logger + Lifecycle cell.Lifecycle + Config Config + IPMasqMap *ipmasqmaps.IPMasqBPFMap +} + +func newIPMasqAgentCell(params ipMasqAgentParams) (*ipmasq.IPMasqAgent, error) { + if !params.Config.EnableIPMasqAgent { + return nil, nil + } + + agent := ipmasq.NewIPMasqAgent(params.Config.IPMasqAgentConfigPath, params.IPMasqMap) + + params.Lifecycle.Append(cell.Hook{ + OnStart: func(cell.HookContext) error { + params.Logger.Info("Starting ip-masq-agent") + if err := agent.Start(); err != nil { + return fmt.Errorf("failed to start ip-masq-agent: %w", err) + } + return nil + }, + OnStop: func(cell.HookContext) error { + params.Logger.Info("Stopping ip-masq-agent") + agent.Stop() + return nil + }, + }) + + return agent, nil +} diff --git a/pkg/ipmasq/cell/cell_test.go b/pkg/ipmasq/cell/cell_test.go new file mode 100644 index 0000000000000..d64c1ff13248f --- /dev/null +++ b/pkg/ipmasq/cell/cell_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package cell + +import ( + "context" + "path" + "testing" + + "github.com/cilium/cilium/pkg/testutils" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cilium/hive/cell" + "github.com/cilium/hive/hivetest" + + "github.com/cilium/cilium/pkg/hive" + "github.com/cilium/cilium/pkg/ipmasq" + ipmasqmaps "github.com/cilium/cilium/pkg/maps/ipmasq" + "github.com/cilium/cilium/pkg/metrics" + "github.com/cilium/cilium/pkg/option" +) + +func TestPrivileged_TestIPMasqAgentCell(t *testing.T) { + testutils.PrivilegedTest(t) + + var agent *ipmasq.IPMasqAgent + + testHive := hive.New( + // Needed for the metrics.Cell + cell.Provide(func() *option.DaemonConfig { return &option.DaemonConfig{} }), + // Needed for the IPMasqBPFMap + metrics.Cell, + ipmasqmaps.Cell, + Cell, + cell.Invoke(func(a *ipmasq.IPMasqAgent) { + agent = a + }), + ) + + hive.AddConfigOverride(testHive, func(cfg *Config) { + cfg.EnableIPMasqAgent = true + cfg.IPMasqAgentConfigPath = path.Join(t.TempDir(), "placeholder.yaml") + }) + + // Start the hive + ctx := context.Background() + tlog := hivetest.Logger(t) + err := testHive.Start(tlog, ctx) + require.NoError(t, err) + + // Verify that the agent was successfully created + assert.NotNil(t, agent) + + // Stop the hive + err = testHive.Stop(tlog, ctx) + require.NoError(t, err) +} + +func TestPrivileged_TestIPMasqAgentCellDisabled(t *testing.T) { + testutils.PrivilegedTest(t) + + var agent *ipmasq.IPMasqAgent + + testHive := hive.New( + // Needed for the metrics.Cell + cell.Provide(func() *option.DaemonConfig { return &option.DaemonConfig{} }), + // Needed for the IPMasqBPFMap + metrics.Cell, + ipmasqmaps.Cell, + Cell, + cell.Invoke(func(a *ipmasq.IPMasqAgent) { + agent = a + }), + ) + + // Disable via config + hive.AddConfigOverride(testHive, func(cfg *Config) { + cfg.EnableIPMasqAgent = false + }) + + // Start the hive + ctx := context.Background() + tlog := hivetest.Logger(t) + err := testHive.Start(tlog, ctx) + require.NoError(t, err) + + // Verify that the agent was not created + assert.Nil(t, agent) + + // Stop the hive + err = testHive.Stop(tlog, ctx) + require.NoError(t, err) +} diff --git a/pkg/ipmasq/cell/config.go b/pkg/ipmasq/cell/config.go new file mode 100644 index 0000000000000..463151975b96b --- /dev/null +++ b/pkg/ipmasq/cell/config.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package cell + +import ( + "github.com/spf13/pflag" + + "github.com/cilium/cilium/pkg/option" +) + +type Config struct { + EnableIPMasqAgent bool `mapstructure:"enable-ip-masq-agent"` + IPMasqAgentConfigPath string `mapstructure:"ip-masq-agent-config-path"` +} + +var defaultConfig = Config{ + EnableIPMasqAgent: false, + IPMasqAgentConfigPath: "/etc/config/ip-masq-agent", +} + +func (c Config) Flags(flags *pflag.FlagSet) { + flags.Bool(option.EnableIPMasqAgent, c.EnableIPMasqAgent, "Enable BPF ip-masq-agent") + flags.String(option.IPMasqAgentConfigPath, c.IPMasqAgentConfigPath, "ip-masq-agent configuration file path") +} diff --git a/pkg/ipmasq/ipmasq.go b/pkg/ipmasq/ipmasq.go index fdc7f8d0aacf3..c0127d0cfaa84 100644 --- a/pkg/ipmasq/ipmasq.go +++ b/pkg/ipmasq/ipmasq.go @@ -16,7 +16,6 @@ import ( "github.com/cilium/cilium/pkg/logging" "github.com/cilium/cilium/pkg/logging/logfields" - "github.com/cilium/cilium/pkg/maps/ipmasq" ) var ( @@ -98,38 +97,34 @@ type IPMasqAgent struct { handlerFinished chan struct{} } -func NewIPMasqAgent(configPath string) (*IPMasqAgent, error) { - return newIPMasqAgent(configPath, &ipmasq.IPMasqBPFMap{}) -} - -func newIPMasqAgent(configPath string, ipMasqMap IPMasqMap) (*IPMasqAgent, error) { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, fmt.Errorf("Failed to create fsnotify watcher: %w", err) - } - - configDir := filepath.Dir(configPath) - // The directory of the config should exist at this time, otherwise - // the watcher will fail to add - if err := watcher.Add(configDir); err != nil { - watcher.Close() - return nil, fmt.Errorf("Failed to add %q dir to fsnotify watcher: %w", configDir, err) - } - +func NewIPMasqAgent(configPath string, ipMasqMap IPMasqMap) *IPMasqAgent { a := &IPMasqAgent{ configPath: configPath, nonMasqCIDRsFromConfig: map[string]netip.Prefix{}, nonMasqCIDRsInMap: map[string]netip.Prefix{}, ipMasqMap: ipMasqMap, - watcher: watcher, } - return a, nil + return a } // Start starts the ip-masq-agent goroutine which tracks the config file and // updates the BPF map accordingly. -func (a *IPMasqAgent) Start() { +func (a *IPMasqAgent) Start() error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("failed to create fsnotify watcher: %w", err) + } + a.watcher = watcher + + configDir := filepath.Dir(a.configPath) + // The directory of the config should exist at this time, otherwise + // the watcher will fail to add + if err := a.watcher.Add(configDir); err != nil { + a.watcher.Close() + return fmt.Errorf("failed to add %q dir to fsnotify watcher: %w", configDir, err) + } + if err := a.restore(); err != nil { log.WithError(err).Warn("Failed to restore") } @@ -167,6 +162,8 @@ func (a *IPMasqAgent) Start() { } } }() + + return nil } // Stop stops the ip-masq-agent goroutine and the watcher. diff --git a/pkg/ipmasq/ipmasq_test.go b/pkg/ipmasq/ipmasq_test.go index 54f139a0fda08..f59d43a3fce8c 100644 --- a/pkg/ipmasq/ipmasq_test.go +++ b/pkg/ipmasq/ipmasq_test.go @@ -143,10 +143,7 @@ func setUpTest(tb testing.TB) *IPMasqTestSuite { require.NoError(tb, err) i.configFilePath = configFile.Name() - agent, err := newIPMasqAgent(i.configFilePath, i.ipMasqMap) - require.NoError(tb, err) - i.ipMasqAgent = agent - + i.ipMasqAgent = NewIPMasqAgent(i.configFilePath, i.ipMasqMap) tb.Cleanup(func() { i.ipMasqAgent.Stop() os.Remove(i.configFilePath) @@ -165,7 +162,8 @@ func TestUpdateIPv4(t *testing.T) { i.ipMasqMap.ipv4Enabled = true i.ipMasqMap.ipv6Enabled = false - i.ipMasqAgent.Start() + err := i.ipMasqAgent.Start() + require.NoError(t, err) i.writeConfig(t, "nonMasqueradeCIDRs:\n- 1.1.1.1/32\n- 2.2.2.2/16") time.Sleep(300 * time.Millisecond) @@ -214,7 +212,7 @@ func TestUpdateIPv4(t *testing.T) { require.True(t, ok) // Delete file, should remove the CIDRs and add default nonMasq CIDRs - err := os.Remove(i.configFilePath) + err = os.Remove(i.configFilePath) require.NoError(t, err) time.Sleep(300 * time.Millisecond) ipnets = i.ipMasqMap.dumpToSet() @@ -232,7 +230,8 @@ func TestUpdateIPv6(t *testing.T) { i.ipMasqMap.ipv4Enabled = false i.ipMasqMap.ipv6Enabled = true - i.ipMasqAgent.Start() + err := i.ipMasqAgent.Start() + require.NoError(t, err) i.writeConfig(t, "nonMasqueradeCIDRs:\n- 1:1:1:1::/64\n- 2:2::/32") time.Sleep(300 * time.Millisecond) @@ -279,7 +278,7 @@ func TestUpdateIPv6(t *testing.T) { require.True(t, ok) // Delete file, should remove the CIDRs and add default nonMasq CIDRs - err := os.Remove(i.configFilePath) + err = os.Remove(i.configFilePath) require.NoError(t, err) time.Sleep(300 * time.Millisecond) ipnets = i.ipMasqMap.dumpToSet() @@ -292,7 +291,8 @@ func TestUpdate(t *testing.T) { i := setUpTest(t) i.ipMasqMap.ipv4Enabled = true i.ipMasqMap.ipv6Enabled = true - i.ipMasqAgent.Start() + err := i.ipMasqAgent.Start() + require.NoError(t, err) i.writeConfig(t, "nonMasqueradeCIDRs:\n- 1.1.1.1/32\n- 2:2::/32") time.Sleep(300 * time.Millisecond) @@ -347,7 +347,7 @@ func TestUpdate(t *testing.T) { require.True(t, ok) // Delete file, should remove the CIDRs and add default nonMasq CIDRs - err := os.Remove(i.configFilePath) + err = os.Remove(i.configFilePath) require.NoError(t, err) time.Sleep(300 * time.Millisecond) ipnets = i.ipMasqMap.dumpToSet() @@ -364,7 +364,8 @@ func TestRestoreIPv4(t *testing.T) { i := setUpTest(t) i.ipMasqMap.ipv4Enabled = true i.ipMasqMap.ipv6Enabled = false - i.ipMasqAgent.Start() + err = i.ipMasqAgent.Start() + require.NoError(t, err) // Check that stale entry is removed from the map after restore i.ipMasqAgent.Stop() @@ -374,9 +375,9 @@ func TestRestoreIPv4(t *testing.T) { i.ipMasqMap.cidrsIPv4[cidr.String()] = cidr i.writeConfig(t, "nonMasqueradeCIDRs:\n- 4.4.0.0/16") - i.ipMasqAgent, err = newIPMasqAgent(i.configFilePath, i.ipMasqMap) + i.ipMasqAgent = NewIPMasqAgent(i.configFilePath, i.ipMasqMap) + err = i.ipMasqAgent.Start() require.NoError(t, err) - i.ipMasqAgent.Start() time.Sleep(300 * time.Millisecond) ipnets := i.ipMasqMap.dumpToSet() @@ -396,9 +397,9 @@ func TestRestoreIPv4(t *testing.T) { } i.ipMasqAgent.ipMasqMap = i.ipMasqMap i.writeConfig(t, "nonMasqueradeCIDRs:\n- 3.3.0.0/16\nmasqLinkLocal: true") - i.ipMasqAgent, err = newIPMasqAgent(i.configFilePath, i.ipMasqMap) + i.ipMasqAgent = NewIPMasqAgent(i.configFilePath, i.ipMasqMap) + err = i.ipMasqAgent.Start() require.NoError(t, err) - i.ipMasqAgent.Start() ipnets = i.ipMasqMap.dumpToSet() require.Len(t, ipnets, 1) @@ -412,7 +413,8 @@ func TestRestoreIPv6(t *testing.T) { i := setUpTest(t) i.ipMasqMap.ipv4Enabled = false i.ipMasqMap.ipv6Enabled = true - i.ipMasqAgent.Start() + err = i.ipMasqAgent.Start() + require.NoError(t, err) // Check that stale entry is removed from the map after restore i.ipMasqAgent.Stop() @@ -422,9 +424,9 @@ func TestRestoreIPv6(t *testing.T) { i.ipMasqMap.cidrsIPv6[cidr.String()] = cidr i.writeConfig(t, "nonMasqueradeCIDRs:\n- 4:4::/32") - i.ipMasqAgent, err = newIPMasqAgent(i.configFilePath, i.ipMasqMap) + i.ipMasqAgent = NewIPMasqAgent(i.configFilePath, i.ipMasqMap) + err = i.ipMasqAgent.Start() require.NoError(t, err) - i.ipMasqAgent.Start() time.Sleep(300 * time.Millisecond) ipnets := i.ipMasqMap.dumpToSet() @@ -444,9 +446,9 @@ func TestRestoreIPv6(t *testing.T) { } i.ipMasqAgent.ipMasqMap = i.ipMasqMap i.writeConfig(t, "nonMasqueradeCIDRs:\n- 3:3::/96\nmasqLinkLocalIPv6: true") - i.ipMasqAgent, err = newIPMasqAgent(i.configFilePath, i.ipMasqMap) + i.ipMasqAgent = NewIPMasqAgent(i.configFilePath, i.ipMasqMap) + err = i.ipMasqAgent.Start() require.NoError(t, err) - i.ipMasqAgent.Start() ipnets = i.ipMasqMap.dumpToSet() require.Len(t, ipnets, 1) @@ -460,7 +462,8 @@ func TestRestore(t *testing.T) { i := setUpTest(t) i.ipMasqMap.ipv4Enabled = true i.ipMasqMap.ipv6Enabled = true - i.ipMasqAgent.Start() + err = i.ipMasqAgent.Start() + require.NoError(t, err) // Check that stale entry is removed from the map after restore i.ipMasqAgent.Stop() @@ -474,9 +477,9 @@ func TestRestore(t *testing.T) { i.ipMasqMap.cidrsIPv4[cidr.String()] = cidr i.writeConfig(t, "nonMasqueradeCIDRs:\n- 4.4.0.0/16\n- 4:4::/32") - i.ipMasqAgent, err = newIPMasqAgent(i.configFilePath, i.ipMasqMap) + i.ipMasqAgent = NewIPMasqAgent(i.configFilePath, i.ipMasqMap) + err = i.ipMasqAgent.Start() require.NoError(t, err) - i.ipMasqAgent.Start() time.Sleep(300 * time.Millisecond) ipnets := i.ipMasqMap.dumpToSet() @@ -501,9 +504,9 @@ func TestRestore(t *testing.T) { } i.ipMasqAgent.ipMasqMap = i.ipMasqMap i.writeConfig(t, "nonMasqueradeCIDRs:\n- 3.3.0.0/16\n- 3:3:3:3::/96\nmasqLinkLocal: true\nmasqLinkLocalIPv6: true") - i.ipMasqAgent, err = newIPMasqAgent(i.configFilePath, i.ipMasqMap) + i.ipMasqAgent = NewIPMasqAgent(i.configFilePath, i.ipMasqMap) + err = i.ipMasqAgent.Start() require.NoError(t, err) - i.ipMasqAgent.Start() ipnets = i.ipMasqMap.dumpToSet() require.Len(t, ipnets, 2) diff --git a/pkg/maps/ipmasq/cell.go b/pkg/maps/ipmasq/cell.go new file mode 100644 index 0000000000000..0908923310d11 --- /dev/null +++ b/pkg/maps/ipmasq/cell.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package ipmasq + +import ( + "fmt" + + "github.com/cilium/hive/cell" + + "github.com/cilium/cilium/pkg/bpf" + "github.com/cilium/cilium/pkg/metrics" + "github.com/cilium/cilium/pkg/option" +) + +var Cell = cell.Module( + "ip-masq-maps", + "BPF ip-masq-agent maps", + + cell.Provide(newIPMasqMaps), +) + +type ipMasqMapsParams struct { + cell.In + + Lifecycle cell.Lifecycle + MetricsRegistry *metrics.Registry +} + +func newIPMasqMaps(p ipMasqMapsParams) bpf.MapOut[*IPMasqBPFMap] { + m := &IPMasqBPFMap{} + + p.Lifecycle.Append(cell.Hook{ + OnStart: func(cell.HookContext) error { + if option.Config.EnableIPMasqAgent { + if option.Config.EnableIPv4Masquerade { + if err := IPMasq4Map().OpenOrCreate(); err != nil { + return fmt.Errorf("initializing IPv4 masquerading map: %w", err) + } + } + if option.Config.EnableIPv6Masquerade { + if err := IPMasq6Map().OpenOrCreate(); err != nil { + return fmt.Errorf("initializing IPv6 masquerading map: %w", err) + } + } + } + return nil + }, + OnStop: func(cell.HookContext) error { + // No clean-up required for the ip-masq-agent maps at shutdown. + return nil + }, + }) + + return bpf.NewMapOut(m) +} diff --git a/pkg/option/config.go b/pkg/option/config.go index 2a6c3a6f0f20c..655410e49b985 100644 --- a/pkg/option/config.go +++ b/pkg/option/config.go @@ -1654,7 +1654,6 @@ type DaemonConfig struct { EnableBPFMasquerade bool EnableMasqueradeRouteSource bool EnableIPMasqAgent bool - IPMasqAgentConfigPath string EnableBPFClockProbe bool EnableIPv4EgressGateway bool @@ -2954,7 +2953,6 @@ func (c *DaemonConfig) Populate(vp *viper.Viper) { c.EnableIPMasqAgent = vp.GetBool(EnableIPMasqAgent) c.EnableIPv4EgressGateway = vp.GetBool(EnableIPv4EgressGateway) c.EnableEnvoyConfig = vp.GetBool(EnableEnvoyConfig) - c.IPMasqAgentConfigPath = vp.GetString(IPMasqAgentConfigPath) c.InstallIptRules = vp.GetBool(InstallIptRules) c.IPSecKeyFile = vp.GetString(IPSecKeyFileName) c.IPsecKeyRotationDuration = vp.GetDuration(IPsecKeyRotationDuration) From d4c3738011684043a62315b32e45b523cc512f16 Mon Sep 17 00:00:00 2001 From: Anton Ippolitov Date: Fri, 30 May 2025 17:00:18 +0200 Subject: [PATCH 2/2] Ensure ip rules match the BPF ip-masq-agent configuration in AWS ENI mode Signed-off-by: Anton Ippolitov --- daemon/cmd/daemon_main.go | 4 +- pkg/ipam/allocator_test.go | 6 +-- pkg/ipam/cell/cell.go | 4 +- pkg/ipam/crd.go | 26 ++++++++++-- pkg/ipam/crd_test.go | 84 +++++++++++++++++++++++++++++++++++--- pkg/ipam/ipam.go | 7 +++- pkg/ipam/ipam_test.go | 8 ++-- pkg/ipam/types.go | 2 + pkg/ipmasq/ipmasq.go | 6 +++ pkg/ipmasq/ipmasq_test.go | 21 ++++++++++ 10 files changed, 146 insertions(+), 22 deletions(-) diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go index db224e0867a4d..e1a42e7f608a1 100644 --- a/daemon/cmd/daemon_main.go +++ b/daemon/cmd/daemon_main.go @@ -65,7 +65,6 @@ import ( "github.com/cilium/cilium/pkg/ipam" ipamOption "github.com/cilium/cilium/pkg/ipam/option" "github.com/cilium/cilium/pkg/ipcache" - "github.com/cilium/cilium/pkg/ipmasq" "github.com/cilium/cilium/pkg/k8s" k8sClient "github.com/cilium/cilium/pkg/k8s/client" k8sSynced "github.com/cilium/cilium/pkg/k8s/synced" @@ -1625,7 +1624,6 @@ type daemonParams struct { Hubble hubblecell.HubbleIntegration LRPManager *redirectpolicy.Manager MaglevConfig maglev.Config - IPMasqAgent *ipmasq.IPMasqAgent } func newDaemonPromise(params daemonParams) promise.Promise[*Daemon] { @@ -1798,7 +1796,7 @@ func startDaemon(d *Daemon, restoredEndpoints *endpointRestoreState, cleaner *da } } } - + go func() { if d.endpointRestoreComplete != nil { select { diff --git a/pkg/ipam/allocator_test.go b/pkg/ipam/allocator_test.go index dac21a23b38b8..e201addc7cf72 100644 --- a/pkg/ipam/allocator_test.go +++ b/pkg/ipam/allocator_test.go @@ -59,7 +59,7 @@ var mtuMock = fakeMTU{} func TestAllocatedIPDump(t *testing.T) { fakeAddressing := fakeTypes.NewNodeAddressing() localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) - ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil) + ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil) ipam.ConfigureAllocator() allocv4, allocv6, status := ipam.Dump() @@ -80,7 +80,7 @@ func TestExpirationTimer(t *testing.T) { fakeAddressing := fakeTypes.NewNodeAddressing() localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) - ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil) + ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil) ipam.ConfigureAllocator() err := ipam.AllocateIP(ip, "foo", PoolDefault()) @@ -148,7 +148,7 @@ func TestAllocateNextWithExpiration(t *testing.T) { fakeAddressing := fakeTypes.NewNodeAddressing() localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) fakeMetadata := fakeMetadataFunc(func(owner string, family Family) (pool string, err error) { return "some-pool", nil }) - ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil) + ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil, nil) ipam.ConfigureAllocator() // Allocate IPs and test expiration timer. 'pool' is empty in order to test diff --git a/pkg/ipam/cell/cell.go b/pkg/ipam/cell/cell.go index 650e04bac1819..f41bc85dbe702 100644 --- a/pkg/ipam/cell/cell.go +++ b/pkg/ipam/cell/cell.go @@ -15,6 +15,7 @@ import ( "github.com/cilium/cilium/pkg/ipam" ipamapi "github.com/cilium/cilium/pkg/ipam/api" ipamMetadata "github.com/cilium/cilium/pkg/ipam/metadata" + "github.com/cilium/cilium/pkg/ipmasq" k8sClient "github.com/cilium/cilium/pkg/k8s/client" "github.com/cilium/cilium/pkg/k8s/watchers" "github.com/cilium/cilium/pkg/mtu" @@ -49,10 +50,11 @@ type ipamParams struct { IPAMMetadataManager ipamMetadata.Manager NodeDiscovery *nodediscovery.NodeDiscovery Sysctl sysctl.Sysctl + IPMasqAgent *ipmasq.IPMasqAgent } func newIPAddressManager(params ipamParams) *ipam.IPAM { - return ipam.NewIPAM(params.NodeAddressing, params.AgentConfig, params.NodeDiscovery, params.LocalNodeStore, params.K8sEventReporter, params.NodeResource, params.MTU, params.Clientset, params.IPAMMetadataManager, params.Sysctl) + return ipam.NewIPAM(params.NodeAddressing, params.AgentConfig, params.NodeDiscovery, params.LocalNodeStore, params.K8sEventReporter, params.NodeResource, params.MTU, params.Clientset, params.IPAMMetadataManager, params.Sysctl, params.IPMasqAgent) } type ipamAPIHandlerParams struct { diff --git a/pkg/ipam/crd.go b/pkg/ipam/crd.go index 8267772070cf0..f8fb93ac656f3 100644 --- a/pkg/ipam/crd.go +++ b/pkg/ipam/crd.go @@ -27,6 +27,7 @@ import ( "github.com/cilium/cilium/pkg/ip" ipamOption "github.com/cilium/cilium/pkg/ipam/option" ipamTypes "github.com/cilium/cilium/pkg/ipam/types" + "github.com/cilium/cilium/pkg/ipmasq" ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" "github.com/cilium/cilium/pkg/k8s/client" "github.com/cilium/cilium/pkg/k8s/informer" @@ -692,6 +693,8 @@ type crdAllocator struct { family Family conf *option.DaemonConfig + + ipMasqAgent *ipmasq.IPMasqAgent } // newCRDAllocator creates a new CRD-backed IP allocator @@ -704,16 +707,18 @@ func newCRDAllocator( k8sEventReg K8sEventRegister, mtuConfig MtuConfiguration, sysctl sysctl.Sysctl, + ipMasqAgent *ipmasq.IPMasqAgent, ) Allocator { initNodeStore.Do(func() { sharedNodeStore = newNodeStore(nodeTypes.GetName(), c, owner, localNodeStore, clientset, k8sEventReg, mtuConfig, sysctl) }) allocator := &crdAllocator{ - allocated: ipamTypes.AllocationMap{}, - family: family, - store: sharedNodeStore, - conf: c, + allocated: ipamTypes.AllocationMap{}, + family: family, + store: sharedNodeStore, + conf: c, + ipMasqAgent: ipMasqAgent, } sharedNodeStore.addAllocator(allocator) @@ -759,6 +764,19 @@ func (a *crdAllocator) buildAllocationResult(ip net.IP, ipInfo *ipamTypes.Alloca if a.conf.IPv4NativeRoutingCIDR != nil { result.CIDRs = append(result.CIDRs, a.conf.IPv4NativeRoutingCIDR.String()) } + // If the ip-masq-agent is enabled, get the CIDRs that are not masqueraded. + // Note that the resulting ip rules will not be dynamically regenerated if the + // ip-masq-agent configuration changes. + if a.conf.EnableIPMasqAgent { + nonMasqCidrs := a.ipMasqAgent.NonMasqCIDRsFromConfig() + for _, prefix := range nonMasqCidrs { + if ip.To4() != nil && prefix.Addr().Is4() { + result.CIDRs = append(result.CIDRs, prefix.String()) + } else if ip.To4() == nil && prefix.Addr().Is6() { + result.CIDRs = append(result.CIDRs, prefix.String()) + } + } + } if eni.Subnet.CIDR != "" { // The gateway for a subnet and VPC is always x.x.x.1 // Ref: https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Route_Tables.html diff --git a/pkg/ipam/crd_test.go b/pkg/ipam/crd_test.go index 2178a0b62d21e..d9f592a2c85b6 100644 --- a/pkg/ipam/crd_test.go +++ b/pkg/ipam/crd_test.go @@ -18,6 +18,7 @@ import ( fakeTypes "github.com/cilium/cilium/pkg/datapath/fake/types" ipamOption "github.com/cilium/cilium/pkg/ipam/option" ipamTypes "github.com/cilium/cilium/pkg/ipam/types" + "github.com/cilium/cilium/pkg/ipmasq" ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" "github.com/cilium/cilium/pkg/node" "github.com/cilium/cilium/pkg/option" @@ -95,12 +96,12 @@ func TestMarkForReleaseNoAllocate(t *testing.T) { fakeAddressing := fakeTypes.NewNodeAddressing() conf := testConfigurationCRD - initNodeStore.Do(func() { - sharedNodeStore = newFakeNodeStore(conf, t) - sharedNodeStore.ownNode = cn - }) + initNodeStore.Do(func() {}) // Ensure the real initNodeStore is not called + sharedNodeStore = newFakeNodeStore(conf, t) + sharedNodeStore.ownNode = cn + localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) - ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil) + ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil) ipam.ConfigureAllocator() sharedNodeStore.updateLocalNodeResource(cn) @@ -127,6 +128,79 @@ func TestMarkForReleaseNoAllocate(t *testing.T) { require.Equal(t, ipamOption.IPAMDoNotRelease, string(cn.Status.IPAM.ReleaseIPs["1.1.1.3"])) } +type ipMasqMapDummy struct{} + +func (m ipMasqMapDummy) Update(netip.Prefix) error { return nil } +func (m ipMasqMapDummy) Delete(netip.Prefix) error { return nil } +func (m ipMasqMapDummy) Dump() ([]netip.Prefix, error) { return []netip.Prefix{}, nil } + +func TestIPMasq(t *testing.T) { + cn := newCiliumNode("node1", 4, 4, 0) + dummyResource := ipamTypes.AllocationIP{Resource: "eni-1"} + cn.Spec.IPAM.Pool["10.1.1.226"] = dummyResource + cn.Status.ENI.ENIs = map[string]eniTypes.ENI{ + "eni-1": { + ID: "eni-1", + Addresses: []string{ + "10.1.1.226", + "10.1.1.229", + }, + VPC: eniTypes.AwsVPC{ + ID: "vpc-1", + PrimaryCIDR: "10.1.0.0/16", + CIDRs: []string{ + "10.2.0.0/16", + }, + }, + }, + } + + fakeAddressing := fakeTypes.NewNodeAddressing() + conf := testConfigurationCRD + conf.IPAM = ipamOption.IPAMENI + conf.EnableIPMasqAgent = true + ipMasqAgent := ipmasq.NewIPMasqAgent("", ipMasqMapDummy{}) + err := ipMasqAgent.Start() + require.NoError(t, err) + + initNodeStore.Do(func() {}) // Ensure the real initNodeStore is not called + sharedNodeStore = newFakeNodeStore(conf, t) + sharedNodeStore.ownNode = cn + + localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) + ipam := NewIPAM(fakeAddressing, conf, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, ipMasqAgent) + ipam.ConfigureAllocator() + + epipv4 := netip.MustParseAddr("10.1.1.226") + result, err := ipam.IPv4Allocator.Allocate(epipv4.AsSlice(), "test1", PoolDefault()) + require.NoError(t, err) + // The resulting CIDRs should contain the VPC CIDRs and the default ip-masq-agent CIDRs from pkg/ipmasq/ipmasq.go + require.ElementsMatch( + t, + []string{ + // VPC CIDRs + "10.1.0.0/16", + "10.2.0.0/16", + // Default ip-masq-agent CIDRs + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "100.64.0.0/10", + "192.0.0.0/24", + "192.0.2.0/24", + "192.88.99.0/24", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "240.0.0.0/4", + "169.254.0.0/16", + }, + result.CIDRs, + ) + + ipMasqAgent.Stop() +} + func Test_validateENIConfig(t *testing.T) { type args struct { node *ciliumv2.CiliumNode diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index d8f88e6468df3..d857952ac2912 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -12,6 +12,7 @@ import ( "github.com/cilium/cilium/pkg/datapath/linux/sysctl" "github.com/cilium/cilium/pkg/datapath/types" ipamOption "github.com/cilium/cilium/pkg/ipam/option" + "github.com/cilium/cilium/pkg/ipmasq" "github.com/cilium/cilium/pkg/k8s/client" "github.com/cilium/cilium/pkg/logging" "github.com/cilium/cilium/pkg/logging/logfields" @@ -79,6 +80,7 @@ func NewIPAM( clientset client.Clientset, metadata Metadata, sysctl sysctl.Sysctl, + ipMasqAgent *ipmasq.IPMasqAgent, ) *IPAM { return &IPAM{ nodeAddressing: nodeAddressing, @@ -95,6 +97,7 @@ func NewIPAM( nodeDiscovery: nodeDiscovery, metadata: metadata, sysctl: sysctl, + ipMasqAgent: ipMasqAgent, } } @@ -129,11 +132,11 @@ func (ipam *IPAM) ConfigureAllocator() { case ipamOption.IPAMCRD, ipamOption.IPAMENI, ipamOption.IPAMAzure, ipamOption.IPAMAlibabaCloud: log.Info("Initializing CRD-based IPAM") if ipam.config.IPv6Enabled() { - ipam.IPv6Allocator = newCRDAllocator(IPv6, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl) + ipam.IPv6Allocator = newCRDAllocator(IPv6, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl, ipam.ipMasqAgent) } if ipam.config.IPv4Enabled() { - ipam.IPv4Allocator = newCRDAllocator(IPv4, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl) + ipam.IPv4Allocator = newCRDAllocator(IPv4, ipam.config, ipam.nodeDiscovery, ipam.localNodeStore, ipam.clientset, ipam.k8sEventReg, ipam.mtuConfig, ipam.sysctl, ipam.ipMasqAgent) } case ipamOption.IPAMDelegatedPlugin: log.Info("Initializing no-op IPAM since we're using a CNI delegated plugin") diff --git a/pkg/ipam/ipam_test.go b/pkg/ipam/ipam_test.go index 3ebd79ea86bd9..9dea1a4b1f93e 100644 --- a/pkg/ipam/ipam_test.go +++ b/pkg/ipam/ipam_test.go @@ -122,7 +122,7 @@ func (f fakePoolAllocator) RestoreFinished() {} func TestLock(t *testing.T) { fakeAddressing := fakeTypes.NewNodeAddressing() localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) - ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil) + ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil) ipam.ConfigureAllocator() // Since the IPs we have allocated to the endpoints might or might not @@ -146,7 +146,7 @@ func TestLock(t *testing.T) { func TestExcludeIP(t *testing.T) { fakeAddressing := fakeTypes.NewNodeAddressing() localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) - ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil) + ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, nil, nil, nil) ipam.ConfigureAllocator() ipv4 := fakeIPv4AllocCIDRIP(fakeAddressing) @@ -194,7 +194,7 @@ func TestIPAMMetadata(t *testing.T) { } }) - ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil) + ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil, nil) ipam.ConfigureAllocator() ipam.IPv4Allocator = newFakePoolAllocator(map[string]string{ "default": "10.10.0.0/16", @@ -253,7 +253,7 @@ func TestLegacyAllocatorIPAMMetadata(t *testing.T) { fakeAddressing := fakeTypes.NewNodeAddressing() localNodeStore := node.NewTestLocalNodeStore(node.LocalNode{}) fakeMetadata := fakeMetadataFunc(func(owner string, family Family) (pool string, err error) { return "some-pool", nil }) - ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil) + ipam := NewIPAM(fakeAddressing, testConfiguration, &ownerMock{}, localNodeStore, &ownerMock{}, &resourceMock{}, &mtuMock, nil, fakeMetadata, nil, nil) ipam.ConfigureAllocator() // AllocateIP requires explicit pool diff --git a/pkg/ipam/types.go b/pkg/ipam/types.go index 6abe6fee6e43c..8385eef1bc445 100644 --- a/pkg/ipam/types.go +++ b/pkg/ipam/types.go @@ -11,6 +11,7 @@ import ( agentK8s "github.com/cilium/cilium/daemon/k8s" "github.com/cilium/cilium/pkg/datapath/linux/sysctl" "github.com/cilium/cilium/pkg/datapath/types" + "github.com/cilium/cilium/pkg/ipmasq" "github.com/cilium/cilium/pkg/k8s/client" "github.com/cilium/cilium/pkg/lock" "github.com/cilium/cilium/pkg/node" @@ -117,6 +118,7 @@ type IPAM struct { clientset client.Clientset nodeDiscovery Owner sysctl sysctl.Sysctl + ipMasqAgent *ipmasq.IPMasqAgent } // DebugStatus implements debug.StatusObject to provide debug status collection diff --git a/pkg/ipmasq/ipmasq.go b/pkg/ipmasq/ipmasq.go index c0127d0cfaa84..45576e97e57f3 100644 --- a/pkg/ipmasq/ipmasq.go +++ b/pkg/ipmasq/ipmasq.go @@ -6,9 +6,11 @@ package ipmasq import ( "encoding/json" "fmt" + "maps" "net/netip" "os" "path/filepath" + "slices" "strings" "github.com/fsnotify/fsnotify" @@ -259,6 +261,10 @@ func (a *IPMasqAgent) readConfig() (bool, error) { return false, nil } +func (a *IPMasqAgent) NonMasqCIDRsFromConfig() []netip.Prefix { + return slices.Collect(maps.Values(a.nonMasqCIDRsFromConfig)) +} + // restore dumps the ipmasq BPF map and populates IPMasqAgent.nonMasqCIDRsInMap // with the CIDRs from the map. func (a *IPMasqAgent) restore() error { diff --git a/pkg/ipmasq/ipmasq_test.go b/pkg/ipmasq/ipmasq_test.go index f59d43a3fce8c..f2a9374a88b84 100644 --- a/pkg/ipmasq/ipmasq_test.go +++ b/pkg/ipmasq/ipmasq_test.go @@ -5,8 +5,10 @@ package ipmasq import ( "fmt" + "maps" "net/netip" "os" + "slices" "testing" "time" @@ -554,3 +556,22 @@ func TestParseCIDR(t *testing.T) { }) } } + +func TestNonMasqCIDRsFromConfig(t *testing.T) { + var err error + + i := setUpTest(t) + err = i.ipMasqAgent.Start() + require.NoError(t, err) + + // When the config is empty, the default non-masquerade CIDRs should be returned + defaultCidrs := slices.Collect(maps.Values(defaultNonMasqCIDRs)) + defaultCidrs = append(defaultCidrs, linkLocalCIDRIPv4, linkLocalCIDRIPv6) + require.ElementsMatch(t, defaultCidrs, i.ipMasqAgent.NonMasqCIDRsFromConfig()) + + // When the config is populated, the configured CIDRs should be returned + i.writeConfig(t, "nonMasqueradeCIDRs:\n- 3.3.0.0/16\nmasqLinkLocal: true\nmasqLinkLocalIPv6: true") + err = i.ipMasqAgent.Update() + require.NoError(t, err) + require.ElementsMatch(t, []netip.Prefix{netip.MustParsePrefix("3.3.0.0/16")}, i.ipMasqAgent.NonMasqCIDRsFromConfig()) +}