Skip to content

Commit 7273f80

Browse files
authored
fix: configure docker / iptables rules on the host (#115)
1 parent d31e28a commit 7273f80

File tree

14 files changed

+866
-15
lines changed

14 files changed

+866
-15
lines changed

components/linux/action.pb.go

Lines changed: 446 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

components/linux/action.proto

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,32 @@ message ConfigureBaseOSSpec {
1818
}
1919

2020
message ConfigureBaseOSStatus {
21+
}
22+
23+
message DisableDocker {
24+
api.Metadata metadata = 1;
25+
26+
DisableDockerSpec spec = 2;
27+
28+
DisableDockerStatus status = 3;
29+
}
30+
31+
message DisableDockerSpec {
32+
}
33+
34+
message DisableDockerStatus {
35+
}
36+
37+
message ConfigureIPTables {
38+
api.Metadata metadata = 1;
39+
40+
ConfigureIPTablesSpec spec = 2;
41+
42+
ConfigureIPTablesStatus status = 3;
43+
}
44+
45+
message ConfigureIPTablesSpec {
46+
}
47+
48+
message ConfigureIPTablesStatus {
2149
}

components/linux/redact.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,9 @@ package linux
22

33
func (x *ConfigureBaseOS) Redact() {
44
}
5+
6+
func (x *DisableDocker) Redact() {
7+
}
8+
9+
func (x *ConfigureIPTables) Redact() {
10+
}
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
*mangle
2+
:PREROUTING ACCEPT [0:0]
3+
:INPUT ACCEPT [0:0]
4+
:FORWARD ACCEPT [0:0]
5+
:OUTPUT ACCEPT [0:0]
6+
:POSTROUTING ACCEPT [0:0]
7+
COMMIT
8+
*raw
9+
:PREROUTING ACCEPT [0:0]
10+
:OUTPUT ACCEPT [0:0]
11+
COMMIT
12+
*filter
13+
:INPUT ACCEPT [0:0]
14+
:FORWARD ACCEPT [0:0]
15+
:OUTPUT ACCEPT [0:0]
16+
COMMIT
17+
*security
18+
:INPUT ACCEPT [0:0]
19+
:FORWARD ACCEPT [0:0]
20+
:OUTPUT ACCEPT [0:0]
21+
COMMIT
22+
*nat
23+
:PREROUTING ACCEPT [0:0]
24+
:INPUT ACCEPT [0:0]
25+
:OUTPUT ACCEPT [0:0]
26+
:POSTROUTING ACCEPT [0:0]
27+
COMMIT
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=Flush iptables rules to a clean ACCEPT state
3+
Before=kubelet.service
4+
5+
[Service]
6+
Type=oneshot
7+
ExecStart=/usr/sbin/iptables-restore -cv /etc/aks-flex-node/iptables-clear
8+
RemainAfterExit=no
9+
10+
[Install]
11+
RequiredBy=kubelet.service
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
package v20260301
2+
3+
import (
4+
"context"
5+
_ "embed"
6+
"fmt"
7+
8+
"google.golang.org/protobuf/types/known/anypb"
9+
10+
"github.com/Azure/AKSFlexNode/components/linux"
11+
"github.com/Azure/AKSFlexNode/components/services/actions"
12+
"github.com/Azure/AKSFlexNode/pkg/config"
13+
"github.com/Azure/AKSFlexNode/pkg/systemd"
14+
"github.com/Azure/AKSFlexNode/pkg/utils/utilio"
15+
"github.com/Azure/AKSFlexNode/pkg/utils/utilpb"
16+
)
17+
18+
const (
19+
iptablesFlushUnit = "iptables-flush.service"
20+
iptablesClearPath = config.ConfigDir + "/iptables-clear"
21+
)
22+
23+
//go:embed assets/iptables-flush.service
24+
var iptablesFlushServiceUnit []byte
25+
26+
//go:embed assets/iptables-clear
27+
var iptablesClearRules []byte
28+
29+
// configureIPTablesAction installs a oneshot systemd unit that flushes all
30+
// iptables rules to a clean ACCEPT state before kubelet starts. This ensures
31+
// stale rules (e.g. left behind by Docker) do not interfere with Kubernetes
32+
// networking.
33+
type configureIPTablesAction struct {
34+
systemd systemd.Manager
35+
}
36+
37+
func newConfigureIPTablesAction() (actions.Server, error) {
38+
return &configureIPTablesAction{
39+
systemd: systemd.New(),
40+
}, nil
41+
}
42+
43+
var _ actions.Server = (*configureIPTablesAction)(nil)
44+
45+
func (a *configureIPTablesAction) ApplyAction(
46+
ctx context.Context,
47+
req *actions.ApplyActionRequest,
48+
) (*actions.ApplyActionResponse, error) {
49+
settings, err := utilpb.AnyTo[*linux.ConfigureIPTables](req.GetItem())
50+
if err != nil {
51+
return nil, err
52+
}
53+
54+
if err := a.ensureIPTablesClearRules(); err != nil {
55+
return nil, fmt.Errorf("installing iptables-clear rules: %w", err)
56+
}
57+
58+
if err := a.ensureIPTablesFlushUnit(ctx); err != nil {
59+
return nil, fmt.Errorf("configuring iptables-flush service: %w", err)
60+
}
61+
62+
item, err := anypb.New(settings)
63+
if err != nil {
64+
return nil, err
65+
}
66+
67+
return actions.ApplyActionResponse_builder{Item: item}.Build(), nil
68+
}
69+
70+
// ensureIPTablesClearRules writes the iptables-restore rules file
71+
// to <config.ConfigDir>/iptables-clear. The file resets all tables (mangle, raw, filter,
72+
// security, nat) to their default ACCEPT policies with empty chains.
73+
func (a *configureIPTablesAction) ensureIPTablesClearRules() error {
74+
return a.ensureIPTablesClearRulesAt(iptablesClearPath)
75+
}
76+
77+
// ensureIPTablesClearRulesAt writes the iptables-clear rules to the given path.
78+
func (a *configureIPTablesAction) ensureIPTablesClearRulesAt(path string) error {
79+
return utilio.WriteFile(path, iptablesClearRules, 0600)
80+
}
81+
82+
// ensureIPTablesFlushUnit installs and enables the
83+
// iptables-flush.service oneshot unit. The unit runs iptables-restore with the
84+
// clean rules file before kubelet.service starts.
85+
func (a *configureIPTablesAction) ensureIPTablesFlushUnit(ctx context.Context) error {
86+
unitUpdated, err := a.systemd.EnsureUnitFile(ctx, iptablesFlushUnit, iptablesFlushServiceUnit)
87+
if err != nil {
88+
return err
89+
}
90+
91+
if err := a.systemd.EnableUnit(ctx, iptablesFlushUnit); err != nil {
92+
return err
93+
}
94+
95+
return systemd.EnsureUnitRunning(ctx, a.systemd, iptablesFlushUnit, unitUpdated, unitUpdated)
96+
}
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package v20260301
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"os"
9+
10+
"google.golang.org/protobuf/types/known/anypb"
11+
12+
"github.com/Azure/AKSFlexNode/components/linux"
13+
"github.com/Azure/AKSFlexNode/components/services/actions"
14+
"github.com/Azure/AKSFlexNode/pkg/systemd"
15+
"github.com/Azure/AKSFlexNode/pkg/utils/utilio"
16+
"github.com/Azure/AKSFlexNode/pkg/utils/utilpb"
17+
)
18+
19+
const (
20+
dockerServiceUnit = "docker.service"
21+
dockerSocketUnit = "docker.socket"
22+
daemonConfigPath = "/etc/docker/daemon.json"
23+
)
24+
25+
// disableDockerAction disables the docker service and configures the docker
26+
// daemon with "iptables": false. This prevents docker from manipulating
27+
// iptables rules, which would conflict with Kubernetes networking.
28+
type disableDockerAction struct {
29+
systemd systemd.Manager
30+
}
31+
32+
func newDisableDockerAction() (actions.Server, error) {
33+
return &disableDockerAction{
34+
systemd: systemd.New(),
35+
}, nil
36+
}
37+
38+
var _ actions.Server = (*disableDockerAction)(nil)
39+
40+
func (a *disableDockerAction) ApplyAction(
41+
ctx context.Context,
42+
req *actions.ApplyActionRequest,
43+
) (*actions.ApplyActionResponse, error) {
44+
settings, err := utilpb.AnyTo[*linux.DisableDocker](req.GetItem())
45+
if err != nil {
46+
return nil, err
47+
}
48+
49+
if err := a.ensureDockerDisabled(ctx); err != nil {
50+
return nil, fmt.Errorf("disabling docker: %w", err)
51+
}
52+
53+
if err := a.ensureDaemonConfig(); err != nil {
54+
return nil, fmt.Errorf("configuring docker daemon: %w", err)
55+
}
56+
57+
item, err := anypb.New(settings)
58+
if err != nil {
59+
return nil, err
60+
}
61+
62+
return actions.ApplyActionResponse_builder{Item: item}.Build(), nil
63+
}
64+
65+
// ensureDockerDisabled idempotently stops, disables, and masks the docker
66+
// service and socket units so docker cannot be started.
67+
func (a *disableDockerAction) ensureDockerDisabled(ctx context.Context) error {
68+
if err := systemd.EnsureUnitMasked(ctx, a.systemd, dockerSocketUnit); err != nil {
69+
return fmt.Errorf("masking %s: %w", dockerSocketUnit, err)
70+
}
71+
72+
if err := systemd.EnsureUnitMasked(ctx, a.systemd, dockerServiceUnit); err != nil {
73+
return fmt.Errorf("masking %s: %w", dockerServiceUnit, err)
74+
}
75+
76+
return nil
77+
}
78+
79+
// ensureDaemonConfig ensures /etc/docker/daemon.json contains
80+
// "iptables": false.
81+
func (a *disableDockerAction) ensureDaemonConfig() error {
82+
return a.ensureDaemonConfigAt(daemonConfigPath)
83+
}
84+
85+
// ensureDaemonConfigAt ensures the daemon.json at the given path
86+
// contains "iptables": false. If the file already exists, the existing
87+
// configuration is preserved and only the iptables key is set. If the file does
88+
// not exist, a new one is created.
89+
func (a *disableDockerAction) ensureDaemonConfigAt(path string) error {
90+
config := map[string]any{}
91+
92+
existing, err := os.ReadFile(path) //#nosec G304 -- trusted path
93+
switch {
94+
case errors.Is(err, os.ErrNotExist):
95+
// file does not exist, will create with defaults
96+
case err != nil:
97+
return fmt.Errorf("reading %s: %w", path, err)
98+
default:
99+
if err := json.Unmarshal(existing, &config); err != nil {
100+
return fmt.Errorf("parsing %s: %w", path, err)
101+
}
102+
}
103+
104+
if val, ok := config["iptables"]; ok {
105+
if enabled, isBool := val.(bool); isBool && !enabled {
106+
// iptables is already set to false, nothing to do
107+
return nil
108+
}
109+
}
110+
111+
config["iptables"] = false
112+
113+
content, err := marshalDaemonConfig(config)
114+
if err != nil {
115+
return err
116+
}
117+
118+
return utilio.WriteFile(path, content, 0644)
119+
}
120+
121+
// marshalDaemonConfig serializes a daemon config map to indented JSON with a
122+
// trailing newline, matching the conventional format for daemon.json.
123+
func marshalDaemonConfig(config map[string]any) ([]byte, error) {
124+
data, err := json.MarshalIndent(config, "", " ")
125+
if err != nil {
126+
return nil, fmt.Errorf("marshaling daemon config: %w", err)
127+
}
128+
data = append(data, '\n')
129+
return data, nil
130+
}

components/linux/v20260301/exports.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,12 @@ func init() {
1010
newConfigureBaseOSAction,
1111
&linux.ConfigureBaseOS{},
1212
)
13+
actions.MustRegister(
14+
newDisableDockerAction,
15+
&linux.DisableDocker{},
16+
)
17+
actions.MustRegister(
18+
newConfigureIPTablesAction,
19+
&linux.ConfigureIPTables{},
20+
)
1321
}

pkg/bootstrapper/bootstrapper.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ func (b *Bootstrapper) Bootstrap(ctx context.Context) (*ExecutionResult, error)
3636
arc.NewInstaller(b.logger), // Setup Arc
3737

3838
configureSystem.Executor("configure-os", b.componentsAPIConn),
39+
// Some environments might have docker pre-installed which can interfere with Kubernetes networking.
40+
// This step disables the docker services and configures the docker daemon to not manage iptables rules.
41+
disableDocker.Executor("disable-docker", b.componentsAPIConn),
3942

4043
// Fetch serverURL and caCertData from AKS cluster admin credentials for
4144
// non-bootstrap-token auth modes (Arc, SP, MI). Must run before startKubelet
@@ -50,6 +53,8 @@ func (b *Bootstrapper) Bootstrap(ctx context.Context) (*ExecutionResult, error)
5053

5154
configureCNI.Executor("configure-cni", b.componentsAPIConn),
5255
startContainerdService.Executor("start-containerd", b.componentsAPIConn),
56+
// Configure iptables rules before kubelet starts to prevent conflicts with Kubernetes networking
57+
configureIPTables.Executor("configure-iptables", b.componentsAPIConn),
5358
startKubelet.Executor("start-kubelet", b.componentsAPIConn),
5459
startNPD.Executor("start-npd", b.componentsAPIConn),
5560
}

pkg/bootstrapper/components.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,30 @@ var configureSystem resolveActionFunc[*linux.ConfigureBaseOS] = func(
9191
}.Build(), nil
9292
}
9393

94+
var disableDocker resolveActionFunc[*linux.DisableDocker] = func(
95+
name string,
96+
cfg *config.Config,
97+
) (*linux.DisableDocker, error) {
98+
spec := linux.DisableDockerSpec_builder{}.Build()
99+
100+
return linux.DisableDocker_builder{
101+
Metadata: componentAction(name),
102+
Spec: spec,
103+
}.Build(), nil
104+
}
105+
106+
var configureIPTables resolveActionFunc[*linux.ConfigureIPTables] = func(
107+
name string,
108+
cfg *config.Config,
109+
) (*linux.ConfigureIPTables, error) {
110+
spec := linux.ConfigureIPTablesSpec_builder{}.Build()
111+
112+
return linux.ConfigureIPTables_builder{
113+
Metadata: componentAction(name),
114+
Spec: spec,
115+
}.Build(), nil
116+
}
117+
94118
var downloadCRIBinaries resolveActionFunc[*cri.DownloadCRIBinaries] = func(
95119
name string,
96120
cfg *config.Config,

0 commit comments

Comments
 (0)