integration: load br_netfilter so pod ClusterIP DNS works on arm64

The arm64 CI diagnostics showed CoreDNS healthy and the kube-dns Service backed
by a ready endpoint, yet the operator pod's lookups to the kube-dns ClusterIP
timed out. That is the signature of bridge netfilter being off: with
br_netfilter unloaded, bridged pod-to-pod traffic skips the kube-proxy iptables
DNAT rules, so every ClusterIP — kube-dns included — is unreachable from pods,
even though the service and its endpoints are perfectly healthy. The amd64 dev
box does not hit this because Docker loads br_netfilter for its bridge networks.

Bind the host kernel modules into the k3s container and, before k3s programs
kube-proxy, load br_netfilter and turn on the bridge-nf-call-iptables and
ip_forward sysctls. Both are no-ops where the module is already present, so the
amd64 path (which passes locally) is unaffected.

Extend the diagnostics with the loaded modules, the bridge/forward sysctls, and
the kube-dns NAT rules, so a ClusterIP-unreachable failure shows whether the
fault is the module, the sysctls, or a missing kube-proxy rule rather than
leaving it to inference.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: kradalby

integration/k3sic/k3sic.go

@@ -94,6 +94,9 @@ const (
 	// kubectlBin is the in-container kubectl the k3s image ships on PATH.
 	kubectlBin = "kubectl"
 
+	// shellBin is the in-container shell used for compound commands.
+	shellBin = "/bin/sh"
+
 	// tailscaleNamespace is where the operator and its proxies are installed.
 	tailscaleNamespace = "tailscale"
 
@@ -245,6 +248,10 @@ func New(
 
 	k.container = container
 
+	// Make ClusterIP DNAT work before k3s programs kube-proxy rules; without it
+	// in-cluster DNS times out on hosts where br_netfilter is not preloaded.
+	k.ensureBridgeNetfilter()
+
 	// Drop the Headscale CA cert(s) into the trust store so helm and the
 	// operator trust the test server's TLS certificate.
 	for i, cert := range k.caCerts {
@@ -276,6 +283,35 @@ func withK3sHostConfig(config *docker.HostConfig) {
 		"/run":     "",
 		"/var/run": "",
 	}
+
+	// Bind the host kernel modules read-only so the container can load
+	// br_netfilter (see ensureBridgeNetfilter). Without bridge netfilter,
+	// kube-proxy's ClusterIP DNAT rules do not apply to bridged pod-to-pod
+	// traffic, so kube-dns (and every other Service) is unreachable from pods —
+	// the in-cluster DNS timeout seen on the arm64 CI runner. The container
+	// shares the host kernel, so the modules match.
+	config.Binds = append(config.Binds, "/lib/modules:/lib/modules:ro")
+}
+
+// ensureBridgeNetfilter loads br_netfilter and enables the sysctls that make
+// kube-proxy's ClusterIP DNAT apply to bridged pod-to-pod traffic. On a host
+// where the module is already loaded (e.g. the amd64 dev box, where Docker
+// loads it for bridge networks) these are no-ops; on the arm64 CI runner the
+// module is absent and pods cannot reach any Service IP — kube-dns included —
+// so in-cluster DNS times out. Best-effort: k3s also loads the module, and a
+// genuinely missing module surfaces in DumpDiagnostics rather than here.
+func (k *K3sInContainer) ensureBridgeNetfilter() {
+	for _, cmd := range []string{
+		"modprobe br_netfilter || true",
+		"sysctl -w net.bridge.bridge-nf-call-iptables=1 || true",
+		"sysctl -w net.bridge.bridge-nf-call-ip6tables=1 || true",
+		"sysctl -w net.ipv4.ip_forward=1 || true",
+	} {
+		out, stderr, err := k.Execute([]string{shellBin, "-c", cmd})
+		if err != nil {
+			log.Printf("[k3s] %q failed: %v (stdout: %s, stderr: %s)", cmd, err, out, stderr)
+		}
+	}
 }
 
 // Hostname returns the hostname of the [K3sInContainer].
@@ -643,6 +679,14 @@ func (k *K3sInContainer) DumpDiagnostics() {
 		{kubectlBin, "-n", kubeSystemNamespace, "logs", "-l", "k8s-app=kube-dns", "--tail=100"},
 		{kubectlBin, "-n", kubeSystemNamespace, "get", "endpoints", "kube-dns", "-o", "wide"},
 		{kubectlBin, "-n", kubeSystemNamespace, "get", "configmap", "coredns", "coredns-custom", "-o", "yaml"},
+		// Host network state behind a ClusterIP-unreachable DNS timeout: whether
+		// br_netfilter is loaded and the call-iptables/forward sysctls are on, and
+		// whether kube-proxy actually programmed the kube-dns DNAT rule. If CoreDNS
+		// is healthy (above) but these are missing, the fault is the Service DNAT
+		// path, not DNS.
+		{shellBin, "-c", "lsmod | grep -E 'br_netfilter|nf_conntrack' || echo 'br_netfilter NOT loaded'"},
+		{shellBin, "-c", "sysctl net.bridge.bridge-nf-call-iptables net.ipv4.ip_forward 2>&1 || true"},
+		{shellBin, "-c", "iptables-save -t nat 2>/dev/null | grep -iE 'KUBE-SERVICES|kube-dns|10.43.0.10' | head -40 || echo 'no kube-dns nat rules'"},
 	} {
 		out, stderr, err := k.Execute(c)
 		label := strings.Join(c, " ")
@@ -715,7 +759,7 @@ func (k *K3sInContainer) Shutdown() error {
 	// k3s spawns containerd and a tree of child processes inside this
 	// container; the bundled k3s-killall.sh tears them down. Best-effort: the
 	// Purge below removes the container regardless.
-	_, _, err = k.Execute([]string{"/bin/sh", "-c", "k3s-killall.sh || true"})
+	_, _, err = k.Execute([]string{shellBin, "-c", "k3s-killall.sh || true"})
 	if err != nil {
 		log.Printf("running k3s-killall in %s: %s", k.hostname, err)
 	}