integration: gate operator install on cluster DNS and dump CoreDNS state

TestK8sOperator failed on the arm64 CI runner with the operator crashlooping
on "lookup kubernetes.default.svc ... i/o timeout": in-cluster DNS was not
servable when the operator started, but the only signal was a downstream
crashloop with no view of CoreDNS itself.

Wait for CoreDNS to roll out and the kube-dns Service to have a ready endpoint
before installing the operator, so a DNS-dependent workload never starts ahead
of name resolution. If DNS never becomes servable the test now fails at that
exact boundary instead of as an opaque operator crash.

Extend the diagnostics dump with CoreDNS pod status, CoreDNS logs (Corefile
parse errors surface there), the kube-dns endpoints, and the Corefile plus
custom config, so a DNS failure shows whether CoreDNS is down or the Service
has no backend rather than leaving it to guesswork.

The test passes locally; this targets the environment-specific DNS failure
seen only on the arm64 runner.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: kradalby

integration/k3sic/k3sic.go

@@ -97,6 +97,9 @@ const (
 	// tailscaleNamespace is where the operator and its proxies are installed.
 	tailscaleNamespace = "tailscale"
 
+	// kubeSystemNamespace holds CoreDNS and the rest of the k3s system addons.
+	kubeSystemNamespace = "kube-system"
+
 	// dockerImportTimeout bounds the host-side build/export and the in-container
 	// containerd import, which move multi-hundred-MB image tarballs.
 	dockerImportTimeout = 600 * time.Second
@@ -107,6 +110,8 @@ var (
 	errHelmNotInTarball = errors.New("helm binary not found in release tarball")
 	errNoCACert         = errors.New("no CA certificate configured; pass k3sic.WithCACert")
 	errImageBuild       = errors.New("building CA-baked image failed")
+
+	errNoKubeDNSEndpoints = errors.New("kube-dns Service has no ready endpoints yet")
 )
 
 // K3sInContainer represents a k3s cluster running in a single privileged
@@ -525,16 +530,17 @@ RUN update-ca-certificates
 //
 // It installs a coredns-custom ConfigMap (a k3s-native extension point: keys
 // ending in .server are imported as additional server blocks) holding a hosts
-// block for the Headscale name, then restarts CoreDNS so the imported file is
-// loaded (CoreDNS's reload watches only the Corefile, not imported files).
+// block for the Headscale name, which CoreDNS's reload plugin picks up. It then
+// waits for in-cluster DNS to be servable so a DNS-dependent workload (the
+// operator) does not start before resolution works.
 func (k *K3sInContainer) ConfigureCoreDNSHost(hostname, ip string) error {
 	// k3s deploys CoreDNS via its addon manager shortly after the node reports
 	// Ready, so WaitForRunning can return before the deployment exists. Wait for
 	// it before patching, otherwise the restart below races and fails with
 	// "deployments.apps \"coredns\" not found".
 	err := k.pool.Retry(func() error {
 		_, stderr, err := k.Execute([]string{
-			kubectlBin, "-n", "kube-system", "get", "deployment", "coredns",
+			kubectlBin, "-n", kubeSystemNamespace, "get", "deployment", "coredns",
 		})
 		if err != nil {
 			return fmt.Errorf("coredns deployment not present yet (stderr: %s): %w", stderr, err)
@@ -573,14 +579,48 @@ data:
 		return fmt.Errorf("applying coredns-custom (stderr: %s): %w", stderr, err)
 	}
 
-	// Deliberately NO `rollout restart deployment/coredns`: on a slower CI runner
-	// that restart can leave CoreDNS unable to serve in-cluster DNS
-	// (kubernetes.default.svc) for minutes, which crashes the operator before it
-	// ever reaches Headscale. k3s's CoreDNS runs the `reload` plugin and the
-	// ConfigMap is mounted into the pod, so the custom block is picked up
-	// automatically (kubelet sync + reload, well within the operator's install
-	// and registration windows) without disrupting the existing resolver.
-	return nil
+	// Deliberately NO `rollout restart deployment/coredns`: k3s's CoreDNS runs the
+	// `reload` plugin and the ConfigMap is mounted into the pod, so the imported
+	// block is picked up automatically without disrupting the running resolver.
+	//
+	// Gate on in-cluster DNS being servable before returning. The operator's very
+	// first action is a kube API call resolved via kubernetes.default.svc, so if
+	// CoreDNS is not Ready or the kube-dns Service has no endpoints, the operator
+	// crashloops on a DNS timeout with no hint as to why. Failing here instead
+	// pins the cause at its source (DumpDiagnostics then dumps CoreDNS state).
+	return k.waitForClusterDNS()
+}
+
+// waitForClusterDNS blocks until CoreDNS is rolled out and the kube-dns Service
+// has at least one ready endpoint — i.e. in-cluster name resolution is actually
+// servable, which every workload (starting with the operator) depends on.
+func (k *K3sInContainer) waitForClusterDNS() error {
+	_, stderr, err := k.Execute([]string{
+		kubectlBin, "-n", kubeSystemNamespace, "rollout", "status",
+		"deployment/coredns", "--timeout=150s",
+	})
+	if err != nil {
+		return fmt.Errorf("coredns did not become available (stderr: %s): %w", stderr, err)
+	}
+
+	return k.pool.Retry(func() error {
+		// A populated endpoint set means a CoreDNS pod is serving :53 and
+		// kube-proxy has a backend to DNAT the kube-dns ClusterIP to; empty means
+		// in-cluster lookups will time out no matter how long a client waits.
+		out, stderr, err := k.Execute([]string{
+			kubectlBin, "-n", kubeSystemNamespace, "get", "endpoints", "kube-dns",
+			"-o", "jsonpath={.subsets[*].addresses[*].ip}",
+		})
+		if err != nil {
+			return fmt.Errorf("reading kube-dns endpoints (stderr: %s): %w", stderr, err)
+		}
+
+		if strings.TrimSpace(out) == "" {
+			return errNoKubeDNSEndpoints
+		}
+
+		return nil
+	})
 }
 
 // DumpDiagnostics logs cluster state useful for debugging a failed operator
@@ -595,6 +635,14 @@ func (k *K3sInContainer) DumpDiagnostics() {
 		// A crashlooping operator's fatal error is in the previous container.
 		{kubectlBin, "-n", tailscaleNamespace, "logs", "deployment/operator", "--previous", "--tail=200"},
 		{kubectlBin, "-n", tailscaleNamespace, "get", "statefulsets,pods", "-o", "wide"},
+		// In-cluster DNS: the operator's first dependency. A "lookup
+		// kubernetes.default.svc ... i/o timeout" crash means CoreDNS is not
+		// serving, so capture its pod state, logs (Corefile parse errors land
+		// here), the Service endpoints, and the Corefile + custom config.
+		{kubectlBin, "-n", kubeSystemNamespace, "get", "pods", "-l", "k8s-app=kube-dns", "-o", "wide"},
+		{kubectlBin, "-n", kubeSystemNamespace, "logs", "-l", "k8s-app=kube-dns", "--tail=100"},
+		{kubectlBin, "-n", kubeSystemNamespace, "get", "endpoints", "kube-dns", "-o", "wide"},
+		{kubectlBin, "-n", kubeSystemNamespace, "get", "configmap", "coredns", "coredns-custom", "-o", "yaml"},
 	} {
 		out, stderr, err := k.Execute(c)
 		label := strings.Join(c, " ")