fix(cluster-agent): gate pprof/expvar debug endpoints to loopback (#54508)

### What does this PR do?

The Cluster Agent's metrics server (`0.0.0.0:metrics_port`, default `5000`) served the entire `http.DefaultServeMux`, exposing the `pprof` and `expvar` debug endpoints (registered via blank imports in `cmd/cluster-agent/main.go`) unauthenticated to anything that can reach the pod. This routes only `/metrics` on the public mux and gates everything under `/debug/` to loopback callers, returning `404` to non-loopback requests.

### Motivation

Solves #CONTP-1771 and #VULN-87907.
`/metrics` must stay reachable off-pod so the node Agent can scrape Cluster Agent telemetry, so a blanket localhost bind (like the core Agent's expvar server) isn't viable — hence a per-path loopback gate. This removes an unauthenticated, network-reachable debug/DoS surface while preserving telemetry scraping and local flare tooling.

### Describe how you validated your changes

- Unit tests (`TestLoopbackOnly`, `TestMetricsMuxRouting`) covering loopback vs off-host for `/metrics` and `/debug/*` — 16/16 pass.
- `dda inv cluster-agent.build` links clean; `dda inv linter.go` clean; release-note lint clean.
- Generated a flare.
- Deployed the built cluster-agent image to a local minikube cluster and curled the live DCA pod:
  - **In-pod** (loopback `127.0.0.1:5000`): `/metrics`, `/debug/vars`, `/debug/pprof/*` all `200` — flare/profiler unaffected.
  - **Off-pod** (from a separate pod → DCA pod IP): `/debug/*` = `404`, `/metrics` = `200`.
- Verified nothing else is registered on the DCA's `DefaultServeMux` (only `pprof`/`expvar`), so no routes are dropped.

### Additional Notes

Brings the DCA in line with the core Agent, which already binds its equivalent `pprof`/`expvar` server to `127.0.0.1`, while keeping `/metrics` public.

Co-authored-by: wassim.dhif <wassim.dhif@datadoghq.com>

Author: wdhif

cmd/cluster-agent/subcommands/start/BUILD.bazel

@@ -2163,66 +2163,79 @@ dd_agent_go_test(
         "@rules_go//go/platform:aix": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:android": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:dragonfly": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:freebsd": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:illumos": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:js": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:linux": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:netbsd": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:openbsd": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:osx": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:plan9": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:qnx": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "@rules_go//go/platform:solaris": [
             "//cmd/cluster-agent/command",
             "//pkg/util/fxutil",
+            "@com_github_stretchr_testify//assert",
             "@com_github_stretchr_testify//require",
         ],
         "//conditions:default": [],

cmd/cluster-agent/subcommands/start/command.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -332,12 +333,19 @@ func start(log log.Component,
 		return errors.New("no API key configured, exiting")
 	}
 
-	// Expose the registered metrics via HTTP.
-	http.Handle("/metrics", telemetry.Handler())
+	// Expose the registered metrics via HTTP. /metrics must stay reachable on
+	// all interfaces so the node agent can scrape cluster-agent telemetry, but
+	// the pprof/expvar handlers that main.go registers on http.DefaultServeMux
+	// must not: gate everything under /debug/ to loopback callers only.
+	metricsMux := http.NewServeMux()
+	metricsMux.Handle("/metrics", telemetry.Handler())
+	debugHandler := loopbackOnly(http.DefaultServeMux)
+	metricsMux.Handle("/debug", debugHandler)
+	metricsMux.Handle("/debug/", debugHandler)
 	metricsPort := config.GetInt("metrics_port")
 	metricsServer := &http.Server{
 		Addr:    fmt.Sprintf("0.0.0.0:%d", metricsPort),
-		Handler: http.DefaultServeMux,
+		Handler: metricsMux,
 	}
 
 	go func() {
@@ -776,6 +784,25 @@ func start(log log.Component,
 	return nil
 }
 
+// loopbackOnly serves h only for requests originating from a loopback address;
+// any other client receives a 404. The check uses the transport-level
+// RemoteAddr, not forwardable headers, so it cannot be spoofed by an off-host
+// caller. This keeps the pprof/expvar debug handlers reachable for local
+// tooling (e.g. the cluster-agent flare) while hiding them on the network.
+func loopbackOnly(h http.Handler) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		host, _, err := net.SplitHostPort(r.RemoteAddr)
+		if err != nil {
+			host = r.RemoteAddr
+		}
+		if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
+			http.NotFound(w, r)
+			return
+		}
+		h.ServeHTTP(w, r)
+	}
+}
+
 func setupInstrumentationCRDHandler(le *leaderelection.LeaderEngine, ac autodiscovery.Component, serviceTemplateStore *instrumentationhandlers.ServiceCheckTemplateStore) []instrumentation.Handler {
 	checkStore := instrumentationhandlers.NewCheckStore()
 	instrHandlers := instrumentationhandlers.DefaultHandlers(&instrumentationhandlers.Deps{

cmd/cluster-agent/subcommands/start/command_test.go

@@ -8,10 +8,14 @@
 package start
 
 import (
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/DataDog/datadog-agent/cmd/cluster-agent/command"
@@ -38,3 +42,93 @@ func newGlobalParamsTest(t *testing.T) *command.GlobalParams {
 		ConfFilePath: config,
 	}
 }
+
+func TestLoopbackOnly(t *testing.T) {
+	handler := loopbackOnly(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = io.WriteString(w, "served")
+	}))
+
+	tests := []struct {
+		name       string
+		remoteAddr string
+		wantServed bool
+	}{
+		{"IPv4 loopback", "127.0.0.1:34567", true},
+		{"IPv4 loopback range", "127.0.0.5:34567", true},
+		{"IPv6 loopback", "[::1]:34567", true},
+		{"RemoteAddr without port", "127.0.0.1", true},
+		{"private IPv4 off-pod", "10.0.0.5:34567", false},
+		{"docker bridge gateway", "172.17.0.1:34567", false},
+		{"pod IP", "192.168.49.2:34567", false},
+		{"public IPv4", "8.8.8.8:34567", false},
+		{"unparseable RemoteAddr", "garbage", false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/debug/vars", nil)
+			req.RemoteAddr = tc.remoteAddr
+			rec := httptest.NewRecorder()
+			handler.ServeHTTP(rec, req)
+			if tc.wantServed {
+				assert.Equal(t, http.StatusOK, rec.Code)
+				assert.Equal(t, "served", rec.Body.String())
+			} else {
+				assert.Equal(t, http.StatusNotFound, rec.Code)
+				assert.NotContains(t, rec.Body.String(), "served")
+			}
+		})
+	}
+}
+
+// TestMetricsMuxRouting mirrors the mux wired in start(): /metrics is public on
+// all interfaces, everything under /debug/ is loopback-only.
+func TestMetricsMuxRouting(t *testing.T) {
+	debug := http.NewServeMux()
+	debug.HandleFunc("/debug/vars", func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = io.WriteString(w, "vars")
+	})
+	debug.HandleFunc("/debug/pprof/", func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = io.WriteString(w, "pprof")
+	})
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/metrics", func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = io.WriteString(w, "metrics")
+	})
+	debugHandler := loopbackOnly(debug)
+	mux.Handle("/debug", debugHandler)
+	mux.Handle("/debug/", debugHandler)
+
+	const offPod = "10.0.0.5:41000"
+	const loopback = "127.0.0.1:41000"
+
+	tests := []struct {
+		name       string
+		path       string
+		remoteAddr string
+		wantCode   int
+		wantBody   string
+	}{
+		{"metrics reachable off-pod", "/metrics", offPod, http.StatusOK, "metrics"},
+		{"metrics reachable loopback", "/metrics", loopback, http.StatusOK, "metrics"},
+		{"debug root hidden off-pod", "/debug", offPod, http.StatusNotFound, ""},
+		{"expvar hidden off-pod", "/debug/vars", offPod, http.StatusNotFound, ""},
+		{"expvar served loopback", "/debug/vars", loopback, http.StatusOK, "vars"},
+		{"pprof hidden off-pod", "/debug/pprof/heap", offPod, http.StatusNotFound, ""},
+		{"pprof served loopback", "/debug/pprof/heap", loopback, http.StatusOK, "pprof"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, tc.path, nil)
+			req.RemoteAddr = tc.remoteAddr
+			rec := httptest.NewRecorder()
+			mux.ServeHTTP(rec, req)
+			assert.Equal(t, tc.wantCode, rec.Code)
+			if tc.wantBody != "" {
+				assert.Equal(t, tc.wantBody, rec.Body.String())
+			}
+		})
+	}
+}

releasenotes-dca/notes/cluster-agent-debug-endpoints-loopback-700c3c5e57f93e6d.yaml

@@ -0,0 +1,12 @@
+---
+security:
+  - |
+    The Cluster Agent no longer exposes the Go ``pprof`` profiling and
+    ``expvar`` debug endpoints on its metrics port (``metrics_port``, default
+    ``5000``) to remote callers. These ``/debug/`` endpoints were previously
+    served on all network interfaces without authentication; they are now
+    restricted to loopback callers, and requests originating from any other
+    address receive a ``404``. The ``/metrics`` endpoint is unchanged and
+    remains reachable off-host so the node Agent can continue to scrape
+    Cluster Agent telemetry. Local tooling such as the Cluster Agent flare,
+    which connects over loopback, is unaffected.