[USM] Add check-maps subcommand for eBPF map leak detection (#43103)

### What does this PR do?
This PR adds a new `usm check-maps` subcommand to system-probe that detects eBPF map leaks in USM TLS monitoring. It also refactors PID-keyed TLS map name management to use a single source of truth with reflection-based automatic discovery.

### Motivation
USM's TLS monitoring uses several eBPF maps keyed by pid_tgid (process ID + thread ID). When processes terminate unexpectedly or the cleanup mechanisms fail, entries can leak in these maps. These leaks:
  - Consume kernel memory
  - Degrade performance as maps grow
  - Are difficult to detect and diagnose in production

The solution involves a a diagnostic tool that enumerates all PID-keyed TLS maps, validates each entry against running processes, and reports detailed leak statistics. 

### Describe how you validated your changes

### Additional Notes
This approach can be extended to other map types beyond PID-keyed maps. For example, maps keyed by connection tuples could use a similar pattern with validation logic that checks if the connection still exists in the system.


Co-authored-by: daniel.lavie <daniel.lavie@datadoghq.com>

Author: Daniel Lavie

cmd/system-probe/subcommands/usm/checkmaps.go

@@ -0,0 +1,53 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2025-present Datadog, Inc.
+
+//go:build linux_bpf
+
+package usm
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/DataDog/datadog-agent/cmd/system-probe/command"
+	"github.com/DataDog/datadog-agent/pkg/network/usm/maps"
+)
+
+func makeCheckMapsCommand(_ *command.GlobalParams) *cobra.Command {
+	return &cobra.Command{
+		Use:   "check-maps",
+		Short: "Check USM eBPF maps for leaked entries",
+		Long: `Check USM eBPF maps for leaked entries by validating map keys against system state.
+
+For PID-keyed maps (TLS/SSL argument storage), this command:
+  - Extracts PIDs from map keys
+  - Checks if processes still exist in /proc
+  - Reports entries where the process no longer exists (leaked entries)
+
+This is useful for diagnosing memory leaks in customer environments where
+eBPF map entries are not being properly cleaned up.`,
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return runCheckMaps()
+		},
+	}
+}
+
+func runCheckMaps() error {
+	report, err := maps.CheckPIDKeyedMaps()
+	if err != nil {
+		return fmt.Errorf("failed to check maps: %w", err)
+	}
+
+	if report.TotalMapsChecked == 0 {
+		fmt.Println("No USM eBPF maps found. Is system-probe running with USM enabled?")
+		return nil
+	}
+
+	// Print the report
+	fmt.Print(report.String())
+
+	return nil
+}

cmd/system-probe/subcommands/usm/checkmaps_unsupported.go

@@ -0,0 +1,19 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2025-present Datadog, Inc.
+
+//go:build !linux_bpf
+
+package usm
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/DataDog/datadog-agent/cmd/system-probe/command"
+)
+
+// makeCheckMapsCommand returns nil on platforms without eBPF support
+func makeCheckMapsCommand(_ *command.GlobalParams) *cobra.Command {
+	return nil
+}

cmd/system-probe/subcommands/usm/command.go

@@ -27,5 +27,10 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 		usmCmd.AddCommand(sysinfoCmd)
 	}
 
+	// Add check-maps command if available on this platform
+	if checkMapsCmd := makeCheckMapsCommand(globalParams); checkMapsCmd != nil {
+		usmCmd.AddCommand(checkMapsCmd)
+	}
+
 	return []*cobra.Command{usmCmd}
 }

pkg/network/usm/ebpf_ssl.go

@@ -10,6 +10,7 @@ package usm
 import (
 	"fmt"
 	"io"
+	"reflect"
 	"regexp"
 	"time"
 	"unsafe"
@@ -81,6 +82,46 @@ const (
 	fdBySSLBioMap       = "fd_by_ssl_bio"
 )
 
+// pidKeyedTLSMaps is the single source of truth for all TLS eBPF maps that use pid_tgid as keys.
+//
+// IMPORTANT: Map names must be unique within their first 15 characters due to kernel truncation
+// (BPF_OBJ_NAME_LEN - 1). The leak detection system searches maps by truncated names, so names
+// like "hash_map_name_10" and "hash_map_name_11" would collide as both truncate to "hash_map_name_1".
+//
+// When adding a new PID-keyed map:
+// 1. Add a new field to this struct with the map name as the value
+// 2. Ensure the name is unique within the first 15 characters
+// 3. Add the corresponding map cleaner field to sslProgram struct
+// 4. Initialize it in initAllMapCleaners() with uint64 key type
+// The GetPIDKeyedTLSMapNames() function will automatically include it via reflection.
+var pidKeyedTLSMaps = struct {
+	SSLReadArgs      string
+	SSLReadExArgs    string
+	SSLWriteArgs     string
+	SSLWriteExArgs   string
+	BioNewSocketArgs string
+	SSLCtxByPIDTGID  string
+}{
+	SSLReadArgs:      "ssl_read_args",
+	SSLReadExArgs:    "ssl_read_ex_args",
+	SSLWriteArgs:     "ssl_write_args",
+	SSLWriteExArgs:   "ssl_write_ex_args",
+	BioNewSocketArgs: "bio_new_socket_args",
+	SSLCtxByPIDTGID:  "ssl_ctx_by_pid_tgid",
+}
+
+// GetPIDKeyedTLSMapNames returns the names of all TLS eBPF maps that use pid_tgid as keys.
+// It uses reflection to extract all field values from pidKeyedTLSMaps struct.
+// This ensures the list is automatically kept in sync with the struct definition.
+func GetPIDKeyedTLSMapNames() []string {
+	v := reflect.ValueOf(pidKeyedTLSMaps)
+	names := make([]string, v.NumField())
+	for i := 0; i < v.NumField(); i++ {
+		names[i] = v.Field(i).String()
+	}
+	return names
+}
+
 var openSSLProbes = []manager.ProbesSelector{
 	&manager.BestEffort{
 		Selectors: []manager.ProbesSelector{
@@ -254,25 +295,25 @@ var sharedLibrariesMaps = []*manager.Map{
 		Name: sslCtxByTupleMap,
 	},
 	{
-		Name: sslReadArgsMap,
+		Name: pidKeyedTLSMaps.SSLReadArgs,
 	},
 	{
-		Name: sslReadExArgsMap,
+		Name: pidKeyedTLSMaps.SSLReadExArgs,
 	},
 	{
-		Name: sslWriteArgsMap,
+		Name: pidKeyedTLSMaps.SSLWriteArgs,
 	},
 	{
-		Name: sslWriteExArgsMap,
+		Name: pidKeyedTLSMaps.SSLWriteExArgs,
 	},
 	{
-		Name: bioNewSocketArgsMap,
+		Name: pidKeyedTLSMaps.BioNewSocketArgs,
 	},
 	{
 		Name: fdBySSLBioMap,
 	},
 	{
-		Name: sslCtxByPIDTGIDMap,
+		Name: pidKeyedTLSMaps.SSLCtxByPIDTGID,
 	},
 }
 
@@ -538,7 +579,7 @@ func sharedLibrariesConfigureOptions(options *manager.Options, cfg *config.Confi
 		MaxEntries: cfg.MaxTrackedConnections,
 		EditorFlag: manager.EditMaxEntries,
 	}
-	options.MapSpecEditors[sslCtxByPIDTGIDMap] = manager.MapSpecEditor{
+	options.MapSpecEditors[pidKeyedTLSMaps.SSLCtxByPIDTGID] = manager.MapSpecEditor{
 		MaxEntries: cfg.MaxTrackedConnections,
 		EditorFlag: manager.EditMaxEntries,
 	}
@@ -570,32 +611,32 @@ func initMapCleaner[K, V interface{}](mgr *manager.Manager, mapName, attacherNam
 func (o *sslProgram) initAllMapCleaners() error {
 	var err error
 
-	o.sslReadArgsMapCleaner, err = initMapCleaner[uint64, http.SslReadArgs](o.ebpfManager, sslReadArgsMap, UsmTLSAttacherName)
+	o.sslReadArgsMapCleaner, err = initMapCleaner[uint64, http.SslReadArgs](o.ebpfManager, pidKeyedTLSMaps.SSLReadArgs, UsmTLSAttacherName)
 	if err != nil {
 		return err
 	}
 
-	o.sslReadExArgsMapCleaner, err = initMapCleaner[uint64, http.SslReadExArgs](o.ebpfManager, sslReadExArgsMap, UsmTLSAttacherName)
+	o.sslReadExArgsMapCleaner, err = initMapCleaner[uint64, http.SslReadExArgs](o.ebpfManager, pidKeyedTLSMaps.SSLReadExArgs, UsmTLSAttacherName)
 	if err != nil {
 		return err
 	}
 
-	o.sslWriteArgsMapCleaner, err = initMapCleaner[uint64, http.SslWriteArgs](o.ebpfManager, sslWriteArgsMap, UsmTLSAttacherName)
+	o.sslWriteArgsMapCleaner, err = initMapCleaner[uint64, http.SslWriteArgs](o.ebpfManager, pidKeyedTLSMaps.SSLWriteArgs, UsmTLSAttacherName)
 	if err != nil {
 		return err
 	}
 
-	o.sslWriteExArgsMapCleaner, err = initMapCleaner[uint64, http.SslWriteExArgs](o.ebpfManager, sslWriteExArgsMap, UsmTLSAttacherName)
+	o.sslWriteExArgsMapCleaner, err = initMapCleaner[uint64, http.SslWriteExArgs](o.ebpfManager, pidKeyedTLSMaps.SSLWriteExArgs, UsmTLSAttacherName)
 	if err != nil {
 		return err
 	}
 
-	o.bioNewSocketArgsMapCleaner, err = initMapCleaner[uint64, uint32](o.ebpfManager, bioNewSocketArgsMap, UsmTLSAttacherName)
+	o.bioNewSocketArgsMapCleaner, err = initMapCleaner[uint64, uint32](o.ebpfManager, pidKeyedTLSMaps.BioNewSocketArgs, UsmTLSAttacherName)
 	if err != nil {
 		return err
 	}
 
-	o.sslCtxByPIDTGIDMapCleaner, err = initMapCleaner[uint64, uint64](o.ebpfManager, sslCtxByPIDTGIDMap, UsmTLSAttacherName)
+	o.sslCtxByPIDTGIDMapCleaner, err = initMapCleaner[uint64, uint64](o.ebpfManager, pidKeyedTLSMaps.SSLCtxByPIDTGID, UsmTLSAttacherName)
 	if err != nil {
 		return err
 	}
@@ -651,7 +692,7 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map)
 			spew.Fdump(w, key, value)
 		}
 
-	case sslReadArgsMap: // maps/ssl_read_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_read_args_t
+	case pidKeyedTLSMaps.SSLReadArgs: // maps/ssl_read_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_read_args_t
 		io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_read_args_t'\n")
 		iter := currentMap.Iterate()
 		var key uint64
@@ -660,7 +701,7 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map)
 			spew.Fdump(w, key, value)
 		}
 
-	case sslReadExArgsMap: // maps/ssl_read_ex_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_read_ex_args_t
+	case pidKeyedTLSMaps.SSLReadExArgs: // maps/ssl_read_ex_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_read_ex_args_t
 		io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_read_ex_args_t'\n")
 		iter := currentMap.Iterate()
 		var key uint64
@@ -669,7 +710,7 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map)
 			spew.Fdump(w, key, value)
 		}
 
-	case sslWriteArgsMap: // maps/ssl_write_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_write_args_t
+	case pidKeyedTLSMaps.SSLWriteArgs: // maps/ssl_write_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_write_args_t
 		io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_write_args_t'\n")
 		iter := currentMap.Iterate()
 		var key uint64
@@ -678,7 +719,7 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map)
 			spew.Fdump(w, key, value)
 		}
 
-	case sslWriteExArgsMap: // maps/ssl_write_ex_args_t (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_write_args_t
+	case pidKeyedTLSMaps.SSLWriteExArgs: // maps/ssl_write_ex_args_t (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_write_args_t
 		io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_write_ex_args_t'\n")
 		iter := currentMap.Iterate()
 		var key uint64
@@ -687,7 +728,7 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map)
 			spew.Fdump(w, key, value)
 		}
 
-	case bioNewSocketArgsMap: // maps/bio_new_socket_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.__u32
+	case pidKeyedTLSMaps.BioNewSocketArgs: // maps/bio_new_socket_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.__u32
 		io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.__u32'\n")
 		iter := currentMap.Iterate()
 		var key uint64
@@ -705,7 +746,7 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map)
 			spew.Fdump(w, key, value)
 		}
 
-	case sslCtxByPIDTGIDMap: // maps/ssl_ctx_by_pid_tgid (BPF_MAP_TYPE_HASH), key C.__u64, value uintptr // C.void *
+	case pidKeyedTLSMaps.SSLCtxByPIDTGID: // maps/ssl_ctx_by_pid_tgid (BPF_MAP_TYPE_HASH), key C.__u64, value uintptr // C.void *
 		io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'uintptr // C.void *'\n")
 		iter := currentMap.Iterate()
 		var key uint64
@@ -797,7 +838,7 @@ func (o *sslProgram) cleanupDeadPids(alivePIDs map[uint32]struct{}) {
 	})
 
 	if err := o.deleteDeadPidsInSSLCtxMap(alivePIDs); err != nil {
-		log.Debugf("SSL map %q cleanup error: %v", sslCtxByPIDTGIDMap, err)
+		log.Debugf("SSL map %q cleanup error: %v", pidKeyedTLSMaps.SSLCtxByPIDTGID, err)
 	}
 }
 

pkg/network/usm/ebpf_ssl_test.go

@@ -311,3 +311,47 @@ func TestSSLMapsCleanup(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+// TestPIDKeyedMapNameUniqueness verifies that all PID-keyed TLS map names are unique
+// within their first 15 characters to prevent collisions from kernel truncation.
+//
+// eBPF map names are limited to 15 characters by the kernel (BPF_OBJ_NAME_LEN - 1).
+// The leak detection system searches maps by truncated names, so names like
+// "hash_map_name_10" and "hash_map_name_11" would collide as both truncate to
+// "hash_map_name_1".
+//
+// This test ensures we catch such collisions at compile/test time rather than
+// discovering them in production.
+func TestPIDKeyedMapNameUniqueness(t *testing.T) {
+	names := GetPIDKeyedTLSMapNames()
+	require.NotEmpty(t, names, "No PID-keyed map names found")
+
+	truncated := make(map[string]string)
+	for _, name := range names {
+		truncName := name
+		if len(name) > 15 {
+			truncName = name[:15]
+		}
+
+		if existing, found := truncated[truncName]; found {
+			t.Errorf("Map name collision detected:\n"+
+				"  Map 1: %q\n"+
+				"  Map 2: %q\n"+
+				"  Both truncate to: %q\n"+
+				"Map names must be unique within their first 15 characters due to kernel limitation (BPF_OBJ_NAME_LEN - 1).",
+				existing, name, truncName)
+		}
+		truncated[truncName] = name
+	}
+
+	// Log all truncated names for reference
+	t.Logf("Current PID-keyed TLS map names and their truncated forms:")
+	for _, name := range names {
+		if len(name) > 15 {
+			truncName := name[:15]
+			t.Logf("  %q -> %q (truncated)", name, truncName)
+		} else {
+			t.Logf("  %q (no truncation)", name)
+		}
+	}
+}