kmt: add ubuntu 25.10 (#42172)

### What does this PR do?

- Adds Ubuntu 25.10 to KMT
- Removes Ubuntu 24.10
- Fixes flake detection to support multiple levels of nesting
- Adds a bunch of flakes for tests that newly fail on 25.10
- Adds flakes for other failing tests

### Motivation

Ensuring latest kernel compatibility

### Describe how you validated your changes

Existing tests.

### Additional Notes


Co-authored-by: bryce.kahle <bryce.kahle@datadoghq.com>

Author: brycekahle

.gitlab/test/kernel_matrix_testing/security_agent.yml

@@ -144,7 +144,7 @@ kmt_run_secagent_tests_x64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_4.14"
           - "amazon_5.4"
           - "amazon_5.10"
@@ -187,7 +187,7 @@ kmt_run_secagent_tests_x64_peds:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_4.14"
           - "amazon_5.4"
           - "amazon_5.10"
@@ -226,7 +226,7 @@ kmt_run_secagent_tests_x64_required:
   parallel:
     matrix:
       - TAG:
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
         TEST_SET: [cws_req]
   after_script:
     - !reference [.collect_outcomes_kmt]
@@ -291,7 +291,7 @@ kmt_run_secagent_tests_x64_docker:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_4.14"
           - "amazon_5.4"
           - "amazon_5.10"
@@ -328,7 +328,7 @@ kmt_run_secagent_tests_arm64:
       - TAG:
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_5.4"
           - "amazon_5.10"
           - "amazon_2023"
@@ -364,7 +364,7 @@ kmt_run_secagent_tests_arm64_peds:
       - TAG:
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_5.4"
           - "amazon_5.10"
           - "amazon_2023"
@@ -442,7 +442,7 @@ kmt_run_secagent_tests_arm64_docker:
       - TAG:
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_5.4"
           - "amazon_5.10"
           - "amazon_2023"

.gitlab/test/kernel_matrix_testing/system_probe.yml

@@ -229,7 +229,7 @@ kmt_run_sysprobe_tests_x64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_4.14"
           - "amazon_5.4"
           - "amazon_5.10"
@@ -269,7 +269,7 @@ kmt_run_sysprobe_tests_arm64:
           - "ubuntu_20.04"
           - "ubuntu_22.04"
           - "ubuntu_24.04"
-          - "ubuntu_24.10"
+          - "ubuntu_25.10"
           - "amazon_4.14"
           - "amazon_5.4"
           - "amazon_5.10"

pkg/collector/corechecks/ebpf/probe/oomkill/oom_kill_test.go

@@ -99,7 +99,7 @@ func TestOOMKillProbe(t *testing.T) {
 			return false
 		}, 10*time.Second, 500*time.Millisecond, "failed to find an OOM killed process with pid %d", cmd.Process.Pid)
 
-		assert.Regexp(t, regexp.MustCompile("run-([0-9|a-z]*).scope"), result.CgroupName, "cgroup name")
+		assert.Regexp(t, regexp.MustCompile(`run-.+\.scope`), result.CgroupName, "cgroup name")
 		assert.Equal(t, result.TriggerPid, result.VictimPid, "tpid == pid")
 		assert.NotZero(t, result.Score, "score")
 		assert.Equal(t, int16(42), result.ScoreAdj, "score adj")

pkg/ebpf/lockcontention_test.go

@@ -10,10 +10,11 @@ package ebpf
 import (
 	"testing"
 
-	"github.com/DataDog/datadog-agent/pkg/util/kernel"
-
 	"github.com/cilium/ebpf"
 	"github.com/stretchr/testify/require"
+
+	"github.com/DataDog/datadog-agent/pkg/util/kernel"
+	"github.com/DataDog/datadog-agent/pkg/util/testutil/flake"
 )
 
 type checkMap struct {
@@ -174,6 +175,7 @@ func TestLoadWithMaxTrackedRanges(t *testing.T) {
 	if !lockContentionCollectorSupported() {
 		t.Skip("EBPF lock contention collector not supported")
 	}
+	flake.MarkOnJobName(t, "ubuntu_25.10")
 
 	l := NewLockContentionCollector()
 	require.NotNil(t, l)

pkg/network/tracer/tracer_linux_test.go

@@ -2001,7 +2001,7 @@ func (s *TracerSuite) TestShortWrite() {
 		require.True(collect, ok)
 
 		require.Equal(collect, sent, conn.Monotonic.SentBytes)
-	}, 3*time.Second, 100*time.Millisecond, "couldn't find connection used by short write")
+	}, 10*time.Second, 100*time.Millisecond, "couldn't find connection used by short write")
 }
 
 func (s *TracerSuite) TestKprobeAttachWithKprobeEvents() {

pkg/network/usm/monitor_test.go

@@ -48,6 +48,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/process/util"
 	"github.com/DataDog/datadog-agent/pkg/util/kernel"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
+	"github.com/DataDog/datadog-agent/pkg/util/testutil/flake"
 )
 
 func TestMain(m *testing.M) {
@@ -202,6 +203,7 @@ func (s *HTTPTestSuite) TestHTTPMonitorLoadWithIncompleteBuffers() {
 
 func (s *HTTPTestSuite) TestHTTPMonitorIntegrationWithResponseBody() {
 	t := s.T()
+	flake.MarkOnJobName(t, "ubuntu_25.10")
 	serverAddr := "localhost:8080"
 
 	tests := []struct {

pkg/network/usm/usm_grpc_monitor_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/network/usm/testutil/grpc"
 	"github.com/DataDog/datadog-agent/pkg/network/usm/utils"
 	"github.com/DataDog/datadog-agent/pkg/util/kernel"
+	"github.com/DataDog/datadog-agent/pkg/util/testutil/flake"
 )
 
 const (
@@ -438,6 +439,7 @@ func (s *usmGRPCSuite) TestLargeBodiesGRPCScenarios() {
 	if s.isTLS {
 		t.Skip("Skipping TestLargeBodiesGRPCScenarios for TLS due to flakiness")
 	}
+	flake.MarkOnJobName(t, "ubuntu_25.10")
 
 	srv, cancel := grpc.NewGRPCTLSServer(t, srvAddr, s.isTLS)
 	t.Cleanup(cancel)

pkg/process/monitor/process_monitor_test.go

@@ -8,7 +8,6 @@
 package monitor
 
 import (
-	"os"
 	"os/exec"
 	"sync"
 	"testing"
@@ -21,7 +20,6 @@ import (
 
 	"github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers/testutil"
 	"github.com/DataDog/datadog-agent/pkg/network/protocols/telemetry"
-	"github.com/DataDog/datadog-agent/pkg/util/filesystem"
 	netnsutil "github.com/DataDog/datadog-agent/pkg/util/kernel/netns"
 )
 
@@ -113,17 +111,6 @@ func registerCallback(t *testing.T, pm *ProcessMonitor, isExec bool, callback *P
 	return unsubscribe
 }
 
-func getTestBinaryPath(t *testing.T) string {
-	tmpFile, err := os.CreateTemp("", "echo")
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		os.Remove(tmpFile.Name())
-	})
-	require.NoError(t, filesystem.CopyFile("/bin/echo", tmpFile.Name()))
-
-	return tmpFile.Name()
-}
-
 func TestProcessMonitorSingleton(t *testing.T) {
 	// Making sure we get the same process monitor if we call it twice.
 	pm := getProcessMonitor(t)
@@ -140,7 +127,6 @@ type processMonitorSuite struct {
 func (s *processMonitorSuite) TestProcessMonitorSanity() {
 	t := s.T()
 	pm := getProcessMonitor(t)
-	testBinaryPath := getTestBinaryPath(t)
 
 	execRecorder := newPidRecorder()
 	registerCallback(t, pm, true, getProcessCallback(execRecorder))
@@ -149,7 +135,7 @@ func (s *processMonitorSuite) TestProcessMonitorSanity() {
 	registerCallback(t, pm, false, getProcessCallback(exitRecorder))
 
 	initializePM(t, pm, s.useEventStream)
-	cmd := exec.Command(testBinaryPath, "test")
+	cmd := exec.Command("/bin/echo", "test")
 	require.NoError(t, cmd.Run())
 	require.EventuallyWithT(t, func(ct *assert.CollectT) {
 		assert.Truef(ct, execRecorder.has(uint32(cmd.Process.Pid)), "didn't capture exec event %d", cmd.Process.Pid)

pkg/security/tests/action_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/security/secl/model"
 	"github.com/DataDog/datadog-agent/pkg/security/secl/rules"
 	"github.com/DataDog/datadog-agent/pkg/security/utils"
+	"github.com/DataDog/datadog-agent/pkg/util/testutil/flake"
 )
 
 func TestActionKill(t *testing.T) {
@@ -749,6 +750,7 @@ func TestActionHash(t *testing.T) {
 	})
 
 	t.Run("exec", func(t *testing.T) {
+		flake.MarkOnJobName(t, "ubuntu_25.10")
 		test.msgSender.flush()
 		test.WaitSignalFromRule(t, func() error {
 			cmd := exec.Command(testExecutable, "/tmp/aaa")
@@ -950,6 +952,7 @@ func TestActionKillWithSignature(t *testing.T) {
 
 func TestActionKillContainerWithSignature(t *testing.T) {
 	SkipIfNotAvailable(t)
+	flake.MarkOnJobName(t, "cws_host")
 
 	if testEnvironment == DockerEnvironment {
 		t.Skip("Skip test spawning docker containers on docker")
@@ -1156,6 +1159,7 @@ func TestActionKillContainerWithSignature(t *testing.T) {
 
 func TestActionKillContainerWithSignatureBroadRule(t *testing.T) {
 	SkipIfNotAvailable(t)
+	flake.MarkOnJobName(t, "cws_host")
 
 	if testEnvironment == DockerEnvironment {
 		t.Skip("Skip test spawning docker containers on docker")

pkg/security/tests/mount_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/security/secl/rules"
 	"github.com/DataDog/datadog-agent/pkg/security/utils"
 	"github.com/DataDog/datadog-agent/pkg/util/kernel"
+	"github.com/DataDog/datadog-agent/pkg/util/testutil/flake"
 )
 
 func TestMount(t *testing.T) {
@@ -537,6 +538,7 @@ func TestMountEvent(t *testing.T) {
 
 	t.Run("mount-in-container-root", func(t *testing.T) {
 		SkipIfNotAvailable(t)
+		flake.MarkOnJobName(t, "ubuntu_25.10")
 
 		if _, err := whichNonFatal("docker"); err != nil {
 			t.Skip("Skip test where docker is unavailable")