[9.1] (backport #11254) Use unique index for TestOtelLogsIngestion

.gitignore

@@ -59,4 +59,5 @@ fleet.yml
 fleet.yml.lock
 fleet.yml.old
 pkg/component/fake/component/component
+pkg/core/process/testsignal/testsignal
 internal/pkg/agent/install/testblocking/testblocking

magefile.go

@@ -2246,7 +2246,9 @@ func (Integration) Local(ctx context.Context, testName string) error {
 	// run the integration tests but only run test that can run locally
 	params := devtools.DefaultGoTestIntegrationArgs()
 	params.Tags = append(params.Tags, "local")
-	params.Packages = []string{"github.com/elastic/elastic-agent/testing/integration"}
+	params.Packages = []string{
+		"github.com/elastic/elastic-agent/testing/integration/...",
+	}
 
 	var goTestFlags []string
 	rawTestFlags := os.Getenv("GOTEST_FLAGS")

testing/integration/ess/otel_test.go

@@ -463,6 +463,15 @@ exporters:
   elasticsearch:
     api_key: {{.ESApiKey}}
     endpoint: {{.ESEndpoint}}
+    logs_index: {{.TestId}}
+    sending_queue:
+      wait_for_result: true
+      block_on_overflow: true
+      enabled: true
+      batch:
+        min_size: 2000
+        max_size: 10000
+        flush_timeout: 1s
     mapping:
       mode: none
 
@@ -489,6 +498,13 @@ service:
         - resource/add-test-id
       receivers:
         - filelog
+  telemetry:
+    logs:
+      level: DEBUG
+      encoding: json
+      disable_stacktrace: true
+      output_paths:
+        - {{.OTelLogFile}}
 `
 
 func TestOtelLogsIngestion(t *testing.T) {
@@ -506,8 +522,11 @@ func TestOtelLogsIngestion(t *testing.T) {
 	// Prepare the OTel config.
 	testId := info.Namespace
 
-	tempDir := t.TempDir()
+	// Ensure everything is saved in case of test failure
+	// this folder is also collected on CI.
+	tempDir := aTesting.TempDir(t, "..", "..", "..", "build")
 	inputFilePath := filepath.Join(tempDir, "input.log")
+	otelLogFilePath := filepath.Join(tempDir, "elastic-agent.ndjson")
 
 	esHost, err := integration.GetESHost()
 	require.NoError(t, err, "failed to get ES host")
@@ -524,6 +543,7 @@ func TestOtelLogsIngestion(t *testing.T) {
 	logsIngestionConfig = strings.ReplaceAll(logsIngestionConfig, "{{.ESEndpoint}}", esHost)
 	logsIngestionConfig = strings.ReplaceAll(logsIngestionConfig, "{{.InputFilePath}}", inputFilePath)
 	logsIngestionConfig = strings.ReplaceAll(logsIngestionConfig, "{{.TestId}}", testId)
+	logsIngestionConfig = strings.ReplaceAll(logsIngestionConfig, "{{.OTelLogFile}}", otelLogFilePath)
 
 	cfgFilePath := filepath.Join(tempDir, "otel.yml")
 	require.NoError(t, os.WriteFile(cfgFilePath, []byte(logsIngestionConfig), 0o600))
@@ -557,26 +577,30 @@ func TestOtelLogsIngestion(t *testing.T) {
 		require.NoError(t, err)
 	}
 	inputFile.Close()
-	t.Cleanup(func() {
-		_ = os.Remove(inputFilePath)
-	})
 
-	actualHits := &struct{ Hits int }{}
-	require.Eventually(t,
-		func() bool {
-			findCtx, findCancel := context.WithTimeout(context.Background(), 10*time.Second)
+	// It takes about 45s to ingest all files on local tests,
+	// so set the timeout to 5min to be on the safe side.
+	require.EventuallyWithT(
+		t,
+		func(c *assert.CollectT) {
+			findCtx, findCancel := context.WithTimeout(t.Context(), 10*time.Second)
 			defer findCancel()
 
-			docs, err := estools.GetLogsForIndexWithContext(findCtx, esClient, ".ds-logs-generic-default*", map[string]interface{}{
-				"Resource.test.id": testId,
-			})
-			require.NoError(t, err)
-
-			actualHits.Hits = docs.Hits.Total.Value
-			return actualHits.Hits == logsCount
+			docs, err := estools.GetAllLogsForIndexWithContext(
+				findCtx,
+				esClient,
+				testId)
+			require.NoError(c, err)
+			require.Equalf(
+				c,
+				logsCount,
+				docs.Hits.Total.Value,
+				"expecting %d events",
+				logsCount)
 		},
-		2*time.Minute, 1*time.Second,
-		"Expected %v logs, got %v", logsCount, actualHits)
+		5*time.Minute,
+		time.Second,
+		"did not find the expected number of events")
 
 	cancel()
 	fixtureWg.Wait()