test: update TestBeatDiagnostics test

Signed-off-by: Dominik Rosiek <dominik.rosiek@elastic.co>

Author: macdewee

testing/integration/ess/diagnostics_test.go

@@ -15,6 +15,8 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"path/filepath"
@@ -371,7 +373,14 @@ func TestBeatDiagnostics(t *testing.T) {
 
 	esURL := integration.StartMockES(t, 0, 0, 0, 0)
 
-	configTemplate := `
+	// Mock HTTP server used by the httpjson tracer test case.
+	httpjsonServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintln(w, `{"message":"hello"}`)
+	}))
+	t.Cleanup(httpjsonServer.Close)
+
+	fileStreamConfigTemplate := `
 inputs:
   - id: filestream-filebeat
     type: filestream
@@ -390,34 +399,67 @@ agent.monitoring.enabled: {{ .MonitoringEnabled }}
 agent.internal.runtime.filebeat.filestream: {{ .Runtime }}
 `
 
-	var filebeatSetup = map[string]integrationtest.ComponentState{
+	// httpjsonTracerConfigTemplate runs httpjson inside the filebeat OTel receiver with request
+	// tracing enabled. The receiver writes trace files under {versionedHome}/components/logs/httpjson/,
+	// which is the path the diagnostic bundle fix now collects.
+	httpjsonTracerConfigTemplate := `
+inputs:
+  - id: httpjson-trace-test
+    type: httpjson
+    request.url: {{ .MockServerURL }}
+    interval: 1s
+    request.tracer.filename: httpjson/logs/http-request-trace-*.ndjson
+    use_output: default
+outputs:
+  default:
+    type: elasticsearch
+    hosts: [{{ .ESHost }}]
+    api_key: placeholder
+agent.monitoring.enabled: false
+agent.internal.runtime.filebeat.httpjson: otel
+`
+
+	fileStreamAgentCompState := map[string]integrationtest.ComponentState{
 		"filestream-default": {
 			State: integrationtest.NewClientState(client.Healthy),
+			Units: map[integrationtest.ComponentUnitKey]integrationtest.ComponentUnitState{
+				integrationtest.ComponentUnitKey{UnitType: client.UnitTypeOutput, UnitID: "filestream-default"}: {
+					State: integrationtest.NewClientState(client.Healthy),
+				},
+				integrationtest.ComponentUnitKey{UnitType: client.UnitTypeInput, UnitID: "filestream-default-filestream-filebeat"}: {
+					State: integrationtest.NewClientState(client.Healthy),
+				},
+			},
 		},
 	}
 
-	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
-	defer cancel()
-	expectedComponentState := map[string]integrationtest.ComponentState{
-		"filestream-default": {
+	httpjsonAgentCompState := map[string]integrationtest.ComponentState{
+		"httpjson-default": {
 			State: integrationtest.NewClientState(client.Healthy),
 			Units: map[integrationtest.ComponentUnitKey]integrationtest.ComponentUnitState{
-				integrationtest.ComponentUnitKey{UnitType: client.UnitTypeOutput, UnitID: "filestream-default"}: {
+				integrationtest.ComponentUnitKey{UnitType: client.UnitTypeOutput, UnitID: "httpjson-default"}: {
 					State: integrationtest.NewClientState(client.Healthy),
 				},
-				integrationtest.ComponentUnitKey{UnitType: client.UnitTypeInput, UnitID: "filestream-default-filestream-filebeat"}: {
+				integrationtest.ComponentUnitKey{UnitType: client.UnitTypeInput, UnitID: "httpjson-default-httpjson-trace-test"}: {
 					State: integrationtest.NewClientState(client.Healthy),
 				},
 			},
 		},
 	}
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
 	expectedAgentState := integrationtest.NewClientState(client.Healthy)
 
 	testCases := []struct {
 		name                         string
 		runtime                      string
 		monitoringEnabled            bool
 		expectedCompDiagnosticsFiles []string
+		agentCompState               map[string]integrationtest.ComponentState
+		diagCompSetup                map[string]integrationtest.ComponentState
+		configTemplate               string
+		extraPatterns                []filePattern
 	}{
 		{
 			name:              "filebeat process",
@@ -430,6 +472,11 @@ agent.internal.runtime.filebeat.filestream: {{ .Runtime }}
 				"beat-rendered-config.yml",
 				"global_processors.txt",
 			),
+			agentCompState: fileStreamAgentCompState,
+			diagCompSetup: map[string]integrationtest.ComponentState{
+				"filestream-default": {State: integrationtest.NewClientState(client.Healthy)},
+			},
+			configTemplate: fileStreamConfigTemplate,
 		},
 		{
 			name:              "filebeat receiver",
@@ -440,6 +487,32 @@ agent.internal.runtime.filebeat.filestream: {{ .Runtime }}
 				"beat_metrics.json",
 				"input_metrics.json",
 			},
+			agentCompState: fileStreamAgentCompState,
+			diagCompSetup: map[string]integrationtest.ComponentState{
+				"filestream-default": {State: integrationtest.NewClientState(client.Healthy)},
+			},
+			configTemplate: fileStreamConfigTemplate,
+		},
+		{
+			// Verifies that beat receiver trace logs written to {versionedHome}/components/logs/
+			// are included in the diagnostic bundle under logs/<commit>/httpjson/.
+			name:    "filebeat receiver with trace logs",
+			runtime: "otel",
+			expectedCompDiagnosticsFiles: []string{
+				"beat_metrics.json",
+				"input_metrics.json",
+			},
+			agentCompState: httpjsonAgentCompState,
+			diagCompSetup: map[string]integrationtest.ComponentState{
+				"httpjson-default": {State: integrationtest.NewClientState(client.Healthy)},
+			},
+			configTemplate: httpjsonTracerConfigTemplate,
+			extraPatterns: []filePattern{
+				{
+					pattern:  path.Join("logs", "*", "httpjson", "http-request-trace-*.ndjson"),
+					optional: false,
+				},
+			},
 		},
 	}
 
@@ -458,14 +531,15 @@ agent.internal.runtime.filebeat.filestream: {{ .Runtime }}
 
 			var configBuffer bytes.Buffer
 			require.NoError(t,
-				template.Must(template.New("config").Parse(configTemplate)).Execute(&configBuffer, map[string]any{
+				template.Must(template.New("config").Parse(tc.configTemplate)).Execute(&configBuffer, map[string]any{
 					"Runtime":           tc.runtime,
 					"InputFile":         inputFilePath,
 					"ESHost":            esURL.Host,
 					"MonitoringEnabled": tc.monitoringEnabled,
+					"MockServerURL":     httpjsonServer.URL,
 				}))
 			expDiagFiles := append([]string{}, diagnosticsFiles...)
-			var extraPatterns []filePattern
+			extraPatterns := append([]filePattern{}, tc.extraPatterns...)
 			if tc.runtime == "otel" {
 				// EDOT adds these extra files.
 				// TestBeatDiagnostics is quite strict about what it expects to see in the archive.
@@ -490,8 +564,8 @@ agent.internal.runtime.filebeat.filestream: {{ .Runtime }}
 			err = f.Run(ctx, integrationtest.State{
 				Configure:  configBuffer.String(),
 				AgentState: expectedAgentState,
-				Components: expectedComponentState,
-				After:      testDiagnosticsFactory(t, filebeatSetup, expDiagFiles, tc.expectedCompDiagnosticsFiles, f, []string{"diagnostics", "collect"}, extraPatterns...),
+				Components: tc.agentCompState,
+				After:      testDiagnosticsFactory(t, tc.diagCompSetup, expDiagFiles, tc.expectedCompDiagnosticsFiles, f, []string{"diagnostics", "collect"}, extraPatterns...),
 			})
 			assert.NoError(t, err)
 		})