Improve TestKubernetesJournaldInput (#11256) (#11315)

- Log as ndjson so logs can easily be parsed/ingested into
Elasticsearch
- Set the log level to debug so we can see the events
- If the test fails, log the index and search condition used
- Improve `FindESDocs` to not log expected failures/errors

(cherry picked from commit 3062264)

# Conflicts:
#	testing/integration/k8s/testdata/journald-otel.yml

Co-authored-by: Tiago Queiroz <[email protected]>

Author: mergify[bot]

testing/integration/common.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/elastic-agent-libs/testing/estools"
@@ -33,21 +34,17 @@ func GetESHost() (string, error) {
 // FindESDocs runs `findFn` until at least one document is returned and there is no error
 func FindESDocs(t *testing.T, findFn func() (estools.Documents, error)) estools.Documents {
 	var docs estools.Documents
-	require.Eventually(
+	require.EventuallyWithT(
 		t,
-		func() bool {
+		func(c *assert.CollectT) {
 			var err error
 			docs, err = findFn()
-			if err != nil {
-				t.Logf("got an error querying ES, retrying. Error: %s", err)
-				return false
-			}
-
-			return docs.Hits.Total.Value != 0
+			require.NoErrorf(c, err, "got an error querying ES, retrying. Error: %s", err)
+			require.NotEqualValues(c, 0, docs.Hits.Total.Value, "expecting at least one document returned by 'findFn'")
 		},
 		3*time.Minute,
 		15*time.Second,
-	)
+		"did not find the expected documents on ES")
 
 	return docs
 }

testing/integration/k8s/journald_test.go

@@ -136,6 +136,12 @@ func journaldTest(
 		t.Errorf("context error: %v", ctx.Err())
 	}
 
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Logf("Index: %q, filter condition: %q=%q", index, field, value)
+		}
+	})
+
 	// Query the index and filter by the input type
 	docs := integration.FindESDocs(t, func() (estools.Documents, error) {
 		return estools.GetLogsForIndexWithContext(

testing/integration/k8s/testdata/journald-otel.yml

@@ -0,0 +1,41 @@
+receivers:
+  filebeatreceiver:
+    filebeat:
+      inputs:
+        - type: journald
+          id: journald-input
+          paths:
+            - /opt/journal/
+          merge: true
+
+    output:
+      otelconsumer:
+    logging:
+      level: debug
+      selectors:
+        - '*'
+
+processors:
+  resource:
+    attributes:
+    - key: data_stream.namespace
+      action: insert
+      value: "${EA_POLICY_NAMESPACE}"
+
+exporters:
+  elasticsearch:
+    endpoint: "${ES_HOST}"
+    api_key: "${ES_API_KEY_ENCODED}"
+
+service:
+  pipelines:
+    logs:
+      receivers: [filebeatreceiver]
+      processors: [resource]
+      exporters:
+        - elasticsearch
+  telemetry:
+    logs:
+      level: DEBUG
+      encoding: json
+      disable_stacktrace: true