Make sure e2e tests reports are uploaded on failure (#6607)

This makes sure e2e tests reports are uploaded on failure with 2 changes:
* the JSON go test results are uploaded regardless of the status of the make e2e-run command
* a new step is run to download all test results and convert them to JUnit XML reports
JUnit XML reports are uploaded temporarily waiting we build a Buildkite annotation to summarize the test failures.

Author: thbkrkr

.buildkite/e2e/pipeline-gen/pipeline.tpl.yaml

@@ -66,13 +66,11 @@ steps:
       {{- end }}
 
       {{- if $test.Dind }}
-      - make -C .ci TARGET="{{ $deployerCommand }} e2e-run e2e-generate-xml" ci
+      - make -C .ci TARGET="{{ $deployerCommand }} e2e-run" ci
       {{- else }}
-      - make {{ $deployerCommand }} e2e-run e2e-generate-xml
+      - make {{ $deployerCommand }} e2e-run
       {{- end }}
 
-      - mkdir tests-report && mv e2e-tests.xml tests-report/e2e-tests-{{ $test.SlugName }}.xml
-
     agents:
       {{- if $test.Dind }}
       provider: "gcp"
@@ -85,7 +83,7 @@ steps:
       {{- end }}
 
     artifact_paths:
-      - tests-report/*.xml
+      - e2e-tests-*.json
       - "eck-diagnostic*.zip"
 
 {{- end }}
@@ -123,6 +121,17 @@ steps:
           image: "family/elastic-buildkite-agent-ubuntu-2004-lts"
         {{- end }}
 
+
+
     {{- end }}
 {{- end }}
 
+  - wait: ~
+    continue_on_failure: true
+
+  - label: aggregate tests-results
+    commands:
+      - buildkite-agent artifact download e2e-tests-*.json .
+      - .buildkite/scripts/test/generate-tests-reports.sh
+    artifact_paths:
+      - e2e-tests-*.xml

.buildkite/scripts/test/generate-tests-reports.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0;
+# you may not use this file except in compliance with the Elastic License 2.0.
+
+# Script to generate e2e tests reports in JUnit XML format from JSON go test results.
+
+set -eu
+
+gen_junit_report() {
+    local input_file=${1:-"input file"}
+    local xml_report=${input_file%.*}.xml
+
+    # exit without error when there is no input file
+    if [[ ! -f $input_file ]]; then
+        echo "No $input_file to generate a JUnit XML report."
+        exit 0
+    fi
+
+    # temporary filter out lines containing a space in the timestamp,
+    # see https://github.com/elastic/cloud-on-k8s/issues/3560.
+    gotestsum \
+        --junitfile "$xml_report" \
+        --raw-command grep -v '"Time":"[^"]*\s[^"]*"' "$input_file" || \
+    ( \
+        echo "Failed to generate a JUnit XML report."
+        # print the input file for further debugging
+        echo " --- $input_file - START ---"
+        cat "$input_file"
+        echo " --- $input_file - END   ---"
+        exit 1
+    )
+}
+
+for f in e2e-tests-*.json; do
+    gen_junit_report "$f"
+done

test/e2e/cmd/run/run.go

@@ -35,12 +35,12 @@ import (
 )
 
 const (
-	jobTimeout           = 600 * time.Minute // time to wait for the test job to finish
-	kubePollInterval     = 10 * time.Second  // Kube API polling interval
-	testRunLabel         = "test-run"        // name of the label applied to resources
-	logStreamLabel       = "stream-logs"     // name of the label enabling log streaming to e2e runner
-	testsLogFile         = "e2e-tests.json"  // name of file to keep all test logs in JSON format
-	operatorReadyTimeout = 3 * time.Minute   // time to wait for the operator pod to be ready
+	jobTimeout           = 600 * time.Minute   // time to wait for the test job to finish
+	kubePollInterval     = 10 * time.Second    // Kube API polling interval
+	testRunLabel         = "test-run"          // name of the label applied to resources
+	logStreamLabel       = "stream-logs"       // name of the label enabling log streaming to e2e runner
+	testsLogFilePattern  = "e2e-tests-%s.json" // name of file to keep all test logs in JSON format
+	operatorReadyTimeout = 3 * time.Minute     // time to wait for the operator pod to be ready
 
 	TestNameLabel = "test-name" // name of the label applied to resources during each test
 )
@@ -602,7 +602,7 @@ func (h *helper) startAndMonitorTestJobs(client *kubernetes.Clientset) error {
 
 	outputs := []io.Writer{os.Stdout}
 	if h.logToFile {
-		jl, err := newJSONLogToFile(testsLogFile)
+		jl, err := newJSONLogToFile(fmt.Sprintf(testsLogFilePattern, h.testContext.ClusterName))
 		if err != nil {
 			log.Error(err, "Failed to create log file for test output")
 			return err