diff --git a/.buildkite/scripts/steps/test/ftr_configs.sh b/.buildkite/scripts/steps/test/ftr_configs.sh
index a14283ef5c7df..ea96c61f71c2d 100755
--- a/.buildkite/scripts/steps/test/ftr_configs.sh
+++ b/.buildkite/scripts/steps/test/ftr_configs.sh
@@ -3,6 +3,7 @@
set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/test/ftr_job_annotation.sh
BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-}
@@ -20,6 +21,9 @@ FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
# a FTR failure will result in the script returning an exit code of 10
exitCode=0
+annotation_rows=()
+failure_detail_lines=()
+retry_recovered=false
configs="${FTR_CONFIG:-}"
@@ -47,6 +51,12 @@ fi
failedConfigs=""
results=()
+# Capture which configs failed in the previous attempt before the meta-data key is overwritten below.
+prevRunFailedConfigs=""
+if [[ "${BUILDKITE_RETRY_COUNT:-0}" -ge "1" ]]; then
+ prevRunFailedConfigs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '' 2>/dev/null || true)
+fi
+
while read -r config; do
if [[ ! "$config" ]]; then
continue;
@@ -62,6 +72,7 @@ while read -r config; do
if [[ "$IS_CONFIG_EXECUTION" == "true" && "$IS_FLAKY_TEST_RUN" == "false" ]]; then
echo "--- [ already-tested ] $FULL_COMMAND"
+ annotation_rows+=("| [\`${config}\`](https://github.com/elastic/kibana/blob/${BUILDKITE_COMMIT:-main}/${config}) | — | skipped (already-tested) |")
continue
else
echo "--- $ $FULL_COMMAND"
@@ -87,6 +98,8 @@ while read -r config; do
"""
fi
+ node scripts/ftr_annotation_helper snapshot "target/junit/$JOB" 2>/dev/null || true
+
# prevent non-zero exit code from breaking the loop
set +e;
node ./scripts/functional_tests \
@@ -121,9 +134,15 @@ while read -r config; do
duration: ${duration}
result: ${lastCode}")
+ config_link="[\`${config}\`](https://github.com/elastic/kibana/blob/${BUILDKITE_COMMIT:-main}/${config})"
if [ $lastCode -eq 0 ]; then
# Test was successful, so mark it as executed
buildkite-agent meta-data set "$CONFIG_EXECUTION_KEY" "true"
+ if [[ -n "$prevRunFailedConfigs" ]] && grep -qxF "$config" <<< "$prevRunFailedConfigs"; then
+ annotation_rows+=("| ${config_link} | ${duration} | recovered |")
+ else
+ annotation_rows+=("| ${config_link} | ${duration} | passed |")
+ fi
else
exitCode=10
echo "FTR exited with code $lastCode"
@@ -134,6 +153,23 @@ while read -r config; do
else
failedConfigs="$config"
fi
+
+ if [[ -n "$prevRunFailedConfigs" ]] && grep -qxF "$config" <<< "$prevRunFailedConfigs"; then
+ annotation_rows+=("| ${config_link} | ${duration} | **still failing** |")
+ elif [[ -n "$prevRunFailedConfigs" ]]; then
+ annotation_rows+=("| ${config_link} | ${duration} | **new failure** (was passing) |")
+ else
+ annotation_rows+=("| ${config_link} | ${duration} | **failed** |")
+ fi
+
+ config_failures=$(node scripts/ftr_annotation_helper list-new-failures "target/junit/$JOB" 2>/dev/null || true)
+ if [[ -n "$config_failures" ]]; then
+ failure_detail_lines+=("**Failing tests — \`${config}\`:**" "")
+ while IFS= read -r t; do
+ [[ -n "$t" ]] && failure_detail_lines+=("- ${t}")
+ done <<< "$config_failures"
+ failure_detail_lines+=("")
+ fi
fi
done <<< "$configs"
@@ -145,4 +181,6 @@ echo "--- FTR configs complete"
printf "%s\n" "${results[@]}"
echo ""
+write_job_annotation
+
exit $exitCode
diff --git a/.buildkite/scripts/steps/test/ftr_job_annotation.sh b/.buildkite/scripts/steps/test/ftr_job_annotation.sh
new file mode 100644
index 0000000000000..5c45652302f2d
--- /dev/null
+++ b/.buildkite/scripts/steps/test/ftr_job_annotation.sh
@@ -0,0 +1,35 @@
+# Sourced by ftr_configs.sh — do not execute directly.
+# Reads globals: exitCode, retry_recovered, annotation_rows, failure_detail_lines,
+# JOB, BUILDKITE_RETRY_COUNT, BUILDKITE_COMMIT.
+
+write_job_annotation() {
+ local attempt_num style
+ attempt_num=$((${BUILDKITE_RETRY_COUNT:-0} + 1))
+ style=$([[ "$exitCode" == "0" ]] && echo "success" || echo "error")
+
+ {
+ echo "### FTR Configs — \`${JOB}\` (attempt ${attempt_num})"
+ echo ""
+
+ if [[ "$retry_recovered" == "true" ]]; then
+ echo "**Recovered on retry** — all originally-failing tests passed; step marked green."
+ echo ""
+ echo "> Configs shown as 'still failing' below introduced *new* failures on retry that were not part of the original failure set and are not counted against recovery."
+ echo ""
+ fi
+
+ if [[ ${#annotation_rows[@]} -gt 0 ]]; then
+ echo "| Config | Duration | Status |"
+ echo "| --- | --- | --- |"
+ printf "%s\n" "${annotation_rows[@]}"
+ fi
+
+ if [[ ${#failure_detail_lines[@]} -gt 0 ]]; then
+ echo ""
+ printf "%s\n" "${failure_detail_lines[@]}"
+ fi
+ } | buildkite-agent annotate \
+ --scope job \
+ --context "ftr-summary" \
+ --style "${style}" || true
+}
diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.test.ts
new file mode 100644
index 0000000000000..ca9affaf5a7fe
--- /dev/null
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.test.ts
@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import Fs from 'fs';
+import Os from 'os';
+import Path from 'path';
+
+import { collectNewFailedTestNames, snapshotJunitDir } from './ftr_annotation_helper';
+
+const buildXml = (testcases: string) => `
+
+ ${testcases}
+`;
+
+const failedCase = (name: string) =>
+ `error`;
+
+const passedCase = (name: string) =>
+ ``;
+
+describe('snapshotJunitDir + collectNewFailedTestNames', () => {
+ let tmpDir: string;
+
+ beforeEach(() => {
+ tmpDir = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'annotation-helper-test-'));
+ });
+
+ afterEach(() => {
+ Fs.rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ it('returns only failures from XMLs written after the snapshot', async () => {
+ Fs.writeFileSync(Path.join(tmpDir, 'TEST-existing.xml'), buildXml(failedCase('old failure')));
+ await snapshotJunitDir(tmpDir);
+ Fs.writeFileSync(Path.join(tmpDir, 'TEST-new.xml'), buildXml(failedCase('new failure')));
+
+ const names = await collectNewFailedTestNames(tmpDir);
+ expect([...names]).toEqual(['new failure']);
+ });
+
+ it('deletes the snapshot file after reading', async () => {
+ await snapshotJunitDir(tmpDir);
+ await collectNewFailedTestNames(tmpDir);
+
+ expect(Fs.existsSync(Path.join(tmpDir, '.ftr_annotation_snapshot'))).toBe(false);
+ });
+
+ it('treats all XMLs as new when no snapshot exists', async () => {
+ Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(failedCase('test A')));
+
+ const names = await collectNewFailedTestNames(tmpDir);
+ expect([...names]).toEqual(['test A']);
+ });
+
+ it('returns empty set when no new XMLs were written after the snapshot', async () => {
+ Fs.writeFileSync(Path.join(tmpDir, 'TEST-existing.xml'), buildXml(failedCase('old failure')));
+ await snapshotJunitDir(tmpDir);
+
+ const names = await collectNewFailedTestNames(tmpDir);
+ expect(names.size).toBe(0);
+ });
+
+ it('ignores passing tests in new XMLs', async () => {
+ await snapshotJunitDir(tmpDir);
+ Fs.writeFileSync(
+ Path.join(tmpDir, 'TEST-new.xml'),
+ buildXml(passedCase('test A') + failedCase('test B'))
+ );
+
+ const names = await collectNewFailedTestNames(tmpDir);
+ expect([...names]).toEqual(['test B']);
+ });
+});
diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.ts
new file mode 100644
index 0000000000000..95859542ce6d7
--- /dev/null
+++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.ts
@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import Fs from 'fs';
+import Path from 'path';
+
+import { createFlagError } from '@kbn/dev-cli-errors';
+import { run } from '@kbn/dev-cli-runner';
+import globby from 'globby';
+import normalize from 'normalize-path';
+
+import { makeFailedTestCaseIter, readTestReport } from './test_report';
+
+const SNAPSHOT_FILE = '.ftr_annotation_snapshot';
+
+/**
+ * Writes the current XML file list in junitDir to a snapshot file.
+ * Call this before running a config so list-new-failures can diff against it.
+ */
+export async function snapshotJunitDir(junitDir: string): Promise {
+ const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true });
+ Fs.mkdirSync(junitDir, { recursive: true });
+ Fs.writeFileSync(Path.join(junitDir, SNAPSHOT_FILE), JSON.stringify(xmlPaths.sort()));
+}
+
+/**
+ * Reads the snapshot written by snapshotJunitDir, diffs against the current XML files,
+ * and returns the failing test names from XMLs produced after the snapshot.
+ * Deletes the snapshot file after reading.
+ */
+export async function collectNewFailedTestNames(junitDir: string): Promise> {
+ const snapshotPath = Path.join(junitDir, SNAPSHOT_FILE);
+ let before = new Set();
+ if (Fs.existsSync(snapshotPath)) {
+ before = new Set(JSON.parse(Fs.readFileSync(snapshotPath, 'utf8')));
+ Fs.unlinkSync(snapshotPath);
+ }
+ const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true });
+ const newPaths = xmlPaths.filter((p) => !before.has(p));
+ const names = new Set();
+ for (const xmlPath of newPaths) {
+ const report = await readTestReport(xmlPath);
+ for (const tc of makeFailedTestCaseIter(report)) {
+ names.add(tc.$.name.trim());
+ }
+ }
+ return names;
+}
+
+export function runAnnotationHelperCli() {
+ run(
+ async ({ flags }) => {
+ const [command, ...rest] = flags._;
+
+ if (command === 'snapshot') {
+ const [junitDir] = rest;
+ if (!junitDir) {
+ throw createFlagError('Usage: snapshot ');
+ }
+ await snapshotJunitDir(junitDir);
+ return;
+ }
+
+ if (command === 'list-new-failures') {
+ const [junitDir] = rest;
+ if (!junitDir) {
+ throw createFlagError('Usage: list-new-failures ');
+ }
+ const names = await collectNewFailedTestNames(junitDir);
+ if (names.size > 0) {
+ process.stdout.write([...names].join('\n') + '\n');
+ }
+ return;
+ }
+
+ throw createFlagError(
+ `Unknown command: ${command}. Valid commands: snapshot, list-new-failures`
+ );
+ },
+ {
+ description: `
+ Per-config JUnit attribution helpers for FTR job annotations.
+
+ Commands:
+ snapshot
+ Writes the current list of *.xml files in to a snapshot file.
+ Call this before running a config.
+
+ list-new-failures
+ Diffs the current *.xml files against the snapshot and prints failing test
+ names from XMLs that are new since the snapshot. Deletes the snapshot file.
+ `,
+ }
+ );
+}
diff --git a/packages/kbn-failed-test-reporter-cli/index.ts b/packages/kbn-failed-test-reporter-cli/index.ts
index fd33d523318a9..dd8b02a4ebea5 100644
--- a/packages/kbn-failed-test-reporter-cli/index.ts
+++ b/packages/kbn-failed-test-reporter-cli/index.ts
@@ -8,3 +8,4 @@
*/
import './failed_tests_reporter/failed_tests_reporter_cli';
+export { runAnnotationHelperCli } from './failed_tests_reporter/ftr_annotation_helper';
diff --git a/scripts/ftr_annotation_helper.js b/scripts/ftr_annotation_helper.js
new file mode 100644
index 0000000000000..8115016483b76
--- /dev/null
+++ b/scripts/ftr_annotation_helper.js
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+require('@kbn/setup-node-env');
+require('@kbn/failed-test-reporter-cli').runAnnotationHelperCli();