feat(ci): per-config failing-test names in FTR job annotation

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/test/ftr_job_annotation.sh
 
 BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
 FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-}
@@ -20,6 +21,9 @@ FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
 
 # a FTR failure will result in the script returning an exit code of 10
 exitCode=0
+annotation_rows=()
+failure_detail_lines=()
+retry_recovered=false
 
 configs="${FTR_CONFIG:-}"
 
@@ -47,6 +51,12 @@ fi
 failedConfigs=""
 results=()
 
+# Capture which configs failed in the previous attempt before the meta-data key is overwritten below.
+prevRunFailedConfigs=""
+if [[ "${BUILDKITE_RETRY_COUNT:-0}" -ge "1" ]]; then
+  prevRunFailedConfigs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '' 2>/dev/null || true)
+fi
+
 while read -r config; do
   if [[ ! "$config" ]]; then
     continue;
@@ -62,6 +72,7 @@ while read -r config; do
 
   if [[ "$IS_CONFIG_EXECUTION" == "true" && "$IS_FLAKY_TEST_RUN" == "false" ]]; then
     echo "--- [ already-tested ] $FULL_COMMAND"
+    annotation_rows+=("| [\`${config}\`](https://github.com/elastic/kibana/blob/${BUILDKITE_COMMIT:-main}/${config}) | — | skipped (already-tested) |")
     continue
   else
     echo "--- $ $FULL_COMMAND"
@@ -87,6 +98,8 @@ while read -r config; do
   """
   fi
 
+  node scripts/ftr_annotation_helper snapshot "target/junit/$JOB" 2>/dev/null || true
+
   # prevent non-zero exit code from breaking the loop
   set +e;
   node ./scripts/functional_tests \
@@ -121,9 +134,15 @@ while read -r config; do
     duration: ${duration}
     result: ${lastCode}")
 
+  config_link="[\`${config}\`](https://github.com/elastic/kibana/blob/${BUILDKITE_COMMIT:-main}/${config})"
   if [ $lastCode -eq 0 ]; then
     # Test was successful, so mark it as executed
     buildkite-agent meta-data set "$CONFIG_EXECUTION_KEY" "true"
+    if [[ -n "$prevRunFailedConfigs" ]] && grep -qxF "$config" <<< "$prevRunFailedConfigs"; then
+      annotation_rows+=("| ${config_link} | ${duration} | recovered |")
+    else
+      annotation_rows+=("| ${config_link} | ${duration} | passed |")
+    fi
   else
     exitCode=10
     echo "FTR exited with code $lastCode"
@@ -134,6 +153,23 @@ while read -r config; do
     else
       failedConfigs="$config"
     fi
+
+    if [[ -n "$prevRunFailedConfigs" ]] && grep -qxF "$config" <<< "$prevRunFailedConfigs"; then
+      annotation_rows+=("| ${config_link} | ${duration} | **still failing** |")
+    elif [[ -n "$prevRunFailedConfigs" ]]; then
+      annotation_rows+=("| ${config_link} | ${duration} | **new failure** (was passing) |")
+    else
+      annotation_rows+=("| ${config_link} | ${duration} | **failed** |")
+    fi
+
+    config_failures=$(node scripts/ftr_annotation_helper list-new-failures "target/junit/$JOB" 2>/dev/null || true)
+    if [[ -n "$config_failures" ]]; then
+      failure_detail_lines+=("**Failing tests — \`${config}\`:**" "")
+      while IFS= read -r t; do
+        [[ -n "$t" ]] && failure_detail_lines+=("- ${t}")
+      done <<< "$config_failures"
+      failure_detail_lines+=("")
+    fi
   fi
 done <<< "$configs"
 
@@ -145,4 +181,6 @@ echo "--- FTR configs complete"
 printf "%s\n" "${results[@]}"
 echo ""
 
+write_job_annotation
+
 exit $exitCode

.buildkite/scripts/steps/test/ftr_job_annotation.sh

@@ -0,0 +1,35 @@
+# Sourced by ftr_configs.sh — do not execute directly.
+# Reads globals: exitCode, retry_recovered, annotation_rows, failure_detail_lines,
+# JOB, BUILDKITE_RETRY_COUNT, BUILDKITE_COMMIT.
+
+write_job_annotation() {
+  local attempt_num style
+  attempt_num=$((${BUILDKITE_RETRY_COUNT:-0} + 1))
+  style=$([[ "$exitCode" == "0" ]] && echo "success" || echo "error")
+
+  {
+    echo "### FTR Configs — \`${JOB}\` (attempt ${attempt_num})"
+    echo ""
+
+    if [[ "$retry_recovered" == "true" ]]; then
+      echo "**Recovered on retry** — all originally-failing tests passed; step marked green."
+      echo ""
+      echo "> Configs shown as 'still failing' below introduced *new* failures on retry that were not part of the original failure set and are not counted against recovery."
+      echo ""
+    fi
+
+    if [[ ${#annotation_rows[@]} -gt 0 ]]; then
+      echo "| Config | Duration | Status |"
+      echo "| --- | --- | --- |"
+      printf "%s\n" "${annotation_rows[@]}"
+    fi
+
+    if [[ ${#failure_detail_lines[@]} -gt 0 ]]; then
+      echo ""
+      printf "%s\n" "${failure_detail_lines[@]}"
+    fi
+  } | buildkite-agent annotate \
+        --scope job \
+        --context "ftr-summary" \
+        --style "${style}" || true
+}

packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.test.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import Fs from 'fs';
+import Os from 'os';
+import Path from 'path';
+
+import { collectNewFailedTestNames, snapshotJunitDir } from './ftr_annotation_helper';
+
+const buildXml = (testcases: string) => `<?xml version="1.0" encoding="utf-8"?>
+<testsuites name="ftr">
+  <testsuite>${testcases}</testsuite>
+</testsuites>`;
+
+const failedCase = (name: string) =>
+  `<testcase name="${name}" classname="suite.file" time="1"><failure>error</failure></testcase>`;
+
+const passedCase = (name: string) =>
+  `<testcase name="${name}" classname="suite.file" time="1"></testcase>`;
+
+describe('snapshotJunitDir + collectNewFailedTestNames', () => {
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'annotation-helper-test-'));
+  });
+
+  afterEach(() => {
+    Fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('returns only failures from XMLs written after the snapshot', async () => {
+    Fs.writeFileSync(Path.join(tmpDir, 'TEST-existing.xml'), buildXml(failedCase('old failure')));
+    await snapshotJunitDir(tmpDir);
+    Fs.writeFileSync(Path.join(tmpDir, 'TEST-new.xml'), buildXml(failedCase('new failure')));
+
+    const names = await collectNewFailedTestNames(tmpDir);
+    expect([...names]).toEqual(['new failure']);
+  });
+
+  it('deletes the snapshot file after reading', async () => {
+    await snapshotJunitDir(tmpDir);
+    await collectNewFailedTestNames(tmpDir);
+
+    expect(Fs.existsSync(Path.join(tmpDir, '.ftr_annotation_snapshot'))).toBe(false);
+  });
+
+  it('treats all XMLs as new when no snapshot exists', async () => {
+    Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(failedCase('test A')));
+
+    const names = await collectNewFailedTestNames(tmpDir);
+    expect([...names]).toEqual(['test A']);
+  });
+
+  it('returns empty set when no new XMLs were written after the snapshot', async () => {
+    Fs.writeFileSync(Path.join(tmpDir, 'TEST-existing.xml'), buildXml(failedCase('old failure')));
+    await snapshotJunitDir(tmpDir);
+
+    const names = await collectNewFailedTestNames(tmpDir);
+    expect(names.size).toBe(0);
+  });
+
+  it('ignores passing tests in new XMLs', async () => {
+    await snapshotJunitDir(tmpDir);
+    Fs.writeFileSync(
+      Path.join(tmpDir, 'TEST-new.xml'),
+      buildXml(passedCase('test A') + failedCase('test B'))
+    );
+
+    const names = await collectNewFailedTestNames(tmpDir);
+    expect([...names]).toEqual(['test B']);
+  });
+});

packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.ts

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import Fs from 'fs';
+import Path from 'path';
+
+import { createFlagError } from '@kbn/dev-cli-errors';
+import { run } from '@kbn/dev-cli-runner';
+import globby from 'globby';
+import normalize from 'normalize-path';
+
+import { makeFailedTestCaseIter, readTestReport } from './test_report';
+
+const SNAPSHOT_FILE = '.ftr_annotation_snapshot';
+
+/**
+ * Writes the current XML file list in junitDir to a snapshot file.
+ * Call this before running a config so list-new-failures can diff against it.
+ */
+export async function snapshotJunitDir(junitDir: string): Promise<void> {
+  const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true });
+  Fs.mkdirSync(junitDir, { recursive: true });
+  Fs.writeFileSync(Path.join(junitDir, SNAPSHOT_FILE), JSON.stringify(xmlPaths.sort()));
+}
+
+/**
+ * Reads the snapshot written by snapshotJunitDir, diffs against the current XML files,
+ * and returns the failing test names from XMLs produced after the snapshot.
+ * Deletes the snapshot file after reading.
+ */
+export async function collectNewFailedTestNames(junitDir: string): Promise<Set<string>> {
+  const snapshotPath = Path.join(junitDir, SNAPSHOT_FILE);
+  let before = new Set<string>();
+  if (Fs.existsSync(snapshotPath)) {
+    before = new Set<string>(JSON.parse(Fs.readFileSync(snapshotPath, 'utf8')));
+    Fs.unlinkSync(snapshotPath);
+  }
+  const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true });
+  const newPaths = xmlPaths.filter((p) => !before.has(p));
+  const names = new Set<string>();
+  for (const xmlPath of newPaths) {
+    const report = await readTestReport(xmlPath);
+    for (const tc of makeFailedTestCaseIter(report)) {
+      names.add(tc.$.name.trim());
+    }
+  }
+  return names;
+}
+
+export function runAnnotationHelperCli() {
+  run(
+    async ({ flags }) => {
+      const [command, ...rest] = flags._;
+
+      if (command === 'snapshot') {
+        const [junitDir] = rest;
+        if (!junitDir) {
+          throw createFlagError('Usage: snapshot <junit-dir>');
+        }
+        await snapshotJunitDir(junitDir);
+        return;
+      }
+
+      if (command === 'list-new-failures') {
+        const [junitDir] = rest;
+        if (!junitDir) {
+          throw createFlagError('Usage: list-new-failures <junit-dir>');
+        }
+        const names = await collectNewFailedTestNames(junitDir);
+        if (names.size > 0) {
+          process.stdout.write([...names].join('\n') + '\n');
+        }
+        return;
+      }
+
+      throw createFlagError(
+        `Unknown command: ${command}. Valid commands: snapshot, list-new-failures`
+      );
+    },
+    {
+      description: `
+        Per-config JUnit attribution helpers for FTR job annotations.
+
+        Commands:
+          snapshot <junit-dir>
+            Writes the current list of *.xml files in <junit-dir> to a snapshot file.
+            Call this before running a config.
+
+          list-new-failures <junit-dir>
+            Diffs the current *.xml files against the snapshot and prints failing test
+            names from XMLs that are new since the snapshot. Deletes the snapshot file.
+      `,
+    }
+  );
+}

packages/kbn-failed-test-reporter-cli/index.ts

@@ -8,3 +8,4 @@
  */
 
 import './failed_tests_reporter/failed_tests_reporter_cli';
+export { runAnnotationHelperCli } from './failed_tests_reporter/ftr_annotation_helper';

scripts/ftr_annotation_helper.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+require('@kbn/setup-node-env');
+require('@kbn/failed-test-reporter-cli').runAnnotationHelperCli();