diff --git a/.buildkite/scripts/steps/test/ftr_configs.sh b/.buildkite/scripts/steps/test/ftr_configs.sh index a14283ef5c7df..ea96c61f71c2d 100755 --- a/.buildkite/scripts/steps/test/ftr_configs.sh +++ b/.buildkite/scripts/steps/test/ftr_configs.sh @@ -3,6 +3,7 @@ set -euo pipefail source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/test/ftr_job_annotation.sh BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-} FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-} @@ -20,6 +21,9 @@ FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}" # a FTR failure will result in the script returning an exit code of 10 exitCode=0 +annotation_rows=() +failure_detail_lines=() +retry_recovered=false configs="${FTR_CONFIG:-}" @@ -47,6 +51,12 @@ fi failedConfigs="" results=() +# Capture which configs failed in the previous attempt before the meta-data key is overwritten below. +prevRunFailedConfigs="" +if [[ "${BUILDKITE_RETRY_COUNT:-0}" -ge "1" ]]; then + prevRunFailedConfigs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '' 2>/dev/null || true) +fi + while read -r config; do if [[ ! "$config" ]]; then continue; @@ -62,6 +72,7 @@ while read -r config; do if [[ "$IS_CONFIG_EXECUTION" == "true" && "$IS_FLAKY_TEST_RUN" == "false" ]]; then echo "--- [ already-tested ] $FULL_COMMAND" + annotation_rows+=("| [\`${config}\`](https://github.com/elastic/kibana/blob/${BUILDKITE_COMMIT:-main}/${config}) | — | skipped (already-tested) |") continue else echo "--- $ $FULL_COMMAND" @@ -87,6 +98,8 @@ while read -r config; do """ fi + node scripts/ftr_annotation_helper snapshot "target/junit/$JOB" 2>/dev/null || true + # prevent non-zero exit code from breaking the loop set +e; node ./scripts/functional_tests \ @@ -121,9 +134,15 @@ while read -r config; do duration: ${duration} result: ${lastCode}") + config_link="[\`${config}\`](https://github.com/elastic/kibana/blob/${BUILDKITE_COMMIT:-main}/${config})" if [ $lastCode -eq 0 ]; then # Test was successful, so mark it as executed buildkite-agent meta-data set "$CONFIG_EXECUTION_KEY" "true" + if [[ -n "$prevRunFailedConfigs" ]] && grep -qxF "$config" <<< "$prevRunFailedConfigs"; then + annotation_rows+=("| ${config_link} | ${duration} | recovered |") + else + annotation_rows+=("| ${config_link} | ${duration} | passed |") + fi else exitCode=10 echo "FTR exited with code $lastCode" @@ -134,6 +153,23 @@ while read -r config; do else failedConfigs="$config" fi + + if [[ -n "$prevRunFailedConfigs" ]] && grep -qxF "$config" <<< "$prevRunFailedConfigs"; then + annotation_rows+=("| ${config_link} | ${duration} | **still failing** |") + elif [[ -n "$prevRunFailedConfigs" ]]; then + annotation_rows+=("| ${config_link} | ${duration} | **new failure** (was passing) |") + else + annotation_rows+=("| ${config_link} | ${duration} | **failed** |") + fi + + config_failures=$(node scripts/ftr_annotation_helper list-new-failures "target/junit/$JOB" 2>/dev/null || true) + if [[ -n "$config_failures" ]]; then + failure_detail_lines+=("**Failing tests — \`${config}\`:**" "") + while IFS= read -r t; do + [[ -n "$t" ]] && failure_detail_lines+=("- ${t}") + done <<< "$config_failures" + failure_detail_lines+=("") + fi fi done <<< "$configs" @@ -145,4 +181,6 @@ echo "--- FTR configs complete" printf "%s\n" "${results[@]}" echo "" +write_job_annotation + exit $exitCode diff --git a/.buildkite/scripts/steps/test/ftr_job_annotation.sh b/.buildkite/scripts/steps/test/ftr_job_annotation.sh new file mode 100644 index 0000000000000..5c45652302f2d --- /dev/null +++ b/.buildkite/scripts/steps/test/ftr_job_annotation.sh @@ -0,0 +1,35 @@ +# Sourced by ftr_configs.sh — do not execute directly. +# Reads globals: exitCode, retry_recovered, annotation_rows, failure_detail_lines, +# JOB, BUILDKITE_RETRY_COUNT, BUILDKITE_COMMIT. + +write_job_annotation() { + local attempt_num style + attempt_num=$((${BUILDKITE_RETRY_COUNT:-0} + 1)) + style=$([[ "$exitCode" == "0" ]] && echo "success" || echo "error") + + { + echo "### FTR Configs — \`${JOB}\` (attempt ${attempt_num})" + echo "" + + if [[ "$retry_recovered" == "true" ]]; then + echo "**Recovered on retry** — all originally-failing tests passed; step marked green." + echo "" + echo "> Configs shown as 'still failing' below introduced *new* failures on retry that were not part of the original failure set and are not counted against recovery." + echo "" + fi + + if [[ ${#annotation_rows[@]} -gt 0 ]]; then + echo "| Config | Duration | Status |" + echo "| --- | --- | --- |" + printf "%s\n" "${annotation_rows[@]}" + fi + + if [[ ${#failure_detail_lines[@]} -gt 0 ]]; then + echo "" + printf "%s\n" "${failure_detail_lines[@]}" + fi + } | buildkite-agent annotate \ + --scope job \ + --context "ftr-summary" \ + --style "${style}" || true +} diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.test.ts new file mode 100644 index 0000000000000..ca9affaf5a7fe --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import Fs from 'fs'; +import Os from 'os'; +import Path from 'path'; + +import { collectNewFailedTestNames, snapshotJunitDir } from './ftr_annotation_helper'; + +const buildXml = (testcases: string) => ` + + ${testcases} +`; + +const failedCase = (name: string) => + `error`; + +const passedCase = (name: string) => + ``; + +describe('snapshotJunitDir + collectNewFailedTestNames', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'annotation-helper-test-')); + }); + + afterEach(() => { + Fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('returns only failures from XMLs written after the snapshot', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-existing.xml'), buildXml(failedCase('old failure'))); + await snapshotJunitDir(tmpDir); + Fs.writeFileSync(Path.join(tmpDir, 'TEST-new.xml'), buildXml(failedCase('new failure'))); + + const names = await collectNewFailedTestNames(tmpDir); + expect([...names]).toEqual(['new failure']); + }); + + it('deletes the snapshot file after reading', async () => { + await snapshotJunitDir(tmpDir); + await collectNewFailedTestNames(tmpDir); + + expect(Fs.existsSync(Path.join(tmpDir, '.ftr_annotation_snapshot'))).toBe(false); + }); + + it('treats all XMLs as new when no snapshot exists', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(failedCase('test A'))); + + const names = await collectNewFailedTestNames(tmpDir); + expect([...names]).toEqual(['test A']); + }); + + it('returns empty set when no new XMLs were written after the snapshot', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-existing.xml'), buildXml(failedCase('old failure'))); + await snapshotJunitDir(tmpDir); + + const names = await collectNewFailedTestNames(tmpDir); + expect(names.size).toBe(0); + }); + + it('ignores passing tests in new XMLs', async () => { + await snapshotJunitDir(tmpDir); + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-new.xml'), + buildXml(passedCase('test A') + failedCase('test B')) + ); + + const names = await collectNewFailedTestNames(tmpDir); + expect([...names]).toEqual(['test B']); + }); +}); diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.ts new file mode 100644 index 0000000000000..95859542ce6d7 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/ftr_annotation_helper.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import Fs from 'fs'; +import Path from 'path'; + +import { createFlagError } from '@kbn/dev-cli-errors'; +import { run } from '@kbn/dev-cli-runner'; +import globby from 'globby'; +import normalize from 'normalize-path'; + +import { makeFailedTestCaseIter, readTestReport } from './test_report'; + +const SNAPSHOT_FILE = '.ftr_annotation_snapshot'; + +/** + * Writes the current XML file list in junitDir to a snapshot file. + * Call this before running a config so list-new-failures can diff against it. + */ +export async function snapshotJunitDir(junitDir: string): Promise { + const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true }); + Fs.mkdirSync(junitDir, { recursive: true }); + Fs.writeFileSync(Path.join(junitDir, SNAPSHOT_FILE), JSON.stringify(xmlPaths.sort())); +} + +/** + * Reads the snapshot written by snapshotJunitDir, diffs against the current XML files, + * and returns the failing test names from XMLs produced after the snapshot. + * Deletes the snapshot file after reading. + */ +export async function collectNewFailedTestNames(junitDir: string): Promise> { + const snapshotPath = Path.join(junitDir, SNAPSHOT_FILE); + let before = new Set(); + if (Fs.existsSync(snapshotPath)) { + before = new Set(JSON.parse(Fs.readFileSync(snapshotPath, 'utf8'))); + Fs.unlinkSync(snapshotPath); + } + const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true }); + const newPaths = xmlPaths.filter((p) => !before.has(p)); + const names = new Set(); + for (const xmlPath of newPaths) { + const report = await readTestReport(xmlPath); + for (const tc of makeFailedTestCaseIter(report)) { + names.add(tc.$.name.trim()); + } + } + return names; +} + +export function runAnnotationHelperCli() { + run( + async ({ flags }) => { + const [command, ...rest] = flags._; + + if (command === 'snapshot') { + const [junitDir] = rest; + if (!junitDir) { + throw createFlagError('Usage: snapshot '); + } + await snapshotJunitDir(junitDir); + return; + } + + if (command === 'list-new-failures') { + const [junitDir] = rest; + if (!junitDir) { + throw createFlagError('Usage: list-new-failures '); + } + const names = await collectNewFailedTestNames(junitDir); + if (names.size > 0) { + process.stdout.write([...names].join('\n') + '\n'); + } + return; + } + + throw createFlagError( + `Unknown command: ${command}. Valid commands: snapshot, list-new-failures` + ); + }, + { + description: ` + Per-config JUnit attribution helpers for FTR job annotations. + + Commands: + snapshot + Writes the current list of *.xml files in to a snapshot file. + Call this before running a config. + + list-new-failures + Diffs the current *.xml files against the snapshot and prints failing test + names from XMLs that are new since the snapshot. Deletes the snapshot file. + `, + } + ); +} diff --git a/packages/kbn-failed-test-reporter-cli/index.ts b/packages/kbn-failed-test-reporter-cli/index.ts index fd33d523318a9..dd8b02a4ebea5 100644 --- a/packages/kbn-failed-test-reporter-cli/index.ts +++ b/packages/kbn-failed-test-reporter-cli/index.ts @@ -8,3 +8,4 @@ */ import './failed_tests_reporter/failed_tests_reporter_cli'; +export { runAnnotationHelperCli } from './failed_tests_reporter/ftr_annotation_helper'; diff --git a/scripts/ftr_annotation_helper.js b/scripts/ftr_annotation_helper.js new file mode 100644 index 0000000000000..8115016483b76 --- /dev/null +++ b/scripts/ftr_annotation_helper.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +require('@kbn/setup-node-env'); +require('@kbn/failed-test-reporter-cli').runAnnotationHelperCli();