diff --git a/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml b/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml index 255debc0d98cb..60da014f75b6d 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml @@ -26,6 +26,7 @@ spec: KIBANA_SLACK_NOTIFICATIONS_ENABLED: 'true' SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' SCOUT_REPORTER_ENABLED: 'true' + FTR_SMART_RETRY_ENABLED: 'false' allow_rebuilds: true branch_configuration: main 9.4 9.3 8.19 default_branch: main diff --git a/.buildkite/pipeline-resource-definitions/kibana-pr.yml b/.buildkite/pipeline-resource-definitions/kibana-pr.yml index bf1f884c5c9e6..4326dfa09dd5a 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-pr.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-pr.yml @@ -23,6 +23,7 @@ spec: KIBANA_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true' GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci SCOUT_REPORTER_ENABLED: 'true' + FTR_SMART_RETRY_ENABLED: 'false' allow_rebuilds: true branch_configuration: '' cancel_intermediate_builds: true diff --git a/.buildkite/scripts/steps/test/ftr_configs.sh b/.buildkite/scripts/steps/test/ftr_configs.sh index a14283ef5c7df..fe5450c0296d3 100755 --- a/.buildkite/scripts/steps/test/ftr_configs.sh +++ b/.buildkite/scripts/steps/test/ftr_configs.sh @@ -3,6 +3,7 @@ set -euo pipefail source .buildkite/scripts/steps/functional/common.sh +source .buildkite/scripts/steps/test/ftr_smart_retry.sh BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-} FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-} @@ -11,6 +12,11 @@ if [ "$FTR_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; the exit 1 fi +BAIL_ARG="--bail" +if [[ "${FTR_SMART_RETRY_ENABLED:-}" =~ ^(1|true)$ ]]; then + BAIL_ARG="" +fi + EXTRA_ARGS=${FTR_EXTRA_ARGS:-} test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS" @@ -52,7 +58,7 @@ while read -r config; do continue; fi - FULL_COMMAND="node scripts/functional_tests --bail --config $config $EXTRA_ARGS" + FULL_COMMAND="node scripts/functional_tests $BAIL_ARG --config $config $EXTRA_ARGS" # see if this config has already been executed successfully CONFIG_EXECUTION_KEY="${config}_executed" @@ -90,9 +96,9 @@ while read -r config; do # prevent non-zero exit code from breaking the loop set +e; node ./scripts/functional_tests \ - --bail \ --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ --config="$config" \ + $BAIL_ARG \ "$EXTRA_ARGS" lastCode=$? set -e; @@ -141,6 +147,12 @@ if [[ "$failedConfigs" ]]; then buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs" fi + +if [[ "${FTR_SMART_RETRY_ENABLED:-}" =~ ^(1|true)$ ]]; then + store_failing_tests # attempt 1: record what failed so the retry can verify recovery + apply_smart_retry # attempt 2: mark green if all previously-failing tests explicitly passed +fi + echo "--- FTR configs complete" printf "%s\n" "${results[@]}" echo "" diff --git a/.buildkite/scripts/steps/test/ftr_smart_retry.sh b/.buildkite/scripts/steps/test/ftr_smart_retry.sh new file mode 100644 index 0000000000000..9342b5679f5d3 --- /dev/null +++ b/.buildkite/scripts/steps/test/ftr_smart_retry.sh @@ -0,0 +1,61 @@ +# Sourced by ftr_configs.sh — do not execute directly. +# Reads/writes globals: exitCode, failedConfigs, +# FAILED_CONFIGS_KEY, JOB, BUILDKITE_RETRY_COUNT. + +FAILED_TESTS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}_failed_tests" +retry_recovered=false + +# Called after attempt 1: stores failing test names so the retry can verify recovery. +store_failing_tests() { + [[ -n "${KIBANA_FLAKY_TEST_RUNNER_CONFIG:-}" ]] && return + [[ "${BUILDKITE_RETRY_COUNT:-0}" != "0" ]] && return + [[ "$exitCode" == "0" ]] && return + + local junitDir="target/junit/$JOB" + [[ -d "$junitDir" ]] || return + + local failedTestNames + failedTestNames=$(node scripts/ftr_check_retry_result list-failures "$junitDir" 2>/dev/null || true) + if [[ "$failedTestNames" ]]; then + buildkite-agent meta-data set "$FAILED_TESTS_KEY" "$failedTestNames" + echo "Stored $(echo "$failedTestNames" | wc -l | tr -d ' ') previously-failing test name(s) for retry evaluation" + fi +} + +# Called after attempt 2: marks the step green if all previously-failing tests explicitly passed. +# On a third-or-later manual retry, logs that smart-retry is inactive. +apply_smart_retry() { + [[ -n "${KIBANA_FLAKY_TEST_RUNNER_CONFIG:-}" ]] && return + [[ "$exitCode" == "0" ]] && return + + local retryCount="${BUILDKITE_RETRY_COUNT:-0}" + + if [[ "$retryCount" -ge "2" ]]; then + echo "--- [smart-retry] inactive on attempt $((retryCount + 1)) — only applies to the first automatic retry" + return + fi + + [[ "$retryCount" != "1" ]] && return + + local prevFailedTests + prevFailedTests=$(buildkite-agent meta-data get "$FAILED_TESTS_KEY" --default '' 2>/dev/null || true) + [[ "$prevFailedTests" ]] || return + + local junitDir="target/junit/$JOB" + + local intersectionCode + set +e + printf '%s' "$prevFailedTests" | node scripts/ftr_check_retry_result check-intersection \ + --junit-dir "$junitDir" \ + --prev-failures-stdin + intersectionCode=$? + set -e + + if [[ "$intersectionCode" == "0" ]]; then + echo "--- [smart-retry] All previously-failing tests recovered on retry — marking step green" + exitCode=0 + failedConfigs="" + retry_recovered=true + buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "" 2>/dev/null || true + fi +} diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index 9f06def545352..01e7f2fb18262 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -27,133 +27,135 @@ const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')]; const DISABLE_MISSING_TEST_REPORT_ERRORS = process.env.DISABLE_MISSING_TEST_REPORT_ERRORS === 'true'; -run( - async ({ log, flags }) => { - const indexInEs = Boolean(flags['index-errors']); - const reportUpdate = Boolean(flags['report-update']); - - let updateGithub = Boolean(flags['github-update']); - if (updateGithub && !process.env.GITHUB_TOKEN) { - throw createFailError( - 'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag' - ); - } - - let branch: string = ''; - let pipeline: string = ''; - let prependTitle: string = ''; - if (updateGithub) { - branch = process.env.BUILDKITE_BRANCH || ''; - pipeline = process.env.BUILDKITE_PIPELINE_SLUG || ''; - updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true'; - prependTitle = process.env.PREPEND_FAILURE_TITLE || ''; - - if (!branch) { +export function runFailedTestsReporterCli() { + run( + async ({ log, flags }) => { + const indexInEs = Boolean(flags['index-errors']); + const reportUpdate = Boolean(flags['report-update']); + + let updateGithub = Boolean(flags['github-update']); + if (updateGithub && !process.env.GITHUB_TOKEN) { throw createFailError( - 'Unable to determine originating branch from job name or other environment variables' + 'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag' ); } - } - const githubApi = new GithubApi({ - log, - token: process.env.GITHUB_TOKEN, - dryRun: !updateGithub, - }); - - const bkMeta = getBuildkiteMetadata(); - - try { - const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl'); - if (typeof buildUrl !== 'string' || !buildUrl) { - throw createFlagError('Missing --build-url or process.env.BUILD_URL'); + let branch: string = ''; + let pipeline: string = ''; + let prependTitle: string = ''; + if (updateGithub) { + branch = process.env.BUILDKITE_BRANCH || ''; + pipeline = process.env.BUILDKITE_PIPELINE_SLUG || ''; + updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true'; + prependTitle = process.env.PREPEND_FAILURE_TITLE || ''; + + if (!branch) { + throw createFailError( + 'Unable to determine originating branch from job name or other environment variables' + ); + } } - const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) => - normalize(Path.resolve(p)) - ); - log.info('Searching for reports at', patterns); - const reportPaths = await globby(patterns, { - absolute: true, + const githubApi = new GithubApi({ + log, + token: process.env.GITHUB_TOKEN, + dryRun: !updateGithub, }); - if (!reportPaths.length && DISABLE_MISSING_TEST_REPORT_ERRORS) { - // it is fine for code coverage to not have test results - return; - } - - if (reportPaths.length) { - log.info('found', reportPaths.length, 'reports', reportPaths); + const bkMeta = getBuildkiteMetadata(); - // Separate JUnit and Scout reports - const junitReports = reportPaths.filter((p) => p.endsWith('.xml')); - const scoutReports = reportPaths.filter((p) => p.endsWith('.ndjson')); + try { + const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl'); + if (typeof buildUrl !== 'string' || !buildUrl) { + throw createFlagError('Missing --build-url or process.env.BUILD_URL'); + } - log.info( - 'Processing', - junitReports.length, - 'JUnit reports and', - scoutReports.length, - 'Scout reports' + const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) => + normalize(Path.resolve(p)) ); - - const existingIssues = new ExistingFailedTestIssues(log); - - const processParams: ProcessReportsParams = { - log, - existingIssues, - buildUrl, - githubApi, - branch, - pipeline, - prependTitle, - updateGithub, - indexInEs, - reportUpdate, - bkMeta, - }; - - // Process FTR JUnit reports - await processJUnitReports(junitReports, processParams); - - // Process Scout reports - await processScoutReports(scoutReports, processParams); - - // Generate Scout test failure artifacts after reports are updated (GH issue info, html reports, etc.) - await generateScoutTestFailureArtifacts({ log, bkMeta }); + log.info('Searching for reports at', patterns); + const reportPaths = await globby(patterns, { + absolute: true, + }); + + if (!reportPaths.length && DISABLE_MISSING_TEST_REPORT_ERRORS) { + // it is fine for code coverage to not have test results + return; + } + + if (reportPaths.length) { + log.info('found', reportPaths.length, 'reports', reportPaths); + + // Separate JUnit and Scout reports + const junitReports = reportPaths.filter((p) => p.endsWith('.xml')); + const scoutReports = reportPaths.filter((p) => p.endsWith('.ndjson')); + + log.info( + 'Processing', + junitReports.length, + 'JUnit reports and', + scoutReports.length, + 'Scout reports' + ); + + const existingIssues = new ExistingFailedTestIssues(log); + + const processParams: ProcessReportsParams = { + log, + existingIssues, + buildUrl, + githubApi, + branch, + pipeline, + prependTitle, + updateGithub, + indexInEs, + reportUpdate, + bkMeta, + }; + + // Process FTR JUnit reports + await processJUnitReports(junitReports, processParams); + + // Process Scout reports + await processScoutReports(scoutReports, processParams); + + // Generate Scout test failure artifacts after reports are updated (GH issue info, html reports, etc.) + await generateScoutTestFailureArtifacts({ log, bkMeta }); + } + } finally { + await CiStatsReporter.fromEnv(log).metrics([ + { + group: 'github api request count', + id: `failed test reporter`, + value: githubApi.getRequestCount(), + meta: Object.fromEntries( + Object.entries(bkMeta).map( + ([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const + ) + ), + }, + ]); } - } finally { - await CiStatsReporter.fromEnv(log).metrics([ - { - group: 'github api request count', - id: `failed test reporter`, - value: githubApi.getRequestCount(), - meta: Object.fromEntries( - Object.entries(bkMeta).map( - ([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const - ) - ), + }, + { + description: `a cli that opens issues or updates existing issues based on junit reports`, + flags: { + boolean: ['github-update', 'report-update'], + string: ['build-url'], + default: { + 'github-update': true, + 'report-update': true, + 'index-errors': true, + 'build-url': process.env.BUILD_URL, }, - ]); - } - }, - { - description: `a cli that opens issues or updates existing issues based on junit reports`, - flags: { - boolean: ['github-update', 'report-update'], - string: ['build-url'], - default: { - 'github-update': true, - 'report-update': true, - 'index-errors': true, - 'build-url': process.env.BUILD_URL, - }, - help: ` + help: ` --no-github-update Execute the CLI without writing to Github --no-report-update Execute the CLI without writing to the JUnit reports --no-index-errors Execute the CLI without indexing failures into Elasticsearch --build-url URL of the failed build, defaults to process.env.BUILD_URL `, - }, - } -); + }, + } + ); +} diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/retry_result_checker.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/retry_result_checker.test.ts new file mode 100644 index 0000000000000..15c0e642f2cfa --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/retry_result_checker.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import Fs from 'fs'; +import Os from 'os'; +import Path from 'path'; + +import { collectFailedTestNames, collectPassedTestNames } from './retry_result_checker'; + +// Minimal JUnit XML helpers +const buildXml = (testcases: string) => ` + + ${testcases} +`; + +const failedCase = (name: string) => + `error`; + +const passedCase = (name: string) => + ``; + +const skippedCase = (name: string) => + ``; + +const hookFailure = (hookName: string) => + `error`; + +describe('collectFailedTestNames', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'retry-checker-test-')); + }); + + afterEach(() => { + Fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('returns names of failed tests from a single XML', async () => { + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-report.xml'), + buildXml(failedCase('suite myTest') + passedCase('suite otherTest')) + ); + const names = await collectFailedTestNames(tmpDir); + expect([...names]).toEqual(['suite myTest']); + }); + + it('aggregates failures across multiple XML files', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(failedCase('test A'))); + Fs.writeFileSync(Path.join(tmpDir, 'TEST-b.xml'), buildXml(failedCase('test B'))); + const names = await collectFailedTestNames(tmpDir); + expect([...names].sort()).toEqual(['test A', 'test B']); + }); + + it('returns empty set when all tests pass', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(passedCase('test A'))); + const names = await collectFailedTestNames(tmpDir); + expect(names.size).toBe(0); + }); + + it('returns empty set when no XML files exist', async () => { + const names = await collectFailedTestNames(tmpDir); + expect(names.size).toBe(0); + }); + + it('captures hook failure names verbatim', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(hookFailure('before all'))); + const names = await collectFailedTestNames(tmpDir); + expect([...names]).toEqual(['suite "before all" hook']); + }); +}); + +describe('collectPassedTestNames', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = Fs.mkdtempSync(Path.join(Os.tmpdir(), 'retry-checker-test-')); + }); + + afterEach(() => { + Fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('returns only tests that passed (no failure, no skipped)', async () => { + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-report.xml'), + buildXml(passedCase('test A') + failedCase('test B') + skippedCase('test C')) + ); + const names = await collectPassedTestNames(tmpDir); + expect([...names]).toEqual(['test A']); + }); + + it('does not count skipped tests as passed (beforeAll hook scenario)', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(skippedCase('test A'))); + const names = await collectPassedTestNames(tmpDir); + expect(names.size).toBe(0); + }); + + it('returns empty set when no XML files exist (runner crash scenario)', async () => { + const names = await collectPassedTestNames(tmpDir); + expect(names.size).toBe(0); + }); + + it('aggregates passed tests across multiple XML files', async () => { + Fs.writeFileSync(Path.join(tmpDir, 'TEST-a.xml'), buildXml(passedCase('test A'))); + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-b.xml'), + buildXml(passedCase('test B') + failedCase('test C')) + ); + const names = await collectPassedTestNames(tmpDir); + expect([...names].sort()).toEqual(['test A', 'test B']); + }); + + it('finds a recovered test even when a stale attempt-1 XML is present', async () => { + // Stale file from attempt 1 where the test failed + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-attempt1-bk__OLD.xml'), + buildXml(failedCase('test A')) + ); + // New file from attempt 2 where the test passes + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-attempt2-bk__NEW.xml'), + buildXml(passedCase('test A')) + ); + const names = await collectPassedTestNames(tmpDir); + expect(names.has('test A')).toBe(true); + }); + + it('does not count a test as passed when it fails in both attempts (stale XMLs present)', async () => { + // Stale file from attempt 1: test A failed + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-attempt1-bk__OLD.xml'), + buildXml(failedCase('test A')) + ); + // New file from attempt 2: test A still fails + Fs.writeFileSync( + Path.join(tmpDir, 'TEST-attempt2-bk__NEW.xml'), + buildXml(failedCase('test A')) + ); + const names = await collectPassedTestNames(tmpDir); + expect(names.has('test A')).toBe(false); + }); +}); diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/retry_result_checker.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/retry_result_checker.ts new file mode 100644 index 0000000000000..c40322b82f9a9 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/retry_result_checker.ts @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import Fs from 'fs'; +import Path from 'path'; + +import { createFlagError } from '@kbn/dev-cli-errors'; +import { run } from '@kbn/dev-cli-runner'; +import globby from 'globby'; +import normalize from 'normalize-path'; + +import { makeFailedTestCaseIter, makeTestCaseIter, readTestReport } from './test_report'; + +export async function collectFailedTestNames(junitDir: string): Promise> { + const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true }); + const names = new Set(); + for (const xmlPath of xmlPaths) { + const report = await readTestReport(xmlPath); + for (const testCase of makeFailedTestCaseIter(report)) { + names.add(testCase.$.name.trim()); + } + } + return names; +} + +/** + * Returns the names of test cases that completed without failure and without being skipped. + * Used on retry to verify previously-failing tests explicitly passed, not merely that they + * were absent from results (e.g. runner crash, beforeAll hook failure, or stale XML files + * from the previous attempt coexisting on a persistent-workspace agent). + */ +export async function collectPassedTestNames(junitDir: string): Promise> { + const xmlPaths = await globby(normalize(Path.resolve(junitDir, '*.xml')), { absolute: true }); + const names = new Set(); + for (const xmlPath of xmlPaths) { + const report = await readTestReport(xmlPath); + for (const testCase of makeTestCaseIter(report)) { + if (!testCase.failure && !testCase.skipped) { + names.add(testCase.$.name.trim()); + } + } + } + return names; +} + +const readStdin = (): Promise => + new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + process.stdin.on('data', (chunk) => + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + ); + process.stdin.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + process.stdin.on('error', reject); + }); + +export function runRetryResultCheckerCli() { + run( + async ({ log, flags }) => { + const [command, ...rest] = flags._; + + if (command === 'list-failures') { + const [junitDir] = rest; + if (!junitDir) { + throw createFlagError('Usage: list-failures '); + } + const names = await collectFailedTestNames(junitDir); + if (names.size > 0) { + process.stdout.write([...names].join('\n') + '\n'); + } + return; + } + + if (command === 'check-intersection') { + const junitDir = flags['junit-dir']; + const prevFailuresFile = flags['prev-failures-file']; + const prevFailuresStdin = flags['prev-failures-stdin']; + + if (typeof junitDir !== 'string' || !junitDir) { + throw createFlagError('--junit-dir is required'); + } + + let prevContent: string; + if (prevFailuresStdin) { + prevContent = await readStdin(); + } else if (typeof prevFailuresFile === 'string' && prevFailuresFile) { + prevContent = Fs.readFileSync(prevFailuresFile, 'utf8'); + } else { + throw createFlagError('Either --prev-failures-file or --prev-failures-stdin is required'); + } + + const prevFailed = new Set( + prevContent + .split('\n') + .map((l) => l.trim()) + .filter(Boolean) + ); + + if (prevFailed.size === 0) { + log.info('No previously-failing tests found — nothing to intersect'); + return; + } + + // Require every previously-failing test to appear as an explicit pass on retry. + // Checking for explicit passes (rather than absence of failure) guards against + // three false-green scenarios: (a) the runner crashes before reaching the test + // leaving an empty JUnit directory, (b) a beforeAll hook failure causes the test + // to be reported as skipped rather than failed, and (c) stale XML files from the + // previous attempt persist in the directory on a persistent-workspace agent. + const currentPassed = await collectPassedTestNames(junitDir); + const notRecovered = [...prevFailed].filter((name) => !currentPassed.has(name)); + + if (notRecovered.length === 0) { + log.success(`All ${prevFailed.size} previously-failing test(s) passed on retry`); + return; + } + + log.error(`${notRecovered.length} test(s) did not pass on retry:`); + for (const name of notRecovered) { + log.error(` ${name}`); + } + process.exit(1); + } + + throw createFlagError( + `Unknown command: ${command}. Valid commands: list-failures, check-intersection` + ); + }, + { + description: ` + Utilities for evaluating FTR retry results. + + Commands: + list-failures + Lists all failed test names (one per line) found in *.xml files under + the given directory. Used to capture attempt-1 failures before retry. + + check-intersection --junit-dir --prev-failures-file |--prev-failures-stdin + Checks whether every test named in (or stdin) appears as an explicit + pass in . Exits 0 if all previously-failing tests passed (step can be + marked green). Exits 1 if any previously-failing test did not pass + (still failing, skipped, or absent). + `, + flags: { + string: ['junit-dir', 'prev-failures-file'], + boolean: ['prev-failures-stdin'], + help: ` + --junit-dir Directory containing JUnit XML files for the current attempt + --prev-failures-file File with newline-separated test names that failed in attempt 1 + --prev-failures-stdin Read prev-failures from stdin instead of a file + `, + }, + } + ); +} diff --git a/packages/kbn-failed-test-reporter-cli/index.ts b/packages/kbn-failed-test-reporter-cli/index.ts index fd33d523318a9..c10d3adb34c1e 100644 --- a/packages/kbn-failed-test-reporter-cli/index.ts +++ b/packages/kbn-failed-test-reporter-cli/index.ts @@ -7,4 +7,5 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import './failed_tests_reporter/failed_tests_reporter_cli'; +export { runFailedTestsReporterCli } from './failed_tests_reporter/failed_tests_reporter_cli'; +export { runRetryResultCheckerCli } from './failed_tests_reporter/retry_result_checker'; diff --git a/scripts/ftr_check_retry_result.js b/scripts/ftr_check_retry_result.js new file mode 100644 index 0000000000000..ac377c7013b01 --- /dev/null +++ b/scripts/ftr_check_retry_result.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +require('@kbn/setup-node-env'); +require('@kbn/failed-test-reporter-cli').runRetryResultCheckerCli(); diff --git a/scripts/report_failed_tests.js b/scripts/report_failed_tests.js index 2b5eb9ffc4821..14226105dda10 100644 --- a/scripts/report_failed_tests.js +++ b/scripts/report_failed_tests.js @@ -8,4 +8,4 @@ */ require('@kbn/setup-node-env'); -require('@kbn/failed-test-reporter-cli'); +require('@kbn/failed-test-reporter-cli').runFailedTestsReporterCli(); diff --git a/src/platform/test/api_integration/apis/unused_urls_task/index.ts b/src/platform/test/api_integration/apis/unused_urls_task/index.ts index c6210ed44023d..eae3bf1227ce4 100644 --- a/src/platform/test/api_integration/apis/unused_urls_task/index.ts +++ b/src/platform/test/api_integration/apis/unused_urls_task/index.ts @@ -11,6 +11,8 @@ import type { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('unused_urls_task', () => { + // TEMPORARY: validates FTR retry intersection logic. Delete before merging this PR. + loadTestFile(require.resolve('./retry_validation_delete_before_merge')); loadTestFile(require.resolve('./run')); }); } diff --git a/src/platform/test/api_integration/apis/unused_urls_task/retry_validation_delete_before_merge.ts b/src/platform/test/api_integration/apis/unused_urls_task/retry_validation_delete_before_merge.ts new file mode 100644 index 0000000000000..6a7efea7dcbf4 --- /dev/null +++ b/src/platform/test/api_integration/apis/unused_urls_task/retry_validation_delete_before_merge.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +// TEMPORARY: validates the FTR retry intersection logic from this PR. Delete before merge. +// +// Scenario: +// - Attempt 1 (BUILDKITE_RETRY_COUNT unset / '0'): TEST_A fails. --bail stops the run. +// JUnit XML records TEST_A as failed; TEST_B is never reached. +// - Attempt 2 (BUILDKITE_RETRY_COUNT == '1'): TEST_A passes (recovered). +// TEST_B now fails (simulates an unrelated flake on retry). --bail stops the run. +// JUnit XML records TEST_B as failed. +// +// Stored prev failures: {TEST_A}. Current failures: {TEST_B}. Intersection: ∅. +// Expected: ftr_configs.sh overrides exit code to 0 and the step turns green. + +import type { FtrProviderContext } from '../../ftr_provider_context'; + +const isFirstAttempt = + !process.env.BUILDKITE_RETRY_COUNT || process.env.BUILDKITE_RETRY_COUNT === '0'; + +export default function ({}: FtrProviderContext) { + describe('retry-validation', () => { + it('TEST_A: intentionally fails on attempt 1, passes on attempt 2', () => { + if (isFirstAttempt) { + throw new Error('Intentional first-attempt failure (retry validation)'); + } + }); + + it('TEST_B: passes on attempt 1, intentionally fails on attempt 2', () => { + if (!isFirstAttempt) { + throw new Error('Intentional second-attempt failure (retry validation)'); + } + }); + }); +}