feat(ci): mark FTR retry green when previously-failing tests recover

.buildkite/scripts/steps/functional/common.sh

@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-# Note, changes here might also need to be made in other scripts, e.g. uptime.sh
-
 source .buildkite/scripts/common/util.sh
 
 # All functional/integration test steps run Kibana from the distributable,

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/test/ftr_smart_retry.sh
 
 BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
 FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-}
@@ -17,14 +18,14 @@ test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_
 export JOB="$FTR_CONFIG_GROUP_KEY"
 
 FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
+FAILED_TESTS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}_failed_tests"
 
-# a FTR failure will result in the script returning an exit code of 10
 exitCode=0
 
 configs="${FTR_CONFIG:-}"
 
-# The first retry should only run the configs that failed in the previous attempt
-# Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
+# The first retry should only run the configs that failed in the previous attempt.
+# Any subsequent retries (generally triggered manually) will run everything.
 if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
   configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
   if [[ "$configs" ]]; then
@@ -52,7 +53,7 @@ while read -r config; do
     continue;
   fi
 
-  FULL_COMMAND="node scripts/functional_tests --bail --config $config $EXTRA_ARGS"
+  FULL_COMMAND="node scripts/functional_tests --config $config $EXTRA_ARGS"
 
   # see if this config has already been executed successfully
   CONFIG_EXECUTION_KEY="${config}_executed"
@@ -90,20 +91,25 @@ while read -r config; do
   # prevent non-zero exit code from breaking the loop
   set +e;
   node ./scripts/functional_tests \
-    --bail \
     --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
     --config="$config" \
     "$EXTRA_ARGS"
   lastCode=$?
   set -e;
 
-  # Scout reporter
+  # Scout reporter — run under set+e so a failure here does not abort the config loop
   if [[ "${SCOUT_REPORTER_ENABLED:-}" =~ ^(1|true)$ ]]; then
-    # Upload events after running each config
     echo "Upload Scout reporter events to AppEx QA's team cluster for config $config"
+    set +e
     node scripts/scout upload-events --dontFailOnError
-    echo "Upload successful, removing local events at .scout/reports"
-    rm -rf .scout/reports
+    scout_upload_code=$?
+    set -e
+    if [[ $scout_upload_code -ne 0 ]]; then
+      echo "Scout reporter upload exited $scout_upload_code (continuing)"
+    else
+      echo "Upload successful, removing local events at .scout/reports"
+      rm -rf .scout/reports
+    fi
   else
     echo "SCOUT_REPORTER_ENABLED=$SCOUT_REPORTER_ENABLED, skipping event upload."
   fi
@@ -122,25 +128,22 @@ while read -r config; do
     result: ${lastCode}")
 
   if [ $lastCode -eq 0 ]; then
-    # Test was successful, so mark it as executed
     buildkite-agent meta-data set "$CONFIG_EXECUTION_KEY" "true"
   else
     exitCode=10
     echo "FTR exited with code $lastCode"
     echo "^^^ +++"
-
-    if [[ "$failedConfigs" ]]; then
-      failedConfigs="${failedConfigs}"$'\n'"$config"
-    else
-      failedConfigs="$config"
-    fi
+    failedConfigs="${failedConfigs:+${failedConfigs}$'\n'}$config"
   fi
 done <<< "$configs"
 
 if [[ "$failedConfigs" ]]; then
   buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
 fi
 
+store_failing_tests  # attempt 1: record what failed so the retry can verify recovery
+apply_smart_retry    # attempt 2: mark green if all previously-failing tests explicitly passed
+
 echo "--- FTR configs complete"
 printf "%s\n" "${results[@]}"
 echo ""

.buildkite/scripts/steps/test/ftr_smart_retry.sh

@@ -0,0 +1,58 @@
+# Sourced by ftr_configs.sh — do not execute directly.
+# Reads/writes globals: exitCode, failedConfigs, retry_recovered,
+# FAILED_TESTS_KEY, FAILED_CONFIGS_KEY, JOB, BUILDKITE_RETRY_COUNT.
+
+# Called after attempt 1: stores failing test names so the retry can verify recovery.
+store_failing_tests() {
+  [[ -n "${KIBANA_FLAKY_TEST_RUNNER_CONFIG:-}" ]] && return
+  [[ "${BUILDKITE_RETRY_COUNT:-0}" != "0" ]] && return
+  [[ "$exitCode" == "0" ]] && return
+
+  local junitDir="target/junit/$JOB"
+  [[ -d "$junitDir" ]] || return
+
+  local failedTestNames
+  failedTestNames=$(node scripts/ftr_check_retry_result list-failures "$junitDir" 2>/dev/null || true)
+  if [[ "$failedTestNames" ]]; then
+    buildkite-agent meta-data set "$FAILED_TESTS_KEY" "$failedTestNames"
+    echo "Stored $(echo "$failedTestNames" | wc -l | tr -d ' ') previously-failing test name(s) for retry evaluation"
+  fi
+}
+
+# Called after attempt 2: marks the step green if all previously-failing tests explicitly passed.
+# On a third-or-later manual retry, logs that smart-retry is inactive.
+apply_smart_retry() {
+  [[ -n "${KIBANA_FLAKY_TEST_RUNNER_CONFIG:-}" ]] && return
+  [[ "$exitCode" == "0" ]] && return
+
+  local retryCount="${BUILDKITE_RETRY_COUNT:-0}"
+
+  if [[ "$retryCount" -ge "2" ]]; then
+    echo "--- [smart-retry] inactive on attempt $((retryCount + 1)) — only applies to the first automatic retry"
+    return
+  fi
+
+  [[ "$retryCount" != "1" ]] && return
+
+  local prevFailedTests
+  prevFailedTests=$(buildkite-agent meta-data get "$FAILED_TESTS_KEY" --default '' 2>/dev/null || true)
+  [[ "$prevFailedTests" ]] || return
+
+  local junitDir="target/junit/$JOB"
+
+  local intersectionCode
+  set +e
+  printf '%s' "$prevFailedTests" | node scripts/ftr_check_retry_result check-intersection \
+    --junit-dir "$junitDir" \
+    --prev-failures-stdin
+  intersectionCode=$?
+  set -e
+
+  if [[ "$intersectionCode" == "0" ]]; then
+    echo "--- [smart-retry] All previously-failing tests recovered on retry — marking step green"
+    exitCode=0
+    failedConfigs=""
+    retry_recovered=true
+    buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "" 2>/dev/null || true
+  fi
+}