put this all behind an env flag

Author: TamerlanG

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -26,6 +26,7 @@ spec:
         KIBANA_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true'
         SCOUT_REPORTER_ENABLED: 'true'
+        FTR_SMART_RETRY_ENABLED: 'false'
       allow_rebuilds: true
       branch_configuration: main 9.4 9.3 8.19
       default_branch: main

.buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -23,6 +23,7 @@ spec:
         KIBANA_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
         GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
         SCOUT_REPORTER_ENABLED: 'true'
+        FTR_SMART_RETRY_ENABLED: 'false'
       allow_rebuilds: true
       branch_configuration: ''
       cancel_intermediate_builds: true

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -141,8 +141,11 @@ if [[ "$failedConfigs" ]]; then
   buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"
 fi
 
-store_failing_tests  # attempt 1: record what failed so the retry can verify recovery
-apply_smart_retry    # attempt 2: mark green if all previously-failing tests explicitly passed
+
+if [[ "${FTR_SMART_RETRY_ENABLED:-}" =~ ^(1|true)$ ]]; then
+  store_failing_tests  # attempt 1: record what failed so the retry can verify recovery
+  apply_smart_retry    # attempt 2: mark green if all previously-failing tests explicitly passed
+fi
 
 echo "--- FTR configs complete"
 printf "%s\n" "${results[@]}"