Change healthcheck statuses for clearer semantic meanings (facebook#5020)

Summary:

Update healthcheck status levels to better reflect their semantic meaning and actionability:

1. Early Stopping Analysis:
   - ESS explicitly disabled: PASS → INFO (surface configuration choice)
   - ESS enabled, working correctly: PASS → INFO (informational, no action needed)
   - ESS not enabled but could save ≥10%: WARNING → INFO (nudge for feature upsell; not blocking the experiment)

2. Complexity Rating:
   - Advanced tier: WARNING → INFO (not blocking, informational)
   - Unsupported tier: FAIL → WARNING (concerning but not blocking)

3. Can Generate Candidates:
   - can_generate=False (recent trial): WARNING → INFO (expected behavior)

4. Baseline Improvement:
   - All objectives improved: PASS → INFO (valuable experiment takeaway)
   - Any objective not improved: WARNING → INFO (informational, not actionable)

Differential Revision: D96412166

Author: shrutipatel31

ax/analysis/healthcheck/baseline_improvement.py

@@ -230,12 +230,6 @@ def compute(
         num_improved = len(improved)
         num_total = len(comparison_list)
 
-        status = (
-            HealthcheckStatus.PASS
-            if num_improved == num_total
-            else HealthcheckStatus.WARNING
-        )
-
         # Build subtitle
         subtitle = self._build_subtitle(
             num_improved=num_improved,
@@ -256,7 +250,7 @@ def compute(
             title="Baseline Improvement Healthcheck",
             subtitle=subtitle,
             df=pd.DataFrame(details),
-            status=status,
+            status=HealthcheckStatus.INFO,
             num_objectives_improved=num_improved,
             num_objectives_total=num_total,
             baseline_arm_name=baseline_arm_name,

ax/analysis/healthcheck/can_generate_candidates.py

@@ -76,7 +76,7 @@ def compute(
                     status = HealthcheckStatus.FAIL
                     title_status = "Failure"
                 else:
-                    status = HealthcheckStatus.WARNING
+                    status = HealthcheckStatus.INFO
                     title_status = "Warning"
                 subtitle += self.LAST_RUN_TEMPLATE.format(days=days_since_last_run)
         else:

ax/analysis/healthcheck/complexity_rating.py

@@ -142,9 +142,9 @@ def compute(
         if tier == "Standard":
             status = HealthcheckStatus.PASS
         elif tier == "Advanced":
-            status = HealthcheckStatus.WARNING
+            status = HealthcheckStatus.INFO
         else:  # Unsupported
-            status = HealthcheckStatus.FAIL
+            status = HealthcheckStatus.WARNING
 
         # Create dataframe with experiment summary
         df = pd.DataFrame(

ax/analysis/healthcheck/early_stopping_healthcheck.py

@@ -233,7 +233,7 @@ def _report_early_stopping_disabled(
                 "auto_early_stopping_config='standard' or configure an "
                 "early_stopping_strategy_config."
             ),
-            status=HealthcheckStatus.PASS,
+            status=HealthcheckStatus.INFO,
             df=df,
         )
 
@@ -391,7 +391,7 @@ def _report_early_stopping_status(
             title=EARLY_STOPPING_SAVINGS_TITLE,
             subtitle=subtitle,
             df=df,
-            status=HealthcheckStatus.PASS,
+            status=HealthcheckStatus.INFO,
         )
 
     def _report_early_stopping_nudge(
@@ -467,7 +467,7 @@ def _report_early_stopping_nudge(
             title=title,
             subtitle=subtitle,
             df=df,
-            status=HealthcheckStatus.WARNING,
+            status=HealthcheckStatus.INFO,
             potential_savings=savings_pct,
             best_metric=metric.name,
         )

ax/analysis/healthcheck/tests/test_baseline_improvement.py

@@ -61,12 +61,12 @@ def _attach_data(
         exp.attach_data(Data(df=pd.DataFrame(rows)))
 
     def test_status_outcomes(self) -> None:
-        """Test PASS/WARNING status based on improvement."""
+        """Test INFO status for all improvement outcomes."""
         # minimize=True: lower is better
         test_cases = [
             # (baseline_mean, comparison_mean, expected_status, description)
-            (100.0, 50.0, HealthcheckStatus.PASS, "improved (lower)"),
-            (50.0, 100.0, HealthcheckStatus.WARNING, "not improved (higher)"),
+            (100.0, 50.0, HealthcheckStatus.INFO, "improved (lower)"),
+            (50.0, 100.0, HealthcheckStatus.INFO, "not improved (higher)"),
         ]
 
         for baseline_mean, comparison_mean, expected_status, desc in test_cases:
@@ -83,7 +83,7 @@ def test_status_outcomes(self) -> None:
                 self.assertEqual(card.get_status(), expected_status)
 
     def test_multi_objective_partial_improvement(self) -> None:
-        """Test WARNING status when only some objectives improve."""
+        """Test INFO status when only some objectives improve."""
         # minimize=True for both objectives (lower is better):
         # branin_a: 100 -> 50, improved (decreased)
         # branin_b: 50 -> 100, NOT improved (increased)
@@ -102,7 +102,7 @@ def test_multi_objective_partial_improvement(self) -> None:
         )
         card = analysis.compute(experiment=self.moo_experiment)
 
-        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertIn("1 out of 2", card.subtitle)
 
     def test_documentation_link(self) -> None:
@@ -178,7 +178,7 @@ def test_dataframe_structure(self) -> None:
         self.assertEqual(len(card.df), 1)
 
     def test_no_improvement_message_parameter(self) -> None:
-        """Test custom no_improvement_message is displayed on WARNING status."""
+        """Test custom no_improvement_message is displayed on INFO status."""
         self._attach_data(
             {"branin": [(50.0, 0.1), (100.0, 0.1)]}, arm_names=["0_0", "0_1"]
         )
@@ -192,5 +192,5 @@ def test_no_improvement_message_parameter(self) -> None:
         )
         card = analysis.compute(experiment=self.experiment)
 
-        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertIn(custom_message, card.subtitle)

ax/analysis/healthcheck/tests/test_can_generate_candidates.py

@@ -57,8 +57,8 @@ def test_warns_if_a_trial_was_recently_run(self) -> None:
             reason="The data is borked.",
             days_till_fail=2,
         ).compute(experiment=experiment, generation_strategy=None)
-        # THEN it is a WARNING
-        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+        # THEN it is INFO
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertEqual(card.name, "CanGenerateCandidatesAnalysis")
         self.assertEqual(card.title, "Ax Candidate Generation Warning")
         self.assertEqual(
@@ -71,11 +71,11 @@ def test_warns_if_a_trial_was_recently_run(self) -> None:
                 "LAST TRIAL RUN: 1 day(s) ago"
             ),
         )
-        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertDictEqual(
             card.get_aditional_attrs(),
             {
-                "status": HealthcheckStatus.WARNING,
+                "status": HealthcheckStatus.INFO,
                 "reason": "The data is borked.",
             },
         )

ax/analysis/healthcheck/tests/test_complexity_rating.py

@@ -65,8 +65,8 @@ def test_standard_configuration(self) -> None:
 
     def test_parameter_counts(self) -> None:
         test_cases = [
-            (60, HealthcheckStatus.WARNING, "Advanced", "60 tunable parameter(s)"),
-            (250, HealthcheckStatus.FAIL, "Unsupported", "250 tunable parameter(s)"),
+            (60, HealthcheckStatus.INFO, "Advanced", "60 tunable parameter(s)"),
+            (250, HealthcheckStatus.WARNING, "Unsupported", "250 tunable parameter(s)"),
         ]
 
         for num_params, expected_status, expected_tier, expected_msg in test_cases:
@@ -92,8 +92,8 @@ def test_parameter_counts(self) -> None:
 
     def test_objectives_count(self) -> None:
         test_cases = [
-            (3, HealthcheckStatus.WARNING, "Advanced", "3 objectives"),
-            (5, HealthcheckStatus.FAIL, "Unsupported", "5 objectives"),
+            (3, HealthcheckStatus.INFO, "Advanced", "3 objectives"),
+            (5, HealthcheckStatus.WARNING, "Unsupported", "5 objectives"),
         ]
 
         for num_objs, expected_status, expected_tier, expected_msg in test_cases:
@@ -128,7 +128,7 @@ def test_constraints(self) -> None:
                 options=self.options, tier_metadata=self.tier_metadata
             ).compute(experiment=self.experiment)
 
-            self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+            self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
             self.assertIn("Advanced", card.subtitle)
             self.assertIn("3 outcome constraints", card.subtitle)
 
@@ -152,7 +152,7 @@ def test_constraints(self) -> None:
                 options=self.options, tier_metadata=self.tier_metadata
             ).compute(experiment=self.experiment)
 
-            self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+            self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
             self.assertIn("Advanced", card.subtitle)
             self.assertIn("3 parameter constraints", card.subtitle)
 
@@ -174,14 +174,14 @@ def test_stopping_strategies(self) -> None:
                     options=options, tier_metadata=self.tier_metadata
                 ).compute(experiment=self.experiment)
 
-                self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+                self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
                 self.assertIn("Advanced", card.subtitle)
                 self.assertIn(expected_msg, card.subtitle)
 
     def test_trial_counts(self) -> None:
         test_cases = [
-            (300, HealthcheckStatus.WARNING, "Advanced", "300 total trials"),
-            (600, HealthcheckStatus.FAIL, "Unsupported", "600 total trials"),
+            (300, HealthcheckStatus.INFO, "Advanced", "300 total trials"),
+            (600, HealthcheckStatus.WARNING, "Unsupported", "600 total trials"),
         ]
 
         for max_trials, expected_status, expected_tier, expected_msg in test_cases:
@@ -229,7 +229,7 @@ def test_unsupported_configurations(self) -> None:
                     options=options, tier_metadata=tier_metadata
                 ).compute(experiment=self.experiment)
 
-                self.assertEqual(card.get_status(), HealthcheckStatus.FAIL)
+                self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
                 self.assertIn("Unsupported", card.subtitle)
                 self.assertIn(expected_msg, card.subtitle)
 
@@ -253,7 +253,7 @@ def test_unordered_choice_parameters(self) -> None:
             options=self.options, tier_metadata=self.tier_metadata
         ).compute(experiment=self.experiment)
 
-        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertIn("Advanced", card.subtitle)
         self.assertIn("unordered choice parameter(s)", card.subtitle)
 
@@ -275,7 +275,7 @@ def test_binary_parameters_count(self) -> None:
             options=self.options, tier_metadata=self.tier_metadata
         ).compute(experiment=self.experiment)
 
-        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertIn("Advanced", card.subtitle)
         self.assertIn("60 binary tunable parameter(s)", card.subtitle)
 
@@ -298,7 +298,7 @@ def test_multiple_violations(self) -> None:
             options=options, tier_metadata=tier_metadata
         ).compute(experiment=experiment)
 
-        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertIn("Advanced", card.subtitle)
         self.assertIn("60 tunable parameter(s)", card.subtitle)
         self.assertIn("300 total trials", card.subtitle)

ax/analysis/healthcheck/tests/test_early_stopping_healthcheck.py

@@ -177,7 +177,7 @@ def test_early_stopping_not_enabled(self) -> None:
                 return_value=mock_savings,
             ):
                 card = healthcheck.compute(experiment=self.experiment)
-                self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+                self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
                 self.assertIn("25%", card.subtitle)
 
     def test_validate_applicable_state(self) -> None:
@@ -261,15 +261,15 @@ def test_auto_early_stopping_config(self) -> None:
             healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="disabled")
             card = healthcheck.compute(experiment=self.experiment)
 
-            self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
+            self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
             self.assertIn("auto_early_stopping_config='disabled'", card.subtitle)
 
         with self.subTest("standard"):
             self._mark_trial_completed()
             healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="standard")
             card = healthcheck.compute(experiment=self.experiment)
 
-            self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
+            self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
             self.assertIn("0 trials were early stopped", card.subtitle)
 
         with self.subTest("strategy_override"):
@@ -280,7 +280,7 @@ def test_auto_early_stopping_config(self) -> None:
             )
             card = healthcheck.compute(experiment=self.experiment)
 
-            self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
+            self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
             self.assertEqual(healthcheck.early_stopping_strategy, custom_strategy)
 
     def test_multiple_metrics_note_in_subtitle(self) -> None:
@@ -294,7 +294,7 @@ def test_multiple_metrics_note_in_subtitle(self) -> None:
         ):
             card = self.healthcheck.compute(experiment=self.experiment)
 
-        self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
+        self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
         self.assertIn("2 metrics are", card.subtitle)
 
     def test_get_problem_type_via_disabled_config(self) -> None:
@@ -355,7 +355,7 @@ def test_hypothetical_savings_nudge(self) -> None:
             ):
                 card = healthcheck.compute(experiment=self.experiment)
 
-            self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+            self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
             self.assertIn("25%", card.subtitle)
 
         with self.subTest("with_additional_info"):
@@ -375,5 +375,5 @@ def test_hypothetical_savings_nudge(self) -> None:
             ):
                 card = healthcheck_with_info.compute(experiment=self.experiment)
 
-            self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
+            self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
             self.assertIn(nudge_info, card.subtitle)