Use WARNING instead of FAIL for Baseline Improvement Healthcheck (#4883)

Summary:

The baseline improvement healthcheck currently marks the result as FAIL when no objectives improve over baseline, which can be overly alarming and may confuse our users unnecessarily. Lack of improvement is informational the experiment is still valid, just not yet showing gains so WARNING is more appropriate.

Also updated the subtitle logic to branch on `num_improved` counts instead of status, since both partial and no improvement now share the same WARNING status. This also allowed simplifying the status assignment to a single ternary (PASS if all improved, WARNING otherwise) and removing the now-unnecessary status parameter from `_build_subtitle`.

Reviewed By: bernardbeckerman

Differential Revision: D92871561

Author: shrutipatel31

ax/analysis/healthcheck/baseline_improvement.py

@@ -41,8 +41,7 @@ class BaselineImprovementAnalysis(Analysis):
 
     Status Logic:
         - PASS: All objectives improved over baseline
-        - WARNING: Some objectives improved over baseline
-        - FAIL: No objectives improved over baseline
+        - WARNING: One or more objectives did not improve over baseline
 
     The healthcheck evaluates improvement by:
         1. Identifying a baseline arm (explicit, status quo, or first trial)
@@ -231,20 +230,18 @@ def compute(
         num_improved = len(improved)
         num_total = len(comparison_list)
 
-        if num_improved == num_total:
-            status = HealthcheckStatus.PASS
-        elif num_improved > 0:
-            status = HealthcheckStatus.WARNING
-        else:
-            status = HealthcheckStatus.FAIL
+        status = (
+            HealthcheckStatus.PASS
+            if num_improved == num_total
+            else HealthcheckStatus.WARNING
+        )
 
         # Build subtitle
         subtitle = self._build_subtitle(
             num_improved=num_improved,
             num_total=num_total,
             not_improved=not_improved,
             details=details,
-            status=status,
             baseline_arm_name=baseline_arm_name,
             auto_selected_from_first_arm=auto_selected_from_first_arm,
             baseline_in_design=experiment.search_space.check_membership(
@@ -325,7 +322,6 @@ def _build_subtitle(
         num_total: int,
         not_improved: list[str],
         details: list[dict[str, str]],
-        status: HealthcheckStatus,
         baseline_arm_name: str,
         auto_selected_from_first_arm: bool,
         baseline_in_design: bool,
@@ -334,9 +330,9 @@ def _build_subtitle(
         parts: list[str] = []
 
         # Status summary
-        if status == HealthcheckStatus.PASS:
+        if num_improved == num_total:
             parts.append(f"All {num_total} objective(s) improved over baseline.")
-        elif status == HealthcheckStatus.WARNING:
+        elif num_improved > 0:
             parts.append(
                 f"{num_improved} out of {num_total} objective(s) "
                 "improved over baseline. The following metrics were not improved: "

ax/analysis/healthcheck/tests/test_baseline_improvement.py

@@ -61,12 +61,12 @@ def _attach_data(
         exp.attach_data(Data(df=pd.DataFrame(rows)))
 
     def test_status_outcomes(self) -> None:
-        """Test PASS/FAIL status based on improvement."""
+        """Test PASS/WARNING status based on improvement."""
         # minimize=True: lower is better
         test_cases = [
             # (baseline_mean, comparison_mean, expected_status, description)
             (100.0, 50.0, HealthcheckStatus.PASS, "improved (lower)"),
-            (50.0, 100.0, HealthcheckStatus.FAIL, "not improved (higher)"),
+            (50.0, 100.0, HealthcheckStatus.WARNING, "not improved (higher)"),
         ]
 
         for baseline_mean, comparison_mean, expected_status, desc in test_cases:
@@ -107,7 +107,7 @@ def test_multi_objective_partial_improvement(self) -> None:
 
     def test_documentation_link(self) -> None:
         """Test documentation_link is appended correctly."""
-        # minimize=True: baseline=50, comparison=100 -> NOT improved (FAIL status)
+        # minimize=True: baseline=50, comparison=100 -> NOT improved (WARNING status)
         self._attach_data(
             {"branin": [(50.0, 0.1), (100.0, 0.1)]}, arm_names=["0_0", "0_1"]
         )
@@ -178,7 +178,7 @@ def test_dataframe_structure(self) -> None:
         self.assertEqual(len(card.df), 1)
 
     def test_no_improvement_message_parameter(self) -> None:
-        """Test custom no_improvement_message is displayed on FAIL status."""
+        """Test custom no_improvement_message is displayed on WARNING status."""
         self._attach_data(
             {"branin": [(50.0, 0.1), (100.0, 0.1)]}, arm_names=["0_0", "0_1"]
         )
@@ -192,5 +192,5 @@ def test_no_improvement_message_parameter(self) -> None:
         )
         card = analysis.compute(experiment=self.experiment)
 
-        self.assertEqual(card.get_status(), HealthcheckStatus.FAIL)
+        self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
         self.assertIn(custom_message, card.subtitle)