From 1c24f7bb21e151f3ab081314b0192ae84cf4b1da Mon Sep 17 00:00:00 2001 From: Shruti Patel Date: Tue, 17 Mar 2026 12:48:08 -0700 Subject: [PATCH] Update healthcheck statuses (#5020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Update healthcheck status levels to better reflect their meaning and actionability: 1. Early Stopping Analysis: - ESS explicitly disabled: PASS → INFO (surface configuration choice) - ESS enabled, working correctly: PASS → INFO (informational, no action needed) - ESS not enabled but could save ≥10%: WARNING → INFO (nudge for feature upsell; not blocking the experiment) 2. Complexity Rating: - Advanced tier: WARNING → INFO (not blocking, informational) - Unsupported tier: FAIL → WARNING (concerning but not blocking) 3. Can Generate Candidates: - can_generate=False (recent trial): WARNING → INFO (expected behavior) 4. Baseline Improvement: - All objectives improved: PASS → INFO (valuable experiment takeaway) - Any objective not improved: WARNING → INFO (informational, not actionable) 5. Should Generate Candidates: - should_generate=False : WARNING → INFO Reviewed By: bernardbeckerman Differential Revision: D96412166 --- .../healthcheck/baseline_improvement.py | 4 +-- .../healthcheck/can_generate_candidates.py | 4 +-- ax/analysis/healthcheck/complexity_rating.py | 4 +-- .../healthcheck/early_stopping_healthcheck.py | 6 ++--- .../healthcheck/should_generate_candidates.py | 4 +-- .../tests/test_baseline_improvement.py | 8 +++--- .../tests/test_can_generate_candidates.py | 10 +++---- .../tests/test_complexity_rating.py | 26 +++++++++---------- .../tests/test_early_stopping_healthcheck.py | 14 +++++----- .../tests/test_should_generate_candidates.py | 2 +- 10 files changed, 39 insertions(+), 43 deletions(-) diff --git a/ax/analysis/healthcheck/baseline_improvement.py b/ax/analysis/healthcheck/baseline_improvement.py index fba97fd3935..30b0e3ea6d2 100644 --- a/ax/analysis/healthcheck/baseline_improvement.py +++ b/ax/analysis/healthcheck/baseline_improvement.py @@ -231,9 +231,7 @@ def compute( num_total = len(comparison_list) status = ( - HealthcheckStatus.PASS - if num_improved == num_total - else HealthcheckStatus.WARNING + HealthcheckStatus.INFO if num_improved > 0 else HealthcheckStatus.WARNING ) # Build subtitle diff --git a/ax/analysis/healthcheck/can_generate_candidates.py b/ax/analysis/healthcheck/can_generate_candidates.py index 4576f8650ae..689fe31a9b2 100644 --- a/ax/analysis/healthcheck/can_generate_candidates.py +++ b/ax/analysis/healthcheck/can_generate_candidates.py @@ -76,8 +76,8 @@ def compute( status = HealthcheckStatus.FAIL title_status = "Failure" else: - status = HealthcheckStatus.WARNING - title_status = "Warning" + status = HealthcheckStatus.INFO + title_status = "Info" subtitle += self.LAST_RUN_TEMPLATE.format(days=days_since_last_run) else: subtitle += f"{self.reason}" diff --git a/ax/analysis/healthcheck/complexity_rating.py b/ax/analysis/healthcheck/complexity_rating.py index 77843262e46..d43c101d740 100644 --- a/ax/analysis/healthcheck/complexity_rating.py +++ b/ax/analysis/healthcheck/complexity_rating.py @@ -142,9 +142,9 @@ def compute( if tier == "Standard": status = HealthcheckStatus.PASS elif tier == "Advanced": - status = HealthcheckStatus.WARNING + status = HealthcheckStatus.INFO else: # Unsupported - status = HealthcheckStatus.FAIL + status = HealthcheckStatus.WARNING # Create dataframe with experiment summary df = pd.DataFrame( diff --git a/ax/analysis/healthcheck/early_stopping_healthcheck.py b/ax/analysis/healthcheck/early_stopping_healthcheck.py index 9af56cb297f..13dd29f68ab 100644 --- a/ax/analysis/healthcheck/early_stopping_healthcheck.py +++ b/ax/analysis/healthcheck/early_stopping_healthcheck.py @@ -233,7 +233,7 @@ def _report_early_stopping_disabled( "auto_early_stopping_config='standard' or configure an " "early_stopping_strategy_config." ), - status=HealthcheckStatus.PASS, + status=HealthcheckStatus.INFO, df=df, ) @@ -391,7 +391,7 @@ def _report_early_stopping_status( title=EARLY_STOPPING_SAVINGS_TITLE, subtitle=subtitle, df=df, - status=HealthcheckStatus.PASS, + status=HealthcheckStatus.INFO, ) def _report_early_stopping_nudge( @@ -467,7 +467,7 @@ def _report_early_stopping_nudge( title=title, subtitle=subtitle, df=df, - status=HealthcheckStatus.WARNING, + status=HealthcheckStatus.INFO, potential_savings=savings_pct, best_metric=metric.name, ) diff --git a/ax/analysis/healthcheck/should_generate_candidates.py b/ax/analysis/healthcheck/should_generate_candidates.py index b1ee907123c..5243e04bf4e 100644 --- a/ax/analysis/healthcheck/should_generate_candidates.py +++ b/ax/analysis/healthcheck/should_generate_candidates.py @@ -40,9 +40,7 @@ def compute( adapter: Adapter | None = None, ) -> HealthcheckAnalysisCard: status = ( - HealthcheckStatus.PASS - if self.should_generate - else HealthcheckStatus.WARNING + HealthcheckStatus.PASS if self.should_generate else HealthcheckStatus.INFO ) return create_healthcheck_analysis_card( name=self.__class__.__name__, diff --git a/ax/analysis/healthcheck/tests/test_baseline_improvement.py b/ax/analysis/healthcheck/tests/test_baseline_improvement.py index 87e8dd1643b..4fb13636fed 100644 --- a/ax/analysis/healthcheck/tests/test_baseline_improvement.py +++ b/ax/analysis/healthcheck/tests/test_baseline_improvement.py @@ -61,11 +61,11 @@ def _attach_data( exp.attach_data(Data(df=pd.DataFrame(rows))) def test_status_outcomes(self) -> None: - """Test PASS/WARNING status based on improvement.""" + """Test INFO/WARNING status based on improvement.""" # minimize=True: lower is better test_cases = [ # (baseline_mean, comparison_mean, expected_status, description) - (100.0, 50.0, HealthcheckStatus.PASS, "improved (lower)"), + (100.0, 50.0, HealthcheckStatus.INFO, "improved (lower)"), (50.0, 100.0, HealthcheckStatus.WARNING, "not improved (higher)"), ] @@ -83,7 +83,7 @@ def test_status_outcomes(self) -> None: self.assertEqual(card.get_status(), expected_status) def test_multi_objective_partial_improvement(self) -> None: - """Test WARNING status when only some objectives improve.""" + """Test INFO status when only some objectives improve.""" # minimize=True for both objectives (lower is better): # branin_a: 100 -> 50, improved (decreased) # branin_b: 50 -> 100, NOT improved (increased) @@ -102,7 +102,7 @@ def test_multi_objective_partial_improvement(self) -> None: ) card = analysis.compute(experiment=self.moo_experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("1 out of 2", card.subtitle) def test_documentation_link(self) -> None: diff --git a/ax/analysis/healthcheck/tests/test_can_generate_candidates.py b/ax/analysis/healthcheck/tests/test_can_generate_candidates.py index c4ac7e9efc2..d69894fe82b 100644 --- a/ax/analysis/healthcheck/tests/test_can_generate_candidates.py +++ b/ax/analysis/healthcheck/tests/test_can_generate_candidates.py @@ -57,10 +57,10 @@ def test_warns_if_a_trial_was_recently_run(self) -> None: reason="The data is borked.", days_till_fail=2, ).compute(experiment=experiment, generation_strategy=None) - # THEN it is a WARNING - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + # THEN it is INFO + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertEqual(card.name, "CanGenerateCandidatesAnalysis") - self.assertEqual(card.title, "Ax Candidate Generation Warning") + self.assertEqual(card.title, "Ax Candidate Generation Info") self.assertEqual( card.subtitle, ( @@ -71,11 +71,11 @@ def test_warns_if_a_trial_was_recently_run(self) -> None: "LAST TRIAL RUN: 1 day(s) ago" ), ) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertDictEqual( card.get_aditional_attrs(), { - "status": HealthcheckStatus.WARNING, + "status": HealthcheckStatus.INFO, "reason": "The data is borked.", }, ) diff --git a/ax/analysis/healthcheck/tests/test_complexity_rating.py b/ax/analysis/healthcheck/tests/test_complexity_rating.py index d632d3a625d..a22422153cb 100644 --- a/ax/analysis/healthcheck/tests/test_complexity_rating.py +++ b/ax/analysis/healthcheck/tests/test_complexity_rating.py @@ -65,8 +65,8 @@ def test_standard_configuration(self) -> None: def test_parameter_counts(self) -> None: test_cases = [ - (60, HealthcheckStatus.WARNING, "Advanced", "60 tunable parameter(s)"), - (250, HealthcheckStatus.FAIL, "Unsupported", "250 tunable parameter(s)"), + (60, HealthcheckStatus.INFO, "Advanced", "60 tunable parameter(s)"), + (250, HealthcheckStatus.WARNING, "Unsupported", "250 tunable parameter(s)"), ] for num_params, expected_status, expected_tier, expected_msg in test_cases: @@ -92,8 +92,8 @@ def test_parameter_counts(self) -> None: def test_objectives_count(self) -> None: test_cases = [ - (3, HealthcheckStatus.WARNING, "Advanced", "3 objectives"), - (5, HealthcheckStatus.FAIL, "Unsupported", "5 objectives"), + (3, HealthcheckStatus.INFO, "Advanced", "3 objectives"), + (5, HealthcheckStatus.WARNING, "Unsupported", "5 objectives"), ] for num_objs, expected_status, expected_tier, expected_msg in test_cases: @@ -135,7 +135,7 @@ def test_constraints(self) -> None: options=self.options, tier_metadata=self.tier_metadata ).compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("Advanced", card.subtitle) self.assertIn("3 outcome constraints", card.subtitle) @@ -159,7 +159,7 @@ def test_constraints(self) -> None: options=self.options, tier_metadata=self.tier_metadata ).compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("Advanced", card.subtitle) self.assertIn("3 parameter constraints", card.subtitle) @@ -181,14 +181,14 @@ def test_stopping_strategies(self) -> None: options=options, tier_metadata=self.tier_metadata ).compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("Advanced", card.subtitle) self.assertIn(expected_msg, card.subtitle) def test_trial_counts(self) -> None: test_cases = [ - (300, HealthcheckStatus.WARNING, "Advanced", "300 total trials"), - (600, HealthcheckStatus.FAIL, "Unsupported", "600 total trials"), + (300, HealthcheckStatus.INFO, "Advanced", "300 total trials"), + (600, HealthcheckStatus.WARNING, "Unsupported", "600 total trials"), ] for max_trials, expected_status, expected_tier, expected_msg in test_cases: @@ -236,7 +236,7 @@ def test_unsupported_configurations(self) -> None: options=options, tier_metadata=tier_metadata ).compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.FAIL) + self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) self.assertIn("Unsupported", card.subtitle) self.assertIn(expected_msg, card.subtitle) @@ -260,7 +260,7 @@ def test_unordered_choice_parameters(self) -> None: options=self.options, tier_metadata=self.tier_metadata ).compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("Advanced", card.subtitle) self.assertIn("unordered choice parameter(s)", card.subtitle) @@ -282,7 +282,7 @@ def test_binary_parameters_count(self) -> None: options=self.options, tier_metadata=self.tier_metadata ).compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("Advanced", card.subtitle) self.assertIn("60 binary tunable parameter(s)", card.subtitle) @@ -305,7 +305,7 @@ def test_multiple_violations(self) -> None: options=options, tier_metadata=tier_metadata ).compute(experiment=experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("Advanced", card.subtitle) self.assertIn("60 tunable parameter(s)", card.subtitle) self.assertIn("300 total trials", card.subtitle) diff --git a/ax/analysis/healthcheck/tests/test_early_stopping_healthcheck.py b/ax/analysis/healthcheck/tests/test_early_stopping_healthcheck.py index ff1b669a5fd..bae7cc8b37a 100644 --- a/ax/analysis/healthcheck/tests/test_early_stopping_healthcheck.py +++ b/ax/analysis/healthcheck/tests/test_early_stopping_healthcheck.py @@ -177,7 +177,7 @@ def test_early_stopping_not_enabled(self) -> None: return_value=mock_savings, ): card = healthcheck.compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("25%", card.subtitle) def test_validate_applicable_state(self) -> None: @@ -261,7 +261,7 @@ def test_auto_early_stopping_config(self) -> None: healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="disabled") card = healthcheck.compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.PASS) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("auto_early_stopping_config='disabled'", card.subtitle) with self.subTest("standard"): @@ -269,7 +269,7 @@ def test_auto_early_stopping_config(self) -> None: healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="standard") card = healthcheck.compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.PASS) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("0 trials were early stopped", card.subtitle) with self.subTest("strategy_override"): @@ -280,7 +280,7 @@ def test_auto_early_stopping_config(self) -> None: ) card = healthcheck.compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.PASS) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertEqual(healthcheck.early_stopping_strategy, custom_strategy) def test_multiple_metrics_note_in_subtitle(self) -> None: @@ -294,7 +294,7 @@ def test_multiple_metrics_note_in_subtitle(self) -> None: ): card = self.healthcheck.compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.PASS) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("2 metrics are", card.subtitle) def test_get_problem_type_via_disabled_config(self) -> None: @@ -355,7 +355,7 @@ def test_hypothetical_savings_nudge(self) -> None: ): card = healthcheck.compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn("25%", card.subtitle) with self.subTest("with_additional_info"): @@ -375,5 +375,5 @@ def test_hypothetical_savings_nudge(self) -> None: ): card = healthcheck_with_info.compute(experiment=self.experiment) - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertIn(nudge_info, card.subtitle) diff --git a/ax/analysis/healthcheck/tests/test_should_generate_candidates.py b/ax/analysis/healthcheck/tests/test_should_generate_candidates.py index 0ea4086ef3d..2a89e3a89b9 100644 --- a/ax/analysis/healthcheck/tests/test_should_generate_candidates.py +++ b/ax/analysis/healthcheck/tests/test_should_generate_candidates.py @@ -30,5 +30,5 @@ def test_should_not(self) -> None: reason="Something concerning", trial_index=trial_index, ).compute() - self.assertEqual(card.get_status(), HealthcheckStatus.WARNING) + self.assertEqual(card.get_status(), HealthcheckStatus.INFO) self.assertEqual(card.subtitle, "Something concerning")