Skip to content

Commit 1c24f7b

Browse files
shrutipatel31facebook-github-bot
shrutipatel31
authored and
facebook-github-bot
committed
Update healthcheck statuses (#5020)
Summary: Update healthcheck status levels to better reflect their meaning and actionability: 1. Early Stopping Analysis: - ESS explicitly disabled: PASS → INFO (surface configuration choice) - ESS enabled, working correctly: PASS → INFO (informational, no action needed) - ESS not enabled but could save ≥10%: WARNING → INFO (nudge for feature upsell; not blocking the experiment) 2. Complexity Rating: - Advanced tier: WARNING → INFO (not blocking, informational) - Unsupported tier: FAIL → WARNING (concerning but not blocking) 3. Can Generate Candidates: - can_generate=False (recent trial): WARNING → INFO (expected behavior) 4. Baseline Improvement: - All objectives improved: PASS → INFO (valuable experiment takeaway) - Any objective not improved: WARNING → INFO (informational, not actionable) 5. Should Generate Candidates: - should_generate=False : WARNING → INFO Reviewed By: bernardbeckerman Differential Revision: D96412166
1 parent d016302 commit 1c24f7b

10 files changed

Lines changed: 39 additions & 43 deletions

‎ax/analysis/healthcheck/baseline_improvement.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -231,9 +231,7 @@ def compute(
231231
num_total = len(comparison_list)
232232

233233
status = (
234-
HealthcheckStatus.PASS
235-
if num_improved == num_total
236-
else HealthcheckStatus.WARNING
234+
HealthcheckStatus.INFO if num_improved > 0 else HealthcheckStatus.WARNING
237235
)
238236

239237
# Build subtitle

‎ax/analysis/healthcheck/can_generate_candidates.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ def compute(
7676
status = HealthcheckStatus.FAIL
7777
title_status = "Failure"
7878
else:
79-
status = HealthcheckStatus.WARNING
80-
title_status = "Warning"
79+
status = HealthcheckStatus.INFO
80+
title_status = "Info"
8181
subtitle += self.LAST_RUN_TEMPLATE.format(days=days_since_last_run)
8282
else:
8383
subtitle += f"{self.reason}"

‎ax/analysis/healthcheck/complexity_rating.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,9 @@ def compute(
142142
if tier == "Standard":
143143
status = HealthcheckStatus.PASS
144144
elif tier == "Advanced":
145-
status = HealthcheckStatus.WARNING
145+
status = HealthcheckStatus.INFO
146146
else: # Unsupported
147-
status = HealthcheckStatus.FAIL
147+
status = HealthcheckStatus.WARNING
148148

149149
# Create dataframe with experiment summary
150150
df = pd.DataFrame(

‎ax/analysis/healthcheck/early_stopping_healthcheck.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ def _report_early_stopping_disabled(
233233
"auto_early_stopping_config='standard' or configure an "
234234
"early_stopping_strategy_config."
235235
),
236-
status=HealthcheckStatus.PASS,
236+
status=HealthcheckStatus.INFO,
237237
df=df,
238238
)
239239

@@ -391,7 +391,7 @@ def _report_early_stopping_status(
391391
title=EARLY_STOPPING_SAVINGS_TITLE,
392392
subtitle=subtitle,
393393
df=df,
394-
status=HealthcheckStatus.PASS,
394+
status=HealthcheckStatus.INFO,
395395
)
396396

397397
def _report_early_stopping_nudge(
@@ -467,7 +467,7 @@ def _report_early_stopping_nudge(
467467
title=title,
468468
subtitle=subtitle,
469469
df=df,
470-
status=HealthcheckStatus.WARNING,
470+
status=HealthcheckStatus.INFO,
471471
potential_savings=savings_pct,
472472
best_metric=metric.name,
473473
)

‎ax/analysis/healthcheck/should_generate_candidates.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,7 @@ def compute(
4040
adapter: Adapter | None = None,
4141
) -> HealthcheckAnalysisCard:
4242
status = (
43-
HealthcheckStatus.PASS
44-
if self.should_generate
45-
else HealthcheckStatus.WARNING
43+
HealthcheckStatus.PASS if self.should_generate else HealthcheckStatus.INFO
4644
)
4745
return create_healthcheck_analysis_card(
4846
name=self.__class__.__name__,

‎ax/analysis/healthcheck/tests/test_baseline_improvement.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,11 @@ def _attach_data(
6161
exp.attach_data(Data(df=pd.DataFrame(rows)))
6262

6363
def test_status_outcomes(self) -> None:
64-
"""Test PASS/WARNING status based on improvement."""
64+
"""Test INFO/WARNING status based on improvement."""
6565
# minimize=True: lower is better
6666
test_cases = [
6767
# (baseline_mean, comparison_mean, expected_status, description)
68-
(100.0, 50.0, HealthcheckStatus.PASS, "improved (lower)"),
68+
(100.0, 50.0, HealthcheckStatus.INFO, "improved (lower)"),
6969
(50.0, 100.0, HealthcheckStatus.WARNING, "not improved (higher)"),
7070
]
7171

@@ -83,7 +83,7 @@ def test_status_outcomes(self) -> None:
8383
self.assertEqual(card.get_status(), expected_status)
8484

8585
def test_multi_objective_partial_improvement(self) -> None:
86-
"""Test WARNING status when only some objectives improve."""
86+
"""Test INFO status when only some objectives improve."""
8787
# minimize=True for both objectives (lower is better):
8888
# branin_a: 100 -> 50, improved (decreased)
8989
# branin_b: 50 -> 100, NOT improved (increased)
@@ -102,7 +102,7 @@ def test_multi_objective_partial_improvement(self) -> None:
102102
)
103103
card = analysis.compute(experiment=self.moo_experiment)
104104

105-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
105+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
106106
self.assertIn("1 out of 2", card.subtitle)
107107

108108
def test_documentation_link(self) -> None:

‎ax/analysis/healthcheck/tests/test_can_generate_candidates.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,10 @@ def test_warns_if_a_trial_was_recently_run(self) -> None:
5757
reason="The data is borked.",
5858
days_till_fail=2,
5959
).compute(experiment=experiment, generation_strategy=None)
60-
# THEN it is a WARNING
61-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
60+
# THEN it is INFO
61+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
6262
self.assertEqual(card.name, "CanGenerateCandidatesAnalysis")
63-
self.assertEqual(card.title, "Ax Candidate Generation Warning")
63+
self.assertEqual(card.title, "Ax Candidate Generation Info")
6464
self.assertEqual(
6565
card.subtitle,
6666
(
@@ -71,11 +71,11 @@ def test_warns_if_a_trial_was_recently_run(self) -> None:
7171
"LAST TRIAL RUN: 1 day(s) ago"
7272
),
7373
)
74-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
74+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
7575
self.assertDictEqual(
7676
card.get_aditional_attrs(),
7777
{
78-
"status": HealthcheckStatus.WARNING,
78+
"status": HealthcheckStatus.INFO,
7979
"reason": "The data is borked.",
8080
},
8181
)

‎ax/analysis/healthcheck/tests/test_complexity_rating.py‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ def test_standard_configuration(self) -> None:
6565

6666
def test_parameter_counts(self) -> None:
6767
test_cases = [
68-
(60, HealthcheckStatus.WARNING, "Advanced", "60 tunable parameter(s)"),
69-
(250, HealthcheckStatus.FAIL, "Unsupported", "250 tunable parameter(s)"),
68+
(60, HealthcheckStatus.INFO, "Advanced", "60 tunable parameter(s)"),
69+
(250, HealthcheckStatus.WARNING, "Unsupported", "250 tunable parameter(s)"),
7070
]
7171

7272
for num_params, expected_status, expected_tier, expected_msg in test_cases:
@@ -92,8 +92,8 @@ def test_parameter_counts(self) -> None:
9292

9393
def test_objectives_count(self) -> None:
9494
test_cases = [
95-
(3, HealthcheckStatus.WARNING, "Advanced", "3 objectives"),
96-
(5, HealthcheckStatus.FAIL, "Unsupported", "5 objectives"),
95+
(3, HealthcheckStatus.INFO, "Advanced", "3 objectives"),
96+
(5, HealthcheckStatus.WARNING, "Unsupported", "5 objectives"),
9797
]
9898

9999
for num_objs, expected_status, expected_tier, expected_msg in test_cases:
@@ -135,7 +135,7 @@ def test_constraints(self) -> None:
135135
options=self.options, tier_metadata=self.tier_metadata
136136
).compute(experiment=self.experiment)
137137

138-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
138+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
139139
self.assertIn("Advanced", card.subtitle)
140140
self.assertIn("3 outcome constraints", card.subtitle)
141141

@@ -159,7 +159,7 @@ def test_constraints(self) -> None:
159159
options=self.options, tier_metadata=self.tier_metadata
160160
).compute(experiment=self.experiment)
161161

162-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
162+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
163163
self.assertIn("Advanced", card.subtitle)
164164
self.assertIn("3 parameter constraints", card.subtitle)
165165

@@ -181,14 +181,14 @@ def test_stopping_strategies(self) -> None:
181181
options=options, tier_metadata=self.tier_metadata
182182
).compute(experiment=self.experiment)
183183

184-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
184+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
185185
self.assertIn("Advanced", card.subtitle)
186186
self.assertIn(expected_msg, card.subtitle)
187187

188188
def test_trial_counts(self) -> None:
189189
test_cases = [
190-
(300, HealthcheckStatus.WARNING, "Advanced", "300 total trials"),
191-
(600, HealthcheckStatus.FAIL, "Unsupported", "600 total trials"),
190+
(300, HealthcheckStatus.INFO, "Advanced", "300 total trials"),
191+
(600, HealthcheckStatus.WARNING, "Unsupported", "600 total trials"),
192192
]
193193

194194
for max_trials, expected_status, expected_tier, expected_msg in test_cases:
@@ -236,7 +236,7 @@ def test_unsupported_configurations(self) -> None:
236236
options=options, tier_metadata=tier_metadata
237237
).compute(experiment=self.experiment)
238238

239-
self.assertEqual(card.get_status(), HealthcheckStatus.FAIL)
239+
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
240240
self.assertIn("Unsupported", card.subtitle)
241241
self.assertIn(expected_msg, card.subtitle)
242242

@@ -260,7 +260,7 @@ def test_unordered_choice_parameters(self) -> None:
260260
options=self.options, tier_metadata=self.tier_metadata
261261
).compute(experiment=self.experiment)
262262

263-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
263+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
264264
self.assertIn("Advanced", card.subtitle)
265265
self.assertIn("unordered choice parameter(s)", card.subtitle)
266266

@@ -282,7 +282,7 @@ def test_binary_parameters_count(self) -> None:
282282
options=self.options, tier_metadata=self.tier_metadata
283283
).compute(experiment=self.experiment)
284284

285-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
285+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
286286
self.assertIn("Advanced", card.subtitle)
287287
self.assertIn("60 binary tunable parameter(s)", card.subtitle)
288288

@@ -305,7 +305,7 @@ def test_multiple_violations(self) -> None:
305305
options=options, tier_metadata=tier_metadata
306306
).compute(experiment=experiment)
307307

308-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
308+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
309309
self.assertIn("Advanced", card.subtitle)
310310
self.assertIn("60 tunable parameter(s)", card.subtitle)
311311
self.assertIn("300 total trials", card.subtitle)

‎ax/analysis/healthcheck/tests/test_early_stopping_healthcheck.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ def test_early_stopping_not_enabled(self) -> None:
177177
return_value=mock_savings,
178178
):
179179
card = healthcheck.compute(experiment=self.experiment)
180-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
180+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
181181
self.assertIn("25%", card.subtitle)
182182

183183
def test_validate_applicable_state(self) -> None:
@@ -261,15 +261,15 @@ def test_auto_early_stopping_config(self) -> None:
261261
healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="disabled")
262262
card = healthcheck.compute(experiment=self.experiment)
263263

264-
self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
264+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
265265
self.assertIn("auto_early_stopping_config='disabled'", card.subtitle)
266266

267267
with self.subTest("standard"):
268268
self._mark_trial_completed()
269269
healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="standard")
270270
card = healthcheck.compute(experiment=self.experiment)
271271

272-
self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
272+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
273273
self.assertIn("0 trials were early stopped", card.subtitle)
274274

275275
with self.subTest("strategy_override"):
@@ -280,7 +280,7 @@ def test_auto_early_stopping_config(self) -> None:
280280
)
281281
card = healthcheck.compute(experiment=self.experiment)
282282

283-
self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
283+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
284284
self.assertEqual(healthcheck.early_stopping_strategy, custom_strategy)
285285

286286
def test_multiple_metrics_note_in_subtitle(self) -> None:
@@ -294,7 +294,7 @@ def test_multiple_metrics_note_in_subtitle(self) -> None:
294294
):
295295
card = self.healthcheck.compute(experiment=self.experiment)
296296

297-
self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
297+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
298298
self.assertIn("2 metrics are", card.subtitle)
299299

300300
def test_get_problem_type_via_disabled_config(self) -> None:
@@ -355,7 +355,7 @@ def test_hypothetical_savings_nudge(self) -> None:
355355
):
356356
card = healthcheck.compute(experiment=self.experiment)
357357

358-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
358+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
359359
self.assertIn("25%", card.subtitle)
360360

361361
with self.subTest("with_additional_info"):
@@ -375,5 +375,5 @@ def test_hypothetical_savings_nudge(self) -> None:
375375
):
376376
card = healthcheck_with_info.compute(experiment=self.experiment)
377377

378-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
378+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
379379
self.assertIn(nudge_info, card.subtitle)

‎ax/analysis/healthcheck/tests/test_should_generate_candidates.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,5 @@ def test_should_not(self) -> None:
3030
reason="Something concerning",
3131
trial_index=trial_index,
3232
).compute()
33-
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
33+
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
3434
self.assertEqual(card.subtitle, "Something concerning")

0 commit comments

Comments
 (0)