Skip to content

Update healthcheck statuses (#5020) #5020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shrutipatel31 wants to merge 1 commit into from
Closed

Update healthcheck statuses (#5020) #5020

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions ax/analysis/healthcheck/baseline_improvement.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,9 +231,7 @@ def compute(
num_total = len(comparison_list)

status = (
HealthcheckStatus.PASS
if num_improved == num_total
else HealthcheckStatus.WARNING
HealthcheckStatus.INFO if num_improved > 0 else HealthcheckStatus.WARNING
)

# Build subtitle
Expand Down
4 changes: 2 additions & 2 deletions ax/analysis/healthcheck/can_generate_candidates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ def compute(
status = HealthcheckStatus.FAIL
title_status = "Failure"
else:
status = HealthcheckStatus.WARNING
title_status = "Warning"
status = HealthcheckStatus.INFO
title_status = "Info"
subtitle += self.LAST_RUN_TEMPLATE.format(days=days_since_last_run)
else:
subtitle += f"{self.reason}"
Expand Down
4 changes: 2 additions & 2 deletions ax/analysis/healthcheck/complexity_rating.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,9 @@ def compute(
if tier == "Standard":
status = HealthcheckStatus.PASS
elif tier == "Advanced":
status = HealthcheckStatus.WARNING
status = HealthcheckStatus.INFO
else: # Unsupported
status = HealthcheckStatus.FAIL
status = HealthcheckStatus.WARNING

# Create dataframe with experiment summary
df = pd.DataFrame(
Expand Down
6 changes: 3 additions & 3 deletions ax/analysis/healthcheck/early_stopping_healthcheck.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ def _report_early_stopping_disabled(
"auto_early_stopping_config='standard' or configure an "
"early_stopping_strategy_config."
),
status=HealthcheckStatus.PASS,
status=HealthcheckStatus.INFO,
df=df,
)

Expand Down Expand Up @@ -391,7 +391,7 @@ def _report_early_stopping_status(
title=EARLY_STOPPING_SAVINGS_TITLE,
subtitle=subtitle,
df=df,
status=HealthcheckStatus.PASS,
status=HealthcheckStatus.INFO,
)

def _report_early_stopping_nudge(
Expand Down Expand Up @@ -467,7 +467,7 @@ def _report_early_stopping_nudge(
title=title,
subtitle=subtitle,
df=df,
status=HealthcheckStatus.WARNING,
status=HealthcheckStatus.INFO,
potential_savings=savings_pct,
best_metric=metric.name,
)
Expand Down
4 changes: 1 addition & 3 deletions ax/analysis/healthcheck/should_generate_candidates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@ def compute(
adapter: Adapter | None = None,
) -> HealthcheckAnalysisCard:
status = (
HealthcheckStatus.PASS
if self.should_generate
else HealthcheckStatus.WARNING
HealthcheckStatus.PASS if self.should_generate else HealthcheckStatus.INFO
)
return create_healthcheck_analysis_card(
name=self.__class__.__name__,
Expand Down
8 changes: 4 additions & 4 deletions ax/analysis/healthcheck/tests/test_baseline_improvement.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,11 @@ def _attach_data(
exp.attach_data(Data(df=pd.DataFrame(rows)))

def test_status_outcomes(self) -> None:
"""Test PASS/WARNING status based on improvement."""
"""Test INFO/WARNING status based on improvement."""
# minimize=True: lower is better
test_cases = [
# (baseline_mean, comparison_mean, expected_status, description)
(100.0, 50.0, HealthcheckStatus.PASS, "improved (lower)"),
(100.0, 50.0, HealthcheckStatus.INFO, "improved (lower)"),
(50.0, 100.0, HealthcheckStatus.WARNING, "not improved (higher)"),
]

Expand All @@ -83,7 +83,7 @@ def test_status_outcomes(self) -> None:
self.assertEqual(card.get_status(), expected_status)

def test_multi_objective_partial_improvement(self) -> None:
"""Test WARNING status when only some objectives improve."""
"""Test INFO status when only some objectives improve."""
# minimize=True for both objectives (lower is better):
# branin_a: 100 -> 50, improved (decreased)
# branin_b: 50 -> 100, NOT improved (increased)
Expand All @@ -102,7 +102,7 @@ def test_multi_objective_partial_improvement(self) -> None:
)
card = analysis.compute(experiment=self.moo_experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("1 out of 2", card.subtitle)

def test_documentation_link(self) -> None:
Expand Down
10 changes: 5 additions & 5 deletions ax/analysis/healthcheck/tests/test_can_generate_candidates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@ def test_warns_if_a_trial_was_recently_run(self) -> None:
reason="The data is borked.",
days_till_fail=2,
).compute(experiment=experiment, generation_strategy=None)
# THEN it is a WARNING
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
# THEN it is INFO
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertEqual(card.name, "CanGenerateCandidatesAnalysis")
self.assertEqual(card.title, "Ax Candidate Generation Warning")
self.assertEqual(card.title, "Ax Candidate Generation Info")
self.assertEqual(
card.subtitle,
(
Expand All @@ -71,11 +71,11 @@ def test_warns_if_a_trial_was_recently_run(self) -> None:
"LAST TRIAL RUN: 1 day(s) ago"
),
)
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertDictEqual(
card.get_aditional_attrs(),
{
"status": HealthcheckStatus.WARNING,
"status": HealthcheckStatus.INFO,
"reason": "The data is borked.",
},
)
Expand Down
26 changes: 13 additions & 13 deletions ax/analysis/healthcheck/tests/test_complexity_rating.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ def test_standard_configuration(self) -> None:

def test_parameter_counts(self) -> None:
test_cases = [
(60, HealthcheckStatus.WARNING, "Advanced", "60 tunable parameter(s)"),
(250, HealthcheckStatus.FAIL, "Unsupported", "250 tunable parameter(s)"),
(60, HealthcheckStatus.INFO, "Advanced", "60 tunable parameter(s)"),
(250, HealthcheckStatus.WARNING, "Unsupported", "250 tunable parameter(s)"),
]

for num_params, expected_status, expected_tier, expected_msg in test_cases:
Expand All @@ -92,8 +92,8 @@ def test_parameter_counts(self) -> None:

def test_objectives_count(self) -> None:
test_cases = [
(3, HealthcheckStatus.WARNING, "Advanced", "3 objectives"),
(5, HealthcheckStatus.FAIL, "Unsupported", "5 objectives"),
(3, HealthcheckStatus.INFO, "Advanced", "3 objectives"),
(5, HealthcheckStatus.WARNING, "Unsupported", "5 objectives"),
]

for num_objs, expected_status, expected_tier, expected_msg in test_cases:
Expand Down Expand Up @@ -135,7 +135,7 @@ def test_constraints(self) -> None:
options=self.options, tier_metadata=self.tier_metadata
).compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("Advanced", card.subtitle)
self.assertIn("3 outcome constraints", card.subtitle)

Expand All @@ -159,7 +159,7 @@ def test_constraints(self) -> None:
options=self.options, tier_metadata=self.tier_metadata
).compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("Advanced", card.subtitle)
self.assertIn("3 parameter constraints", card.subtitle)

Expand All @@ -181,14 +181,14 @@ def test_stopping_strategies(self) -> None:
options=options, tier_metadata=self.tier_metadata
).compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("Advanced", card.subtitle)
self.assertIn(expected_msg, card.subtitle)

def test_trial_counts(self) -> None:
test_cases = [
(300, HealthcheckStatus.WARNING, "Advanced", "300 total trials"),
(600, HealthcheckStatus.FAIL, "Unsupported", "600 total trials"),
(300, HealthcheckStatus.INFO, "Advanced", "300 total trials"),
(600, HealthcheckStatus.WARNING, "Unsupported", "600 total trials"),
]

for max_trials, expected_status, expected_tier, expected_msg in test_cases:
Expand Down Expand Up @@ -236,7 +236,7 @@ def test_unsupported_configurations(self) -> None:
options=options, tier_metadata=tier_metadata
).compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.FAIL)
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertIn("Unsupported", card.subtitle)
self.assertIn(expected_msg, card.subtitle)

Expand All @@ -260,7 +260,7 @@ def test_unordered_choice_parameters(self) -> None:
options=self.options, tier_metadata=self.tier_metadata
).compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("Advanced", card.subtitle)
self.assertIn("unordered choice parameter(s)", card.subtitle)

Expand All @@ -282,7 +282,7 @@ def test_binary_parameters_count(self) -> None:
options=self.options, tier_metadata=self.tier_metadata
).compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("Advanced", card.subtitle)
self.assertIn("60 binary tunable parameter(s)", card.subtitle)

Expand All @@ -305,7 +305,7 @@ def test_multiple_violations(self) -> None:
options=options, tier_metadata=tier_metadata
).compute(experiment=experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("Advanced", card.subtitle)
self.assertIn("60 tunable parameter(s)", card.subtitle)
self.assertIn("300 total trials", card.subtitle)
Expand Down
14 changes: 7 additions & 7 deletions ax/analysis/healthcheck/tests/test_early_stopping_healthcheck.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def test_early_stopping_not_enabled(self) -> None:
return_value=mock_savings,
):
card = healthcheck.compute(experiment=self.experiment)
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("25%", card.subtitle)

def test_validate_applicable_state(self) -> None:
Expand Down Expand Up @@ -261,15 +261,15 @@ def test_auto_early_stopping_config(self) -> None:
healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="disabled")
card = healthcheck.compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("auto_early_stopping_config='disabled'", card.subtitle)

with self.subTest("standard"):
self._mark_trial_completed()
healthcheck = EarlyStoppingAnalysis(auto_early_stopping_config="standard")
card = healthcheck.compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("0 trials were early stopped", card.subtitle)

with self.subTest("strategy_override"):
Expand All @@ -280,7 +280,7 @@ def test_auto_early_stopping_config(self) -> None:
)
card = healthcheck.compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertEqual(healthcheck.early_stopping_strategy, custom_strategy)

def test_multiple_metrics_note_in_subtitle(self) -> None:
Expand All @@ -294,7 +294,7 @@ def test_multiple_metrics_note_in_subtitle(self) -> None:
):
card = self.healthcheck.compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.PASS)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("2 metrics are", card.subtitle)

def test_get_problem_type_via_disabled_config(self) -> None:
Expand Down Expand Up @@ -355,7 +355,7 @@ def test_hypothetical_savings_nudge(self) -> None:
):
card = healthcheck.compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn("25%", card.subtitle)

with self.subTest("with_additional_info"):
Expand All @@ -375,5 +375,5 @@ def test_hypothetical_savings_nudge(self) -> None:
):
card = healthcheck_with_info.compute(experiment=self.experiment)

self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertIn(nudge_info, card.subtitle)
2 changes: 1 addition & 1 deletion ax/analysis/healthcheck/tests/test_should_generate_candidates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,5 @@ def test_should_not(self) -> None:
reason="Something concerning",
trial_index=trial_index,
).compute()
self.assertEqual(card.get_status(), HealthcheckStatus.WARNING)
self.assertEqual(card.get_status(), HealthcheckStatus.INFO)
self.assertEqual(card.subtitle, "Something concerning")
Loading