78
78
OPTIMIZATION_COMPLETION_MSG = """Optimization completed with total of {num_trials}
79
79
trials attached to the underlying Ax experiment '{experiment_name}'.
80
80
"""
81
+ FAILURE_EXCEEDED_MSG = (
82
+ "Failure rate exceeds the tolerated trial failure rate of {f_rate} (at least "
83
+ "{n_failed} out of first {n_ran} trials failed). Checks are triggered both at "
84
+ "the end of a optimization and if at least {min_failed} trials have failed."
85
+ )
81
86
82
87
83
88
# Wait time b/w reports will not exceed 15 mins.
@@ -95,7 +100,9 @@ class SchedulerInternalError(AxError):
95
100
96
101
97
102
class FailureRateExceededError (AxError ):
98
- """Error that indicates the optimization was aborted due to excessive failure rate."""
103
+ """Error that indicates the optimization was aborted due to excessive
104
+ failure rate.
105
+ """
99
106
100
107
pass
101
108
@@ -156,8 +163,8 @@ class Scheduler(WithDBSettingsBase, BestPointMixin):
156
163
logger : LoggerAdapter
157
164
# Mapping of form {short string identifier -> message to show in reported
158
165
# results}. This is a mapping and not a list to allow for changing of
159
- # some optimization messages throughout the course of the optimization (e.g. progress
160
- # report of the optimization).
166
+ # some optimization messages throughout the course of the optimization
167
+ # (e.g. progress report of the optimization).
161
168
markdown_messages : Dict [str , str ]
162
169
163
170
# Number of trials that existed on the scheduler's experiment before
@@ -684,8 +691,8 @@ def error_if_failure_rate_exceeded(self, force_check: bool = False) -> None:
684
691
Args:
685
692
force_check: Indicates whether to force a failure-rate check
686
693
regardless of the number of trials that have been executed. If False
687
- (default), the check will be skipped if the optimization has fewer than five
688
- failed trials. If True, the check will be performed unless there
694
+ (default), the check will be skipped if the optimization has fewer than
695
+ five failed trials. If True, the check will be performed unless there
689
696
are 0 failures.
690
697
"""
691
698
failed_idcs = self .experiment .trial_indices_by_status [TrialStatus .FAILED ]
@@ -718,12 +725,12 @@ def error_if_failure_rate_exceeded(self, force_check: bool = False) -> None:
718
725
719
726
if failure_rate_exceeded :
720
727
raise FailureRateExceededError (
721
- "Failure rate exceeds the tolerated trial failure rate of "
722
- f" { self .options .tolerated_trial_failure_rate } (at least "
723
- f" { num_failed_in_scheduler } out of first { num_ran_in_scheduler } trials "
724
- "failed). Checks are triggered both at the end of a optimization and if "
725
- f"at least { self .options .min_failed_trials_for_failure_rate_check } "
726
- "trials have failed."
728
+ FAILURE_EXCEEDED_MSG . format (
729
+ f_rate = self .options .tolerated_trial_failure_rate ,
730
+ n_failed = num_failed_in_scheduler ,
731
+ n_ran = num_ran_in_scheduler ,
732
+ min_failed = self .options .min_failed_trials_for_failure_rate_check ,
733
+ )
727
734
)
728
735
729
736
def run_trials_and_yield_results (
@@ -1213,7 +1220,8 @@ def _complete_optimization(
1213
1220
res = self .wait_for_completed_trials_and_report_results (
1214
1221
idle_callback = idle_callback
1215
1222
)
1216
- # raise an error if the failure rate exceeds tolerance at the end of the optimization
1223
+ # Raise an error if the failure rate exceeds tolerance at the
1224
+ # end of the optimization.
1217
1225
self .error_if_failure_rate_exceeded (force_check = True )
1218
1226
self ._record_run_trials_status (
1219
1227
num_preexisting_trials = num_preexisting_trials ,
0 commit comments