avoid scheduler failure due to missing metric value (#1061)

Summary:
Pull Request resolved: #1061

wrap results-fetching in try catch so as not to fail the scheduler when a single metric fails to fetch; instead mark the trial as "abandoned" and log a warning. if this works we can maybe do something similar for EarlyStopping metric fetching as well.

Reviewed By: lena-kashtelyan

Differential Revision: D38160234

fbshipit-source-id: 2609b40e07964675cb3a5f64a8d2f7bd696dd0a3

Author: bernardbeckerman

ax/service/scheduler.py

@@ -1055,7 +1055,7 @@ def poll_and_process_results(self, poll_all_trial_statuses: bool = False) -> boo
                 f"Fetching data for trials: {idcs} because some metrics "
                 "on experiment are available while trials are running."
             )
-            self.experiment.fetch_trials_data(
+            self._fetch_trials_data(
                 trial_indices=running_trial_indices,
                 overwrite_existing_data=True,
             )
@@ -1101,7 +1101,7 @@ def poll_and_process_results(self, poll_all_trial_statuses: bool = False) -> boo
                 # fetch it during candidate generation.
                 idcs = make_indices_str(indices=newly_completed)
                 self.logger.info(f"Fetching data for trials: {idcs}.")
-                self.experiment.fetch_trials_data(trial_indices=newly_completed)
+                self._fetch_trials_data(trial_indices=newly_completed)
 
             updated_trials.extend(trials)
 
@@ -1524,3 +1524,11 @@ def _append_to_experiment_properties(self, to_append: Dict[str, Any]) -> None:
         self._update_experiment_properties_in_db(
             experiment_with_updated_properties=self.experiment
         )
+
+    def _fetch_trials_data(
+        self, trial_indices: Iterable[int], overwrite_existing_data: bool = False
+    ) -> None:
+        """Fetches data from experiment."""
+        self.experiment.fetch_trials_data(
+            trial_indices=trial_indices, overwrite_existing_data=overwrite_existing_data
+        )