Raise UnsupportedError for scalarized objectives in best_point_mixin and report_utils (#5065)

Summary:
Pull Request resolved: #5065

Several methods in `best_point_mixin.py` and `report_utils.py` use
`metric_names[0]` as the objective metric and call `.minimize`, which is
semantically wrong for scalarized objectives (where the objective is a
combination of metrics). Add early guards raising `UnsupportedError`.

Affected methods:
- `_get_trace_by_progression`
- `get_improvement_over_baseline`
- `_get_objective_trace_plot`
- `maybe_extract_baseline_comparison_values`

Reviewed By: Balandat

Differential Revision: D97123523

fbshipit-source-id: 664f3284cf3c39cc3bb1ec30c5a4d2f985f64123

Author: saitcakmak

ax/service/tests/test_best_point.py

@@ -24,14 +24,15 @@
 from ax.core.observation import Observation, ObservationData, ObservationFeatures
 from ax.core.optimization_config import (
     MultiObjectiveOptimizationConfig,
+    OptimizationConfig,
     PreferenceOptimizationConfig,
 )
 from ax.core.outcome_constraint import OutcomeConstraint
 from ax.core.parameter import ParameterType, RangeParameter
 from ax.core.search_space import SearchSpace
 from ax.core.trial import Trial
 from ax.core.types import ComparisonOp
-from ax.exceptions.core import DataRequiredError, UserInputError
+from ax.exceptions.core import DataRequiredError, UnsupportedError, UserInputError
 from ax.service.utils.best_point import (
     get_tensor_converter_adapter,
     get_trace,
@@ -799,3 +800,25 @@ def test_get_tensor_converter_adapter(self) -> None:
             self.assertIsInstance(
                 get_tensor_converter_adapter(experiment=experiment), TorchAdapter
             )
+
+    def test_get_trace_by_progression_scalarized(self) -> None:
+        """_get_trace_by_progression raises UnsupportedError for scalarized."""
+        experiment = get_experiment_with_trial()
+        experiment._optimization_config = OptimizationConfig(
+            objective=Objective(expression="2*m1 + -1*m2"),
+        )
+        with self.assertRaisesRegex(UnsupportedError, "not supported for scalarized"):
+            BestPointMixin._get_trace_by_progression(experiment=experiment)
+
+    def test_get_improvement_over_baseline_scalarized(self) -> None:
+        """get_improvement_over_baseline raises UnsupportedError for scalarized."""
+        experiment = get_experiment_with_trial()
+        experiment._optimization_config = OptimizationConfig(
+            objective=Objective(expression="2*m1 + -1*m2"),
+        )
+        mixin = BestPointMixin.__new__(BestPointMixin)
+        with self.assertRaisesRegex(UnsupportedError, "not supported for scalarized"):
+            mixin.get_improvement_over_baseline(
+                experiment=experiment,
+                generation_strategy=Mock(),
+            )

ax/service/tests/test_report_utils.py

@@ -20,9 +20,13 @@
 from ax.core.arm import Arm
 from ax.core.metric import Metric
 from ax.core.objective import MultiObjective, Objective
-from ax.core.optimization_config import MultiObjectiveOptimizationConfig
+from ax.core.optimization_config import (
+    MultiObjectiveOptimizationConfig,
+    OptimizationConfig,
+)
 from ax.core.outcome_constraint import ObjectiveThreshold
 from ax.core.types import ComparisonOp
+from ax.exceptions.core import UnsupportedError
 from ax.generation_strategy.generation_node import GenerationStep
 from ax.generation_strategy.generation_strategy import GenerationStrategy
 from ax.orchestration.orchestrator import Orchestrator
@@ -49,6 +53,7 @@
     get_branin_experiment,
     get_branin_experiment_with_multi_objective,
     get_branin_experiment_with_timestamp_map_metric,
+    get_branin_metric,
     get_experiment_with_observations,
     get_high_dimensional_branin_experiment,
     get_multi_type_experiment,
@@ -743,3 +748,31 @@ def test_maybe_extract_baseline_comparison_values_metric_missing_moo(self) -> No
             baseline_arm_name=arm_names[0],
         )
         self.assertIsNone(result)
+
+    def test_get_objective_trace_plot_scalarized(self) -> None:
+        """_get_objective_trace_plot raises UnsupportedError for scalarized."""
+        exp = get_branin_experiment(with_completed_trial=True)
+        exp.add_tracking_metric(get_branin_metric(name="branin2"))
+        exp._optimization_config = OptimizationConfig(
+            objective=Objective(expression="2*branin + -1*branin2"),
+        )
+        with self.assertRaisesRegex(UnsupportedError, "not supported for scalarized"):
+            _get_objective_trace_plot(experiment=exp)
+
+    def test_maybe_extract_baseline_comparison_values_scalarized(self) -> None:
+        """maybe_extract_baseline_comparison_values raises UnsupportedError
+        for scalarized."""
+        exp = get_branin_experiment_with_multi_objective(with_batch=True)
+        exp.trials[0].run()
+        exp.fetch_data()
+        arm_names = list(exp.arms_by_name.keys())
+        exp._optimization_config = OptimizationConfig(
+            objective=Objective(expression="2*branin_a + -1*branin_b"),
+        )
+        with self.assertRaisesRegex(UnsupportedError, "not supported for scalarized"):
+            maybe_extract_baseline_comparison_values(
+                experiment=exp,
+                optimization_config=exp.optimization_config,
+                comparison_arm_names=[arm_names[1]],
+                baseline_arm_name=arm_names[0],
+            )

ax/service/utils/best_point_mixin.py

@@ -22,7 +22,7 @@
 )
 from ax.core.trial import Trial
 from ax.core.types import TModelPredictArm, TParameterization
-from ax.exceptions.core import UserInputError
+from ax.exceptions.core import UnsupportedError, UserInputError
 from ax.generation_strategy.generation_strategy import GenerationStrategy
 from ax.service.utils import best_point as best_point_utils
 from ax.service.utils.best_point import get_tensor_converter_adapter
@@ -385,6 +385,12 @@ def _get_trace_by_progression(
         optimization_config = optimization_config or none_throws(
             experiment.optimization_config
         )
+        if optimization_config.objective.is_scalarized_objective:
+            raise UnsupportedError(
+                "`_get_trace_by_progression` is not supported for scalarized "
+                "objectives. The objective is a combination of metrics, not a "
+                "single metric."
+            )
         objective = optimization_config.objective.metric_names[0]
         minimize = optimization_config.objective.minimize
         map_data = experiment.lookup_data()
@@ -458,16 +464,21 @@ def get_improvement_over_baseline(
                 "`get_improvement_over_baseline` not yet implemented"
                 + " for multi-objective problems."
             )
+        optimization_config = experiment.optimization_config
+        if not optimization_config:
+            raise ValueError("No optimization config found.")
+        if optimization_config.objective.is_scalarized_objective:
+            raise UnsupportedError(
+                "`get_improvement_over_baseline` is not supported for "
+                "scalarized objectives. The objective is a combination of "
+                "metrics, not a single metric."
+            )
         if not baseline_arm_name:
             baseline_arm_name, _ = select_baseline_name_default_first_trial(
                 experiment=experiment,
                 baseline_arm_name=baseline_arm_name,
             )
 
-        optimization_config = experiment.optimization_config
-        if not optimization_config:
-            raise ValueError("No optimization config found.")
-
         objective_metric_name = optimization_config.objective.metric_names[0]
 
         # get the baseline trial

ax/service/utils/report_utils.py

@@ -42,7 +42,7 @@
 from ax.core.trial import BaseTrial
 from ax.core.trial_status import TrialStatus
 from ax.early_stopping.strategies.base import BaseEarlyStoppingStrategy
-from ax.exceptions.core import DataRequiredError, UserInputError
+from ax.exceptions.core import DataRequiredError, UnsupportedError, UserInputError
 from ax.generation_strategy.generation_strategy import GenerationStrategy
 from ax.plot.contour import interact_contour_plotly
 from ax.plot.diagnostic import interact_cross_validation_plotly
@@ -123,10 +123,18 @@ def _get_objective_trace_plot(
     if optimization_config is None:
         return []
 
+    objective = optimization_config.objective
+    if objective.is_scalarized_objective:
+        raise UnsupportedError(
+            "`_get_objective_trace_plot` is not supported for scalarized "
+            "objectives. The objective is a combination of metrics, not a "
+            "single metric."
+        )
+
     metric_names = (
         metric_name
         for metric_name in [
-            optimization_config.objective.metric_names[0],
+            objective.metric_names[0],
             true_objective_metric_name,
         ]
         if metric_name is not None
@@ -137,8 +145,8 @@ def _get_objective_trace_plot(
             exp_df=exp_df,
             metric_colname=metric_name,
             minimize=none_throws(
-                optimization_config.objective.minimize
-                if optimization_config.objective.metric_names[0] == metric_name
+                objective.minimize
+                if objective.metric_names[0] == metric_name
                 else experiment.metrics[metric_name].lower_is_better
             ),
             title=f"Best {metric_name} found vs. trial index",
@@ -1372,6 +1380,13 @@ def maybe_extract_baseline_comparison_values(
             result_list.append(result_tuple)
         return result_list if result_list else None
 
+    if optimization_config.objective.is_scalarized_objective:
+        raise UnsupportedError(
+            "`maybe_extract_baseline_comparison_values` is not supported for "
+            "scalarized objectives. The objective is a combination of "
+            "metrics, not a single metric."
+        )
+
     objective_name = optimization_config.objective.metric_names[0]
 
     # Check if metric column exists in both comparison and baseline dataframes