_

Author: jeromedockes

examples/02_data_ops/1130_choices.py

@@ -269,6 +269,20 @@
 # performs better than the ``RidgeClassifier`` in most cases, that the ``StringEncoder``
 # outperforms the ``MinHashEncoder``, and that the choice of the additional ``length``
 # feature does not have a significant impact on the score.
+#
+# When we have many choices in our DataOp, or when we add many scorers with
+# :meth:`.skb.with_scoring() <DataOp.skb.with_scoring>` (not shown in this
+# example), the parallel coordinate plot can get very crowded or even
+# unreadable. To avoid this, :meth:`ParamSearch.plot_results` (and
+# :meth:`OptunaParamSearch.plot_results`) allow us to filter the scores and
+# parameters to include in the plot, and whether to include fitting and scoring
+# durations or not.
+#
+# For example if we wanted to include only the encoder and classifier and hide
+# the times:
+
+# %%
+search.plot_results(show_choices=["encoder", "classifier"], show_times=False)
 
 # %%
 # In this example, we've seen how to use skrub's ``choose_from`` objects to tune

skrub/_data_ops/_estimator.py

@@ -1303,6 +1303,20 @@ def detailed_results_(self):
         except NotFittedError:
             attribute_error(self, "results_")
 
+    def _get_score_names(self):
+        return [
+            k.removeprefix("mean_test_")
+            for k in self.cv_results_.keys()
+            if k.startswith("mean_test_")
+        ]
+
+    def _get_choice_names(self):
+        return [
+            n
+            for c in choice_graph(self.data_op)["choices"].values()
+            if (n := c.name) is not None
+        ]
+
     def _get_cv_results_table(self, detailed=False):
         check_is_fitted(self, "cv_results_")
         data_op_choices = choice_graph(self.data_op)
@@ -1318,30 +1332,20 @@ def _get_cv_results_table(self, detailed=False):
         columns_metadata = []
         for c_id, display_name in data_op_choices["choice_display_names"].items():
             c = data_op_choices["choices"][c_id]
-            columns_metadata.append(
-                {
-                    "type": "choice",
-                    "id": c_id,
-                    "name": c.name,
-                    "display_name": display_name,
-                }
-            )
-        metric_names = [
-            k.removeprefix("mean_test_")
-            for k in self.cv_results_.keys()
-            if k.startswith("mean_test_")
-        ]
+            columns_metadata.append({"type": "choice", "name": c.name})
+        score_names = self._get_score_names()
+
         if isinstance(self.refit_, str):
-            metric_names.insert(0, metric_names.pop(metric_names.index(self.refit_)))
+            score_names.insert(0, score_names.pop(score_names.index(self.refit_)))
         result_keys = [
-            *(f"mean_test_{n}" for n in metric_names),
-            *(f"std_test_{n}" for n in metric_names),
+            *(f"mean_test_{n}" for n in score_names),
+            *(f"std_test_{n}" for n in score_names),
             "mean_fit_time",
             "std_fit_time",
             "mean_score_time",
             "std_score_time",
-            *(f"mean_train_{n}" for n in metric_names),
-            *(f"std_train_{n}" for n in metric_names),
+            *(f"mean_train_{n}" for n in score_names),
+            *(f"std_train_{n}" for n in score_names),
         ]
         new_names = _join_utils.pick_column_names(table.columns, result_keys)
         renaming = dict(zip(table.columns, new_names))
@@ -1351,14 +1355,14 @@ def _get_cv_results_table(self, detailed=False):
             renaming[c] for c in metadata["log_scale_columns"]
         ]
         if detailed:
-            for k in result_keys[len(metric_names) :][::-1]:
+            for k in result_keys[len(score_names) :][::-1]:
                 if k in self.cv_results_:
                     table.insert(table.shape[1], k, self.cv_results_[k])
                     columns_metadata.append({"type": "score", "name": k})
-        for k in result_keys[: len(metric_names)][::-1]:
+        for k in result_keys[: len(score_names)][::-1]:
             table.insert(table.shape[1], k, self.cv_results_[k])
             columns_metadata.append({"type": "score", "name": k})
-        metadata["col_score"] = f"mean_test_{metric_names[0]}"
+        metadata["col_score"] = f"mean_test_{score_names[0]}"
         metadata["columns_metadata"] = dict(zip(table.columns, columns_metadata))
         table = table.sort_values(
             metadata["col_score"],
@@ -1394,11 +1398,13 @@ def plot_results(
             use of the colorscale's range.
 
         show_scores : list of str, optional
-            List of score names to show. By default all are shown.
+            List of score names to show
+            (e.g. "accuracy" in ``.skb.with_scoring(["roc_auc", "accuracy"])``).
+            By default all are shown.
 
         show_choices : list of str, optional
-            List of choice names (e.g. "alpha" in
-            ``choose_float(0.0, 1.0, name="alpha")``) to show.
+            List of choice names to show
+            (e.g. "alpha" in ``choose_float(0.0, 1.0, name="alpha")``).
             By default all are shown.
 
         show_time : bool, optional, default=True
@@ -1408,11 +1414,38 @@ def plot_results(
         -------
         Plotly Figure
         """
-        cv_results, metadata = self._get_cv_results_table(detailed=True)
+
+        # Check that requested show_scores and show_choices (if any) are available
+
         if isinstance(show_scores, str):
             show_scores = [show_scores]
+        if show_scores is not None:
+            available_scores = self._get_score_names()
+            missing_scores = set(show_scores).difference(available_scores)
+            if missing_scores:
+                raise ValueError(
+                    "The following scores were requested in show_scores "
+                    f"but do not exist in the results:\n{missing_scores}.\n"
+                    f"The available scores are:\n{available_scores}."
+                )
         if isinstance(show_choices, str):
             show_choices = [show_choices]
+        if show_choices is not None:
+            available_choices = self._get_choice_names()
+            missing_choices = set(show_choices).difference(available_choices)
+            if missing_choices:
+                raise ValueError(
+                    "The following choices (params) were requested in show_choices "
+                    f"but do not exist in the results:\n{missing_choices}.\n"
+                    f"The available choices are: {available_choices}."
+                )
+
+        # Prepare the figure data
+
+        cv_results, metadata = self._get_cv_results_table(detailed=True)
+
+        # Find columns to show based on show_scores, show_choices and show_time
+
         to_drop = set()
         for col_name, col_meta in metadata["columns_metadata"].items():
             if col_meta["type"] == "score":
@@ -1425,21 +1458,23 @@ def plot_results(
                     and col_meta["name"].removeprefix("mean_test_") not in show_scores
                 ):
                     to_drop.add(col_name)
-            if (
-                show_choices is not None
-                and col_meta["type"] == "choice"
-                and col_meta["name"] not in show_choices
-            ):
-                to_drop.add(col_name)
+            if col_meta["type"] == "choice":
+                if show_choices is not None and col_meta["name"] not in show_choices:
+                    to_drop.add(col_name)
         time_cols = {"mean_fit_time", "mean_score_time"}
         to_drop = (to_drop - time_cols) if show_time else (to_drop | time_cols)
         to_show = set(cv_results.columns) - to_drop
 
+        # Find rows to show based on min_score
+
         if min_score is not None:
             col_score = metadata["col_score"]
             cv_results = cv_results[cv_results[col_score] >= min_score]
         if not cv_results.shape[0]:
             raise ValueError("No results to plot")
+
+        # Make the figure
+
         return plot_parallel_coord(
             cv_results=cv_results,
             show_columns=to_show,

skrub/_data_ops/tests/test_parallel_coord.py

@@ -35,7 +35,8 @@ def test_no_plotly():
         search.plot_results()
 
 
-def test_parallel_coord():
+@pytest.mark.parametrize("specify_show_scores", [False, True])
+def test_parallel_coord(specify_show_scores):
     X_a, y_a = make_classification(n_samples=20, n_features=4, n_informative=2)
     c0 = skrub.choose_from({"a": 0, "b": 1}, name="c0")
     c1 = skrub.choose_from([0, 1], name="c1")
@@ -56,7 +57,7 @@ def test_parallel_coord():
 
     pytest.importorskip("plotly")
 
-    fig = search.plot_results()
+    fig = search.plot_results(show_scores=["score"] if specify_show_scores else None)
     data = iter(fig.data[0]["dimensions"])
     dim = next(data)
     assert dim["label"] == "c0"
@@ -153,7 +154,8 @@ def test_column_preparation(is_log_scale, is_int):
         assert np.all(jittered["values"] == 1.0)
 
 
-def test_multi_scoring():
+@pytest.fixture(scope="module")
+def classif_grid_search():
     pytest.importorskip("plotly")
 
     X, y = make_classification()
@@ -162,19 +164,73 @@ def test_multi_scoring():
     X, y = skrub.X(X), skrub.y(y)
 
     cols = skrub.choose_from([["0"], ["1"]], name="cols")
-    pred = X[cols].skb.apply(DummyClassifier(), y=y)
+    add = skrub.choose_float(0.0, 1.0, name="add", n_steps=3)
+    mul = skrub.choose_float(1.0, 2.0, n_steps=3)  # no name
+    pred = ((X[cols] + add) * mul).skb.apply(DummyClassifier(), y=y)
     search = pred.skb.make_grid_search(
         fitted=True,
         scoring=["accuracy", "neg_brier_score"],
         refit="accuracy",
     )
-    fig = search.plot_results()
+    return search
 
+
+@pytest.mark.parametrize(
+    "show_scores, show_choices, show_time, expected",
+    [
+        (
+            None,
+            None,
+            True,
+            [
+                "cols",
+                "add",
+                "choose_float(1.0,<br>\n2.0, n_steps=3)",
+                "score time",
+                "fit time",
+                "mean_test_neg_brier_<br>\nscore",
+                "mean_test_accuracy",
+            ],
+        ),
+        (
+            "accuracy",
+            ["cols", "add"],
+            False,
+            [
+                "cols",
+                "add",
+                "mean_test_accuracy",
+            ],
+        ),
+        (
+            [],
+            "add",
+            True,
+            [
+                "add",
+                "score time",
+                "fit time",
+            ],
+        ),
+    ],
+)
+def test_multi_scoring_and_filtering(
+    classif_grid_search, show_scores, show_choices, show_time, expected
+):
+    fig = classif_grid_search.plot_results(
+        show_scores=show_scores, show_choices=show_choices, show_time=show_time
+    )
     dimensions = fig.data[0]["dimensions"]
-    assert [d["label"].replace("<br>\n", "") for d in dimensions] == [
-        "cols",
-        "score time",
-        "fit time",
-        "mean_test_neg_brier_score",
-        "mean_test_accuracy",
-    ]
+    assert [d["label"] for d in dimensions] == expected
+
+
+def test_bad_filtering_params(classif_grid_search):
+    # Ask for a score that is not available, available scores are shown
+    with pytest.raises(ValueError, match="['accuracy', 'neg_brier_score']"):
+        classif_grid_search.plot_results(show_scores=["roc_auc"])
+    # Only choices with an actual name can be selected. Here we ask for a
+    # choice name that does not exist, available ones are shown.
+    with pytest.raises(ValueError, match="['cols', 'add']"):
+        classif_grid_search.plot_results(
+            show_choices=["choose_float(1.0, 2.0, n_steps=3)"]
+        )