feat(skore/project)!: Put/get/summarize CrossValidationReport (#2025)

Closes #1944 and
#1923.

Author: thomass-dev

skore/pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
   "plotly",
   "rich",
   "scikit-learn",
-  "skore-local-project",
+  "skore[local]",
   "skrub",
   "seaborn",
 ]
@@ -79,7 +79,8 @@ dev = [
   "nbformat",
   "ipykernel",
 ]
-hub = ["skore-hub-project"]
+local = ["skore-local-project>=0.0.4"]
+hub = ["skore-hub-project>=0.0.10"]
 
 [project.urls]
 Homepage = "https://probabl.ai"

skore/src/skore/project/project.py

@@ -6,7 +6,7 @@
 from importlib.metadata import entry_points
 from typing import Any, Literal
 
-from skore import EstimatorReport
+from skore import CrossValidationReport, EstimatorReport
 from skore.project.summary import Summary
 
 
@@ -95,7 +95,7 @@ class Project:
     >>> from sklearn.datasets import make_classification, make_regression
     >>> from sklearn.linear_model import LinearRegression, LogisticRegression
     >>> from sklearn.model_selection import train_test_split
-    >>> from skore import EstimatorReport
+    >>> from skore import CrossValidationReport, EstimatorReport
     >>>
     >>> X, y = make_classification(random_state=42)
     >>> X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
@@ -118,6 +118,8 @@ class Project:
     >>>     X_test=X_test,
     >>>     y_test=y_test,
     >>> )
+    >>>
+    >>> cv_regressor_report = CrossValidationReport(regressor, X, y)
 
     Construct the project in local mode, persisted in a temporary directory.
 
@@ -132,6 +134,7 @@ class Project:
 
     >>> local_project.put("my-simple-classification", classifier_report)
     >>> local_project.put("my-simple-regression", regressor_report)
+    >>> local_project.put("my-simple-cv_regression", cv_regressor_report)
 
     Investigate metadata/metrics to filter the best reports.
 
@@ -153,6 +156,7 @@ def __setup_plugin(name: str) -> tuple[Literal["local", "hub"], str, Any, dict]:
             raise SystemError("No project plugin found, please install at least one.")
 
         mode: Literal["local", "hub"]
+
         if match := re.match(Project.__HUB_NAME_PATTERN, name):
             mode = "hub"
             name = match["name"]
@@ -162,10 +166,7 @@ def __setup_plugin(name: str) -> tuple[Literal["local", "hub"], str, Any, dict]:
             parameters = {"name": name}
 
         if mode not in PLUGINS.names:
-            raise ValueError(
-                f"Unknown mode `{mode}`. "
-                f"Please install the `skore-{mode}-project` python package."
-            )
+            raise ValueError(f"Unknown mode `{mode}`. Please install `skore[{mode}]`.")
 
         return mode, name, PLUGINS[mode].load(), parameters
 
@@ -214,7 +215,7 @@ def name(self):
         """The name of the project."""
         return self.__name
 
-    def put(self, key: str, report: EstimatorReport):
+    def put(self, key: str, report: EstimatorReport | CrossValidationReport):
         """
         Put a key-report pair to the project.
 
@@ -225,7 +226,7 @@ def put(self, key: str, report: EstimatorReport):
         ----------
         key : str
             The key to associate with ``report`` in the project.
-        report : skore.EstimatorReport
+        report : EstimatorReport | CrossValidationReport
             The report to associate with ``key`` in the project.
 
         Raises
@@ -236,19 +237,20 @@ def put(self, key: str, report: EstimatorReport):
         if not isinstance(key, str):
             raise TypeError(f"Key must be a string (found '{type(key)}')")
 
-        if not isinstance(report, EstimatorReport):
+        if not isinstance(report, EstimatorReport | CrossValidationReport):
             raise TypeError(
-                f"Report must be a `skore.EstimatorReport` (found '{type(report)}')"
+                f"Report must be `EstimatorReport` or `CrossValidationReport` "
+                f"(found '{type(report)}')"
             )
 
         return self.__project.put(key=key, report=report)
 
-    def get(self, id: str) -> EstimatorReport:
+    def get(self, id: str) -> EstimatorReport | CrossValidationReport:
         """
         Get a persisted report by its id.
 
-        Report IDs can be found via :meth:`skore.Project.summarize`, which is
-        also the preferred method of interacting with a ``skore.Project``.
+        Report IDs can be found via :meth:`skore.Project.summarize`, which is also the
+        preferred method of interacting with a ``skore.Project``.
 
         Parameters
         ----------
@@ -260,7 +262,7 @@ def get(self, id: str) -> EstimatorReport:
         KeyError
             If a non-existent ID is passed.
         """
-        return self.__project.reports.get(id)
+        return self.__project.get(id)
 
     def summarize(self) -> Summary:
         """Obtain metadata/metrics for all persisted reports."""

skore/src/skore/project/summary.py

@@ -6,13 +6,13 @@
 
 from pandas import Categorical, DataFrame, Index, MultiIndex, RangeIndex
 
-from skore._sklearn import ComparisonReport
+from skore import ComparisonReport
 from skore.project.widget import ModelExplorerWidget
 
 if TYPE_CHECKING:
     from typing import Literal
 
-    from skore._sklearn import EstimatorReport
+    from skore import CrossValidationReport, EstimatorReport
 
 
 class Summary(DataFrame):
@@ -45,15 +45,15 @@ def factory(project, /):
 
         Parameters
         ----------
-        project : Union[skore_local_project.Project, skore_hub_project.Project]
+        project : ``skore_local_project.Project`` | ``skore_hub_project.Project``
             The project from which the summary object is to be constructed.
 
         Notes
         -----
         This function is not intended for direct use. Instead simply use the accessor
         :meth:`skore.Project.summarize`.
         """
-        summary = DataFrame(project.reports.metadata(), copy=False)
+        summary = DataFrame(project.summarize(), copy=False)
 
         if not summary.empty:
             summary["learner"] = Categorical(summary["learner"])
@@ -79,7 +79,7 @@ def reports(
         *,
         filter: bool = True,
         return_as: Literal["list", "comparison"] = "list",
-    ) -> list[EstimatorReport] | ComparisonReport:
+    ) -> list[EstimatorReport | CrossValidationReport] | ComparisonReport:
         """
         Return the reports referenced by the summary object from the project.
 
@@ -100,9 +100,7 @@ def reports(
         if filter and (querystr := self._query_string_selection()):
             self = self.query(querystr)
 
-        reports = [
-            self.project.reports.get(id) for id in self.index.get_level_values("id")
-        ]
+        reports = [self.project.get(id) for id in self.index.get_level_values("id")]
 
         if return_as == "comparison":
             try:

skore/tests/unit/project/test_project.py

@@ -1,12 +1,13 @@
 from importlib.metadata import EntryPoint, EntryPoints
+from re import escape
 from unittest.mock import Mock
 
 from pandas import DataFrame, MultiIndex, Series
-from pytest import fixture, raises
+from pytest import fixture, mark, param, raises
 from sklearn.datasets import make_regression
 from sklearn.linear_model import LinearRegression
 from sklearn.model_selection import train_test_split
-from skore import EstimatorReport, Project
+from skore import CrossValidationReport, EstimatorReport, Project
 from skore.project.summary import Summary
 
 
@@ -47,7 +48,7 @@ def monkeypatch_entrypoints(monkeypatch, request, FakeLocalProject, FakeHubProje
 
 
 @fixture(scope="module")
-def regression():
+def regression() -> EstimatorReport:
     X, y = make_regression(random_state=42)
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, test_size=0.2, random_state=42
@@ -62,6 +63,11 @@ def regression():
     )
 
 
+@fixture(scope="module")
+def cv_regression() -> CrossValidationReport:
+    return CrossValidationReport(LinearRegression(), *make_regression(random_state=42))
+
+
 class TestProject:
     def test_init_local(self, FakeLocalProject):
         project = Project("<name>", workspace="<workspace>")
@@ -115,10 +121,7 @@ def test_init_exception_unknown_plugin(self, monkeypatch, tmp_path):
 
         with raises(
             ValueError,
-            match=(
-                "Unknown mode `local`. "
-                "Please install the `skore-local-project` python package."
-            ),
+            match=escape("Unknown mode `local`. Please install `skore[local]`."),
         ):
             Project("<name>")
 
@@ -130,24 +133,32 @@ def test_name(self):
         assert Project("<name>").name == "<name>"
         assert Project("hub://<tenant>/<name>").name == "<name>"
 
-    def test_put(self, regression, FakeLocalProject):
+    @mark.parametrize(
+        "report",
+        (
+            param("regression", id="EstimatorReport - regression"),
+            param("cv_regression", id="CrossValidationReport - regression"),
+        ),
+    )
+    def test_put(self, report, FakeLocalProject, request):
+        report = request.getfixturevalue(report)
         project = Project("<name>")
 
-        project.put("<key>", regression)
+        project.put("<key>", report)
 
         assert FakeLocalProject.called
         assert project._Project__project.put.called
         assert not project._Project__project.put.call_args.args
         assert project._Project__project.put.call_args.kwargs == {
             "key": "<key>",
-            "report": regression,
+            "report": report,
         }
 
     def test_put_exception(self):
         with raises(TypeError, match="Key must be a string"):
             Project("<name>").put(None, "<value>")
 
-        with raises(TypeError, match="Report must be a `skore.EstimatorReport`"):
+        with raises(TypeError, match="Report must be `EstimatorReport` or"):
             Project("<name>").put("<key>", "<value>")
 
     def test_get(self, FakeLocalProject):
@@ -156,13 +167,13 @@ def test_get(self, FakeLocalProject):
         project.get("<id>")
 
         assert FakeLocalProject.called
-        assert project._Project__project.reports.get.called
-        assert project._Project__project.reports.get.call_args.args == ("<id>",)
-        assert not project._Project__project.reports.get.call_args.kwargs
+        assert project._Project__project.get.called
+        assert project._Project__project.get.call_args.args == ("<id>",)
+        assert not project._Project__project.get.call_args.kwargs
 
     def test_summarize(self):
         project = Project("<name>")
-        project._Project__project.reports.metadata.return_value = [
+        project._Project__project.summarize.return_value = [
             {
                 "learner": "<learner>",
                 "accuracy": 1.0,
@@ -172,7 +183,7 @@ def test_summarize(self):
 
         summary = project.summarize()
 
-        assert project._Project__project.reports.metadata.called
+        assert project._Project__project.summarize.called
         assert isinstance(summary, DataFrame)
         assert isinstance(summary, Summary)
         assert DataFrame.equals(

skore/tests/unit/project/test_summary.py

@@ -1,5 +1,4 @@
 from copy import deepcopy
-from types import SimpleNamespace
 
 from joblib import hash as joblib_hash
 from pandas import DataFrame, Index, MultiIndex, RangeIndex
@@ -58,7 +57,7 @@ class FakeProject:
     def __init__(self, *reports):
         self.__reports = reports
 
-    def _make_report_metadata(self, index, report):
+    def make_report_metadata(self, index, report):
         return {
             "id": index,
             "run_id": None,
@@ -93,18 +92,14 @@ def _make_report_metadata(self, index, report):
             "predict_time_std": None,
         }
 
-    @property
-    def reports(self):
-        def get(id: str):
-            return self.__reports[int(id)]
+    def get(self, id: str):
+        return self.__reports[int(id)]
 
-        def metadata():
-            return [
-                self._make_report_metadata(index, report)
-                for index, report in enumerate(self.__reports)
-            ]
-
-        return SimpleNamespace(metadata=metadata, get=get)
+    def summarize(self):
+        return [
+            self.make_report_metadata(index, report)
+            for index, report in enumerate(self.__reports)
+        ]
 
 
 class TestSummary: