Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,7 @@ def coefficients(self) -> FeatureImportanceDisplay:
"""
similar_reports = defaultdict(list)

for report, name in zip(
self._parent.reports_, self._parent.report_names_, strict=False
):
for name, report in self._parent.reports_.items():
report = cast(CrossValidationReport | EstimatorReport, report)
feature_names = (
report.feature_importance.coefficients().frame().index.tolist()
Expand Down
30 changes: 14 additions & 16 deletions skore/src/skore/_sklearn/_comparison/metrics_accessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ def _compute_metric_scores(

generator = parallel(
joblib.delayed(getattr(report.metrics, report_metric_name))(**kwargs)
for report in self._parent.reports_
for report in self._parent.reports_.values()
)
individual_results = []
for result in generator:
Expand All @@ -256,15 +256,15 @@ def _compute_metric_scores(
if self._parent._reports_type == "EstimatorReport":
results = _combine_estimator_results(
individual_results,
estimator_names=self._parent.report_names_,
estimator_names=self._parent.reports_.keys(),
indicator_favorability=metric_kwargs.get(
"indicator_favorability", False
),
)
else: # "CrossValidationReport"
results = _combine_cross_validation_results(
individual_results,
estimator_names=self._parent.report_names_,
estimator_names=self._parent.reports_.keys(),
indicator_favorability=metric_kwargs.get(
"indicator_favorability", False
),
Expand Down Expand Up @@ -325,10 +325,10 @@ def timings(
timings: pd.DataFrame = pd.concat(
[
pd.Series(report.metrics.timings())
for report in self._parent.reports_
for report in self._parent.reports_.values()
],
axis=1,
keys=self._parent.report_names_,
keys=self._parent.reports_.keys(),
)
timings.index = timings.index.str.replace("_", " ").str.capitalize()

Expand All @@ -342,19 +342,19 @@ def timings(
else: # "CrossValidationReport"
results = [
report.metrics.timings(aggregate=None)
for report in self._parent.reports_
for report in self._parent.reports_.values()
]

# Put dataframes in the right shape
for i, result in enumerate(results):
result.index.name = "Metric"
result.columns = pd.MultiIndex.from_product(
[[self._parent.report_names_[i]], result.columns]
[[list(self._parent.reports_.keys())[i]], result.columns]
)

timings = _combine_cross_validation_results(
results,
self._parent.report_names_,
self._parent.reports_.keys(),
indicator_favorability=False,
aggregate=aggregate,
)
Expand Down Expand Up @@ -1218,9 +1218,7 @@ def _get_display(
y_pred: list[YPlotData] = []

if self._parent._reports_type == "EstimatorReport":
for report, report_name in zip(
self._parent.reports_, self._parent.report_names_, strict=False
):
for report_name, report in self._parent.reports_.items():
report_X, report_y, _ = (
report.metrics._get_X_y_and_data_source_hash(
data_source=data_source,
Expand Down Expand Up @@ -1264,16 +1262,16 @@ def _get_display(
y_true=y_true,
y_pred=y_pred,
report_type="comparison-estimator",
estimators=[report.estimator_ for report in self._parent.reports_],
estimators=[
report.estimator_ for report in self._parent.reports_.values()
],
ml_task=self._parent._ml_task,
data_source=data_source,
**display_kwargs,
)

else:
for report, report_name in zip(
self._parent.reports_, self._parent.report_names_, strict=False
):
for report_name, report in self._parent.reports_.items():
for split, estimator_report in enumerate(report.estimator_reports_):
report_X, report_y, _ = (
estimator_report.metrics._get_X_y_and_data_source_hash(
Expand Down Expand Up @@ -1321,7 +1319,7 @@ def _get_display(
report_type="comparison-cross-validation",
estimators=[
estimator_report.estimator_
for report in self._parent.reports_
for report in self._parent.reports_.values()
for estimator_report in report.estimator_reports_
],
ml_task=self._parent._ml_task,
Expand Down
43 changes: 19 additions & 24 deletions skore/src/skore/_sklearn/_comparison/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from skore._sklearn._comparison.feature_importance_accessor import (
_FeatureImportanceAccessor,
)
from skore._sklearn._estimator.metrics_accessor import _MetricsAccessor
from skore._sklearn._comparison.metrics_accessor import _MetricsAccessor

ReportType = Literal["EstimatorReport", "CrossValidationReport"]

Expand Down Expand Up @@ -57,15 +57,9 @@ class ComparisonReport(_BaseReport, DirNamesMixin):

Attributes
----------
reports_ : list of :class:`~skore.EstimatorReport` or list of \
:class:`~skore.CrossValidationReport`
reports_ : dict mapping names to reports
The compared reports.

report_names_ : list of str
The names of the compared estimators. If the names are not customized (i.e. the
class names are used), a de-duplication process is used to make sure that the
names are distinct.

See Also
--------
skore.EstimatorReport
Expand All @@ -87,13 +81,13 @@ class names are used), a de-duplication process is used to make sure that the
>>> estimator_2 = LogisticRegression(C=2) # Different regularization
>>> estimator_report_2 = EstimatorReport(estimator_2, **split_data)
>>> report = ComparisonReport([estimator_report_1, estimator_report_2])
>>> report.report_names_
['LogisticRegression_1', 'LogisticRegression_2']
>>> report.reports_
{'LogisticRegression_1': ..., 'LogisticRegression_2': ...}
>>> report = ComparisonReport(
... {"model1": estimator_report_1, "model2": estimator_report_2}
... )
>>> report.report_names_
['model1', 'model2']
>>> report.reports_
{'model1': ..., 'model2': ...}

>>> from sklearn.datasets import make_classification
>>> from sklearn.linear_model import LogisticRegression
Expand Down Expand Up @@ -122,8 +116,7 @@ def _validate_reports(
| list[CrossValidationReport]
| dict[str, CrossValidationReport],
) -> tuple[
list[EstimatorReport] | list[CrossValidationReport],
list[str],
dict[str, EstimatorReport] | dict[str, CrossValidationReport],
ReportType,
PositiveLabel,
]:
Expand All @@ -136,11 +129,8 @@ def _validate_reports(

Returns
-------
list of EstimatorReport or list of CrossValidationReport
dict
The validated reports.
list of str
The report names, either taken from dict keys or computed from the estimator
class names.
{"EstimatorReport", "CrossValidationReport"}
The inferred type of the reports that will be compared.
int, float, bool, str or None
Expand Down Expand Up @@ -222,7 +212,12 @@ class names.
else:
deduped_report_names = report_names

return reports_list, deduped_report_names, reports_type, pos_label
reports_dict = cast(
dict[str, EstimatorReport] | dict[str, CrossValidationReport],
dict(zip(deduped_report_names, reports_list, strict=True)),
)

return reports_dict, reports_type, pos_label

def __init__(
self,
Expand All @@ -244,7 +239,7 @@ def __init__(
- all estimators have non-empty X_test and y_test,
- all estimators have the same X_test and y_test.
"""
self.reports_, self.report_names_, self._reports_type, self._pos_label = (
self.reports_, self._reports_type, self._pos_label = (
ComparisonReport._validate_reports(reports)
)

Expand All @@ -256,7 +251,7 @@ def __init__(
low=np.iinfo(np.int64).min, high=np.iinfo(np.int64).max
)
self._cache: dict[tuple[Any, ...], Any] = {}
self._ml_task = self.reports_[0]._ml_task
self._ml_task = list(self.reports_.values())[0]._ml_task # type: ignore

def clear_cache(self) -> None:
"""Clear the cache.
Expand All @@ -279,7 +274,7 @@ def clear_cache(self) -> None:
>>> report._cache
{}
"""
for report in self.reports_:
for report in self.reports_.values():
report.clear_cache()
self._cache = {}

Expand Down Expand Up @@ -332,7 +327,7 @@ def cache_predictions(
total_estimators = len(self.reports_)
progress.update(main_task, total=total_estimators)

for report in self.reports_:
for report in self.reports_.values():
# Share the parent's progress bar with child report
report._progress_info = {"current_progress": progress}
report.cache_predictions(response_methods=response_methods, n_jobs=n_jobs)
Expand Down Expand Up @@ -418,7 +413,7 @@ def get_predictions(
X=X,
pos_label=pos_label,
)
for report in self.reports_
for report in self.reports_.values()
]

@property
Expand Down
5 changes: 3 additions & 2 deletions skore/src/skore/_utils/_accessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ def check(accessor: Any) -> bool:

parent = accessor._parent
parent_estimators = []
for parent_report in parent.reports_:
for parent_report in parent.reports_.values():
if parent._reports_type == "CrossValidationReport":
parent_report = cast(CrossValidationReport, parent_report)
parent_estimators.append(parent_report.estimator_reports_[0].estimator_)
Expand All @@ -203,7 +203,8 @@ def _check_any_sub_report_has_metric(metric: str) -> Callable[[Any], bool]:

def check(accessor: Any) -> bool:
return any(
hasattr(report.metrics, metric) for report in accessor._parent.reports_
hasattr(report.metrics, metric)
for report in accessor._parent.reports_.values()
)

return check
2 changes: 1 addition & 1 deletion skore/src/skore/_utils/_progress_bar.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def wrapper(*args: Any, **kwargs: Any) -> T:
# assigning progress to child reports
reports_to_cleanup: list[Any] = []
if hasattr(self_obj, "reports_"):
for report in self_obj.reports_:
for report in self_obj.reports_.values():
if hasattr(report, "_progress_info"):
report._progress_info = {"current_progress": progress}
reports_to_cleanup.append(report)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ def test_binary_classification(

pos_label = 1
n_reports = len(report.reports_)
n_splits = len(report.reports_[0].estimator_reports_)
n_splits = len(list(report.reports_.values())[0].estimator_reports_)

display.plot()
assert isinstance(display.lines_, list)
assert len(display.lines_) == n_reports * n_splits
default_colors = sample_mpl_colormap(pyplot.cm.tab10, 10)
for i, estimator_name in enumerate(report.report_names_):
for i, estimator_name in enumerate(report.reports_.keys()):
precision_recall_mpl = display.lines_[i * n_splits]
assert isinstance(precision_recall_mpl, Line2D)

Expand Down Expand Up @@ -76,15 +76,15 @@ def test_multiclass_classification(

labels = display.precision_recall["label"].cat.categories
n_reports = len(report.reports_)
n_splits = len(report.reports_[0].estimator_reports_)
n_splits = len(list(report.reports_.values())[0].estimator_reports_)

display.plot()
assert isinstance(display.lines_, list)
assert len(display.lines_) == n_reports * len(labels) * n_splits

default_colors = sample_mpl_colormap(pyplot.cm.tab10, 10)
for i, ((estimator_idx, estimator_name), label) in enumerate(
product(enumerate(report.report_names_), labels)
product(enumerate(report.reports_.keys()), labels)
):
precision_recall_mpl = display.lines_[i * n_splits]
assert isinstance(precision_recall_mpl, Line2D)
Expand Down Expand Up @@ -219,7 +219,7 @@ def test_binary_classification_constructor(forest_binary_classification_data):
assert df.query("estimator_name == 'estimator_2'")[
"split"
].unique().tolist() == list(range(cv + 1))
assert df["estimator_name"].unique().tolist() == report.report_names_
assert df["estimator_name"].unique().tolist() == list(report.reports_.keys())
assert df["label"].unique() == 1

assert len(display.average_precision) == cv + (cv + 1)
Expand All @@ -245,7 +245,7 @@ def test_multiclass_classification_constructor(forest_multiclass_classification_
assert df.query("estimator_name == 'estimator_2'")[
"split"
].unique().tolist() == list(range(cv + 1))
assert df["estimator_name"].unique().tolist() == report.report_names_
assert df["estimator_name"].unique().tolist() == list(report.reports_.keys())
np.testing.assert_array_equal(df["label"].unique(), classes)

assert len(display.average_precision) == len(classes) * cv + len(classes) * (cv + 1)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def test_binary_classification(pyplot, logistic_binary_classification_with_train
display.plot()
expected_colors = sample_mpl_colormap(pyplot.cm.tab10, 10)
for idx, (estimator_name, line) in enumerate(
zip(report.report_names_, display.lines_, strict=False)
zip(report.reports_.keys(), display.lines_, strict=False)
):
assert isinstance(line, mpl.lines.Line2D)
average_precision = display.average_precision.query(
Expand Down Expand Up @@ -96,14 +96,14 @@ def test_multiclass_classification(
assert isinstance(display, PrecisionRecallCurveDisplay)
check_display_data(display)

class_labels = report.reports_[0].estimator_.classes_
class_labels = list(report.reports_.values())[0].estimator_.classes_

display.plot()
assert isinstance(display.lines_, list)
assert len(display.lines_) == len(class_labels) * 2
default_colors = sample_mpl_colormap(pyplot.cm.tab10, 10)
for idx, (estimator_name, expected_color) in enumerate(
zip(report.report_names_, default_colors, strict=False)
zip(report.reports_.keys(), default_colors, strict=False)
):
for class_label_idx, class_label in enumerate(class_labels):
roc_curve_mpl = display.lines_[idx * len(class_labels) + class_label_idx]
Expand Down Expand Up @@ -350,7 +350,7 @@ def test_binary_classification_constructor(
index_columns = ["estimator_name", "split", "label"]
for df in [display.precision_recall, display.average_precision]:
assert all(col in df.columns for col in index_columns)
assert df["estimator_name"].unique().tolist() == report.report_names_
assert df["estimator_name"].unique().tolist() == list(report.reports_.keys())
assert df["split"].isnull().all()
assert df["label"].unique() == 1

Expand Down Expand Up @@ -378,7 +378,7 @@ def test_multiclass_classification_constructor(
index_columns = ["estimator_name", "split", "label"]
for df in [display.precision_recall, display.average_precision]:
assert all(col in df.columns for col in index_columns)
assert df["estimator_name"].unique().tolist() == report.report_names_
assert df["estimator_name"].unique().tolist() == list(report.reports_.keys())
assert df["split"].isnull().all()
np.testing.assert_array_equal(df["label"].unique(), np.unique(y_train))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ def test_constructor(linear_regression_data):
assert df.query("estimator_name == 'estimator_2'")[
"split"
].unique().tolist() == list(range(cv + 1))
assert df["estimator_name"].unique().tolist() == report.report_names_
assert df["estimator_name"].unique().tolist() == list(report.reports_.keys())


def test_frame(comparison_cross_validation_reports_regression):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -275,5 +275,5 @@ def test_constructor(linear_regression_with_train_test):
index_columns = ["estimator_name", "split"]
df = display._prediction_error
assert all(col in df.columns for col in index_columns)
assert df["estimator_name"].unique().tolist() == report.report_names_
assert df["estimator_name"].unique().tolist() == list(report.reports_.keys())
assert df["split"].isnull().all()
Loading
Loading