diff --git a/CHANGES.md b/CHANGES.md index 189107e0..4c1c1fae 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `ValidSplit` now gives an actionable error message when `stratified=True` + and no `y` is passed together with a `skorch.dataset.Dataset` or + `torch.utils.data.TensorDataset`, naming the dataset type and pointing to + `predefined_split`, `stratified=False`, or `train_split=None` as + workarounds (#1122) + ## [1.4.0] ### Added diff --git a/docs/user/FAQ.rst b/docs/user/FAQ.rst index a3d387da..b470a82a 100644 --- a/docs/user/FAQ.rst +++ b/docs/user/FAQ.rst @@ -264,6 +264,58 @@ The ``RandomDataset`` can be passed directly to net = NeuralNet(MyModule, criterion=torch.nn.MSELoss) net.fit(train_ds) +Why do I get an error about stratified CV when I pass a Dataset? +------------------------------------------------------------------ + +By default, :class:`~skorch.classifier.NeuralNetClassifier` performs a +stratified train/valid split internally (``train_split=ValidSplit(5, +stratified=True)``). Stratification requires ``y`` so that the split can +preserve the class proportions in each fold. If you call ``net.fit(dataset)`` +with a single object that already bundles ``X`` and ``y`` -- e.g. a +:class:`skorch.dataset.Dataset` or a +:class:`torch.utils.data.TensorDataset` -- instead of calling +``net.fit(X, y)``, skorch has no way to extract the labels from +``dataset`` to pass on to scikit-learn's stratified splitter, and raises: + +.. code:: text + + ValueError: Stratified CV requires explicitly passing a suitable y. ... + +skorch will not guess ``y`` from the dataset or silently turn stratification +off, since that could hide a real problem. Instead, pick one of the +following workarounds: + +* Wrap your own validation set with + :func:`skorch.helper.predefined_split` and pass it via ``train_split``, + so skorch doesn't need to split (or stratify) anything itself: + + .. code:: python + + from skorch.helper import predefined_split + + net = NeuralNetClassifier( + MyModule, + train_split=predefined_split(valid_ds), + ) + +* Disable stratification and let skorch split by index instead: + + .. code:: python + + from skorch.dataset import ValidSplit + + net = NeuralNetClassifier( + MyModule, + train_split=ValidSplit(5, stratified=False), + ) + +* Disable the internal train/valid split entirely, e.g. if you handle + validation yourself outside of ``fit``: + + .. code:: python + + net = NeuralNetClassifier(MyModule, train_split=None) + How can I deal with multiple return values from forward? -------------------------------------------------------- diff --git a/skorch/dataset.py b/skorch/dataset.py index 62ac61af..64c48d0f 100644 --- a/skorch/dataset.py +++ b/skorch/dataset.py @@ -34,6 +34,18 @@ "https://skorch.readthedocs.io/en/stable/user/dataset.html).") +ERROR_MSG_BAD_Y = "Stratified CV requires explicitly passing a suitable y." + + +ERROR_MSG_BAD_Y_STRATIFIED_DATASET = ( + ERROR_MSG_BAD_Y + " You passed a {} as dataset, which skorch cannot " + "introspect to obtain labels for stratification. To resolve this, " + "either wrap your validation data with skorch.helper.predefined_split " + "and pass it as train_split, disable stratification with " + "train_split=ValidSplit(5, stratified=False), or disable the internal " + "validation split entirely with train_split=None.") + + def _apply_to_data(data, func, unpack_dict=False): """Apply a function to data, trying to unpack different data types. @@ -301,15 +313,23 @@ def check_cv(self, y): def _is_regular(self, x): return (x is None) or isinstance(x, np.ndarray) or is_pandas_ndframe(x) + def _bad_y_error(self, dataset): + if isinstance(dataset, Dataset): + dataset_type = 'skorch.dataset.Dataset' + elif isinstance(dataset, torch.utils.data.TensorDataset): + dataset_type = 'torch.utils.data.TensorDataset' + else: + return ValueError(ERROR_MSG_BAD_Y) + msg = ERROR_MSG_BAD_Y_STRATIFIED_DATASET.format(dataset_type) + return ValueError(msg) + def __call__(self, dataset, y=None, groups=None): - bad_y_error = ValueError( - "Stratified CV requires explicitly passing a suitable y.") if (y is None) and self.stratified: - raise bad_y_error + raise self._bad_y_error(dataset) cv = self.check_cv(y) if self.stratified and not self._is_stratified(cv): - raise bad_y_error + raise ValueError(ERROR_MSG_BAD_Y) # pylint: disable=invalid-name len_dataset = get_len(dataset) diff --git a/skorch/tests/test_dataset.py b/skorch/tests/test_dataset.py index 68489ab0..b3d74d57 100644 --- a/skorch/tests/test_dataset.py +++ b/skorch/tests/test_dataset.py @@ -861,10 +861,49 @@ def test_y_dict_stratified_raises(self, valid_split_cls, data): @pytest.mark.parametrize('cv', [5, 0.2]) @pytest.mark.parametrize('X', [np.zeros((100, 10)), torch.zeros((100, 10))]) def test_y_none_stratified(self, valid_split_cls, data, cv, X): + # data is a skorch.dataset.Dataset, which should be named in the msg data.X = X with pytest.raises(ValueError) as exc: valid_split_cls(cv, stratified=True)(data, None) + expected = ( + "Stratified CV requires explicitly passing a suitable y. You " + "passed a skorch.dataset.Dataset as dataset, which skorch " + "cannot introspect to obtain labels for stratification. To " + "resolve this, either wrap your validation data with " + "skorch.helper.predefined_split and pass it as train_split, " + "disable stratification with train_split=ValidSplit(5, " + "stratified=False), or disable the internal validation split " + "entirely with train_split=None.") + assert exc.value.args[0] == expected + + def test_y_none_stratified_tensor_dataset(self, valid_split_cls): + X = torch.zeros((100, 10)) + y = torch.zeros(100) + dataset = torch.utils.data.TensorDataset(X, y) + + with pytest.raises(ValueError) as exc: + valid_split_cls(5, stratified=True)(dataset, None) + + expected = ( + "Stratified CV requires explicitly passing a suitable y. You " + "passed a torch.utils.data.TensorDataset as dataset, which " + "skorch cannot introspect to obtain labels for stratification. " + "To resolve this, either wrap your validation data with " + "skorch.helper.predefined_split and pass it as train_split, " + "disable stratification with train_split=ValidSplit(5, " + "stratified=False), or disable the internal validation split " + "entirely with train_split=None.") + assert exc.value.args[0] == expected + + def test_y_none_stratified_generic_dataset(self, valid_split_cls): + # neither a skorch.dataset.Dataset nor a TensorDataset, so the + # error keeps the original, generic message + X = np.zeros((100, 10)) + + with pytest.raises(ValueError) as exc: + valid_split_cls(5, stratified=True)(X, None) + expected = "Stratified CV requires explicitly passing a suitable y." assert exc.value.args[0] == expected diff --git a/skorch/tests/test_net.py b/skorch/tests/test_net.py index 2a180855..5366a581 100644 --- a/skorch/tests/test_net.py +++ b/skorch/tests/test_net.py @@ -2156,7 +2156,16 @@ def test_fit_with_dataset_stratified_without_explicit_y_raises( with pytest.raises(ValueError) as exc: net.fit(ds, None) - msg = "Stratified CV requires explicitly passing a suitable y." + msg = ( + "Stratified CV requires explicitly passing a suitable y. You " + "passed a skorch.dataset.Dataset as dataset, which skorch " + "cannot introspect to obtain labels for stratification. To " + "resolve this, either wrap your validation data with " + "skorch.helper.predefined_split and pass it as train_split, " + "disable stratification with train_split=ValidSplit(5, " + "stratified=False), or disable the internal validation split " + "entirely with train_split=None." + ) assert exc.value.args[0] == msg @pytest.fixture