Improve error message and add FAQ entry for stratified CV with Dataset (#1122)

CHANGES.md

@@ -16,6 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- `ValidSplit` now gives an actionable error message when `stratified=True`
+  and no `y` is passed together with a `skorch.dataset.Dataset` or
+  `torch.utils.data.TensorDataset`, naming the dataset type and pointing to
+  `predefined_split`, `stratified=False`, or `train_split=None` as
+  workarounds (#1122)
+
 ## [1.4.0]
 
 ### Added

docs/user/FAQ.rst

@@ -264,6 +264,58 @@ The ``RandomDataset`` can be passed directly to
     net = NeuralNet(MyModule, criterion=torch.nn.MSELoss)
     net.fit(train_ds)
 
+Why do I get an error about stratified CV when I pass a Dataset?
+------------------------------------------------------------------
+
+By default, :class:`~skorch.classifier.NeuralNetClassifier` performs a
+stratified train/valid split internally (``train_split=ValidSplit(5,
+stratified=True)``). Stratification requires ``y`` so that the split can
+preserve the class proportions in each fold. If you call ``net.fit(dataset)``
+with a single object that already bundles ``X`` and ``y`` -- e.g. a
+:class:`skorch.dataset.Dataset` or a
+:class:`torch.utils.data.TensorDataset` -- instead of calling
+``net.fit(X, y)``, skorch has no way to extract the labels from
+``dataset`` to pass on to scikit-learn's stratified splitter, and raises:
+
+.. code:: text
+
+    ValueError: Stratified CV requires explicitly passing a suitable y. ...
+
+skorch will not guess ``y`` from the dataset or silently turn stratification
+off, since that could hide a real problem. Instead, pick one of the
+following workarounds:
+
+* Wrap your own validation set with
+  :func:`skorch.helper.predefined_split` and pass it via ``train_split``,
+  so skorch doesn't need to split (or stratify) anything itself:
+
+  .. code:: python
+
+      from skorch.helper import predefined_split
+
+      net = NeuralNetClassifier(
+          MyModule,
+          train_split=predefined_split(valid_ds),
+      )
+
+* Disable stratification and let skorch split by index instead:
+
+  .. code:: python
+
+      from skorch.dataset import ValidSplit
+
+      net = NeuralNetClassifier(
+          MyModule,
+          train_split=ValidSplit(5, stratified=False),
+      )
+
+* Disable the internal train/valid split entirely, e.g. if you handle
+  validation yourself outside of ``fit``:
+
+  .. code:: python
+
+      net = NeuralNetClassifier(MyModule, train_split=None)
+
 
 How can I deal with multiple return values from forward?
 --------------------------------------------------------

skorch/dataset.py

@@ -34,6 +34,18 @@
     "https://skorch.readthedocs.io/en/stable/user/dataset.html).")
 
 
+ERROR_MSG_BAD_Y = "Stratified CV requires explicitly passing a suitable y."
+
+
+ERROR_MSG_BAD_Y_STRATIFIED_DATASET = (
+    ERROR_MSG_BAD_Y + " You passed a {} as dataset, which skorch cannot "
+    "introspect to obtain labels for stratification. To resolve this, "
+    "either wrap your validation data with skorch.helper.predefined_split "
+    "and pass it as train_split, disable stratification with "
+    "train_split=ValidSplit(5, stratified=False), or disable the internal "
+    "validation split entirely with train_split=None.")
+
+
 def _apply_to_data(data, func, unpack_dict=False):
     """Apply a function to data, trying to unpack different data
     types.
@@ -301,15 +313,23 @@ def check_cv(self, y):
     def _is_regular(self, x):
         return (x is None) or isinstance(x, np.ndarray) or is_pandas_ndframe(x)
 
+    def _bad_y_error(self, dataset):
+        if isinstance(dataset, Dataset):
+            dataset_type = 'skorch.dataset.Dataset'
+        elif isinstance(dataset, torch.utils.data.TensorDataset):
+            dataset_type = 'torch.utils.data.TensorDataset'
+        else:
+            return ValueError(ERROR_MSG_BAD_Y)
+        msg = ERROR_MSG_BAD_Y_STRATIFIED_DATASET.format(dataset_type)
+        return ValueError(msg)
+
     def __call__(self, dataset, y=None, groups=None):
-        bad_y_error = ValueError(
-            "Stratified CV requires explicitly passing a suitable y.")
         if (y is None) and self.stratified:
-            raise bad_y_error
+            raise self._bad_y_error(dataset)
 
         cv = self.check_cv(y)
         if self.stratified and not self._is_stratified(cv):
-            raise bad_y_error
+            raise ValueError(ERROR_MSG_BAD_Y)
 
         # pylint: disable=invalid-name
         len_dataset = get_len(dataset)

skorch/tests/test_dataset.py

@@ -861,10 +861,49 @@ def test_y_dict_stratified_raises(self, valid_split_cls, data):
     @pytest.mark.parametrize('cv', [5, 0.2])
     @pytest.mark.parametrize('X', [np.zeros((100, 10)), torch.zeros((100, 10))])
     def test_y_none_stratified(self, valid_split_cls, data, cv, X):
+        # data is a skorch.dataset.Dataset, which should be named in the msg
         data.X = X
         with pytest.raises(ValueError) as exc:
             valid_split_cls(cv, stratified=True)(data, None)
 
+        expected = (
+            "Stratified CV requires explicitly passing a suitable y. You "
+            "passed a skorch.dataset.Dataset as dataset, which skorch "
+            "cannot introspect to obtain labels for stratification. To "
+            "resolve this, either wrap your validation data with "
+            "skorch.helper.predefined_split and pass it as train_split, "
+            "disable stratification with train_split=ValidSplit(5, "
+            "stratified=False), or disable the internal validation split "
+            "entirely with train_split=None.")
+        assert exc.value.args[0] == expected
+
+    def test_y_none_stratified_tensor_dataset(self, valid_split_cls):
+        X = torch.zeros((100, 10))
+        y = torch.zeros(100)
+        dataset = torch.utils.data.TensorDataset(X, y)
+
+        with pytest.raises(ValueError) as exc:
+            valid_split_cls(5, stratified=True)(dataset, None)
+
+        expected = (
+            "Stratified CV requires explicitly passing a suitable y. You "
+            "passed a torch.utils.data.TensorDataset as dataset, which "
+            "skorch cannot introspect to obtain labels for stratification. "
+            "To resolve this, either wrap your validation data with "
+            "skorch.helper.predefined_split and pass it as train_split, "
+            "disable stratification with train_split=ValidSplit(5, "
+            "stratified=False), or disable the internal validation split "
+            "entirely with train_split=None.")
+        assert exc.value.args[0] == expected
+
+    def test_y_none_stratified_generic_dataset(self, valid_split_cls):
+        # neither a skorch.dataset.Dataset nor a TensorDataset, so the
+        # error keeps the original, generic message
+        X = np.zeros((100, 10))
+
+        with pytest.raises(ValueError) as exc:
+            valid_split_cls(5, stratified=True)(X, None)
+
         expected = "Stratified CV requires explicitly passing a suitable y."
         assert exc.value.args[0] == expected
 

skorch/tests/test_net.py

@@ -2156,7 +2156,16 @@ def test_fit_with_dataset_stratified_without_explicit_y_raises(
         with pytest.raises(ValueError) as exc:
             net.fit(ds, None)
 
-        msg = "Stratified CV requires explicitly passing a suitable y."
+        msg = (
+            "Stratified CV requires explicitly passing a suitable y. You "
+            "passed a skorch.dataset.Dataset as dataset, which skorch "
+            "cannot introspect to obtain labels for stratification. To "
+            "resolve this, either wrap your validation data with "
+            "skorch.helper.predefined_split and pass it as train_split, "
+            "disable stratification with train_split=ValidSplit(5, "
+            "stratified=False), or disable the internal validation split "
+            "entirely with train_split=None."
+        )
         assert exc.value.args[0] == msg
 
     @pytest.fixture