FIX SMOTENC: clear error for column-dropping categorical encoders (#1035) (#1177)

Author: immu4989

doc/whats_new/v0.15.rst

@@ -11,6 +11,12 @@ Changelog
 Bug fixes
 .........
 
+- Raise an informative error in :class:`~imblearn.over_sampling.SMOTENC` when the
+  provided `categorical_encoder` does not keep one column per category (e.g.
+  `OneHotEncoder(drop=...)` or an encoder merging infrequent categories), instead
+  of failing with an opaque ``zero-size array`` NumPy error.
+  :issue:`1035` by :user:`Imran Ahamed <immu4989>`.
+
 Enhancements
 ............
 

imblearn/over_sampling/_smote/base.py

@@ -619,6 +619,26 @@ def _fit_resample(self, X, y):
         if not sparse.issparse(X_ohe):
             X_ohe = sparse.csr_matrix(X_ohe, dtype=dtype_ohe)
 
+        # SMOTENC reconstructs the categorical features by activating exactly one
+        # column per categorical feature (see `_generate_samples`). This requires
+        # a complete one-hot encoding with one column per category. Encoders that
+        # emit fewer columns -- e.g. ``OneHotEncoder(drop=...)`` or one merging
+        # infrequent categories -- break this assumption and previously raised an
+        # opaque "zero-size array" error or silently produced wrong categories.
+        n_categories = sum(
+            categories.size for categories in self.categorical_encoder_.categories_
+        )
+        if X_ohe.shape[1] != n_categories:
+            raise ValueError(
+                "SMOTENC requires a one-hot encoding with one column per category "
+                "for the categorical features. The provided `categorical_encoder` "
+                f"produced {X_ohe.shape[1]} columns for {n_categories} categories. "
+                "This happens when the encoder drops columns (e.g. "
+                "`OneHotEncoder(drop=...)`) or merges infrequent categories, which "
+                "is not supported. Pass an encoder that keeps all categories, such "
+                "as `OneHotEncoder(handle_unknown='ignore')`."
+            )
+
         X_encoded = sparse.hstack((X_continuous, X_ohe), format="csr", dtype=dtype_ohe)
         X_resampled = [X_encoded.copy()]
         y_resampled = [y.copy()]

imblearn/over_sampling/_smote/tests/test_smote_nc.py

@@ -277,6 +277,39 @@ def test_smotenc_categorical_encoder():
     assert getattr(smote.categorical_encoder_, "sparse_output") is False
 
 
+@pytest.mark.parametrize("drop", ["first", "if_binary"])
+def test_smotenc_categorical_encoder_dropped_columns(drop):
+    """Check that a clear error is raised when the categorical encoder does not
+    keep one column per category (e.g. ``OneHotEncoder(drop=...)``).
+
+    Non-regression test for:
+    https://github.com/scikit-learn-contrib/imbalanced-learn/issues/1035
+    """
+    rng = np.random.RandomState(0)
+    n_samples = 200
+    X = np.hstack(
+        [
+            rng.randn(n_samples, 2),
+            rng.randint(0, 2, size=(n_samples, 1)),  # binary categorical
+            rng.randint(0, 4, size=(n_samples, 1)),
+            rng.randint(0, 3, size=(n_samples, 1)),
+        ]
+    ).astype(object)
+    y = np.array([1] * 40 + [0] * (n_samples - 40))
+    rng.shuffle(y)
+
+    encoder = OneHotEncoder(drop=drop, handle_unknown="ignore")
+    smote = SMOTENC(
+        categorical_features=[2, 3, 4],
+        categorical_encoder=encoder,
+        sampling_strategy="minority",
+        random_state=0,
+    )
+    err_msg = "SMOTENC requires a one-hot encoding with one column per category"
+    with pytest.raises(ValueError, match=err_msg):
+        smote.fit_resample(X, y)
+
+
 def test_smotenc_bool_categorical():
     """Check that we don't try to early convert the full input data to numeric when
     handling a pandas dataframe.