Merge pull request #372 from guillaume-vignal/master

Enable compatibility with new version of category-encoders

Author: guillaume-vignal

requirements.dev.txt

@@ -1,7 +1,7 @@
 pip==21.3.1
 dash==2.3.1
 catboost==0.26.1
-category-encoders==2.1.0
+category-encoders==2.2.2
 dash-bootstrap-components==1.1.0
 dash-core-components==2.0.0
 dash-daq==0.5.0

setup.py

@@ -52,7 +52,7 @@
 extras['lightgbm'] = ['lightgbm>=2.3.0']
 extras['catboost'] = ['catboost>=0.21']
 extras['scikit-learn'] = ['scikit-learn>=0.23.0']
-extras['category_encoders'] = ['category_encoders==2.2.2']
+extras['category_encoders'] = ['category_encoders>=2.2.2']
 extras['acv'] = ['acv-exp==1.1.2']
 extras['lime'] = ['lime']
 

shapash/utils/category_encoder_backend.py

@@ -4,6 +4,7 @@
 
 import pandas as pd
 import numpy as np
+import category_encoders as ce
 
 category_encoder_onehot = "<class 'category_encoders.one_hot.OneHotEncoder'>"
 category_encoder_ordinal = "<class 'category_encoders.ordinal.OrdinalEncoder'>"
@@ -54,8 +55,12 @@ def inv_transform_ce(x_in, encoding):
         rst = inv_transform_ordinal(x, encoding.ordinal_encoder.mapping)
 
     elif str(type(encoding)) == category_encoder_binary:
-        x = reverse_basen(x_in, encoding.base_n_encoder)
-        rst = inv_transform_ordinal(x, encoding.base_n_encoder.ordinal_encoder.mapping)
+        if ce.__version__ <= '2.2.2':
+            x = reverse_basen(x_in, encoding.base_n_encoder)
+            rst = inv_transform_ordinal(x, encoding.base_n_encoder.ordinal_encoder.mapping)
+        else:
+            x = reverse_basen(x_in, encoding)
+            rst = inv_transform_ordinal(x, encoding.ordinal_encoder.mapping)
 
     elif str(type(encoding)) == category_encoder_targetencoder:
         rst = inv_transform_target(x_in, encoding)
@@ -106,8 +111,8 @@ def inv_transform_target(x_in, enc_target):
         rst_target = pd.concat([reverse_target, mapping_ordinal], axis=1, join='inner').fillna(value='NaN')
         aggregate = rst_target.groupby(1)[0].apply(lambda x: ' / '.join(map(str, x)))
         if aggregate.shape[0] != rst_target.shape[0]:
-            raise Exception('Multiple label found for the same value in TargetEncoder on col '+str(name_target) +'.')
-            #print("Warning in inverse TargetEncoder - col " + str(name_target) + ": Multiple label for the same value, "
+            raise Exception('Multiple label found for the same value in TargetEncoder on col '+str(name_target) + '.')
+            # print("Warning in inverse TargetEncoder - col " + str(name_target) + ": Multiple label for the same value, "
             #                                                                   "each label will be separate using : / ")
 
         transco = {'col': name_target,
@@ -138,7 +143,10 @@ def inv_transform_ordinal(x_in, encoding):
         if not col_name in x_in.columns:
             raise Exception(f'Columns {col_name} not in dataframe.')
         column_mapping = switch.get('mapping')
-        inverse = pd.Series(data=column_mapping.index, index=column_mapping.values)
+        if isinstance(column_mapping, dict):
+            inverse = pd.Series(data=column_mapping.keys(), index=column_mapping.values())
+        else:
+            inverse = pd.Series(data=column_mapping.index, index=column_mapping.values)
         x_in[col_name] = x_in[col_name].map(inverse).astype(switch.get('data_type'))
     return x_in
 
@@ -201,7 +209,7 @@ def calc_inv_contrib_ce(x_contrib, encoding, agg_columns):
         The aggregate contributions depending on which processing is apply.
     """
     if str(type(encoding)) in dummies_category_encoder:
-        if str(type(encoding)) in category_encoder_binary:
+        if str(type(encoding)) in category_encoder_binary and ce.__version__ <= '2.2.2':
             encoding = encoding.base_n_encoder
         drop_col = []
         for switch in encoding.mapping:
@@ -218,6 +226,7 @@ def calc_inv_contrib_ce(x_contrib, encoding, agg_columns):
     else:
         return x_contrib
 
+
 def transform_ce(x_in, encoding):
     """
     Choose and apply the transformation for the given encoding.
@@ -242,14 +251,15 @@ def transform_ce(x_in, encoding):
     if str(type(encoding)) in encoder:
         rst = encoding.transform(x_in)
 
-    elif isinstance(encoding,list):
+    elif isinstance(encoding, list):
         rst = transform_ordinal(x_in, encoding)
 
     else:
         raise Exception(f"{encoding.__class__.__name__} not supported, no preprocessing done.")
 
     return rst
 
+
 def transform_ordinal(x_in, encoding):
     """
     Transformation based on ordinal category encoder.
@@ -271,7 +281,10 @@ def transform_ordinal(x_in, encoding):
         if not col_name in x_in.columns:
             raise Exception(f'Columns {col_name} not in dataframe.')
         column_mapping = switch.get('mapping')
-        transform = pd.Series(data=column_mapping.values, index=column_mapping.index)
+        if isinstance(column_mapping, dict):
+            transform = pd.Series(data=column_mapping.values(), index=column_mapping.keys())
+        else:
+            transform = pd.Series(data=column_mapping.values, index=column_mapping.index)
         x_in[col_name] = x_in[col_name].map(transform).astype(switch.get('mapping').values.dtype)
     return x_in
 
@@ -294,7 +307,10 @@ def get_col_mapping_ce(encoder):
                               category_encoder_targetencoder]:
         encoder_mapping = encoder.mapping
     elif str(type(encoder)) == category_encoder_binary:
-        encoder_mapping = encoder.base_n_encoder.mapping
+        if ce.__version__ <= '2.2.2':
+            encoder_mapping = encoder.base_n_encoder.mapping
+        else:
+            encoder_mapping = encoder.mapping
     else:
         raise NotImplementedError(f"{encoder} not supported.")
 

shapash/utils/columntransformer_backend.py

@@ -179,6 +179,7 @@ def inv_transform_sklearn_in_ct(x_in, init, name_encoding, col_encoding, ct_enco
     init += nb_col
     return frame, init
 
+
 def calc_inv_contrib_ct(x_contrib, encoding, agg_columns):
     """
     Reversed contribution when ColumnTransformer is used.
@@ -226,7 +227,10 @@ def calc_inv_contrib_ct(x_contrib, encoding, agg_columns):
                         if str(type(ct_encoding)) == sklearn_onehot:
                             col_origin = ct_encoding.categories_[i_enc]
                         elif str(type(ct_encoding)) == category_encoder_binary:
-                            col_origin = ct_encoding.base_n_encoder.mapping[i_enc].get('mapping').columns.tolist()
+                            try:
+                                col_origin = ct_encoding.base_n_encoder.mapping[i_enc].get('mapping').columns.tolist()
+                            except:
+                                col_origin = ct_encoding.mapping[i_enc].get('mapping').columns.tolist()
                         else:
                             col_origin = ct_encoding.mapping[i_enc].get('mapping').columns.tolist()
                         nb_col = len(col_origin)
@@ -292,8 +296,8 @@ def transform_ct(x_in, model, encoding):
 
         elif str(type(model)) in other_model:
             rst = pd.DataFrame(encoding.transform(x_in),
-                                columns=extract_features_model(model, dict_model_feature[str(type(model))]),
-                                index=x_in.index)
+                               columns=extract_features_model(model, dict_model_feature[str(type(model))]),
+                               index=x_in.index)
         else:
             raise ValueError("Model specified isn't supported by Shapash.")
 
@@ -305,6 +309,7 @@ def transform_ct(x_in, model, encoding):
 
     return rst
 
+
 def get_names(name, trans, column, column_transformer):
     """
     Allow to extract features names from one encoder of the ColumnTransformer.
@@ -347,6 +352,7 @@ def get_names(name, trans, column, column_transformer):
 
     return [name + "__" + f for f in trans.get_feature_names()]
 
+
 def get_feature_names(column_transformer):
     """
     Allow to extract all features names from encoders of the ColumnTransformer once it has been applied.
@@ -370,6 +376,7 @@ def get_feature_names(column_transformer):
 
     return feature_names
 
+
 def get_list_features_names(list_preprocessing, columns_dict):
     """
     Allow to extract all features names from encoders when a list of preprocessing is uesd once it has been applied.

tests/unit_tests/utils/test_category_encoders_backend.py

@@ -6,7 +6,6 @@
 import numpy as np
 import category_encoders as ce
 import catboost as cb
-import sklearn
 import lightgbm
 import xgboost
 from shapash.utils.transform import inverse_transform, apply_preprocessing, get_col_mapping_ce
@@ -87,7 +86,6 @@ def test_inverse_transform_2(self):
 
         pd.testing.assert_frame_equal(expected, original)
 
-
     def test_inverse_transform_3(self):
         """
         Test target encoding
@@ -404,15 +402,15 @@ def test_inverse_transform_26(self):
                              'BaseN1': ['M', 'N', 'N'], 'BaseN2': ['O', 'P', 'ZZ'],
                              'Target1': ['Q', 'R', 'R'], 'Target2': ['S', 'T', 'ZZ'],
                              'other': ['other', '123', np.nan]},
-                             index=['index1', 'index2', 'index3'])
+                            index=['index1', 'index2', 'index3'])
 
         expected = pd.DataFrame({'Onehot1': ['A', 'B', 'A'], 'Onehot2': ['C', 'D', 'missing'],
                                  'Binary1': ['E', 'F', 'F'], 'Binary2': ['G', 'H', 'missing'],
                                  'Ordinal1': ['I', 'J', 'J'], 'Ordinal2': ['K', 'L', 'missing'],
                                  'BaseN1': ['M', 'N', 'N'], 'BaseN2': ['O', 'P', np.nan],
                                  'Target1': ['Q', 'R', 'R'], 'Target2': ['S', 'T', 'NaN'],
                                  'other': ['other', '123', np.nan]},
-                             index=['index1', 'index2', 'index3'])
+                                index=['index1', 'index2', 'index3'])
 
         y = pd.DataFrame(data=[0, 1, 0, 0], columns=['y'])
 
@@ -668,7 +666,7 @@ def test_get_col_mapping_ce_1(self):
         y = pd.DataFrame(data=[0, 1, 1], columns=['y'])
 
         enc = ce.TargetEncoder(cols=['city', 'state'])
-        test_encoded = pd.DataFrame(enc.fit_transform(test, y))
+        enc.fit(test, y)
 
         mapping = get_col_mapping_ce(enc)
         expected_mapping = {'city': ['city'], 'state': ['state']}
@@ -685,7 +683,7 @@ def test_get_col_mapping_ce_2(self):
         y = pd.DataFrame(data=[0, 1, 1], columns=['y'])
 
         enc = ce.OrdinalEncoder(handle_missing='value', handle_unknown='value')
-        test_encoded = pd.DataFrame(enc.fit_transform(test, y))
+        enc.fit(test, y)
 
         mapping = get_col_mapping_ce(enc)
         expected_mapping = {'city': ['city'], 'state': ['state'], 'other': ['other']}
@@ -702,7 +700,7 @@ def test_get_col_mapping_ce_3(self):
         y = pd.DataFrame(data=[0, 1, 1], columns=['y'])
 
         enc = ce.BinaryEncoder(cols=['city', 'state'])
-        test_encoded = pd.DataFrame(enc.fit_transform(test, y))
+        enc.fit(test, y)
 
         mapping = get_col_mapping_ce(enc)
         expected_mapping = {'city': ['city_0', 'city_1'], 'state': ['state_0', 'state_1']}
@@ -719,11 +717,15 @@ def test_get_col_mapping_ce_4(self):
         y = pd.DataFrame(data=[0, 1, 1], columns=['y'])
 
         enc = ce.BaseNEncoder(base=2)
-        test_encoded = pd.DataFrame(enc.fit_transform(test, y))
+        enc.fit(test, y)
 
         mapping = get_col_mapping_ce(enc)
-        expected_mapping = {'city': ['city_0', 'city_1', 'city_2'], 'state': ['state_0', 'state_1'],
-                            'other': ['other_0', 'other_1']}
+        if ce.__version__ <= '2.2.2':
+            expected_mapping = {'city': ['city_0', 'city_1', 'city_2'], 'state': ['state_0', 'state_1'],
+                                'other': ['other_0', 'other_1']}
+        else:
+            expected_mapping = {'city': ['city_0', 'city_1'], 'state': ['state_0', 'state_1'],
+                                'other': ['other_0', 'other_1']}
 
         self.assertDictEqual(mapping, expected_mapping)
 
@@ -737,7 +739,7 @@ def test_get_col_mapping_ce_5(self):
         y = pd.DataFrame(data=[0, 1, 1], columns=['y'])
 
         enc = ce.OneHotEncoder(cols=['city', 'state'], use_cat_names=True)
-        test_encoded = pd.DataFrame(enc.fit_transform(test, y))
+        enc.fit(test, y)
 
         mapping = get_col_mapping_ce(enc)
         expected_mapping = {'city': ['city_chicago', 'city_paris'], 'state': ['state_US', 'state_FR']}