reformatted

Author: Julien Roussel

pyproject.toml

@@ -133,7 +133,7 @@ testpaths = ["tests"]
 norecursedirs = ["_build"]
 
 [tool.ruff]
-line-length = 79
+line-length = 99
 fix = true
 indent-width = 4
 target-version = "py310"

qolmat/analysis/holes_characterization.py

@@ -45,9 +45,7 @@ def __init__(self, random_state: RandomSetting = None):
         self.rng = sku.check_random_state(random_state)
 
     @abstractmethod
-    def test(
-        self, df: Union[pd.DataFrame, np.ndarray]
-    ) -> Union[float, Tuple[float, List[float]]]:
+    def test(self, df: Union[pd.DataFrame, np.ndarray]) -> Union[float, Tuple[float, List[float]]]:
         """Perform the MCAR test on the input data.
 
         Parameters
@@ -99,8 +97,7 @@ def __init__(
         super().__init__()
         if imputer and imputer.model != "multinormal":
             raise AttributeError(
-                "The ImputerEM model must be 'multinormal' "
-                "to use the Little's test"
+                "The ImputerEM model must be 'multinormal' " "to use the Little's test"
             )
         self.imputer = imputer
         self.random_state = random_state
@@ -131,22 +128,16 @@ def test(self, df: pd.DataFrame) -> float:
         # Iterate over the patterns
 
         df_nan = df.notna()
-        for tup_pattern, df_nan_pattern in df_nan.groupby(
-            df_nan.columns.tolist()
-        ):
+        for tup_pattern, df_nan_pattern in df_nan.groupby(df_nan.columns.tolist()):
             n_rows_pattern, _ = df_nan_pattern.shape
             ind_pattern = df_nan_pattern.index
             df_pattern = df.loc[ind_pattern, list(tup_pattern)]
             obs_mean = df_pattern.mean().to_numpy()
 
             diff_means = obs_mean - ml_means[list(tup_pattern)]
-            inv_sigma_pattern = np.linalg.inv(
-                ml_cov[:, tup_pattern][tup_pattern, :]
-            )
+            inv_sigma_pattern = np.linalg.inv(ml_cov[:, tup_pattern][tup_pattern, :])
 
-            d0 += n_rows_pattern * np.dot(
-                np.dot(diff_means, inv_sigma_pattern), diff_means.T
-            )
+            d0 += n_rows_pattern * np.dot(np.dot(diff_means, inv_sigma_pattern), diff_means.T)
             degree_f += tup_pattern.count(True)
 
         return 1 - float(chi2.cdf(d0, degree_f))
@@ -242,9 +233,7 @@ def _encode_dataframe(self, df: pd.DataFrame) -> np.ndarray:
 
         return self.encoder.fit_transform(df)
 
-    def _pklm_preprocessing(
-        self, X: Union[pd.DataFrame, np.ndarray]
-    ) -> np.ndarray:
+    def _pklm_preprocessing(self, X: Union[pd.DataFrame, np.ndarray]) -> np.ndarray:
         """Preprocess the input DataFrame or ndarray for further processing.
 
         Parameters
@@ -295,9 +284,7 @@ def _get_max_draw(p: int) -> int:
         """
         return p * (2 ** (p - 1) - 1)
 
-    def _draw_features_and_target_indexes(
-        self, X: np.ndarray
-    ) -> Tuple[List[int], int]:
+    def _draw_features_and_target_indexes(self, X: np.ndarray) -> Tuple[List[int], int]:
         """Randomly select features and a target from the dataframe.
 
         This corresponds to the Ai and Bi projections of the paper.
@@ -320,9 +307,7 @@ def _draw_features_and_target_indexes(
         return features_idx.tolist(), target_idx
 
     @staticmethod
-    def _check_draw(
-        X: np.ndarray, features_idx: List[int], target_idx: int
-    ) -> bool:
+    def _check_draw(X: np.ndarray, features_idx: List[int], target_idx: int) -> bool:
         """Check if the drawn features and target are valid.
 
         Here we check
@@ -344,16 +329,12 @@ def _check_draw(
             True if the draw is valid, False otherwise.
 
         """
-        target_values = X[~np.isnan(X[:, features_idx]).any(axis=1)][
-            :, target_idx
-        ]
+        target_values = X[~np.isnan(X[:, features_idx]).any(axis=1)][:, target_idx]
         is_nan = np.isnan(target_values).any()
         is_distinct_values = (~np.isnan(target_values)).any()
         return is_nan and is_distinct_values
 
-    def _generate_label_feature_combinations(
-        self, X: np.ndarray
-    ) -> List[Tuple[List[int], int]]:
+    def _generate_label_feature_combinations(self, X: np.ndarray) -> List[Tuple[List[int], int]]:
         """Generate all valid combinations of features and labels.
 
         Parameters
@@ -400,9 +381,7 @@ def _draw_projection(self, X: np.ndarray) -> Tuple[List[int], int]:
         """
         is_checked = False
         while not is_checked:
-            features_idx, target_idx = self._draw_features_and_target_indexes(
-                X
-            )
+            features_idx, target_idx = self._draw_features_and_target_indexes(X)
             is_checked = self._check_draw(X, features_idx, target_idx)
         return features_idx, target_idx
 
@@ -434,13 +413,9 @@ def _build_dataset(
             the target column.
 
         """
-        X_features = X[~np.isnan(X[:, features_idx]).any(axis=1)][
-            :, features_idx
-        ]
+        X_features = X[~np.isnan(X[:, features_idx]).any(axis=1)][:, features_idx]
         y = np.where(
-            np.isnan(
-                X[~np.isnan(X[:, features_idx]).any(axis=1)][:, target_idx]
-            ),
+            np.isnan(X[~np.isnan(X[:, features_idx]).any(axis=1)][:, target_idx]),
             1,
             0,
         )
@@ -479,9 +454,7 @@ def _build_label(
         """
         return perm[~np.isnan(X[:, features_idx]).any(axis=1), target_idx]
 
-    def _get_oob_probabilities(
-        self, X: np.ndarray, y: np.ndarray
-    ) -> np.ndarray:
+    def _get_oob_probabilities(self, X: np.ndarray, y: np.ndarray) -> np.ndarray:
         """Retrieve out-of-bag probabilities.
 
         Train a RandomForestClassifier and retrieves out-of-bag (OOB)
@@ -547,26 +520,14 @@ def _U_hat(oob_probabilities: np.ndarray, labels: np.ndarray) -> float:
         if unique_labels.shape[0] == 1:
             if unique_labels[0] == 0:
                 n0 = labels.shape[0]
-                return (
-                    np.log(p0_0 / (1 - p0_0)).sum() / n0
-                    - np.log(p1_0 / (1 - p1_0)).sum() / n0
-                )
+                return np.log(p0_0 / (1 - p0_0)).sum() / n0 - np.log(p1_0 / (1 - p1_0)).sum() / n0
             else:
                 n1 = labels.shape[0]
-                return (
-                    np.log(p1_1 / (1 - p1_1)).sum() / n1
-                    - np.log(p0_1 / (1 - p0_1)).sum() / n1
-                )
+                return np.log(p1_1 / (1 - p1_1)).sum() / n1 - np.log(p0_1 / (1 - p0_1)).sum() / n1
 
         n0, n1 = label_matrix.sum(axis=0)
-        u_0 = (
-            np.log(p0_0 / (1 - p0_0)).sum() / n0
-            - np.log(p0_1 / (1 - p0_1)).sum() / n1
-        )
-        u_1 = (
-            np.log(p1_1 / (1 - p1_1)).sum() / n1
-            - np.log(p1_0 / (1 - p1_0)).sum() / n0
-        )
+        u_0 = np.log(p0_0 / (1 - p0_0)).sum() / n0 - np.log(p0_1 / (1 - p0_1)).sum() / n1
+        u_1 = np.log(p1_1 / (1 - p1_1)).sum() / n1 - np.log(p1_0 / (1 - p1_0)).sum() / n0
 
         return u_0 + u_1
 
@@ -708,9 +669,7 @@ def _compute_partial_p_value(
 
         return p_v_k / (self.nb_permutation + 1)
 
-    def test(
-        self, X: Union[pd.DataFrame, np.ndarray]
-    ) -> Union[float, Tuple[float, List[float]]]:
+    def test(self, X: Union[pd.DataFrame, np.ndarray]) -> Union[float, Tuple[float, List[float]]]:
         """Apply the PKLM test over a real dataset.
 
         Parameters
@@ -733,21 +692,15 @@ def test(
         if self._get_max_draw(n_cols) <= self.nb_projections_threshold:
             list_proj = self._generate_label_feature_combinations(X)
         else:
-            list_proj = [
-                self._draw_projection(X) for _ in range(self.nb_projections)
-            ]
+            list_proj = [self._draw_projection(X) for _ in range(self.nb_projections)]
 
         M = np.isnan(X).astype(int)
-        list_perm = [
-            self.rng.permutation(M) for _ in range(self.nb_permutation)
-        ]
+        list_perm = [self.rng.permutation(M) for _ in range(self.nb_permutation)]
         U = 0.0
         list_U_sigma = [0.0 for _ in range(self.nb_permutation)]
 
         parallel_results = Parallel(n_jobs=-1)(
-            delayed(self._parallel_process_projection)(
-                X, list_perm, features_idx, target_idx
-            )
+            delayed(self._parallel_process_projection)(X, list_perm, features_idx, target_idx)
             for features_idx, target_idx in list_proj
         )
 
@@ -769,14 +722,9 @@ def test(
             return p_value
         else:
             B = self._build_B(list_proj, n_cols)
-            U_matrix = np.array(
-                [np.atleast_1d(item[0]) for item in parallel_results]
-            )
-            U_sigma = np.array(
-                [np.atleast_1d(item[1]) for item in parallel_results]
-            )
+            U_matrix = np.array([np.atleast_1d(item[0]) for item in parallel_results])
+            U_sigma = np.array([np.atleast_1d(item[1]) for item in parallel_results])
             p_values = [
-                self._compute_partial_p_value(B, U_matrix, U_sigma, k)
-                for k in range(n_cols)
+                self._compute_partial_p_value(B, U_matrix, U_sigma, k) for k in range(n_cols)
             ]
             return p_value, p_values

qolmat/benchmark/comparator.py

@@ -84,15 +84,11 @@ def get_errors(
         dict_errors = {}
         for name_metric in self.metrics:
             fun_metric = metrics.get_metric(name_metric)
-            dict_errors[name_metric] = fun_metric(
-                df_origin, df_imputed, df_mask
-            )
+            dict_errors[name_metric] = fun_metric(df_origin, df_imputed, df_mask)
         df_errors = pd.concat(dict_errors.values(), keys=dict_errors.keys())
         return df_errors
 
-    def process_split(
-        self, split_data: Tuple[int, pd.DataFrame, pd.DataFrame]
-    ) -> pd.DataFrame:
+    def process_split(self, split_data: Tuple[int, pd.DataFrame, pd.DataFrame]) -> pd.DataFrame:
         """Process a split.
 
         Parameters
@@ -119,15 +115,12 @@ def process_split(
         subset = self.generator_holes.subset
         if subset is None:
             raise ValueError(
-                "HoleGenerator `subset` should be overwritten in split "
-                "but it is none!"
+                "HoleGenerator `subset` should be overwritten in split " "but it is none!"
             )
 
         split_results = {}
         for imputer_name, imputer in self.dict_imputers.items():
-            dict_config_opti_imputer = self.dict_config_opti.get(
-                imputer_name, {}
-            )
+            dict_config_opti_imputer = self.dict_config_opti.get(imputer_name, {})
 
             imputer_opti = hyperparameters.optimize(
                 imputer,
@@ -140,9 +133,7 @@ def process_split(
             )
 
             df_imputed = imputer_opti.fit_transform(df_with_holes)
-            errors = self.get_errors(
-                df_origin[subset], df_imputed[subset], df_mask[subset]
-            )
+            errors = self.get_errors(df_origin[subset], df_imputed[subset], df_mask[subset])
             split_results[imputer_name] = errors
 
         return pd.concat(split_results, axis=1)
@@ -168,8 +159,7 @@ def process_imputer(
         subset = self.generator_holes.subset
         if subset is None:
             raise ValueError(
-                "HoleGenerator `subset` should be overwritten in split "
-                "but it is none!"
+                "HoleGenerator `subset` should be overwritten in split " "but it is none!"
             )
 
         dict_config_opti_imputer = self.dict_config_opti.get(imputer_name, {})
@@ -188,14 +178,10 @@ def process_imputer(
             df_with_holes = df_origin.copy()
             df_with_holes[df_mask] = np.nan
             df_imputed = imputer_opti.fit_transform(df_with_holes)
-            errors = self.get_errors(
-                df_origin[subset], df_imputed[subset], df_mask[subset]
-            )
+            errors = self.get_errors(df_origin[subset], df_imputed[subset], df_mask[subset])
             imputer_results.append(errors)
 
-        return imputer_name, pd.concat(imputer_results).groupby(
-            level=[0, 1]
-        ).mean()
+        return imputer_name, pd.concat(imputer_results).groupby(level=[0, 1]).mean()
 
     def compare(
         self,
@@ -229,26 +215,17 @@ def compare(
             1-level index are the column names.
 
         """
-        logging.info(
-            f"Starting comparison for {len(self.dict_imputers)} imputers."
-        )
+        logging.info(f"Starting comparison for {len(self.dict_imputers)} imputers.")
 
         all_splits = list(self.generator_holes.split(df_origin))
 
         if parallel_over == "auto":
-            parallel_over = (
-                "splits"
-                if len(all_splits) > len(self.dict_imputers)
-                else "imputers"
-            )
+            parallel_over = "splits" if len(all_splits) > len(self.dict_imputers) else "imputers"
 
         if use_parallel:
             logging.info(f"Parallelisation over: {parallel_over}...")
             if parallel_over == "splits":
-                split_data = [
-                    (i, df_mask, df_origin)
-                    for i, df_mask in enumerate(all_splits)
-                ]
+                split_data = [(i, df_mask, df_origin) for i, df_mask in enumerate(all_splits)]
                 n_jobs = self.get_optimal_n_jobs(split_data, n_jobs)
                 results = Parallel(n_jobs=n_jobs)(
                     delayed(self.process_split)(data) for data in split_data
@@ -261,22 +238,16 @@ def compare(
                 ]
                 n_jobs = self.get_optimal_n_jobs(imputer_data, n_jobs)
                 results = Parallel(n_jobs=n_jobs)(
-                    delayed(self.process_imputer)(data)
-                    for data in imputer_data
+                    delayed(self.process_imputer)(data) for data in imputer_data
                 )
                 final_results = pd.concat(dict(results), axis=1)
             else:
-                raise ValueError(
-                    "`parallel_over` should be `auto`, `splits` or `imputers`."
-                )
+                raise ValueError("`parallel_over` should be `auto`, `splits` or `imputers`.")
 
         else:
             logging.info("Sequential treatment...")
             if parallel_over == "splits":
-                split_data = [
-                    (i, df_mask, df_origin)
-                    for i, df_mask in enumerate(all_splits)
-                ]
+                split_data = [(i, df_mask, df_origin) for i, df_mask in enumerate(all_splits)]
                 results = [self.process_split(data) for data in split_data]
                 final_results = pd.concat(results).groupby(level=[0, 1]).mean()
             elif parallel_over == "imputers":
@@ -287,9 +258,7 @@ def compare(
                 results = [self.process_imputer(data) for data in imputer_data]
                 final_results = pd.concat(dict(results), axis=1)
             else:
-                raise ValueError(
-                    "`parallel_over` should be `auto`, `splits` or `imputers`."
-                )
+                raise ValueError("`parallel_over` should be `auto`, `splits` or `imputers`.")
 
         logging.info("Comparison successfully terminated.")
         return final_results

qolmat/benchmark/hyperparameters.py

@@ -62,9 +62,7 @@ def fun_obf(args: List[HyperValue]) -> float:
             df_imputed = imputer.fit_transform(df_corrupted)
             subset = generator.subset
             fun_metric = metrics.get_metric(metric)
-            errors = fun_metric(
-                df_origin[subset], df_imputed[subset], df_mask[subset]
-            )
+            errors = fun_metric(df_origin[subset], df_imputed[subset], df_mask[subset])
             list_errors.append(errors)
 
         mean_errors = np.mean(errors)
@@ -120,9 +118,7 @@ def optimize(
         return imputer
     names_hyperparams = list(dict_config.keys())
     values_hyperparams = list(dict_config.values())
-    imputer.imputer_params = tuple(
-        set(imputer.imputer_params) | set(dict_config.keys())
-    )
+    imputer.imputer_params = tuple(set(imputer.imputer_params) | set(dict_config.keys()))
     if verbose and hasattr(imputer, "verbose"):
         setattr(imputer, "verbose", False)
     fun_obj = get_objective(imputer, df, generator, metric, names_hyperparams)