diff --git a/.github/workflows/publish-pypi-workflow.yml b/.github/workflows/publish-pypi-workflow.yml new file mode 100644 index 00000000..545f05f1 --- /dev/null +++ b/.github/workflows/publish-pypi-workflow.yml @@ -0,0 +1,51 @@ +name: Publish PyPI + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APPLICATION_ID }} + private-key: ${{ secrets.APPLICATION_PRIVATE_KEY }} + + - name: Checkout code + uses: actions/checkout@v4.2.2 + with: + fetch-tags: true + fetch-depth: 0 + ref: main + token: ${{ steps.app-token.outputs.token }} # Needed to trigger other actions + + - name: Set up Python + uses: actions/setup-python@v5.3.0 + with: + python-version: "3.9" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch + + - name: Get Latest Release Version + id: get_release + run: | + latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`) + echo "Latest release version: $latest_tag" + echo "tag=$latest_tag" >> $GITHUB_ENV # Save the tag to an environment variable + + - name: Publish to PyPI + env: + HATCH_INDEX_USER: __token__ + HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }} + run: | + hatch build + hatch publish + + + + diff --git a/.github/workflows/publish-workflow.yml b/.github/workflows/publish-slack-workflow.yml similarity index 71% rename from .github/workflows/publish-workflow.yml rename to .github/workflows/publish-slack-workflow.yml index ca09a991..440575e1 100644 --- a/.github/workflows/publish-workflow.yml +++ b/.github/workflows/publish-slack-workflow.yml @@ -1,4 +1,4 @@ -name: Publish and Notification +name: Slack Notification on: workflow_dispatch: @@ -21,16 +21,6 @@ jobs: ref: main token: ${{ steps.app-token.outputs.token }} # Needed to trigger other actions - - name: Set up Python - uses: actions/setup-python@v5.3.0 - with: - python-version: "3.9" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install hatch - - name: Get Latest Release Version id: get_release run: | @@ -38,25 +28,19 @@ jobs: echo "Latest release version: $latest_tag" echo "tag=$latest_tag" >> $GITHUB_ENV # Save the tag to an environment variable - - name: Publish to PyPI - env: - HATCH_INDEX_USER: __token__ - HATCH_INDEX_AUTH: ${{ secrets.PYPI_TOKEN }} - run: | - hatch build - hatch publish - - name: Post to a Slack channel id: slack uses: slackapi/slack-github-action@v2.0.0 with: - channel-id: "${{ vars.SLACK_CHANNEL_ID }}" + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} payload: | { + "channel": "${{ vars.SLACK_CHANNEL_ID }}", + "text": "Release holisticai *v${{env.tag}}* is on the way :rocket:", "attachments": [ { "color": "#36a64f", - "pretext": "Release holisticai *v${{env.tag}}* is on the way :rocket:", "fields": [ { "title": "Release Notes", @@ -77,9 +61,3 @@ jobs: } ] } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - - - - diff --git a/CHANGELOG.md b/CHANGELOG.md index 0db98186..3d501f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,28 @@ All notable changes to this project will be documented in this file. +## [1.0.12] - 2025-02-12 + +### 🐛 Bug Fixes + +- Update publish-workflow.yml (#283) + +### ⚙️ Miscellaneous Tasks + +- Update Workflow Names (#284) + +## [1.0.11] - 2024-12-12 + +### ⚙️ Miscellaneous Tasks + +- Update release workflows (#282) + ## [1.0.7] - 2024-11-01 +### 💼 Other + +- Develop -> main (#260) + ### 📚 Documentation - Update Documentation (#235) @@ -12,10 +32,6 @@ All notable changes to this project will be documented in this file. - Fix documentation (#233) -### Merge - -- Develop -> main (#260) - ## [1.0.6] - 2024-09-23 ### 🚀 Features @@ -216,7 +232,7 @@ All notable changes to this project will be documented in this file. - Release new mitigation techniques (#4) -### Bump +### 💼 Other - Version 0.2.0 → 0.3.0 @@ -226,7 +242,7 @@ All notable changes to this project will be documented in this file. - Test new version -### Bump +### 💼 Other - Version 0.1.3 → 0.1.4 @@ -240,13 +256,13 @@ All notable changes to this project will be documented in this file. - Remove unused imports - Test new version +### 💼 Other + +- Version 0.1.1 → 0.1.2 + ### 🚜 Refactor - Second commit - Remove .idea directory -### Bump - -- Version 0.1.1 → 0.1.2 - diff --git a/README.md b/README.md index 2b769b5a..f1a1a685 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Holistic AI currently focuses on five verticals of AI trustworthiness: ```bash pip install holisticai # Basic installation +pip install holisticai[datasets] # add datasets and plot dependencies pip install holisticai[bias] # Bias mitigation support pip install holisticai[explainability] # For explainability metrics and plots pip install holisticai[all] # Install all packages for bias and explainability diff --git a/docs/source/getting_started/bias/mitigation/postprocessing/rs_fair_topk_fa*ir_algorithm.rst b/docs/source/getting_started/bias/mitigation/postprocessing/rs_fair_topk_fair_algorithm.rst similarity index 100% rename from docs/source/getting_started/bias/mitigation/postprocessing/rs_fair_topk_fa*ir_algorithm.rst rename to docs/source/getting_started/bias/mitigation/postprocessing/rs_fair_topk_fair_algorithm.rst diff --git a/docs/source/getting_started/install.rst b/docs/source/getting_started/install.rst index c5bcfafb..ae99a605 100644 --- a/docs/source/getting_started/install.rst +++ b/docs/source/getting_started/install.rst @@ -9,7 +9,10 @@ You can install the library with `pip` using the following command: # Basic installation of the library pip install holisticai - + + # Additional packages for handle datasets and visualization + pip install holisticai[datasets] + # bias mitigation support pip install holisticai[bias] diff --git a/pyproject.toml b/pyproject.toml index ddcad07e..fcc1ed2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,19 +25,17 @@ classifiers = [ dependencies = [ "scikit-learn>=1.0.2", "pandas", - "matplotlib", - "networkx>=3.1", - "seaborn", "numpy>=1.25", "pybind11>=2.12", "pyarrow" ] [project.optional-dependencies] +datasets = ["networkx>=3.1", "matplotlib", "seaborn", "pybind11>=2.12", "pyarrow", "fastparquet"] bias = ["networkx>=3.1", "pygad==3.3.1", "jax>=0.4.30", "jaxopt>=0.8.3", "optax>=0.2.3", "flax>=0.8.5"] explainability = ["lime>=0.2.0.1", "shap>=0.42.1"] security = ["jax>=0.4.30", "jaxopt>=0.8.3"] -all = ["networkx>=3.1", "pygad==3.3.1", "lime>=0.2.0.1", "shap>=0.42.1", "jax>=0.4.30", "jaxopt>=0.8.3", "optax>=0.2.3", "flax>=0.8.5"] +all = ["networkx>=3.1", "pygad==3.3.1", "lime>=0.2.0.1", "shap>=0.42.1", "jax>=0.4.30", "jaxopt>=0.8.3", "optax>=0.2.3", "flax>=0.8.5", "networkx>=3.1", "matplotlib", "seaborn", "pybind11>=2.12", "pyarrow", "fastparquet"] [project.urls] Documentation = "https://github.com/holistic-ai/holisticai#readme" @@ -118,7 +116,9 @@ dependencies = [ "jax", "jaxopt", "optax", - "flax" + "flax", + "pyarrow==19.0.0", + "fastparquet" ] env-vars = { PYTHONPATH = "src" } @@ -130,4 +130,7 @@ foo = ["which python"] env-vars = { PYTHONPATH = "src" } [tool.hatch.build.targets.wheel] -packages = ["src/holisticai"] +packages = [ + "src/holisticai", +] + diff --git a/ruff_default.toml b/ruff_default.toml index 04218b60..cd476da0 100644 --- a/ruff_default.toml +++ b/ruff_default.toml @@ -168,7 +168,7 @@ select = [ "ICN001", "ICN002", "ICN003", - "INP001", + #"INP001", "INT001", "INT002", "INT003", diff --git a/src/holisticai/__about__.py b/src/holisticai/__about__.py index 39e0411d..b19b12ea 100644 --- a/src/holisticai/__about__.py +++ b/src/holisticai/__about__.py @@ -1 +1 @@ -__version__ = "1.0.9" +__version__ = "1.0.14" diff --git a/src/holisticai/__init__.py b/src/holisticai/__init__.py index e69de29b..8b137891 100644 --- a/src/holisticai/__init__.py +++ b/src/holisticai/__init__.py @@ -0,0 +1 @@ + diff --git a/src/holisticai/bias/__init__.py b/src/holisticai/bias/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/holisticai/bias/metrics/_classification.py b/src/holisticai/bias/metrics/_classification.py index e79728c2..1329ca47 100644 --- a/src/holisticai/bias/metrics/_classification.py +++ b/src/holisticai/bias/metrics/_classification.py @@ -4,6 +4,10 @@ import numpy as np import pandas as pd +# Efficacy metrics +from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score +from sklearn.neighbors import NearestNeighbors + # utils from holisticai.utils._validation import ( _array_like_to_numpy, @@ -13,10 +17,6 @@ _regression_checks, ) -# Efficacy metrics -from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score -from sklearn.neighbors import NearestNeighbors - def _group_success_rate(g, y): """Group success rate. diff --git a/src/holisticai/bias/metrics/_clustering.py b/src/holisticai/bias/metrics/_clustering.py index 9fa84d1f..c492a3ac 100644 --- a/src/holisticai/bias/metrics/_clustering.py +++ b/src/holisticai/bias/metrics/_clustering.py @@ -1,7 +1,5 @@ import numpy as np import pandas as pd -from holisticai.utils._recommender_tools import entropy -from holisticai.utils._validation import _clustering_checks # sklearn imports from sklearn.metrics import ( @@ -10,6 +8,9 @@ silhouette_samples, ) +from holisticai.utils._recommender_tools import entropy +from holisticai.utils._validation import _clustering_checks + def cluster_balance(group_a, group_b, y_pred): """Cluster Balance diff --git a/src/holisticai/bias/metrics/_recommender.py b/src/holisticai/bias/metrics/_recommender.py index f9d10a78..bbc268d8 100644 --- a/src/holisticai/bias/metrics/_recommender.py +++ b/src/holisticai/bias/metrics/_recommender.py @@ -2,6 +2,9 @@ import numpy as np import pandas as pd +# sklearn imports +from sklearn.metrics import mean_absolute_error + # utils from holisticai.utils import mat_to_binary, normalize_tensor @@ -16,9 +19,6 @@ ) from holisticai.utils._validation import _recommender_checks -# sklearn imports -from sklearn.metrics import mean_absolute_error - def aggregate_diversity(mat_pred, top=None, thresh=0.5, normalize=False): r"""Aggregate Diversity diff --git a/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_numerical_repairer.py b/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_numerical_repairer.py index b2f98cb3..9d78fb0b 100644 --- a/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_numerical_repairer.py +++ b/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_numerical_repairer.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Optional - from holisticai.bias.mitigation.commons.disparate_impact_remover._categorical_repairer import CategoricalRepairer from holisticai.bias.mitigation.commons.disparate_impact_remover._utils import ( freedman_diaconis_bin_size as bin_calculator, @@ -46,7 +44,7 @@ def __init__( feature_to_repair: int, repair_level: float, kdd: bool = False, - features_to_ignore: Optional[list[str]] = None, + features_to_ignore: list[str] | None = None, ): if features_to_ignore is None: features_to_ignore = [] diff --git a/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_utils.py b/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_utils.py index 6f6ad5ad..b42b3719 100644 --- a/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_utils.py +++ b/src/holisticai/bias/mitigation/commons/disparate_impact_remover/_utils.py @@ -3,7 +3,6 @@ import math import random from copy import deepcopy -from typing import Union import numpy as np from holisticai.bias.mitigation.commons.disparate_impact_remover._categorical_feature import CategoricalFeature @@ -32,7 +31,7 @@ def get_categories_count_norm(categories, all_stratified_groups, count_dict, gro dict The dictionary containing the normalized count for each category. """ - norm = { + return { cat: SparseList( data=( count_dict[cat][i] * (1.0 / len(group_features[group].data)) if group_features[group].data else 0.0 @@ -41,7 +40,6 @@ def get_categories_count_norm(categories, all_stratified_groups, count_dict, gro ) for cat in categories } - return norm def gen_desired_dist(group_index, cat, col_id, median, repair_level, norm_counts, feature_to_remove, mode): @@ -158,7 +156,7 @@ def get_categories_count(categories, all_stratified_groups, group_feature): dict The dictionary containing the count for each category. """ - count_dict = { + return { cat: SparseList( data=( group_feature[group].category_count[cat] if cat in group_feature[group].category_count else 0 @@ -168,8 +166,6 @@ def get_categories_count(categories, all_stratified_groups, group_feature): for cat in categories } - return count_dict - def gen_desired_count(group_index, group, category, median, group_features, repair_level, categories_count): """ @@ -200,8 +196,7 @@ def gen_desired_count(group_index, group, category, median, group_features, repa med = median[category] size = len(group_features[group].data) count = categories_count[category][group_index] - des_count = math.floor(((1 - repair_level) * count) + (repair_level) * med * size) - return des_count + return math.floor(((1 - repair_level) * count) + (repair_level) * med * size) def flow_on_group_features(all_stratified_groups, group_features, repair_generator): @@ -283,7 +278,7 @@ def get_count_norm(count, group_feature_data): return 0.0 -def get_column_type(values: Union[list, np.ndarray]): +def get_column_type(values: list | np.ndarray): """ Get the type of the column. @@ -322,7 +317,8 @@ def get_median(values, kdd): The median of the list of values. """ if not values: - raise ValueError("Cannot calculate median of list with no values!") + msg = "Cannot calculate median of list with no values!" + raise ValueError(msg) sorted_values = deepcopy(values) sorted_values.sort() # Not calling `sorted` b/c `sorted_values` may not be list. @@ -450,9 +446,7 @@ def make_histogram_bins(bin_size_calculator, data, col_id): index_bins[bin_num].append(row_index) break - index_bins = [b for b in index_bins if b] - - return index_bins + return [b for b in index_bins if b] def freedman_diaconis_bin_size(feature_values): diff --git a/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kcenters.py b/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kcenters.py index b6441d10..5b20eeed 100644 --- a/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kcenters.py +++ b/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kcenters.py @@ -84,7 +84,7 @@ def assign(self): list The mapping of points to their closest centers. """ - mapping = [ + return [ ( i, sorted( @@ -95,5 +95,3 @@ def assign(self): ) for i in range(len(self.data)) ] - - return mapping diff --git a/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kmedoids.py b/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kmedoids.py index 6afac06e..4ca3005a 100644 --- a/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kmedoids.py +++ b/src/holisticai/bias/mitigation/commons/fairlet_clustering/clustering/_kmedoids.py @@ -128,7 +128,8 @@ def _check_nonnegative_int(self, value, desc, strict=True): """ negative = value is None or value <= 0 if strict else value is None or value < 0 if negative or not isinstance(value, (int, np.integer)): - raise ValueError(f"{desc} should be a nonnegative integer. " f"{value} was given") + msg = f"{desc} should be a nonnegative integer. " f"{value} was given" + raise ValueError(msg) def _check_init_args(self): """ @@ -366,9 +367,7 @@ def _compute_inertia(self, distances): # Define inertia as the sum of the sample-distances # to closest cluster centers - inertia = np.sum(np.min(distances, axis=1)) - - return inertia + return np.sum(np.min(distances, axis=1)) def _initialize_medoids(self, D, n_clusters, random_state_): """ @@ -399,7 +398,8 @@ def _initialize_medoids(self, D, n_clusters, random_state_): # to every other point. These are the initial medoids. medoids = np.argpartition(np.sum(D, axis=1), n_clusters - 1)[:n_clusters] else: - raise ValueError(f"init value '{self.init}' not recognized") + msg = f"init value '{self.init}' not recognized" + raise ValueError(msg) return medoids diff --git a/src/holisticai/bias/mitigation/commons/fairlet_clustering/decomposition/_scalable.py b/src/holisticai/bias/mitigation/commons/fairlet_clustering/decomposition/_scalable.py index 47dd0340..ea3d457f 100644 --- a/src/holisticai/bias/mitigation/commons/fairlet_clustering/decomposition/_scalable.py +++ b/src/holisticai/bias/mitigation/commons/fairlet_clustering/decomposition/_scalable.py @@ -165,7 +165,8 @@ def decompose(self, node, dataset, donelist, depth): if sum(R) == 0 or sum(B) == 0: if sum(R) == 0 and sum(B) == 0: - raise ValueError("One color class became empty for this node while the other did not") + msg = "One color class became empty for this node while the other did not" + raise ValueError(msg) return 0 NR = 0 @@ -226,7 +227,8 @@ def decompose(self, node, dataset, donelist, depth): NB += excess_blue if self.balanced(p, q, NR, NB): - raise ValueError("Constructed node sets are unbalanced") + msg = "Constructed node sets are unbalanced" + raise ValueError(msg) reds = [] blues = [] @@ -239,7 +241,8 @@ def decompose(self, node, dataset, donelist, depth): donelist[j] = 1 if len(reds) == NR and len(blues) == NB: - raise ValueError("Something went horribly wrong") + msg = "Something went horribly wrong" + raise ValueError(msg) return super().decompose(blues, reds, dataset) + sum( [self.decompose(child, dataset, donelist, depth + 1) for child in node.children] diff --git a/src/holisticai/bias/mitigation/commons/fairlet_clustering/decompositions/_scalable.py b/src/holisticai/bias/mitigation/commons/fairlet_clustering/decompositions/_scalable.py index 15c186d1..b17699ad 100644 --- a/src/holisticai/bias/mitigation/commons/fairlet_clustering/decompositions/_scalable.py +++ b/src/holisticai/bias/mitigation/commons/fairlet_clustering/decompositions/_scalable.py @@ -222,7 +222,8 @@ def _decompose(self, node, dataset, donelist, depth): NB += excess_blue if self.balanced(p, q, NR, NB): - raise ValueError("Constructed node sets are unbalanced") + msg = "Constructed node sets are unbalanced" + raise ValueError(msg) reds = [] blues = [] @@ -235,7 +236,8 @@ def _decompose(self, node, dataset, donelist, depth): donelist[j] = 1 if len(reds) == NR and len(blues) == NB: - raise ValueError("Something went horribly wrong") + msg = "Something went horribly wrong" + raise ValueError(msg) return super()._decompose(blues, reds, dataset) + sum( [self._decompose(child, dataset, donelist, depth + 1) for child in node.children] diff --git a/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/models.py b/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/models.py index 17c5efec..7be40549 100644 --- a/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/models.py +++ b/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/models.py @@ -71,8 +71,7 @@ def adv_loss_fn(adversarial_params, classifier_params, batch, rng): _, y_logits = cls_state.apply_fn({"params": classifier_params}, x, trainable=True, rngs=rngs) _, z_logits = adv_state.apply_fn({"params": adversarial_params}, y_logits, y, trainable=True, rngs=rngs) - loss_adv = optax.sigmoid_binary_cross_entropy(z_logits, group).mean() - return loss_adv + return optax.sigmoid_binary_cross_entropy(z_logits, group).mean() (loss, (loss_cls, loss_adv)), grads = jax.value_and_grad(loss_fn, argnums=(0), has_aux=True)( cls_state.params, adv_state.params, batch, rng diff --git a/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/transformer.py b/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/transformer.py index bd400b3e..eb854888 100644 --- a/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/adversarial_debiasing/transformer.py @@ -1,12 +1,12 @@ from __future__ import annotations import logging -from typing import Optional import jax import jax.numpy as jnp import numpy as np import pandas as pd + from holisticai.bias.mitigation.inprocessing.adversarial_debiasing.models import ( ADModel, AdversarialModel, @@ -26,7 +26,8 @@ def is_numeric(df): return all(pd.api.types.is_numeric_dtype(df[col]) for col in df.columns) if isinstance(df, np.ndarray): return np.issubdtype(df.dtype, np.number) - raise ValueError("Input must be a pandas DataFrame or numpy array.") + msg = "Input must be a pandas DataFrame or numpy array." + raise ValueError(msg) class AdversarialDebiasing(BMImp): @@ -95,19 +96,19 @@ class AdversarialDebiasing(BMImp): def __init__( self, - features_dim: Optional[int] = None, - keep_prob: Optional[float] = 0.1, - hidden_size: Optional[int] = 128, - batch_size: Optional[int] = 32, - shuffle: Optional[bool] = True, - epochs: Optional[int] = 10, - learning_rate: Optional[float] = 0.01, - use_debias: Optional[bool] = True, - adversary_loss_weight: Optional[float] = 0.1, - verbose: Optional[int] = 1, - print_interval: Optional[int] = 100, - device: Optional[str] = "cpu", - seed: Optional[int] = None, + features_dim: int | None = None, + keep_prob: float | None = 0.1, + hidden_size: int | None = 128, + batch_size: int | None = 32, + shuffle: bool | None = True, + epochs: int | None = 10, + learning_rate: float | None = 0.01, + use_debias: bool | None = True, + adversary_loss_weight: float | None = 0.1, + verbose: int | None = 1, + print_interval: int | None = 100, + device: str | None = "cpu", + seed: int | None = None, ): # default classifier config self.features_dim = features_dim @@ -179,7 +180,8 @@ def fit( params = self._load_data(X=X, y=y, group_a=group_a, group_b=group_b) x = pd.DataFrame(params["X"]) if not is_numeric(x): - raise ValueError("Adversarial Debiasing only works with numeric features.") + msg = "Adversarial Debiasing only works with numeric features." + raise ValueError(msg) y = pd.Series(params["y"]) group_a = pd.Series(params["group_a"]) @@ -245,7 +247,8 @@ def predict(self, X): np.ndarray: Predicted output per sample. """ if not is_numeric(X): - raise ValueError("Adversarial Debiasing only works with numeric features.") + msg = "Adversarial Debiasing only works with numeric features." + raise ValueError(msg) p = self.predict_proba(X) return np.argmax(p, axis=1).ravel() @@ -268,7 +271,8 @@ def predict_proba(self, X): np.ndarray: Predicted matrix probability per sample. """ if not is_numeric(X): - raise ValueError("Adversarial Debiasing only works with numeric features.") + msg = "Adversarial Debiasing only works with numeric features." + raise ValueError(msg) proba = np.empty((X.shape[0], 2)) proba[:, 1] = self._predict_proba(X) @@ -294,7 +298,7 @@ def predict_score(self, X): np.ndarray: Predicted probability per sample. """ if not is_numeric(X): - raise ValueError("Adversarial Debiasing only works with numeric features.") + msg = "Adversarial Debiasing only works with numeric features." + raise ValueError(msg) - p = self._predict(X).reshape([-1]) - return p + return self._predict(X).reshape([-1]) diff --git a/src/holisticai/bias/mitigation/inprocessing/commons/_moments_utils.py b/src/holisticai/bias/mitigation/inprocessing/commons/_moments_utils.py index 4fb0eb75..cbcfd8b5 100644 --- a/src/holisticai/bias/mitigation/inprocessing/commons/_moments_utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/commons/_moments_utils.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd + from holisticai.bias.mitigation.inprocessing.commons._conventions import _EVENT, _GROUP_ID, _LABEL, _SIGNED @@ -59,12 +60,11 @@ def project_lambda(self, lambda_vec): lambda_neg = -lambda_pos lambda_pos[lambda_pos < 0.0] = 0.0 lambda_neg[lambda_neg < 0.0] = 0.0 - lambda_projected = pd.concat( + return pd.concat( [lambda_pos, lambda_neg], keys=["+", "-"], names=[_SIGNED, _EVENT, _GROUP_ID], ) - return lambda_projected return lambda_vec def bound(self): diff --git a/src/holisticai/bias/mitigation/inprocessing/commons/classification/_constraints.py b/src/holisticai/bias/mitigation/inprocessing/commons/classification/_constraints.py index 8c9b3561..5586103a 100644 --- a/src/holisticai/bias/mitigation/inprocessing/commons/classification/_constraints.py +++ b/src/holisticai/bias/mitigation/inprocessing/commons/classification/_constraints.py @@ -1,9 +1,8 @@ from __future__ import annotations -from typing import Optional - import numpy as np import pandas as pd + from holisticai.bias.mitigation.inprocessing.commons._conventions import ( _ALL, _EVENT, @@ -43,7 +42,7 @@ def load_data( y, sensitive_features: pd.Series, event: pd.Series, - utilities: Optional[np.ndarray] = None, + utilities: np.ndarray | None = None, ): """ Description @@ -135,8 +134,7 @@ def get_signed_weight(row): signed_weights = self.tags.apply(get_signed_weight, axis=1) utility_diff = self.utilities[:, 1] - self.utilities[:, 0] - signed_weights = utility_diff.T * signed_weights - return signed_weights + return utility_diff.T * signed_weights def default_objective(self): return ErrorRate() @@ -147,8 +145,7 @@ def gamma(self, predictor): utility_diff = self.utilities[:, 1] - self.utilities[:, 0] predictions = np.squeeze(predictor(self.X)) pred = utility_diff.T * predictions + self.utilities[:, 0] - g_signed = -self.U.T.dot(pred) / self.total_samples - return g_signed # self._gamma_signed(pred) + return -self.U.T.dot(pred) / self.total_samples def _gamma_signed(self, pred): self.tags[_PRED] = pred @@ -157,7 +154,7 @@ def _gamma_signed(self, pred): expect_group_event = tags.groupby(level=[_EVENT, _GROUP_ID]).mean() expect_group_event[_UPPER_BOUND_DIFF] = self.ratio * expect_group_event[_PRED] - expect_event[_PRED] expect_group_event[_LOWER_BOUND_DIFF] = -expect_group_event[_PRED] + self.ratio * expect_event[_PRED] - gamma_signed = pd.concat( + return pd.concat( [ expect_group_event[_UPPER_BOUND_DIFF], expect_group_event[_LOWER_BOUND_DIFF], @@ -165,7 +162,6 @@ def _gamma_signed(self, pred): keys=["+", "-"], names=["signed", _EVENT, _GROUP_ID], ) - return gamma_signed def _get_basis(self): pos_basis = pd.DataFrame() @@ -185,7 +181,7 @@ def _get_basis(self): self.basis = {"+": pos_basis, "-": neg_basis} def _get_index_format(self): - index = ( + return ( pd.DataFrame( [ {_SIGNED: signed, _EVENT: e, _GROUP_ID: g} @@ -197,7 +193,6 @@ def _get_index_format(self): .set_index([_SIGNED, _EVENT, _GROUP_ID]) .index ) - return index class DemographicParity(ClassificationConstraint): diff --git a/src/holisticai/bias/mitigation/inprocessing/commons/classification/_objectives.py b/src/holisticai/bias/mitigation/inprocessing/commons/classification/_objectives.py index 8193e773..f6d2107f 100644 --- a/src/holisticai/bias/mitigation/inprocessing/commons/classification/_objectives.py +++ b/src/holisticai/bias/mitigation/inprocessing/commons/classification/_objectives.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd + from holisticai.bias.mitigation.inprocessing.commons._conventions import _ALL, _LABEL from holisticai.bias.mitigation.inprocessing.commons._moments_utils import BaseMoment @@ -27,5 +28,4 @@ def gamma(self, predictor): if isinstance(pred, np.ndarray): pred = np.squeeze(pred) - error = pd.Series(data=(self.tags[_LABEL] - pred).abs().mean(), index=self.index) - return error + return pd.Series(data=(self.tags[_LABEL] - pred).abs().mean(), index=self.index) diff --git a/src/holisticai/bias/mitigation/inprocessing/commons/regression/_constraints.py b/src/holisticai/bias/mitigation/inprocessing/commons/regression/_constraints.py index 024eca81..7fd26e8d 100644 --- a/src/holisticai/bias/mitigation/inprocessing/commons/regression/_constraints.py +++ b/src/holisticai/bias/mitigation/inprocessing/commons/regression/_constraints.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd + from holisticai.bias.mitigation.inprocessing.commons._conventions import ( _ALL, _EVENT, @@ -97,7 +98,8 @@ def bound(self): A vector of bounds on group-level losses """ if self.upper_bound is None: - raise ValueError("No Upper Bound") + msg = "No Upper Bound" + raise ValueError(msg) return pd.Series(self.upper_bound, index=self.index) def project_lambda(self, lambda_vec): diff --git a/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/_lagrangian.py b/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/_lagrangian.py index 8d6a622a..f1083132 100644 --- a/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/_lagrangian.py +++ b/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/_lagrangian.py @@ -1,9 +1,10 @@ import numpy as np import pandas as pd import scipy.optimize as opt -from holisticai.bias.mitigation.inprocessing.commons._conventions import PRECISION from sklearn import clone +from holisticai.bias.mitigation.inprocessing.commons._conventions import PRECISION + class Lagrangian: """Operations related to the Lagrangian""" diff --git a/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/algorithm.py index 02a9dcf7..0db12d44 100644 --- a/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/algorithm.py @@ -1,10 +1,11 @@ from __future__ import annotations import sys -from typing import Optional import numpy as np import pandas as pd +from sklearn.utils import check_random_state + from holisticai.bias.mitigation.inprocessing.commons._conventions import ( ACCURACY_MUL, MIN_ITER, @@ -15,7 +16,6 @@ SHRINK_REGRET, ) from holisticai.bias.mitigation.inprocessing.exponentiated_gradient._lagrangian import Lagrangian -from sklearn.utils import check_random_state class ExponentiatedGradientAlgorithm: @@ -31,12 +31,12 @@ def __init__( self, estimator, constraints, - eps: Optional[float] = 0.01, - max_iter: Optional[int] = 50, - nu: Optional[float] = None, - eta0: Optional[float] = 2.0, - verbose: Optional[int] = 0, - seed: Optional[int] = None, + eps: float | None = 0.01, + max_iter: int | None = 50, + nu: float | None = None, + eta0: float | None = 2.0, + verbose: int | None = 0, + seed: int | None = None, ): self.estimator = estimator self.constraints = constraints @@ -55,8 +55,7 @@ def fit(self, X, y, **kwargs): def compute_default_nu(h): absolute_error = (h(X) - self.constraints.y_as_series).abs() - nu = ACCURACY_MUL * absolute_error.std() / np.sqrt(self.constraints.total_samples) - return nu + return ACCURACY_MUL * absolute_error.std() / np.sqrt(self.constraints.total_samples) eta = self.eta0 / B gap_LP = np.inf diff --git a/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/transformer.py b/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/transformer.py index 14fbc4b2..f4402d15 100644 --- a/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/exponentiated_gradient/transformer.py @@ -1,14 +1,15 @@ from __future__ import annotations -from typing import Literal, Optional +from typing import Literal import numpy as np +from sklearn.base import BaseEstimator, ClassifierMixin, clone + from holisticai.bias.mitigation.inprocessing.commons.classification import _constraints as cc from holisticai.bias.mitigation.inprocessing.commons.regression import _constraints as rc from holisticai.bias.mitigation.inprocessing.commons.regression import _losses as rl from holisticai.bias.mitigation.inprocessing.exponentiated_gradient.algorithm import ExponentiatedGradientAlgorithm from holisticai.utils.transformers.bias import BMInprocessing as BMImp -from sklearn.base import BaseEstimator, ClassifierMixin, clone class ExponentiatedGradientReduction(BaseEstimator, ClassifierMixin, BMImp): @@ -93,15 +94,15 @@ class ExponentiatedGradientReduction(BaseEstimator, ClassifierMixin, BMImp): def __init__( self, constraints: str = "EqualizedOdds", - eps: Optional[float] = 0.01, - max_iter: Optional[int] = 50, - nu: Optional[float] = None, - eta0: Optional[float] = 2.0, + eps: float | None = 0.01, + max_iter: int | None = 50, + nu: float | None = None, + eta0: float | None = 2.0, loss: str = "ZeroOne", - min_val: Optional[float] = None, - max_val: Optional[float] = None, + min_val: float | None = None, + max_val: float | None = None, upper_bound: float = 0.01, - verbose: Optional[int] = 0, + verbose: int | None = 0, estimator=None, seed: int = 0, ): diff --git a/src/holisticai/bias/mitigation/inprocessing/fair_k_center_clustering/transformer.py b/src/holisticai/bias/mitigation/inprocessing/fair_k_center_clustering/transformer.py index f0067d09..4244ae82 100644 --- a/src/holisticai/bias/mitigation/inprocessing/fair_k_center_clustering/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/fair_k_center_clustering/transformer.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import Optional - import numpy as np +from sklearn.base import BaseEstimator +from sklearn.metrics.pairwise import pairwise_distances, pairwise_distances_argmin + from holisticai.bias.mitigation.inprocessing.fair_k_center_clustering.algorithms import ( fair_k_center_approx, heuristic_greedy_on_each_group, @@ -10,8 +11,6 @@ ) from holisticai.utils.transformers.bias import BMInprocessing as BMImp from holisticai.utils.transformers.bias import SensitiveGroups -from sklearn.base import BaseEstimator -from sklearn.metrics.pairwise import pairwise_distances, pairwise_distances_argmin STRATEGIES_CATALOG = { "Fair K-Center": fair_k_center_approx, @@ -63,10 +62,10 @@ class FairKCenterClustering(BaseEstimator, BMImp): def __init__( self, - req_nr_per_group: Optional[list] = None, - nr_initially_given: Optional[int] = 100, - strategy: Optional[str] = "Fair K-Center", - seed: Optional[int] = None, + req_nr_per_group: list | None = None, + nr_initially_given: int | None = 100, + strategy: str | None = "Fair K-Center", + seed: int | None = None, ): if req_nr_per_group is None: req_nr_per_group = [200, 200] diff --git a/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/algorithm.py index d609bf56..14302bb1 100644 --- a/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/algorithm.py @@ -30,13 +30,11 @@ def _compute_cost(self, centers): for gid in self.group_ids: group_cost = np.mean(np.amin(self.distances[np.ix_(self.p_attr == gid, centers)], axis=1)) group_costs.append(group_cost) - cost = np.max(group_costs) - return cost + return np.max(group_costs) def _compute_new_assigment(self, centers): centers = np.array(centers, dtype=np.int32) - assignment = centers[np.argmin(self.distances[np.ix_(np.arange(self.n), centers)], axis=1)] - return assignment + return centers[np.argmin(self.distances[np.ix_(np.arange(self.n), centers)], axis=1)] def fit(self, X, p_attr): self.n = len(X) diff --git a/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/transformer.py b/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/transformer.py index b28e85ac..26cdfbd5 100644 --- a/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/fair_k_mediam_clustering/transformer.py @@ -1,12 +1,11 @@ from __future__ import annotations -from typing import Optional - import numpy as np +from sklearn.base import BaseEstimator + from holisticai.bias.mitigation.inprocessing.fair_k_mediam_clustering.algorithm import KMediamClusteringAlgorithm from holisticai.utils.transformers.bias import BMInprocessing as BMImp from holisticai.utils.transformers.bias import SensitiveGroups -from sklearn.base import BaseEstimator class FairKMedianClustering(BaseEstimator, BMImp): @@ -54,9 +53,9 @@ def __init__( self, n_clusters: int = 2, max_iter: int = 1000, - seed: Optional[int] = None, - strategy: Optional[str] = "LS", - verbose: Optional[int] = 0, + seed: int | None = None, + strategy: str | None = "LS", + verbose: int | None = 0, ): self.n_clusters = n_clusters self.max_iter = max_iter diff --git a/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/algorithm.py index e64b519a..e70a9ca9 100644 --- a/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/algorithm.py @@ -1,9 +1,9 @@ from __future__ import annotations import logging -from typing import Optional import numpy as np + from holisticai.bias.mitigation.inprocessing.fair_scoring_classifier.utils import get_class_count, get_class_indexes logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def __init__( objectives: str, fairness_groups: list, fairness_labels: list, - constraints: Optional[dict] = None, + constraints: dict | None = None, lambda_bound: int = 9, time_limit: int = 100, verbose: int = 0, diff --git a/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/transformer.py b/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/transformer.py index 19722e88..48b95886 100644 --- a/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/transformer.py @@ -1,13 +1,14 @@ from __future__ import annotations -from typing import Literal, Optional +from typing import Literal import numpy as np import pandas as pd +from sklearn.base import BaseEstimator + from holisticai.bias.mitigation.inprocessing.fair_scoring_classifier.algorithm import FairScoreClassifierAlgorithm from holisticai.utils.transformers.bias import BMInprocessing as BMImp from holisticai.utils.transformers.bias import SensitiveGroups -from sklearn.base import BaseEstimator class FairScoreClassifier(BaseEstimator, BMImp): @@ -48,7 +49,7 @@ class FairScoreClassifier(BaseEstimator, BMImp): def __init__( self, objectives: Literal["a", "ab"], - constraints: Optional[dict] = None, + constraints: dict | None = None, lambda_bound: int = 9, time_limit: int = 100, verbose: int = 0, diff --git a/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/utils.py b/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/utils.py index 5e89ce0c..458be911 100644 --- a/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/fair_scoring_classifier/utils.py @@ -17,8 +17,7 @@ def get_indexes_from_names(df, names): ------- list : list of index """ - indexes = [df.columns.get_loc(item) for item in names] - return indexes + return [df.columns.get_loc(item) for item in names] def process_y(y): @@ -36,8 +35,7 @@ def process_y(y): ------- y_processed (list of labels) """ - y_processed = [i for line in y for i, val in enumerate(line) if val == 1] - return y_processed + return [i for line in y for i, val in enumerate(line) if val == 1] def get_majority_class(y): @@ -210,8 +208,7 @@ def predict(x, l_lists): def format_labels(y): - y_formatted = [i for labels in y for i in range(len(labels)) if labels[i] == 1] - return y_formatted + return [i for labels in y for i in range(len(labels)) if labels[i] == 1] def get_accuracy(x, y, l_lists): @@ -238,15 +235,11 @@ def get_accuracy(x, y, l_lists): y_pred = predict(x, l_lists) y = format_labels(y) y_pred = format_labels(y_pred) - accuracy = accuracy_score(y, y_pred) - - return accuracy + return accuracy_score(y, y_pred) def get_balanced_accuracy(x, y, l_lists): y_pred = predict(x, l_lists) y = format_labels(y) y_pred = format_labels(y_pred) - balanced_accuracy = balanced_accuracy_score(y, y_pred) - - return balanced_accuracy + return balanced_accuracy_score(y, y_pred) diff --git a/src/holisticai/bias/mitigation/inprocessing/fairlet_clustering/transformer.py b/src/holisticai/bias/mitigation/inprocessing/fairlet_clustering/transformer.py index 9c8aba40..989df16e 100644 --- a/src/holisticai/bias/mitigation/inprocessing/fairlet_clustering/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/fairlet_clustering/transformer.py @@ -1,8 +1,8 @@ from __future__ import annotations -from typing import Optional, Union - import numpy as np +from sklearn.base import BaseEstimator + from holisticai.bias.mitigation.commons.fairlet_clustering.decompositions import ( DecompositionMixin, ScalableFairletDecomposition, @@ -13,7 +13,6 @@ ) from holisticai.utils.models.cluster import KCenters, KMedoids from holisticai.utils.transformers.bias import BMInprocessing as BMImp -from sklearn.base import BaseEstimator DECOMPOSITION_CATALOG = { "Scalable": ScalableFairletDecomposition, @@ -65,12 +64,12 @@ class FairletClustering(BaseEstimator, BMImp): def __init__( self, - n_clusters: Optional[int], - decomposition: Union[str, DecompositionMixin] = "Vanilla", - clustering_model: Optional[str] = "KCenters", - p: Optional[str] = 1, - q: Optional[float] = 3, - seed: Optional[int] = None, + n_clusters: int | None, + decomposition: str | DecompositionMixin = "Vanilla", + clustering_model: str | None = "KCenters", + p: str | None = 1, + q: float | None = 3, + seed: int | None = None, ): if decomposition in ["Scalable", "Vanilla"]: self.decomposition = DECOMPOSITION_CATALOG[decomposition](p=p, q=q) diff --git a/src/holisticai/bias/mitigation/inprocessing/grid_search/_grid_generator.py b/src/holisticai/bias/mitigation/inprocessing/grid_search/_grid_generator.py index ce64fb55..ebb3b404 100644 --- a/src/holisticai/bias/mitigation/inprocessing/grid_search/_grid_generator.py +++ b/src/holisticai/bias/mitigation/inprocessing/grid_search/_grid_generator.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from typing import Optional import numpy as np import pandas as pd @@ -18,7 +17,7 @@ def __init__( self, grid_size: int = 5, grid_limit: float = 2.0, - neg_allowed: Optional[np.ndarray] = None, + neg_allowed: np.ndarray | None = None, force_L1_norm: bool = True, ): """ @@ -73,8 +72,7 @@ def _generate_coefs(self): neg_coefs = -pos_coefs.copy() pos_coefs[pos_coefs < 0] = 0.0 neg_coefs[neg_coefs < 0] = 0.0 - lambda_vector = {"+": pos_coefs, "-": neg_coefs} - return lambda_vector + return {"+": pos_coefs, "-": neg_coefs} def _build_grid(self, n_units): """ @@ -93,8 +91,7 @@ def _build_grid(self, n_units): f"Warning: The desired grid size was not reached. {len(self.accumulator)} points were generated." ) - xs = np.array(self.accumulator) * self.grid_limit / n_units - return xs + return np.array(self.accumulator) * self.grid_limit / n_units def _accumulate_integer_grid(self, index, max_val): """ diff --git a/src/holisticai/bias/mitigation/inprocessing/grid_search/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/grid_search/algorithm.py index be31c4a8..d34e1ada 100644 --- a/src/holisticai/bias/mitigation/inprocessing/grid_search/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/grid_search/algorithm.py @@ -2,9 +2,10 @@ import logging from typing import Any -from holisticai.bias.mitigation.inprocessing.grid_search._grid_generator import GridGenerator from joblib import Parallel, delayed +from holisticai.bias.mitigation.inprocessing.grid_search._grid_generator import GridGenerator + logger = logging.getLogger(__name__) @@ -101,7 +102,8 @@ def fit(self, X: Any, y: Any, sensitive_features: Any): ) ) if not results: - raise ValueError("No results were generated. This is likely due to an issue with the grid.") + msg = "No results were generated. This is likely due to an issue with the grid." + raise ValueError(msg) def loss_fct(result): return (1 - self.constraint_weight) * result["objective"] + self.constraint_weight * result["gamma"].max() @@ -164,8 +166,7 @@ def _generate_grid(self) -> Any: # Generar la grid de coeficientes grid = generator.generate_grid(self.constraint) - grid = grid.loc[:, ~(grid == 0).all(axis=0)] - return grid + return grid.loc[:, ~(grid == 0).all(axis=0)] def _fit_estimator(self, X: Any, lambda_vec: Any): weights = self._compute_weights(lambda_vec) diff --git a/src/holisticai/bias/mitigation/inprocessing/grid_search/transformer.py b/src/holisticai/bias/mitigation/inprocessing/grid_search/transformer.py index f3331ddc..5036ed30 100644 --- a/src/holisticai/bias/mitigation/inprocessing/grid_search/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/grid_search/transformer.py @@ -3,12 +3,13 @@ from typing import Literal import numpy as np +from sklearn.base import BaseEstimator, clone + from holisticai.bias.mitigation.inprocessing.commons.classification import _constraints as cc from holisticai.bias.mitigation.inprocessing.commons.regression import _constraints as rc from holisticai.bias.mitigation.inprocessing.commons.regression import _losses as rl from holisticai.bias.mitigation.inprocessing.grid_search.algorithm import GridSearchAlgorithm from holisticai.utils.transformers.bias import BMInprocessing as BMImp -from sklearn.base import BaseEstimator, clone class GridSearchReduction(BMImp, BaseEstimator): diff --git a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/blind_spot_aware.py b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/blind_spot_aware.py index 9b754506..ad042bf9 100644 --- a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/blind_spot_aware.py +++ b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/blind_spot_aware.py @@ -1,9 +1,9 @@ from __future__ import annotations import logging -from typing import Optional import numpy as np + from holisticai.utils.models.recommender._rsbase import RecommenderSystemBase from holisticai.utils.transformers.bias import BMInprocessing as BMImp @@ -54,11 +54,11 @@ class BlindSpotAwareMF(BMImp, RecommenderSystemBase): def __init__( self, K: int = 10, - beta: Optional[float] = 0.002, - steps: Optional[int] = 200, - alpha: Optional[float] = 0.002, - lamda: Optional[float] = 0.2, - verbose: Optional[int] = 0, + beta: float | None = 0.002, + steps: int | None = 200, + alpha: float | None = 0.002, + lamda: float | None = 0.2, + verbose: int | None = 0, ): self.beta = beta self.steps = steps diff --git a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/propensity_utils.py b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/propensity_utils.py index 4c7ef8a5..e02453b1 100644 --- a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/propensity_utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/propensity_utils.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.inprocessing.matrix_factorization.common_utils.utils import calculate_popularity_model @@ -6,8 +7,7 @@ def constant_propensity(rmat, numItems): numObservations = np.ma.count(rmat) numUsers, numItems = np.shape(rmat) scale = numUsers * numItems - inversePropensities = np.ones((numUsers, numItems), dtype=np.longdouble) * scale / numObservations - return inversePropensities + return np.ones((numUsers, numItems), dtype=np.longdouble) * scale / numObservations def popularity_model_propensity(rmat): diff --git a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/utils.py b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/utils.py index 9c2b2573..d8bc0025 100644 --- a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/common_utils/utils.py @@ -8,8 +8,7 @@ def calculate_popularity_model(ratings): propensity_score = [float(np.count_nonzero(ratings[:, i])) / ratings.shape[0] for i in range(ratings.shape[1])] temp = np.array(propensity_score) - temp = temp.reshape((1, len(temp))) - return temp + return temp.reshape((1, len(temp))) def erros(P_s, Q_s, R_s, W_s, lamda, beta): diff --git a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm.py index 502ea87b..b34d81c2 100644 --- a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm.py @@ -1,5 +1,6 @@ import numpy as np import scipy + from holisticai.bias.mitigation.inprocessing.matrix_factorization.debiasing_learning.algorithm_utils import ( ParameterSerializer, Problem, @@ -33,10 +34,9 @@ def train(self, train_observations, inv_propensities, start_vector=None): if self.clip_val >= 0: tempInvPropensities = np.clip(tempInvPropensities, a_min=0, a_max=self.clip_val) - parameters = self.optimize_debiasing_parameters( + return self.optimize_debiasing_parameters( train_observations, tempInvPropensities, self.lamda, start_vec=start_vector ) - return parameters def optimize_debiasing_parameters(self, observed_ratings, inverse_propensities, l2_regularization, start_vec=None): metricMode = get_metric_mode(self.metric) diff --git a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm_utils.py b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm_utils.py index b45fa437..7abff910 100644 --- a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/algorithm_utils.py @@ -16,7 +16,7 @@ def process_propensities(observed_ratings, inverse_propensities): else: inversePropensities = np.array(inverse_propensities, dtype=np.longdouble, copy=True) - inversePropensities = np.ma.array( + return np.ma.array( inversePropensities, dtype=np.longdouble, copy=False, @@ -24,14 +24,12 @@ def process_propensities(observed_ratings, inverse_propensities): fill_value=0, hard_mask=True, ) - return inversePropensities def predict_scores(user_vectors, item_vectors, user_biases, item_biases, global_bias, use_bias=True): rawScores = np.dot(user_vectors, item_vectors.T) if use_bias: - biasedScores = rawScores + user_biases[:, None] + item_biases[None, :] + global_bias - return biasedScores + return rawScores + user_biases[:, None] + item_biases[None, :] + global_bias return rawScores diff --git a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/transformer.py b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/transformer.py index 1a1847fc..2d6bc5f0 100644 --- a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/debiasing_learning/transformer.py @@ -1,9 +1,8 @@ from __future__ import annotations -from typing import Optional - import numpy as np import scipy + from holisticai.bias.mitigation.inprocessing.matrix_factorization.debiasing_learning.algorithm import ( DebiasingLearningAlgorithm, ) @@ -66,14 +65,14 @@ class DebiasingLearningMF(BMImp, RecommenderSystemBase): def __init__( self, - K: Optional[int] = 10, - normalization: Optional[str] = "Vanilla", - lamda: Optional[float] = 0.02, - metric: Optional[str] = "mse", - bias_mode: Optional[str] = None, - clip_val: Optional[float] = -1, - seed: Optional[int] = None, - verbose: Optional[int] = 0, + K: int | None = 10, + normalization: str | None = "Vanilla", + lamda: float | None = 0.02, + metric: str | None = "mse", + bias_mode: str | None = None, + clip_val: float | None = -1, + seed: int | None = None, + verbose: int | None = 0, ): self.seed = seed self.normalization = normalization @@ -94,7 +93,7 @@ def __init__( ) # self.params = [lamda, K, clipVal, bias_mode] - def fit(self, X: Optional[np.ndarray], propensities: Optional[np.ndarray] = None): + def fit(self, X: np.ndarray | None, propensities: np.ndarray | None = None): """ Fit model @@ -144,5 +143,4 @@ def _init_parameters(self, partial_observations): Q = np.transpose(np.multiply(vt, s[:, None])) userBias = np.zeros(numUsers) itemBias = np.zeros(numItems) - params0 = (P, Q, userBias, itemBias, averageObservedRating) - return params0 + return (P, Q, userBias, itemBias, averageObservedRating) diff --git a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/popularity_propensity.py b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/popularity_propensity.py index fb9a238e..fa037485 100644 --- a/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/popularity_propensity.py +++ b/src/holisticai/bias/mitigation/inprocessing/matrix_factorization/popularity_propensity.py @@ -1,9 +1,9 @@ from __future__ import annotations import logging -from typing import Optional import numpy as np + from holisticai.bias.mitigation.inprocessing.matrix_factorization.common_utils.propensity_utils import ( popularity_model_propensity, ) @@ -48,17 +48,17 @@ class PopularityPropensityMF(BMImp, RecommenderSystemBase): def __init__( self, - K: Optional[int] = 10, - beta: Optional[float] = 0.02, - steps: Optional[int] = 100, - verbose: Optional[int] = 0, + K: int | None = 10, + beta: float | None = 0.02, + steps: int | None = 100, + verbose: int | None = 0, ): self.K = K self.beta = beta self.steps = steps self.verbose = verbose - def fit(self, X: Optional[np.ndarray], **kargs): + def fit(self, X: np.ndarray | None, **kargs): """ Fit model diff --git a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm.py index 7b7d3de0..97e12ad3 100644 --- a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm.py @@ -1,10 +1,11 @@ from functools import partial import numpy as np -from holisticai.utils.transformers.bias import SensitiveGroups from scipy.stats import multivariate_normal from sklearn.metrics import accuracy_score +from holisticai.utils.transformers.bias import SensitiveGroups + def prob(dist, x): return dist.pdf(x) @@ -169,5 +170,4 @@ def predict_proba(self, X: np.ndarray): probability output per sample. """ t = self.predictor(X) - scores = ((t + 1) / 2).reshape((-1, 1)) - return scores + return ((t + 1) / 2).reshape((-1, 1)) diff --git a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm_utils.py b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm_utils.py index 551f4174..ae6a54cd 100644 --- a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/algorithm_utils.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.inprocessing.commons import Logging diff --git a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/constraints.py b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/constraints.py index b29fe356..4dad5b15 100644 --- a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/constraints.py +++ b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/constraints.py @@ -30,8 +30,7 @@ def forward(self, P, params, a=None, b=None, return_cs=False): # noqa: ARG002 def _gradient(self, a, b, t, c, l): _t = t * c / np.sqrt(t**2 + self.mu**2) exp = np.mean(_t) - dl = exp - b + (b - a) / 2 + (b - a) * l / (2 * np.sqrt(l**2 + self.mu**2)) - return dl + return exp - b + (b - a) / 2 + (b - a) * l / (2 * np.sqrt(l**2 + self.mu**2)) def expected_gradient(self, P, params, a, b): l_1, l_2 = params diff --git a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/transformer.py b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/transformer.py index eea79d7c..36f5b845 100644 --- a/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/meta_fair_classifier/transformer.py @@ -1,6 +1,7 @@ from __future__ import annotations import numpy as np + from holisticai.bias.mitigation.inprocessing.meta_fair_classifier.algorithm import MetaFairClassifierAlgorithm from holisticai.bias.mitigation.inprocessing.meta_fair_classifier.algorithm_utils import MFLogger from holisticai.bias.mitigation.inprocessing.meta_fair_classifier.constraints import FalseDiscovery, StatisticalRate diff --git a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm.py index cef4b23b..ae8df42e 100644 --- a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm.py @@ -1,7 +1,8 @@ import numpy as np -from holisticai.utils.transformers.bias import SensitiveGroups from scipy.optimize import fmin_cg +from holisticai.utils.transformers.bias import SensitiveGroups + class PrejudiceRemoverAlgorithm: """Two class LogisticRegression with Prejudice Remover""" diff --git a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm_utils.py b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm_utils.py index 19da19df..2a829b61 100644 --- a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/algorithm_utils.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.inprocessing.commons import Logging @@ -34,8 +35,7 @@ def loss(self, coef_, X, y, groups): coef = coef_.reshape(self.estimator.nb_group_values, self.estimator.nb_features).astype(np.float64) X = self.estimator.preprocessing_data(X) sigma = self.estimator.sigmoid(X=X, groups=groups, coef=coef) - loss = self.loss_fn(y=y, sigma=sigma, groups=groups, coef=coef) - return loss + return self.loss_fn(y=y, sigma=sigma, groups=groups, coef=coef) def grad_loss(self, coef_, X, y, groups): """first derivative of loss function diff --git a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/losses.py b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/losses.py index f05fafed..511bce0c 100644 --- a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/losses.py +++ b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/losses.py @@ -13,8 +13,7 @@ def __call__(self, y, sigma, groups, coef): f = self.freg(sigma, groups) # l2 regularizer reg = np.sum(coef * coef) - loss = -L + self.eta * f + 0.5 * self.C * reg - return loss + return -L + self.eta * f + 0.5 * self.C * reg def gradient(self, X, y, sigma, groups, coef): L = self.xent.gradient(X, y, sigma, groups) @@ -22,8 +21,7 @@ def gradient(self, X, y, sigma, groups, coef): # l2 regularizer reg = coef # sum - loss = np.reshape(-L + self.eta * f + self.C * reg, [-1]) - return loss + return np.reshape(-L + self.eta * f + self.C * reg, [-1]) class BinaryFairnessRegularizer: @@ -81,8 +79,7 @@ def gradient(self, X, sigma, groups): f3 = (sigma - pi) / (pi * (1.0 - pi)) f4 = (f1[:, np.newaxis] * d_sigma + f2[:, np.newaxis] * d_rho_s) - np.outer(f3, d_pi) - f = op_by_group(f4, groups, reduce_op=np.sum, axis=0) - return f + return op_by_group(f4, groups, reduce_op=np.sum, axis=0) class BinaryCrossEntropy: @@ -95,8 +92,7 @@ def gradient(self, X, y, sigma, groups): # likelihood # l(si) = \sum_{x,y in D st s=si} (y - sigma(x, si)) x loss = (y - sigma)[:, np.newaxis] * X - L = op_by_group(loss, groups, reduce_op=np.sum, axis=0) - return L + return op_by_group(loss, groups, reduce_op=np.sum, axis=0) def op_by_group(matrix, groups, reduce_op, axis=None): diff --git a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/transformer.py b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/transformer.py index 7fc63688..1ea0f55a 100644 --- a/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/prejudice_remover/transformer.py @@ -1,14 +1,13 @@ from __future__ import annotations -from typing import Optional - import numpy as np +from sklearn.base import BaseEstimator, ClassifierMixin + from holisticai.bias.mitigation.inprocessing.prejudice_remover.algorithm import PrejudiceRemoverAlgorithm from holisticai.bias.mitigation.inprocessing.prejudice_remover.algorithm_utils import ObjectiveFunction, PRLogger from holisticai.bias.mitigation.inprocessing.prejudice_remover.losses import PRBinaryCrossEntropy from holisticai.bias.mitigation.inprocessing.prejudice_remover.model import PRLogiticRegression, PRParamInitializer from holisticai.utils.transformers.bias import BMInprocessing as BMImp -from sklearn.base import BaseEstimator, ClassifierMixin class PrejudiceRemover(BaseEstimator, ClassifierMixin, BMImp): @@ -63,14 +62,14 @@ class PrejudiceRemover(BaseEstimator, ClassifierMixin, BMImp): def __init__( self, - eta: Optional[float] = 1.0, - C: Optional[float] = 1.0, - fit_intercept: Optional[bool] = True, - penalty: Optional[str] = "l2", - init_type: Optional[str] = "Zero", - maxiter: Optional[int] = 1000, - verbose: Optional[int] = 0, - print_interval: Optional[int] = 20, + eta: float | None = 1.0, + C: float | None = 1.0, + fit_intercept: bool | None = True, + penalty: str | None = "l2", + init_type: str | None = "Zero", + maxiter: int | None = 1000, + verbose: int | None = 0, + print_interval: int | None = 20, ): # Default estimator parameters self.eta = eta diff --git a/src/holisticai/bias/mitigation/inprocessing/two_sided_fairness/transformer.py b/src/holisticai/bias/mitigation/inprocessing/two_sided_fairness/transformer.py index 62e951f5..149a30e3 100644 --- a/src/holisticai/bias/mitigation/inprocessing/two_sided_fairness/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/two_sided_fairness/transformer.py @@ -1,9 +1,8 @@ from __future__ import annotations -from typing import Optional - import numpy as np import pandas as pd + from holisticai.bias.mitigation.inprocessing.two_sided_fairness.algorithm import FairRecAlg from holisticai.utils.transformers.bias import BMInprocessing as BMImp @@ -33,7 +32,7 @@ class FairRec(BMImp): recommendations in two-sided platforms." Proceedings of The Web Conference 2020. 2020. """ - def __init__(self, rec_size: Optional[int] = 10, MMS_fraction: Optional[float] = 0.5): + def __init__(self, rec_size: int | None = 10, MMS_fraction: float | None = 0.5): self.rec_size = rec_size self.MMS_fraction = MMS_fraction @@ -54,7 +53,7 @@ def fit(self, X): self.recommendation = algorithm.rank(X) return self - def predict(self, X: Optional[np.ndarray], top_n: Optional[int] = None): + def predict(self, X: np.ndarray | None, top_n: int | None = None): """ Fit model diff --git a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm.py b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm.py index aafdcb4c..e042dee4 100644 --- a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm.py +++ b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd + from holisticai.bias.mitigation.inprocessing.variational_fair_clustering.algorithm_utils._fair_clustering import ( FairClustering, ) diff --git a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_bound_update.py b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_bound_update.py index 505e1de7..8f39ca17 100644 --- a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_bound_update.py +++ b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_bound_update.py @@ -6,20 +6,16 @@ def normalize(S_in): S_in = S_in - maxcol S_out = np.exp(S_in) S_out_sum = np.sum(S_out, axis=1, keepdims=True) - S_out = S_out / S_out_sum - - return S_out + return S_out / S_out_sum def normalize_2(S_in): S_in_sum = np.sum(S_in, axis=1, keepdims=True) - S_in = S_in / S_in_sum - return S_in + return S_in / S_in_sum def bound_energy(S, S_in, a_term, b_term): - E = np.nansum(S * np.log(np.maximum(S, 1e-15)) - S * np.log(np.maximum(S_in, 1e-15)) + a_term * S + b_term * S) - return E + return np.nansum(S * np.log(np.maximum(S, 1e-15)) - S * np.log(np.maximum(S_in, 1e-15)) + a_term * S + b_term * S) def get_S_discrete(L, N, K): # noqa: N802 diff --git a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_fair_clustering.py b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_fair_clustering.py index 0363cc2f..767a1918 100644 --- a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_fair_clustering.py +++ b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_fair_clustering.py @@ -3,6 +3,8 @@ from collections import defaultdict import numpy as np +from sklearn.cluster import KMeans + from holisticai.bias.mitigation.inprocessing.variational_fair_clustering.algorithm_utils._bound_update import ( BoundUpdate, normalize_2, @@ -22,7 +24,6 @@ from holisticai.bias.mitigation.inprocessing.variational_fair_clustering.algorithm_utils._utils import ( get_fair_accuracy_proportional, ) -from sklearn.cluster import KMeans logger = logging.getLogger() diff --git a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_method_utils.py b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_method_utils.py index 9c86f9c2..0cc18b28 100644 --- a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_method_utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_method_utils.py @@ -13,8 +13,7 @@ def check_version(version_str): def reduce_func(D_chunk, start): # noqa: ARG001 - J = np.mean(D_chunk, axis=1) - return J + return np.mean(D_chunk, axis=1) def NormalizedCutEnergy(A, S, clustering): # noqa: N802 @@ -38,9 +37,7 @@ def NormalizedCutEnergy(A, S, clustering): # noqa: N802 elif isinstance(A, sparse.csc_matrix): nassoc_e = nassoc_e + np.dot(np.transpose(S_k), A.dot(S_k)) / np.dot(np.transpose(d), S_k) nassoc_e = nassoc_e[0, 0] - ncut_e = num_cluster - nassoc_e - - return ncut_e + return num_cluster - nassoc_e def NormalizedCutEnergy_discrete(A, clustering): # noqa: N802 @@ -64,9 +61,7 @@ def NormalizedCutEnergy_discrete(A, clustering): # noqa: N802 elif isinstance(A, sparse.csc_matrix): nassoc_e = nassoc_e + np.dot(np.transpose(S_k), A.dot(S_k)) / np.dot(np.transpose(d), S_k) nassoc_e = nassoc_e[0, 0] - ncut_e = num_cluster - nassoc_e - - return ncut_e + return num_cluster - nassoc_e def KernelBound_k(A, d, S_k, N): # noqa: N802 @@ -86,9 +81,7 @@ def km_le(X, M): """ e_dist = ecdist(X, M) - L = e_dist.argmin(axis=1) - - return L + return e_dist.argmin(axis=1) # Fairness term calculation @@ -96,8 +89,7 @@ def fairness_term_V_j(u_j, S, V_j): # noqa: N802 V_j = V_j.astype("float") S_term = np.maximum(np.dot(V_j, S), 1e-20) S_sum = np.maximum(S.sum(0), 1e-20) - S_term = u_j * (np.log(S_sum) - np.log(S_term)) - return S_term + return u_j * (np.log(S_sum) - np.log(S_term)) def km_discrete_energy(e_dist, L, k): @@ -110,8 +102,7 @@ def compute_fairness_energy(group_prob, groups_ids, S, bound_lambda): compute fair clustering energy """ fairness_E = [fairness_term_V_j(group_prob.loc[g], S, groups_ids[f"{g}"]) for g in group_prob.index] - fairness_E = (bound_lambda * sum(fairness_E)).sum() - return fairness_E + return (bound_lambda * sum(fairness_E)).sum() def _is_arraylike(x): diff --git a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_methods.py b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_methods.py index 4c4acea1..5c615a52 100644 --- a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_methods.py +++ b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_methods.py @@ -1,4 +1,6 @@ import numpy as np +from sklearn.metrics import pairwise_distances_chunked as pdist_chunk + from holisticai.bias.mitigation.inprocessing.variational_fair_clustering.algorithm_utils._bound_update import ( get_S_discrete, ) @@ -12,7 +14,6 @@ reduce_func, ) from holisticai.bias.mitigation.inprocessing.variational_fair_clustering.algorithm_utils._utils import create_affinity -from sklearn.metrics import pairwise_distances_chunked as pdist_chunk class KUtils: @@ -39,8 +40,7 @@ def compute_clustering_energy(self, C, L, S): class KmeansUtils(KUtils): def compute_a_p(self, L, C): # noqa: ARG002 - a_p = ecdist(self.X, C, squared=True) - return a_p + return ecdist(self.X, C, squared=True) def step(self, L, C): tmp_list = [np.where(k == L)[0] for k in range(self.K)] @@ -53,15 +53,12 @@ def step(self, L, C): def kmeans_update(self, tmp): """ """ X_tmp = self.X[tmp, :] - c1 = X_tmp.mean(axis=0) - - return c1 + return X_tmp.mean(axis=0) class KmedianUtils(KUtils): def compute_a_p(self, L, C): # noqa: ARG002 - a_p = ecdist(self.X, C) - return a_p + return ecdist(self.X, C) def step(self, L, C): tmp_list = [np.where(k == L)[0] for k in range(self.K)] @@ -77,8 +74,7 @@ def kmedian_update(self, tmp): D = pdist_chunk(X_tmp, reduce_func=reduce_func) J = next(D) j = np.argmin(J) - c1 = X_tmp[j, :] - return c1 + return X_tmp[j, :] class NCutUtils: @@ -101,8 +97,7 @@ def step(self, L, C): # noqa: ARG002 S = get_S_discrete(L, self.N, self.K) sqdist_list = [KernelBound_k(self.A, self.d, S[:, k], self.N) for k in range(self.K)] sqdist = np.asarray(np.vstack(sqdist_list).T) - a_p = sqdist.copy() - return a_p + return sqdist.copy() def update(self, a_p, L, C): # noqa: ARG002 L = a_p.argmin(axis=1) diff --git a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_utils.py b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_utils.py index 3a69b149..0e7076ac 100644 --- a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_utils.py +++ b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/algorithm_utils/_utils.py @@ -26,8 +26,7 @@ def normalizefea(X): L2 normalize """ feanorm = np.maximum(1e-14, np.sum(X**2, axis=1)) - X_out = X / (feanorm[:, None] ** 0.5) - return X_out + return X / (feanorm[:, None] ** 0.5) def get_V_jl(x, L, N, K): # noqa: N802 @@ -35,8 +34,7 @@ def get_V_jl(x, L, N, K): # noqa: N802 temp = np.zeros((N, K)) index_cluster = L[x] temp[(x, index_cluster)] = 1 - temp = temp.sum(0) - return temp + return temp.sum(0) def get_fair_accuracy(group_prob, groups_ids, L, K): diff --git a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/transformer.py b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/transformer.py index 5ef2c93f..d5a09a8e 100644 --- a/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/transformer.py +++ b/src/holisticai/bias/mitigation/inprocessing/variational_fair_clustering/transformer.py @@ -1,12 +1,11 @@ from __future__ import annotations -from typing import Optional - import numpy as np +from sklearn.base import BaseEstimator + from holisticai.bias.mitigation.inprocessing.variational_fair_clustering.algorithm import FairClusteringAlgorithm from holisticai.utils.transformers.bias import BMInprocessing as BMImp from holisticai.utils.transformers.bias import SensitiveGroups -from sklearn.base import BaseEstimator class VariationalFairClustering(BaseEstimator, BMImp): @@ -53,13 +52,13 @@ class VariationalFairClustering(BaseEstimator, BMImp): def __init__( self, - n_clusters: Optional[int], - lipchitz_value: Optional[str] = 1, - lmbda: Optional[float] = 0.7, - method: Optional[str] = "kmeans", - normalize_input: Optional[bool] = True, - seed: Optional[int] = None, - verbose: Optional[int] = 0, + n_clusters: int | None, + lipchitz_value: str | None = 1, + lmbda: float | None = 0.7, + method: str | None = "kmeans", + normalize_input: bool | None = True, + seed: int | None = None, + verbose: int | None = 0, ): # Constant parameters self.algorithm = FairClusteringAlgorithm( diff --git a/src/holisticai/bias/mitigation/postprocessing/calibrated_eq_odds_postprocessing.py b/src/holisticai/bias/mitigation/postprocessing/calibrated_eq_odds_postprocessing.py index 0c9fa290..104bc63b 100644 --- a/src/holisticai/bias/mitigation/postprocessing/calibrated_eq_odds_postprocessing.py +++ b/src/holisticai/bias/mitigation/postprocessing/calibrated_eq_odds_postprocessing.py @@ -3,6 +3,7 @@ from typing import Literal import numpy as np + from holisticai.utils.transformers.bias import BMPostprocessing as BMPost diff --git a/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm.py index d87a652a..9caf870b 100644 --- a/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm.py @@ -1,6 +1,7 @@ import logging import numpy as np + from holisticai.bias.mitigation.postprocessing.debiasing_exposure.algorithm_utils import ( exposure_diff, find_items_per_group_per_query, diff --git a/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm_utils.py b/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm_utils.py index f83dd4b3..a6d34d8a 100644 --- a/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/algorithm_utils.py @@ -9,8 +9,7 @@ def filter_invalid_examples(rankings, query_col, group_col): for _, ranking in rankings.groupby(query_col): if (ranking[group_col].sum() > 0).any(): new_rankings.append(ranking) - new_rankings = pd.concat(new_rankings, axis=0).reset_index(drop=True) - return new_rankings + return pd.concat(new_rankings, axis=0).reset_index(drop=True) def exposure_metric(rankings, group_col: str, query_col: str, score_col: str): @@ -56,9 +55,7 @@ def exposure_diff(data_per_query, prot_idx_per_query): exposure_prot = normalized_exposure(protected_items_per_query, judgments_per_query) exposure_nprot = normalized_exposure(nonprotected_items_per_query, judgments_per_query) - exposure_diff = np.maximum(0, (exposure_nprot - exposure_prot)) - - return exposure_diff + return np.maximum(0, (exposure_nprot - exposure_prot)) def exposure_ratio(data_per_query, prot_idx_per_query): @@ -87,15 +84,14 @@ def exposure_ratio(data_per_query, prot_idx_per_query): exposure_prot = normalized_exposure(protected_items_per_query, judgments_per_query) exposure_nprot = normalized_exposure(nonprotected_items_per_query, judgments_per_query) - exposure_diff = exposure_nprot / exposure_prot - - return exposure_diff + return exposure_nprot / exposure_prot def find_items_per_group_per_query(data, protected_feature): data_per_query = np.array(data).astype(np.float32) if np.any(np.isnan(data_per_query)): - raise ValueError("data has NaN values!!, fix your data or normalize it before training.") + msg = "data has NaN values!!, fix your data or normalize it before training." + raise ValueError(msg) if len(data_per_query.shape) == 1: data_per_query = data_per_query[:, None] protected_feature = np.array(protected_feature) @@ -149,8 +145,7 @@ def normalized_topp_prot_deriv_per_group(group_features, all_features, group_pre numpy array of float values. """ derivative = topp_prot_first_derivative(group_features, all_features, group_predictions, all_predictions) - result = (np.sum(derivative / np.log(2), axis=0)) / group_predictions.size - return result + return (np.sum(derivative / np.log(2), axis=0)) / group_predictions.size def topp_prot(group_items, all_items): diff --git a/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/transformer.py b/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/transformer.py index c77108c2..87bb394b 100644 --- a/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/transformer.py +++ b/src/holisticai/bias/mitigation/postprocessing/debiasing_exposure/transformer.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd + from holisticai.bias.mitigation.postprocessing.debiasing_exposure.algorithm import DELTRAlgorithm from holisticai.bias.mitigation.postprocessing.debiasing_exposure.algorithm_utils import Standarizer @@ -74,9 +75,11 @@ def __init__( # check if mandatory parameters are present if group_col is None: - raise ValueError("The name of column in data `group_col` must be initialized") + msg = "The name of column in data `group_col` must be initialized" + raise ValueError(msg) if gamma is None: - raise ValueError("The `gamma` parameter must be initialized") + msg = "The `gamma` parameter must be initialized" + raise ValueError(msg) # initialize the protected_feature index to -1 @@ -110,8 +113,7 @@ def _filter_invalid_examples(self, rankings): for _, ranking in rankings.groupby(self.query_col): if (ranking[self.group_col].sum() > 0).any(): new_rankings.append(ranking) - new_rankings = pd.concat(new_rankings, axis=0).reset_index(drop=True) - return new_rankings + return pd.concat(new_rankings, axis=0).reset_index(drop=True) def fit(self, rankings: pd.DataFrame): """ @@ -164,7 +166,8 @@ def transform(self, rankings: pd.DataFrame): """ if self._omega is None: - raise SystemError("You need to train a model first!") + msg = "You need to train a model first!" + raise SystemError(msg) # prepare data query_ids, doc_ids, protected_attributes, feature_matrix = self._prepare_data(rankings, has_judgment=False) @@ -187,9 +190,7 @@ def transform(self, rankings: pd.DataFrame): ) # sort by the score in descending order - result = result.sort_values([self.score_col], ascending=[0]) - - return result + return result.sort_values([self.score_col], ascending=[0]) def _prepare_data(self, data, has_judgment=False): """ diff --git a/src/holisticai/bias/mitigation/postprocessing/disparate_impact_remover_rs.py b/src/holisticai/bias/mitigation/postprocessing/disparate_impact_remover_rs.py index 03d01270..90bbdabc 100644 --- a/src/holisticai/bias/mitigation/postprocessing/disparate_impact_remover_rs.py +++ b/src/holisticai/bias/mitigation/postprocessing/disparate_impact_remover_rs.py @@ -61,15 +61,15 @@ def __init__( def _assert_parameters(self, repair_level): if not 0.0 <= repair_level <= 1.0: - raise ValueError("'repair_level' must be between 0.0 and 1.0.") + msg = "'repair_level' must be between 0.0 and 1.0." + raise ValueError(msg) def _filter_invalid_examples(self, rankings): new_rankings = [] for _, ranking in rankings.groupby(self.query_col): if (ranking[self.group_col].sum() > 0).any(): new_rankings.append(ranking) - new_rankings = pd.concat(new_rankings, axis=0).reset_index(drop=True) - return new_rankings + return pd.concat(new_rankings, axis=0).reset_index(drop=True) def fit_transform(self, rankings: pd.DataFrame): """ @@ -142,9 +142,8 @@ def transform_features(self, ranking: pd.DataFrame): new_ranking = ranking.copy() new_ranking[self.score_col] = new_data_matrix new_ranking = new_ranking.reset_index(drop=True) - new_ranking = ( + return ( new_ranking.groupby(self.query_col) .apply(lambda df: df.sort_values(self.score_col, ascending=False)) .reset_index(drop=True) ) - return new_ranking diff --git a/src/holisticai/bias/mitigation/postprocessing/eq_odds_postprocessing.py b/src/holisticai/bias/mitigation/postprocessing/eq_odds_postprocessing.py index dbee5068..2c563495 100644 --- a/src/holisticai/bias/mitigation/postprocessing/eq_odds_postprocessing.py +++ b/src/holisticai/bias/mitigation/postprocessing/eq_odds_postprocessing.py @@ -3,10 +3,11 @@ from typing import Literal import numpy as np -from holisticai.utils.transformers.bias import BMPostprocessing as BMPost from scipy.optimize import linprog from sklearn.metrics import confusion_matrix +from holisticai.utils.transformers.bias import BMPostprocessing as BMPost + class EqualizedOdds(BMPost): """ diff --git a/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/fail_prob.py b/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/fail_prob.py index ec4402ac..9e4b1708 100644 --- a/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/fail_prob.py +++ b/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/fail_prob.py @@ -1,6 +1,7 @@ -from holisticai.bias.mitigation.postprocessing.fair_topk.algorithm_utils import mtable_generator from scipy.stats import binom +from holisticai.bias.mitigation.postprocessing.fair_topk.algorithm_utils import mtable_generator + EPS = 0.0000000000000001 diff --git a/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/mtable_generator.py b/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/mtable_generator.py index 713bec34..485606c1 100644 --- a/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/mtable_generator.py +++ b/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/mtable_generator.py @@ -22,10 +22,12 @@ def mtable_as_dataframe(self): def m(self, k): if k < 1: - raise ValueError("Parameter k must be at least 1") + msg = "Parameter k must be at least 1" + raise ValueError(msg) if k > self.k: - raise ValueError(f"Parameter k must be at most {self.k}") + msg = f"Parameter k must be at most {self.k}" + raise ValueError(msg) result = stats.binom.ppf(self.alpha, k, self.p) return 0 if result < 0 else result @@ -47,7 +49,8 @@ def compute_aux_mtable(mtable): Stores the inverse of an mTable entry and the size of the block with respect to the inverse """ if not (isinstance(mtable, pd.DataFrame)): - raise TypeError("Internal mtable must be a DataFrame") + msg = "Internal mtable must be a DataFrame" + raise TypeError(msg) aux_mtable = pd.DataFrame(columns=["inv", "block"]) last_m_seen = 0 @@ -60,6 +63,7 @@ def compute_aux_mtable(mtable): aux_mtable.loc[position] = [position, position - last_position] last_position = position elif mtable.at[position, "m"] != last_m_seen: - raise RuntimeError("Inconsistent mtable") + msg = "Inconsistent mtable" + raise RuntimeError(msg) return aux_mtable diff --git a/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/valitation_utils.py b/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/valitation_utils.py index c3daf375..38cd3652 100644 --- a/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/valitation_utils.py +++ b/src/holisticai/bias/mitigation/postprocessing/fair_topk/algorithm_utils/valitation_utils.py @@ -26,7 +26,8 @@ def check_ranking(protected, mtable): """ # if the mtable has a different number elements than there are in the top docs return false if len(protected) != len(mtable): - raise ValueError("Number of documents in ranking and mtable length must be equal!") + msg = "Number of documents in ranking and mtable length must be equal!" + raise ValueError(msg) # check number of protected element at each rank return not (protected.cumsum().values < np.array(mtable)).any() @@ -51,14 +52,14 @@ def validate_basic_parameters(k, p, alpha): """ if k < 10 or k > 400: if k < 2: - raise ValueError("Total number of elements `k` should be between 10 and 400") + msg = "Total number of elements `k` should be between 10 and 400" + raise ValueError(msg) logger.warning("Library has not been tested with values outside this range") if p < 0.02 or p > 0.98: if p < 0 or p > 1: - raise ValueError( - "The proportion of protected candidates `p` in the top-k ranking should be between " "0.02 and 0.98" - ) + msg = "The proportion of protected candidates `p` in the top-k ranking should be between " "0.02 and 0.98" + raise ValueError(msg) logger.warning("Library has not been tested with values outside this range") validate_alpha(alpha) @@ -67,5 +68,6 @@ def validate_basic_parameters(k, p, alpha): def validate_alpha(alpha): if alpha < 0.01 or alpha > 0.15: if alpha < 0.001 or alpha > 0.5: - raise ValueError("The significance level `alpha` must be between 0.01 and 0.15") + msg = "The significance level `alpha` must be between 0.01 and 0.15" + raise ValueError(msg) logger.warning("Library has not been tested with values outside this range") diff --git a/src/holisticai/bias/mitigation/postprocessing/fair_topk/transformer.py b/src/holisticai/bias/mitigation/postprocessing/fair_topk/transformer.py index 163683a1..ac188a0b 100644 --- a/src/holisticai/bias/mitigation/postprocessing/fair_topk/transformer.py +++ b/src/holisticai/bias/mitigation/postprocessing/fair_topk/transformer.py @@ -1,8 +1,7 @@ from __future__ import annotations -from typing import Optional - import pandas as pd + from holisticai.bias.mitigation.postprocessing.fair_topk.algorithm_utils.fail_prob import ( RecursiveNumericFailProbabilityCalculator, ) @@ -57,13 +56,13 @@ class FairTopK(BMPost): def __init__( self, - top_n: Optional[int], - p: Optional[float], - alpha: Optional[float], - query_col: Optional[str] = "query_id", - doc_col: Optional[str] = "doc_id", - group_col: Optional[str] = "group_id", - score_col: Optional[str] = "score", + top_n: int | None, + p: float | None, + alpha: float | None, + query_col: str | None = "query_id", + doc_col: str | None = "doc_id", + group_col: str | None = "group_id", + score_col: str | None = "score", ): # check the parameters first validate_basic_parameters(top_n, p, alpha) @@ -97,7 +96,8 @@ def transform(self, rankings, p_attr=None): """ if p_attr is None: if self.group_col not in rankings.columns: - raise ValueError("protected groups must be provided") + msg = "protected groups must be provided" + raise ValueError(msg) new_rankings = rankings else: if self.group_col in rankings.columns: diff --git a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm.py index 72a722de..dad0e6e4 100644 --- a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm.py @@ -84,5 +84,4 @@ def predict(self, p_attr, y_pred=None, y_proba=None, binom=False): y_pred[group_ids[g]] = tools.threshold(probs[group_ids[g]], cut) # Returning the adjusted predictions - adj = tools.pred_from_pya(y_pred, p_attr, self.pya, binom) - return adj + return tools.pred_from_pya(y_pred, p_attr, self.pya, binom) diff --git a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm_utils.py b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm_utils.py index 31550c8f..5acc9fc3 100644 --- a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/algorithm_utils.py @@ -19,8 +19,7 @@ def __init__(self, y_true, y_pred, round=4): # noqa: A002 def from_top(roc_point, round=4): # noqa: A002, ARG001 - d = np.sqrt(roc_point[0] ** 2 + (roc_point[1] - 1) ** 2) - return d + return np.sqrt(roc_point[0] ** 2 + (roc_point[1] - 1) ** 2) def loss_from_roc(y, probs, roc): diff --git a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/constraints.py b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/constraints.py index acb409ff..8f050b1c 100644 --- a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/constraints.py +++ b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/constraints.py @@ -11,8 +11,7 @@ def __call__(self, dr, overall_rates): e = 1 - s # Setting up the coefficients for the objective function - obj_coefs = np.array([[(s - e) * r[0], (e - s) * r[1]] for r in dr]).flatten() - return obj_coefs + return np.array([[(s - e) * r[0], (e - s) * r[1]] for r in dr]).flatten() class ConstraintBase: diff --git a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/transformer.py b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/transformer.py index 76aac3e5..60c9719c 100644 --- a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/transformer.py +++ b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/binary_balancer/transformer.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, Optional +from typing import Literal import numpy as np from holisticai.bias.mitigation.postprocessing.lp_debiaser.binary_balancer.algorithm import BinaryBalancerAlgorithm @@ -35,7 +35,7 @@ class LPDebiaserBinary(BMPost): def __init__( self, - constraint: Optional[CONSTRAINT] = "EqualizedOdds", + constraint: CONSTRAINT | None = "EqualizedOdds", ): self.constraint = constraint self._sensgroups = SensitiveGroups() @@ -45,8 +45,8 @@ def fit( y: np.ndarray, group_a: np.ndarray, group_b: np.ndarray, - y_pred: Optional[np.ndarray] = None, - y_proba: Optional[np.ndarray] = None, + y_pred: np.ndarray | None = None, + y_proba: np.ndarray | None = None, ): """ Compute parameters for Linear Programming Debiaser.\ @@ -105,8 +105,8 @@ def transform( self, group_a: np.ndarray, group_b: np.ndarray, - y_pred: Optional[np.ndarray] = None, - y_proba: Optional[np.ndarray] = None, + y_pred: np.ndarray | None = None, + y_proba: np.ndarray | None = None, ): """ Apply transform function to predictions and likelihoods @@ -148,8 +148,8 @@ def fit_transform( y: np.ndarray, group_a: np.ndarray, group_b: np.ndarray, - y_pred: Optional[np.ndarray] = None, - y_proba: Optional[np.ndarray] = None, + y_pred: np.ndarray | None = None, + y_proba: np.ndarray | None = None, ): """ Fit and transform diff --git a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/algorithm_utils.py b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/algorithm_utils.py index 349228f4..cc8f9712 100644 --- a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/algorithm_utils.py @@ -25,5 +25,4 @@ def p_vec(y, flatten=True): def pars_to_cpmat(opt, n_groups=3, n_classes=3): """Reshapes the LP parameters as an n_group * n_class * n_class array""" shaped = np.reshape(opt.x, (n_groups, n_classes, n_classes)) - flipped = np.array([m.T for m in shaped]) - return flipped + return np.array([m.T for m in shaped]) diff --git a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/constraints.py b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/constraints.py index 45bb532b..b07f670c 100644 --- a/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/constraints.py +++ b/src/holisticai/bias/mitigation/postprocessing/lp_debiaser/multiclass_balancer/constraints.py @@ -7,8 +7,7 @@ class MacroLosses: def __call__(self, cp_mats): off_loss = [[np.delete(a, i, 0).sum(0) for i in range(self.n_classes)] for a in cp_mats] - obj = np.array(off_loss).flatten() - return obj + return np.array(off_loss).flatten() class MicroLosses: @@ -16,8 +15,7 @@ def __call__(self, cp_mats): u = np.array([[np.delete(a, i, 0) for i in range(self.n_classes)] for a in cp_mats]) p = np.array([[np.delete(a, i).reshape(-1, 1) for i in range(self.n_classes)] for a in self.p_vecs]) w = np.array([[p[i, j] * u[i, j] for j in range(self.n_classes)] for i in range(self.n_groups)]) - obj = w.sum(2).flatten() - return obj + return w.sum(2).flatten() class ConstraintBase: diff --git a/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/algorithm.py index aeb2ae0b..be24cf34 100644 --- a/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/algorithm.py @@ -1,10 +1,11 @@ import logging import numpy as np -from holisticai.utils.transformers.bias import SensitiveGroups from scipy.optimize import linprog from scipy.sparse import lil_matrix +from holisticai.utils.transformers.bias import SensitiveGroups + logger = logging.getLogger(__name__) @@ -46,14 +47,12 @@ def penalty_weights(self, X=None, centroids: np.ndarray = None): if self.metric == "L1": norm_p = 1 d = np.linalg.norm(centroids[:, None, :] - X[None, ...], ord=norm_p, axis=-1) - w = (-d).reshape(-1) - return w + return (-d).reshape(-1) if self.metric == "L2": norm_p = 2 d = np.linalg.norm(centroids[:, None, :] - X[None, ...], ord=norm_p, axis=-1) - w = (-d).reshape(-1) - return w + return (-d).reshape(-1) message = f"Penalty Weights not implemented : {self.metric}" raise NotImplementedError(message) @@ -74,7 +73,8 @@ def compute_cost_function(self, centroids, X, z_pred, z_mod): w = np.mean(d, axis=0) - np.sum(d * z_mod, axis=0) return np.sum(w * (1 - np.sum(z_mod * z_pred, axis=0))) - raise NotImplementedError(f"Cost Function not implemented : {self.metric}") + msg = f"Cost Function not implemented : {self.metric}" + raise NotImplementedError(msg) def transform( self, diff --git a/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/utils/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/utils/algorithm.py index fe8a53b5..ffb3ed2c 100644 --- a/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/utils/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/mcmf_clustering/utils/algorithm.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.postprocessing.mcmf_clustering.utils.algorithm_utils import Utils diff --git a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm.py index 201a4a00..683a274b 100644 --- a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.postprocessing.ml_debiaser.randomized_threshold.algorithm_utils import ( FullGDDebiaser, RTLogger, @@ -97,7 +98,8 @@ def fit(self, y_score, groups_num): self.rho = self.avrg_y_score / 2.0 + 0.5 if self.rho <= 0: - raise ValueError("rho must be either None or a strictly positive number.") + msg = "rho must be either None or a strictly positive number." + raise ValueError(msg) num_groups = len(set(groups_num)) eps0 = self.eps / 2.0 diff --git a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm_utils.py b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm_utils.py index 57f35bb2..75f6238b 100644 --- a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/randomized_threshold/algorithm_utils.py @@ -1,6 +1,7 @@ import math import numpy as np + from holisticai.bias.mitigation.inprocessing.commons import Logging @@ -28,8 +29,7 @@ def compute_gradients(self, y_batch): lambda_gradient = self.eps0 + self.rho - mean_xi mu_gradient = self.eps0 - self.rho + mean_xi - gradients = {"lambda": lambda_gradient, "mu": mu_gradient} - return gradients + return {"lambda": lambda_gradient, "mu": mu_gradient} def update_parameters(self, gradients): """ diff --git a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/reduce2binary/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/reduce2binary/algorithm.py index 93a1650d..b6b03bdd 100644 --- a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/reduce2binary/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/reduce2binary/algorithm.py @@ -1,6 +1,7 @@ import copy import numpy as np + from holisticai.bias.mitigation.postprocessing.ml_debiaser.randomized_threshold.algorithm import ( RandomizedThresholdAlgorithm, ) @@ -60,13 +61,16 @@ def __init__( If True, display progress. """ if num_classes < 2: - raise ValueError("Number of classes (must be >= 2).") + msg = "Number of classes (must be >= 2)." + raise ValueError(msg) if eps < 0: - raise ValueError("eps must be non-negative.") + msg = "eps must be non-negative." + raise ValueError(msg) if gamma <= 0: - raise ValueError("gamma must be a strictly positive number.") + msg = "gamma must be a strictly positive number." + raise ValueError(msg) self.num_groups = 1 self.gamma = gamma @@ -127,10 +131,11 @@ def predict(self, y_prob, p_attr): """ if len(y_prob.shape) != 2: - raise ValueError( + msg = ( "Original prob scores must be a 2-dimensional array." "Use RandomizedThreshold for binary classification." ) + raise ValueError(msg) y_prob_scores = copy.deepcopy(y_prob) @@ -160,5 +165,4 @@ def predict(self, y_prob, p_attr): self.logger.update(iteration=iteration + 1, primal_residual=r, dual_residual=s) z_mat = np.maximum(z_mat, 0) - z_mat = z_mat / np.sum(z_mat, axis=1, keepdims=True) - return z_mat + return z_mat / np.sum(z_mat, axis=1, keepdims=True) diff --git a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/transformer.py b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/transformer.py index 8541d442..d0f079ab 100644 --- a/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/transformer.py +++ b/src/holisticai/bias/mitigation/postprocessing/ml_debiaser/transformer.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.postprocessing.ml_debiaser.randomized_threshold.algorithm import ( RandomizedThresholdAlgorithm, ) diff --git a/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm.py index 34af531d..432918a6 100644 --- a/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.postprocessing.plugin_estimator_and_recalibration.algorithm_utils import f_lambda from holisticai.utils.transformers.bias import SensitiveGroups @@ -92,5 +93,4 @@ def transform(self, y_pred: np.ndarray, sensitive_features: np.ndarray): ) min_indices = np.argmin(minimizing_values, axis=1) output_predictions = index_range[min_indices] * self.multiplier / self.length - output_predictions = (output_predictions + 1) / 2 - return output_predictions + return (output_predictions + 1) / 2 diff --git a/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm_utils.py b/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm_utils.py index 97abdc9e..9e473918 100644 --- a/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm_utils.py +++ b/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/algorithm_utils.py @@ -27,9 +27,7 @@ def tot_fn(H, S, i): RISs = tmp1 * ((2 * S[None, :] - 1) * lambda_[i_values, None] + H_i - beta * np.log((2 * L + 1) * tmp1)) RIS = np.sum(RISs, axis=0) - ris = np.mean(RIS[S == 0]) + np.mean(RIS[S == 1]) - - return ris + return np.mean(RIS[S == 0]) + np.mean(RIS[S == 1]) def f_lambda(Y, S, M, L, beta): @@ -39,6 +37,4 @@ def f_lambda(Y, S, M, L, beta): lambda_ = 0.99 * np.ones(shape=(2 * L + 1,)) fun = partial(f_mincon, Y, S, M, L, beta) - lambda_ = minimize(fun, lambda_, bounds=[(0, 4 * M)] * len(lambda_)).x # , method='L-BFGS-B').x - - return lambda_ + return minimize(fun, lambda_, bounds=[(0, 4 * M)] * len(lambda_)).x # , method='L-BFGS-B').x diff --git a/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/transformer.py b/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/transformer.py index c1c98e31..33155b45 100644 --- a/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/transformer.py +++ b/src/holisticai/bias/mitigation/postprocessing/plugin_estimator_and_recalibration/transformer.py @@ -1,6 +1,7 @@ from __future__ import annotations import numpy as np + from holisticai.bias.mitigation.postprocessing.plugin_estimator_and_recalibration.algorithm import ( PluginEstimationAndCalibrationAlgorithm, ) diff --git a/src/holisticai/bias/mitigation/postprocessing/reject_option_classification.py b/src/holisticai/bias/mitigation/postprocessing/reject_option_classification.py index cbc72999..67458efb 100644 --- a/src/holisticai/bias/mitigation/postprocessing/reject_option_classification.py +++ b/src/holisticai/bias/mitigation/postprocessing/reject_option_classification.py @@ -4,11 +4,12 @@ import sys from typing import Literal -import holisticai.bias.metrics as bias_metrics import numpy as np -from holisticai.utils.transformers.bias import BMPostprocessing as BMPost from sklearn.metrics import balanced_accuracy_score +import holisticai.bias.metrics as bias_metrics +from holisticai.utils.transformers.bias import BMPostprocessing as BMPost + def statistical_parity(group_a, group_b, y_pred, _): return bias_metrics.statistical_parity(group_a, group_b, y_pred) diff --git a/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/algorithm.py b/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/algorithm.py index fb05f63a..3a2b3756 100644 --- a/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/algorithm.py +++ b/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/algorithm.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.utils.transformers.bias import SensitiveGroups @@ -42,8 +43,7 @@ def _optimize_ts(self, yt, i1, i2, n1, n2): t_values = np.linspace(self.minY, self.maxY, 100) tmp1 = np.sum(yl_masked1[:, None] < t_values, axis=0) / n1 dist = np.abs(tmp1 - tmp2) - ts = t_values[np.argmin(dist)] - return ts + return t_values[np.argmin(dist)] def _update_yt(self, yt, group): if group == self.group_values[self.im]: @@ -59,5 +59,4 @@ def transform(self, y_pred: np.ndarray, sensitive_groups: np.ndarray): ST = self._sensgroups.transform(sensitive_groups, convert_numeric=True) noise = self.eps * np.random.randn(len(y_pred)).squeeze() YT = y_pred + noise - YF = np.array([self._update_yt(yt, st) for yt, st in zip(YT, ST)]) - return YF + return np.array([self._update_yt(yt, st) for yt, st in zip(YT, ST)]) diff --git a/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/transformer.py b/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/transformer.py index 574084ba..fd6e1c9f 100644 --- a/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/transformer.py +++ b/src/holisticai/bias/mitigation/postprocessing/wasserstein_barycenters/transformer.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.postprocessing.wasserstein_barycenters.algorithm import WassersteinBarycenterAlgorithm from holisticai.utils.transformers.bias import BMPostprocessing as BMPost diff --git a/src/holisticai/bias/mitigation/preprocessing/correlation_remover.py b/src/holisticai/bias/mitigation/preprocessing/correlation_remover.py index 419e2d6c..7f2ad03f 100644 --- a/src/holisticai/bias/mitigation/preprocessing/correlation_remover.py +++ b/src/holisticai/bias/mitigation/preprocessing/correlation_remover.py @@ -1,13 +1,13 @@ import jax.numpy as jnp -from holisticai.utils.transformers.bias import BMPreprocessing as BMPre from jax import jit +from holisticai.utils.transformers.bias import BMPreprocessing as BMPre + @jit def _fit(X, sensitive_features, sensitive_mean): sensitive_features_center = sensitive_features - sensitive_mean - beta = jnp.linalg.lstsq(sensitive_features_center, X, rcond=None)[0] - return beta + return jnp.linalg.lstsq(sensitive_features_center, X, rcond=None)[0] @jit @@ -97,8 +97,7 @@ def transform(self, X: jnp.ndarray, group_a: jnp.ndarray, group_b: jnp.ndarray): group_b = params["group_b"] sensitive_features = jnp.stack([group_a, group_b], axis=1).astype(jnp.int32) - x_filtered = _transform(x, sensitive_features, self.beta_, self.alpha, self.sensitive_mean_) - return x_filtered + return _transform(x, sensitive_features, self.beta_, self.alpha, self.sensitive_mean_) def fit_transform(self, X: jnp.ndarray, group_a: jnp.ndarray, group_b: jnp.ndarray): """ diff --git a/src/holisticai/bias/mitigation/preprocessing/disparate_impact_remover.py b/src/holisticai/bias/mitigation/preprocessing/disparate_impact_remover.py index 066f0837..65b662ce 100644 --- a/src/holisticai/bias/mitigation/preprocessing/disparate_impact_remover.py +++ b/src/holisticai/bias/mitigation/preprocessing/disparate_impact_remover.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.bias.mitigation.commons.disparate_impact_remover._numerical_repairer import ( NumericalRepairer, ) @@ -50,7 +51,8 @@ def _assert_parameters(self, repair_level): If repair_level is not between 0.0 and 1.0 """ if not 0.0 <= repair_level <= 1.0: - raise ValueError("'repair_level' must be between 0.0 and 1.0.") + msg = "'repair_level' must be between 0.0 and 1.0." + raise ValueError(msg) def repair_data(self, X, group_a, group_b): """ diff --git a/src/holisticai/bias/mitigation/preprocessing/fairlet_clustering/transformer.py b/src/holisticai/bias/mitigation/preprocessing/fairlet_clustering/transformer.py index 2e50d91c..387ac482 100644 --- a/src/holisticai/bias/mitigation/preprocessing/fairlet_clustering/transformer.py +++ b/src/holisticai/bias/mitigation/preprocessing/fairlet_clustering/transformer.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import Optional, Union - import numpy as np +from sklearn.base import BaseEstimator +from sklearn.metrics.pairwise import pairwise_distances_argmin + from holisticai.bias.mitigation.commons.fairlet_clustering.decompositions import ( DecompositionMixin, ScalableFairletDecomposition, @@ -10,8 +11,6 @@ ) from holisticai.utils.models.cluster import KCenters, KMedoids from holisticai.utils.transformers.bias import BMPreprocessing as BMPre -from sklearn.base import BaseEstimator -from sklearn.metrics.pairwise import pairwise_distances_argmin DECOMPOSITION_CATALOG = { "Scalable": ScalableFairletDecomposition, @@ -57,10 +56,10 @@ class FairletClusteringPreprocessing(BaseEstimator, BMPre): def __init__( self, - decomposition: Union[str, DecompositionMixin] = "Vanilla", - p: Optional[str] = 1, - q: Optional[float] = 3, - seed: Optional[int] = None, + decomposition: str | DecompositionMixin = "Vanilla", + p: str | None = 1, + q: float | None = 3, + seed: int | None = None, ): self.decomposition = DECOMPOSITION_CATALOG[decomposition](p=p, q=q) self.p = p @@ -72,7 +71,7 @@ def fit_transform( X: np.ndarray, group_a: np.ndarray, group_b: np.ndarray, - sample_weight: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, ): """ Fits the model by learning a fair cluster. diff --git a/src/holisticai/bias/mitigation/preprocessing/learning_fair_representation.py b/src/holisticai/bias/mitigation/preprocessing/learning_fair_representation.py index ce989c9e..0a89253c 100644 --- a/src/holisticai/bias/mitigation/preprocessing/learning_fair_representation.py +++ b/src/holisticai/bias/mitigation/preprocessing/learning_fair_representation.py @@ -1,15 +1,15 @@ from __future__ import annotations import logging -from typing import Optional import jax import jax.numpy as jnp import numpy as np -from holisticai.utils.transformers.bias import BMPreprocessing from jax.nn import softmax from scipy.optimize import minimize +from holisticai.utils.transformers.bias import BMPreprocessing + logger = logging.getLogger(__name__) @@ -73,9 +73,7 @@ def __call__(self, parameters): loss = jnp.array([loss_x, loss_y, loss_z]) - total_loss = jnp.dot(self.A, loss) - - return total_loss + return jnp.dot(self.A, loss) class LearningFairRepresentation(BMPreprocessing): @@ -117,14 +115,14 @@ class LearningFairRepresentation(BMPreprocessing): def __init__( self, - k: Optional[int] = 5, - Ax: Optional[float] = 0.01, - Ay: Optional[float] = 1.0, - Az: Optional[float] = 50.0, - maxiter: Optional[int] = 5000, - maxfun: Optional[int] = 5000, - verbose: Optional[int] = 0, - seed: Optional[int] = None, + k: int | None = 5, + Ax: float | None = 0.01, + Ay: float | None = 1.0, + Az: float | None = 50.0, + maxiter: int | None = 5000, + maxfun: int | None = 5000, + verbose: int | None = 0, + seed: int | None = None, ): self.seed = seed self.k = k diff --git a/src/holisticai/bias/mitigation/preprocessing/reweighing.py b/src/holisticai/bias/mitigation/preprocessing/reweighing.py index 9034b3e5..ec935c95 100644 --- a/src/holisticai/bias/mitigation/preprocessing/reweighing.py +++ b/src/holisticai/bias/mitigation/preprocessing/reweighing.py @@ -1,9 +1,8 @@ from __future__ import annotations -from typing import Optional - import numpy as np import pandas as pd + from holisticai.utils.transformers.bias import BMPreprocessing as BMPre from holisticai.utils.transformers.bias import SensitiveGroups @@ -34,7 +33,7 @@ def fit( y: np.ndarray, group_a: np.ndarray, group_b: np.ndarray, - sample_weight: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, ): """ Fit the Reweighing model to the data. This method calculates the sample weights to ensure that the \ @@ -110,7 +109,7 @@ def fit_transform( y: np.ndarray, group_a: np.ndarray, group_b: np.ndarray, - sample_weight: Optional[np.ndarray] = None, + sample_weight: np.ndarray | None = None, ): """ Fit the Reweighing model to the data. This method calculates the sample weights to ensure that the \ diff --git a/src/holisticai/bias/plots/_bias_classification_plots.py b/src/holisticai/bias/plots/_bias_classification_plots.py index b154f952..65392f9a 100644 --- a/src/holisticai/bias/plots/_bias_classification_plots.py +++ b/src/holisticai/bias/plots/_bias_classification_plots.py @@ -3,13 +3,13 @@ import numpy as np import seaborn as sns +# sklearn imports +from sklearn.metrics import roc_curve + # utils from holisticai.utils import get_colors from holisticai.utils._validation import _check_binary, _regression_checks -# sklearn imports -from sklearn.metrics import roc_curve - def abroca_plot(group_a, group_b, y_score, y_true, ax=None, size=None, title=None): """ diff --git a/src/holisticai/bias/plots/_bias_exploratory_plots.py b/src/holisticai/bias/plots/_bias_exploratory_plots.py index 77616370..63bc457d 100644 --- a/src/holisticai/bias/plots/_bias_exploratory_plots.py +++ b/src/holisticai/bias/plots/_bias_exploratory_plots.py @@ -3,6 +3,7 @@ import numpy as np import pandas as pd import seaborn as sns + from holisticai.bias.metrics import confusion_matrix # utils @@ -40,7 +41,8 @@ def group_pie_plot(y_feat, ax=None, size=None, title=None): labels = value_counts.index.tolist() else: - raise TypeError("input is not a numpy array or pandas series") + msg = "input is not a numpy array or pandas series" + raise TypeError(msg) # calculations n_b = np.sum(value_counts / np.sum(value_counts) > 0.02) diff --git a/src/holisticai/bias/plots/_bias_exploratory_view.py b/src/holisticai/bias/plots/_bias_exploratory_view.py index d9024be1..dc818bc8 100644 --- a/src/holisticai/bias/plots/_bias_exploratory_view.py +++ b/src/holisticai/bias/plots/_bias_exploratory_view.py @@ -1,8 +1,9 @@ import ipywidgets as widgets # type: ignore import matplotlib.pyplot as plt +from IPython.display import display + from holisticai.bias.plots._bias_exploratory_plots import group_pie_plot, histogram_plot from holisticai.bias.plots._bias_multiclass_plots import accuracy_bar_plot, frequency_matrix_plot, frequency_plot -from IPython.display import display def binary_classification_data_exploration(dataset): diff --git a/src/holisticai/bias/plots/_bias_multiclass_plots.py b/src/holisticai/bias/plots/_bias_multiclass_plots.py index 63a9a7ad..01911f1a 100644 --- a/src/holisticai/bias/plots/_bias_multiclass_plots.py +++ b/src/holisticai/bias/plots/_bias_multiclass_plots.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd import seaborn as sns +from matplotlib import pyplot as plt # Import metrics from holisticai.bias.metrics import frequency_matrix @@ -9,7 +10,6 @@ # utils from holisticai.utils._validation import _check_binary, _multiclass_checks -from matplotlib import pyplot as plt def frequency_plot(p_attr, y_pred, ax=None, size=None, title=None): diff --git a/src/holisticai/bias/plots/_bias_recommender_plots.py b/src/holisticai/bias/plots/_bias_recommender_plots.py index 8581c455..00cfd8ff 100644 --- a/src/holisticai/bias/plots/_bias_recommender_plots.py +++ b/src/holisticai/bias/plots/_bias_recommender_plots.py @@ -1,11 +1,11 @@ # Base Imports import numpy as np import seaborn as sns +from matplotlib import pyplot as plt # utils from holisticai.utils import get_colors, mat_to_binary, normalize_tensor from holisticai.utils._validation import _recommender_checks -from matplotlib import pyplot as plt def long_tail_plot(mat_pred, top=None, thresh=0.5, normalize=False, ax=None, size=None, title=None): diff --git a/src/holisticai/bias/plots/_bias_regression_plots.py b/src/holisticai/bias/plots/_bias_regression_plots.py index 4c0b75f5..fee89c17 100644 --- a/src/holisticai/bias/plots/_bias_regression_plots.py +++ b/src/holisticai/bias/plots/_bias_regression_plots.py @@ -1,15 +1,15 @@ # Base Imports import numpy as np import seaborn as sns - -# utils -from holisticai.utils import get_colors -from holisticai.utils._validation import _multiclass_checks, _regression_checks from matplotlib import pyplot as plt # sklearn imports from sklearn.metrics import mean_absolute_error, mean_squared_error +# utils +from holisticai.utils import get_colors +from holisticai.utils._validation import _multiclass_checks, _regression_checks + def success_rate_curve(group_a, group_b, y_pred, ax=None, size=None, title=None): """ @@ -154,7 +154,8 @@ def statistical_parity_curve(group_a, group_b, y_pred, x_axis="score", ax=None, ax.legend() else: - raise ValueError("x_axis is not one of : quantile, score") + msg = "x_axis is not one of : quantile, score" + raise ValueError(msg) return ax @@ -239,7 +240,8 @@ def disparate_impact_curve(group_a, group_b, y_pred, x_axis="score", ax=None, si ax.legend() else: - raise ValueError("x_axis is not one of : score, quantile") + msg = "x_axis is not one of : score, quantile" + raise ValueError(msg) return ax @@ -335,7 +337,8 @@ def success_rate_curves(p_attr, y_pred, groups=None, x_axis="score", ax=None, si ax.legend() else: - raise ValueError("x_axis is not one of : score, quantile") + msg = "x_axis is not one of : score, quantile" + raise ValueError(msg) return ax diff --git a/src/holisticai/bias/plots/_classification.py b/src/holisticai/bias/plots/_classification.py index 10af2285..76d7856b 100644 --- a/src/holisticai/bias/plots/_classification.py +++ b/src/holisticai/bias/plots/_classification.py @@ -1,15 +1,15 @@ # Base Imports import numpy as np import seaborn as sns - -# utils -from holisticai.utils import get_colors -from holisticai.utils._validation import _check_binary, _classification_checks from matplotlib import pyplot as plt # sklearn imports from sklearn.metrics import roc_curve +# utils +from holisticai.utils import get_colors +from holisticai.utils._validation import _check_binary, _classification_checks + def abroca_plot(group_a, group_b, y_pred, y_true, ax=None, size=None, title=None): """ diff --git a/src/holisticai/bias/plots/_multiclass.py b/src/holisticai/bias/plots/_multiclass.py index fddd8790..49afbd82 100644 --- a/src/holisticai/bias/plots/_multiclass.py +++ b/src/holisticai/bias/plots/_multiclass.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd import seaborn as sns +from matplotlib import pyplot as plt # Import metrics from holisticai.bias.metrics import frequency_matrix @@ -9,7 +10,6 @@ # utils from holisticai.utils._validation import _multiclass_checks -from matplotlib import pyplot as plt def frequency_plot(p_attr, y_pred, ax=None, size=None, title=None): diff --git a/src/holisticai/bias/plots/_recommender.py b/src/holisticai/bias/plots/_recommender.py index 8c4c0611..43630af6 100644 --- a/src/holisticai/bias/plots/_recommender.py +++ b/src/holisticai/bias/plots/_recommender.py @@ -1,11 +1,11 @@ # Base Imports import numpy as np import seaborn as sns +from matplotlib import pyplot as plt # utils from holisticai.utils import get_colors, mat_to_binary, normalize_tensor from holisticai.utils._validation import _recommender_checks -from matplotlib import pyplot as plt def long_tail_plot(mat_pred, top=None, thresh=0.5, normalize=False, ax=None, size=None, title=None): diff --git a/src/holisticai/bias/plots/_regression.py b/src/holisticai/bias/plots/_regression.py index 2fc1fc34..92667a31 100644 --- a/src/holisticai/bias/plots/_regression.py +++ b/src/holisticai/bias/plots/_regression.py @@ -1,15 +1,15 @@ # Base Imports import numpy as np import seaborn as sns - -# utils -from holisticai.utils import get_colors -from holisticai.utils._validation import _multiclass_checks, _regression_checks from matplotlib import pyplot as plt # sklearn imports from sklearn.metrics import mean_absolute_error, mean_squared_error +# utils +from holisticai.utils import get_colors +from holisticai.utils._validation import _multiclass_checks, _regression_checks + def success_rate_curve(group_a, group_b, y_pred, ax=None, size=None, title=None): """ diff --git a/src/holisticai/datasets/_dataset.py b/src/holisticai/datasets/_dataset.py index 555c0916..c62bb771 100644 --- a/src/holisticai/datasets/_dataset.py +++ b/src/holisticai/datasets/_dataset.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal, Union +from typing import TYPE_CHECKING, Literal import pandas as pd from sklearn.model_selection import train_test_split @@ -10,9 +10,10 @@ if TYPE_CHECKING: from collections.abc import Iterable + from numpy.random import RandomState + import numpy as np -from numpy.random import RandomState class DatasetDict(dict, DatasetReprObj): @@ -429,7 +430,8 @@ def __getitem__(self, key: str | int | list): def __setitem__(self, key, value): if not isinstance(key, str): - raise TypeError("Key must be a string.") + msg = "Key must be a string." + raise TypeError(msg) feature_exists = key in self.data.columns.get_level_values(0) existing_subfeatures = self.data[key].columns if feature_exists else [] @@ -481,7 +483,7 @@ def apply_fn_to_multilevel_df(df, fn): return result_df -def sample_n(group: pd.DataFrame, n: int, random_state: Union[RandomState, None] = None) -> pd.DataFrame: +def sample_n(group: pd.DataFrame, n: int, random_state: RandomState | None = None) -> pd.DataFrame: if len(group) < n: return group return group.sample(n=n, replace=False, random_state=random_state) diff --git a/src/holisticai/datasets/_load_dataset.py b/src/holisticai/datasets/_load_dataset.py index aa0cb1f8..91f4e61f 100644 --- a/src/holisticai/datasets/_load_dataset.py +++ b/src/holisticai/datasets/_load_dataset.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, Optional +from typing import Literal import numpy as np import pandas as pd @@ -32,7 +32,7 @@ def create_preprocessor(X, numerical_transform: bool = True, categorical_transfo return ColumnTransformer(transformers=transformers) -def load_adult_dataset(protected_attribute: Optional[Literal["race", "sex"]] = "sex", preprocessed: bool = True): +def load_adult_dataset(protected_attribute: Literal["race", "sex"] | None = "sex", preprocessed: bool = True): sensitive_attribute = ["race", "sex"] feature_names = [ "age", @@ -88,7 +88,8 @@ def load_adult_dataset(protected_attribute: Optional[Literal["race", "sex"]] = " group_a = pd.Series(get_protected_values(df, protected_attribute, ga_label), name="group_a") group_b = pd.Series(get_protected_values(df, protected_attribute, gb_label), name="group_b") else: - raise ValueError("The protected attribute must be: race or sex") + msg = "The protected attribute must be: race or sex" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -96,7 +97,7 @@ def load_adult_dataset(protected_attribute: Optional[Literal["race", "sex"]] = " return Dataset(X=xt, y=y, p_attrs=p_attrs) -def load_law_school_dataset(protected_attribute: Optional[Literal["race", "sex"]] = "sex", preprocessed: bool = True): +def load_law_school_dataset(protected_attribute: Literal["race", "sex"] | None = "sex", preprocessed: bool = True): data = load_hai_datasets(dataset_name="law_school") sensitive_attribute = ["race1", "gender", "age"] output_variable = "bar" @@ -122,7 +123,8 @@ def load_law_school_dataset(protected_attribute: Optional[Literal["race", "sex"] group_a = pd.Series(get_protected_values(df, "gender", ga_label), name="group_a") group_b = pd.Series(get_protected_values(df, "gender", gb_label), name="group_b") else: - raise ValueError("The protected attribute must be one of: race or gender") + msg = "The protected attribute must be one of: race or gender" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -130,9 +132,7 @@ def load_law_school_dataset(protected_attribute: Optional[Literal["race", "sex"] return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_student_multiclass_dataset( - protected_attribute: Optional[Literal["sex", "address"]] = "sex", preprocessed=True -): +def load_student_multiclass_dataset(protected_attribute: Literal["sex", "address"] | None = "sex", preprocessed=True): sensitive_attributes = ["sex", "address", "Mjob", "Fjob"] output_column = "G3" drop_columns = ["G1", "G2", "G3", "sex", "address", "Mjob", "Fjob"] @@ -172,7 +172,8 @@ def load_student_multiclass_dataset( group_a = pd.Series(df["address"] == ga_label, name="group_a") group_b = ~group_a else: - raise ValueError("The protected attribute must be one sex or address") + msg = "The protected attribute must be one sex or address" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -181,9 +182,9 @@ def load_student_multiclass_dataset( def load_student_dataset( - target: Optional[Literal["G1", "G2", "G3"]] = None, + target: Literal["G1", "G2", "G3"] | None = None, preprocessed: bool = True, - protected_attribute: Optional[Literal["sex", "address"]] = "sex", + protected_attribute: Literal["sex", "address"] | None = "sex", ): if target is None: target = "G3" @@ -215,7 +216,8 @@ def load_student_dataset( group_a = pd.Series(df["address"] == "U", name="group_a") group_b = pd.Series(df["address"] == "R", name="group_b") else: - raise ValueError("The protected attribute doesn't exist or not implemented") + msg = "The protected attribute doesn't exist or not implemented" + raise ValueError(msg) df = df.drop(drop_columns, axis=1) if preprocessed: @@ -273,7 +275,7 @@ def load_lastfm_dataset(): return Dataset(data_pivot=df_pivot, p_attr=pd.Series(p_attr)) -def load_us_crime_dataset(preprocessed=True, protected_attribute: Optional[Literal["race"]] = "race"): +def load_us_crime_dataset(preprocessed=True, protected_attribute: Literal["race"] | None = "race"): """ Processes the US crime dataset and returns the data, output variable, protected group A and \ protected group B as numerical arrays or as dataframe if needed @@ -316,9 +318,8 @@ def load_us_crime_dataset(preprocessed=True, protected_attribute: Optional[Liter group_a = pd.Series(df[protected_attribute_column] > threshold, name="group_a") group_b = pd.Series(~group_a, name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attributes}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attributes}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -326,7 +327,7 @@ def load_us_crime_dataset(preprocessed=True, protected_attribute: Optional[Liter return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_us_crime_multiclass_dataset(preprocessed=True, protected_attribute: Optional[Literal["race"]] = "race"): +def load_us_crime_multiclass_dataset(preprocessed=True, protected_attribute: Literal["race"] | None = "race"): """ Processes the US crime dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed @@ -370,9 +371,8 @@ def load_us_crime_multiclass_dataset(preprocessed=True, protected_attribute: Opt group_a = pd.Series(df[protected_attribute_column] > threshold, name="group_a") group_b = pd.Series(~group_a, name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -380,7 +380,7 @@ def load_us_crime_multiclass_dataset(preprocessed=True, protected_attribute: Opt return Dataset(X=X, y=y_cat, p_attrs=p_attrs) -def load_clinical_records_dataset(protected_attribute: Optional[Literal["sex"]] = "sex"): +def load_clinical_records_dataset(protected_attribute: Literal["sex"] | None = "sex"): """ Processes the heart dataset and returns the data, output variable, protected group A and protected group B as numerical arrays @@ -410,7 +410,8 @@ def load_clinical_records_dataset(protected_attribute: Optional[Literal["sex"]] group_a = pd.Series(df[protected_attribute] == ga_label, name="group_a") group_b = pd.Series(df[protected_attribute] == gb_label, name="group_b") else: - raise ValueError("The protected attribute doesn't exist or not implemented. Please use: sex") + msg = "The protected attribute doesn't exist or not implemented. Please use: sex" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -418,7 +419,7 @@ def load_clinical_records_dataset(protected_attribute: Optional[Literal["sex"]] return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_german_credit_dataset(preprocessed=True, protected_attribute: Optional[Literal["sex"]] = "sex"): +def load_german_credit_dataset(preprocessed=True, protected_attribute: Literal["sex"] | None = "sex"): """ Processes the german credit dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed @@ -458,9 +459,8 @@ def load_german_credit_dataset(preprocessed=True, protected_attribute: Optional[ group_a = pd.Series(df["Sex"] == "male", name="group_a") group_b = pd.Series(df["Sex"] == "female", name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -468,7 +468,7 @@ def load_german_credit_dataset(preprocessed=True, protected_attribute: Optional[ return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_census_kdd_dataset(preprocessed=True, protected_attribute: Optional[Literal["sex"]] = "sex"): +def load_census_kdd_dataset(preprocessed=True, protected_attribute: Literal["sex"] | None = "sex"): """ Processes the Census KDD dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed @@ -516,9 +516,8 @@ def load_census_kdd_dataset(preprocessed=True, protected_attribute: Optional[Lit group_a = pd.Series(df["race"] == 1, name="group_a") group_b = pd.Series(df["race"] == 0, name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -526,7 +525,7 @@ def load_census_kdd_dataset(preprocessed=True, protected_attribute: Optional[Lit return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_bank_marketing_dataset(preprocessed=True, protected_attribute: Optional[Literal["marital"]] = "marital"): +def load_bank_marketing_dataset(preprocessed=True, protected_attribute: Literal["marital"] | None = "marital"): """ Processes the Banking Marketing dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed @@ -588,9 +587,8 @@ def load_bank_marketing_dataset(preprocessed=True, protected_attribute: Optional group_a = pd.Series(df["marital"] == 1, name="group_a") group_b = pd.Series(df["marital"] == 0, name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -598,9 +596,7 @@ def load_bank_marketing_dataset(preprocessed=True, protected_attribute: Optional return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_compas_two_year_recid_dataset( - preprocessed=True, protected_attribute: Optional[Literal["race", "sex"]] = "race" -): +def load_compas_two_year_recid_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "race"): """ Processes the compas dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed Target: 2-year recidivism @@ -646,9 +642,8 @@ def load_compas_two_year_recid_dataset( group_a = pd.Series(df["race"] == 1, name="group_a") group_b = pd.Series(df["race"] == 2, name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -656,7 +651,7 @@ def load_compas_two_year_recid_dataset( return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_compas_is_recid_dataset(preprocessed=True, protected_attribute: Optional[Literal["race", "sex"]] = "race"): +def load_compas_is_recid_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "race"): """ Processes the compas dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed Target: 2-year recidivism @@ -700,9 +695,8 @@ def load_compas_is_recid_dataset(preprocessed=True, protected_attribute: Optiona group_a = pd.Series(df["race"] == 1, name="group_a") group_b = pd.Series(df["race"] == 2, name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -710,7 +704,7 @@ def load_compas_is_recid_dataset(preprocessed=True, protected_attribute: Optiona return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_diabetes_dataset(preprocessed=True, protected_attribute: Optional[Literal["race", "sex"]] = "sex"): +def load_diabetes_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "sex"): """ Processes the Diabetes dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed @@ -758,9 +752,8 @@ def load_diabetes_dataset(preprocessed=True, protected_attribute: Optional[Liter group_a = pd.Series(df["race"] == "Caucasian", name="group_a") group_b = pd.Series(df["race"] == "Non-Caucasian", name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -768,7 +761,7 @@ def load_diabetes_dataset(preprocessed=True, protected_attribute: Optional[Liter return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_acsincome_dataset(preprocessed=True, protected_attribute: Optional[Literal["race", "sex"]] = "sex"): +def load_acsincome_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "sex"): """ Processes the ACSIncome dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed @@ -814,9 +807,8 @@ def load_acsincome_dataset(preprocessed=True, protected_attribute: Optional[Lite group_a = pd.Series(df["RAC1P"] == "White", name="group_a") group_b = pd.Series(df["RAC1P"] == "Non-White", name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -824,7 +816,7 @@ def load_acsincome_dataset(preprocessed=True, protected_attribute: Optional[Lite return Dataset(X=X, y=y, p_attrs=p_attrs) -def load_acspublic_dataset(preprocessed=True, protected_attribute: Optional[Literal["race", "sex"]] = "sex"): +def load_acspublic_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "sex"): """ Processes the ACSPublicCoverage dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed @@ -869,9 +861,118 @@ def load_acspublic_dataset(preprocessed=True, protected_attribute: Optional[Lite group_a = pd.Series(df["RAC1P"] == "White", name="group_a") group_b = pd.Series(df["RAC1P"] == "Non-White", name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) + + if protected_attribute is not None: + metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" + return Dataset(X=X, y=y, p_attrs=p_attrs, group_a=group_a, group_b=group_b, _metadata=metadata) + return Dataset(X=X, y=y, p_attrs=p_attrs) + + +def load_mw_medium_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "race"): + """ + Processes the Minimum Wage Medium dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed + Target: contracted_hours + + Parameters + ---------- + preprocessed : bool + Whether to return the preprocessed X and y. + protected_attribute : str + If this parameter is set, the dataset will be returned with the protected attribute as a binary column group_a and group_b. + Otherwise, the dataset will be returned with the protected attribute as a column p_attrs. + + Returns + ------- + tuple + A tuple with two lists containing the data, output variable, protected group A and protected group B + """ + data = load_hai_datasets(dataset_name="mw_medium") + protected_attributes = ["race", "sex", "age_group", "education_level", "age"] + output_column = "class" + + df = data.copy() + remove_columns = [*protected_attributes, output_column] + df.reset_index(drop=True, inplace=True) + X = df.drop(columns=remove_columns) + + if preprocessed: + for col in X.select_dtypes(include=["category", "object"]).columns: + X[col] = pd.factorize(X[col])[0] + + p_attrs = df[protected_attributes] + y = df[output_column] + + if protected_attribute is not None: + if protected_attribute == "sex": + ga_label = "Male" + gb_label = "Female" + group_a = pd.Series(df["sex"] == ga_label, name="group_a") + group_b = pd.Series(df["sex"] == gb_label, name="group_b") + elif protected_attribute == "race": + ga_label = "White" + gb_label = "Black" + group_a = pd.Series(df["race"] == ga_label, name="group_a") + group_b = pd.Series(df["race"] == gb_label, name="group_b") + else: + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) + + if protected_attribute is not None: + metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" + return Dataset(X=X, y=y, p_attrs=p_attrs, group_a=group_a, group_b=group_b, _metadata=metadata) + return Dataset(X=X, y=y, p_attrs=p_attrs) + + +def load_mw_small_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "race"): + """ + Processes the Minimum Wage Small dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed + Target: contracted_hours + + Parameters + ---------- + preprocessed : bool + Whether to return the preprocessed X and y. + protected_attribute : str + If this parameter is set, the dataset will be returned with the protected attribute as a binary column group_a and group_b. + Otherwise, the dataset will be returned with the protected attribute as a column p_attrs. + + Returns + ------- + tuple + A tuple with two lists containing the data, output variable, protected group A and protected group B + """ + data = load_hai_datasets(dataset_name="mw_small") + protected_attributes = ["race", "sex", "age_group", "education_level", "age"] + output_column = "class" + + df = data.copy() + remove_columns = [*protected_attributes, output_column] + df.reset_index(drop=True, inplace=True) + X = df.drop(columns=remove_columns) + + if preprocessed: + for col in X.select_dtypes(include=["category", "object"]).columns: + X[col] = pd.factorize(X[col])[0] + + p_attrs = df[protected_attributes] + y = df[output_column] + + if protected_attribute is not None: + if protected_attribute == "sex": + ga_label = "Male" + gb_label = "Female" + group_a = pd.Series(df["sex"] == ga_label, name="group_a") + group_b = pd.Series(df["sex"] == gb_label, name="group_b") + elif protected_attribute == "race": + ga_label = "White" + gb_label = "Black" + group_a = pd.Series(df["race"] == ga_label, name="group_a") + group_b = pd.Series(df["race"] == gb_label, name="group_b") + else: + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -981,9 +1082,118 @@ def load_mw_small_dataset(preprocessed=True, protected_attribute: Optional[Liter group_a = pd.Series(df["race"] == ga_label, name="group_a") group_b = pd.Series(df["race"] == gb_label, name="group_b") else: - raise ValueError( - f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" - ) + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) + + if protected_attribute is not None: + metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" + return Dataset(X=X, y=y, p_attrs=p_attrs, group_a=group_a, group_b=group_b, _metadata=metadata) + return Dataset(X=X, y=y, p_attrs=p_attrs) + + +def load_mw_medium_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "race"): + """ + Processes the Minimum Wage Medium dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed + Target: contracted_hours + + Parameters + ---------- + preprocessed : bool + Whether to return the preprocessed X and y. + protected_attribute : str + If this parameter is set, the dataset will be returned with the protected attribute as a binary column group_a and group_b. + Otherwise, the dataset will be returned with the protected attribute as a column p_attrs. + + Returns + ------- + tuple + A tuple with two lists containing the data, output variable, protected group A and protected group B + """ + data = load_hai_datasets(dataset_name="mw_medium") + protected_attributes = ["race", "sex", "age_group", "education_level", "age"] + output_column = "class" + + df = data.copy() + remove_columns = [*protected_attributes, output_column] + df.reset_index(drop=True, inplace=True) + X = df.drop(columns=remove_columns) + + if preprocessed: + for col in X.select_dtypes(include=["category", "object"]).columns: + X[col] = pd.factorize(X[col])[0] + + p_attrs = df[protected_attributes] + y = df[output_column] + + if protected_attribute is not None: + if protected_attribute == "sex": + ga_label = "Male" + gb_label = "Female" + group_a = pd.Series(df["sex"] == ga_label, name="group_a") + group_b = pd.Series(df["sex"] == gb_label, name="group_b") + elif protected_attribute == "race": + ga_label = "White" + gb_label = "Black" + group_a = pd.Series(df["race"] == ga_label, name="group_a") + group_b = pd.Series(df["race"] == gb_label, name="group_b") + else: + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) + + if protected_attribute is not None: + metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" + return Dataset(X=X, y=y, p_attrs=p_attrs, group_a=group_a, group_b=group_b, _metadata=metadata) + return Dataset(X=X, y=y, p_attrs=p_attrs) + + +def load_mw_small_dataset(preprocessed=True, protected_attribute: Literal["race", "sex"] | None = "race"): + """ + Processes the Minimum Wage Small dataset and returns the data, output variable, protected group A and protected group B as numerical arrays or as dataframe if needed + Target: contracted_hours + + Parameters + ---------- + preprocessed : bool + Whether to return the preprocessed X and y. + protected_attribute : str + If this parameter is set, the dataset will be returned with the protected attribute as a binary column group_a and group_b. + Otherwise, the dataset will be returned with the protected attribute as a column p_attrs. + + Returns + ------- + tuple + A tuple with two lists containing the data, output variable, protected group A and protected group B + """ + data = load_hai_datasets(dataset_name="mw_small") + protected_attributes = ["race", "sex", "age_group", "education_level", "age"] + output_column = "class" + + df = data.copy() + remove_columns = [*protected_attributes, output_column] + df.reset_index(drop=True, inplace=True) + X = df.drop(columns=remove_columns) + + if preprocessed: + for col in X.select_dtypes(include=["category", "object"]).columns: + X[col] = pd.factorize(X[col])[0] + + p_attrs = df[protected_attributes] + y = df[output_column] + + if protected_attribute is not None: + if protected_attribute == "sex": + ga_label = "Male" + gb_label = "Female" + group_a = pd.Series(df["sex"] == ga_label, name="group_a") + group_b = pd.Series(df["sex"] == gb_label, name="group_b") + elif protected_attribute == "race": + ga_label = "White" + gb_label = "Black" + group_a = pd.Series(df["race"] == ga_label, name="group_a") + group_b = pd.Series(df["race"] == gb_label, name="group_b") + else: + msg = f"The protected attribute doesn't exist or not implemented. Please use: {protected_attribute}" + raise ValueError(msg) if protected_attribute is not None: metadata = f"""{protected_attribute}: {{'group_a': '{ga_label}', 'group_b': '{gb_label}'}}""" @@ -1016,8 +1226,8 @@ def load_mw_small_dataset(preprocessed=True, protected_attribute: Optional[Liter def load_dataset( dataset_name: ProcessedDatasets, preprocessed: bool = True, - protected_attribute: Optional[str] = None, - target: Optional[str] = None, + protected_attribute: str | None = None, + target: str | None = None, ) -> Dataset: """ Load a specific dataset based on the given dataset name. diff --git a/src/holisticai/datasets/_load_dataset.pyi b/src/holisticai/datasets/_load_dataset.pyi index 4d5fa63a..5e249f3c 100644 --- a/src/holisticai/datasets/_load_dataset.pyi +++ b/src/holisticai/datasets/_load_dataset.pyi @@ -1,29 +1,29 @@ -from typing import Literal, Union, overload +from typing import Literal, overload @overload def load_dataset( dataset_name: Literal["adult"], - protected_attribute: Union[Literal["race", "sex"], None] = None, + protected_attribute: Literal["race", "sex"] | None = None, preprocessed: bool = True, ): ... @overload def load_dataset( dataset_name: Literal["law_school"], - protected_attribute: Union[Literal["race", "gender"], None] = None, + protected_attribute: Literal["race", "gender"] | None = None, preprocessed: bool = True, ): ... @overload def load_dataset( dataset_name: Literal["student_multiclass"], - protected_attribute: Union[Literal["sex", "address"], None] = None, + protected_attribute: Literal["sex", "address"] | None = None, preprocessed: bool = True, ): ... @overload def load_dataset( dataset_name: Literal["student"], - target: Union[Literal["G1", "G2", "G3"], None] = None, + target: Literal["G1", "G2", "G3"] | None = None, preprocessed: bool = True, - protected_attribute: Union[Literal["sex", "address"], None] = None, + protected_attribute: Literal["sex", "address"] | None = None, ): ... @overload def load_dataset( @@ -33,15 +33,13 @@ def load_dataset( def load_dataset( dataset_name: Literal["us_crime"], preprocessed: bool = True, - protected_attribute: Union[Literal["race"], None] = None, + protected_attribute: Literal["race"] | None = None, ): ... @overload def load_dataset( dataset_name: Literal["us_crime_multiclass"], preprocessed: bool = True, - protected_attribute: Union[Literal["race"], None] = None, + protected_attribute: Literal["race"] | None = None, ): ... @overload -def load_dataset( - dataset_name: Literal["clinical_records"], protected_attribute: Union[Literal["sex"], None] = None -): ... +def load_dataset(dataset_name: Literal["clinical_records"], protected_attribute: Literal["sex"] | None = None): ... diff --git a/src/holisticai/efficacy/metrics/_classification.py b/src/holisticai/efficacy/metrics/_classification.py index f3286239..6080c248 100644 --- a/src/holisticai/efficacy/metrics/_classification.py +++ b/src/holisticai/efficacy/metrics/_classification.py @@ -73,7 +73,8 @@ def confusion_matrix(y_pred, y_true, classes=None, normalize=None): confmat = confmat / np.sum(confmat, axis=0).reshape(1, -1) else: - raise ValueError('normalize should be one of None, "pred" or "true"') + msg = 'normalize should be one of None, "pred" or "true"' + raise ValueError(msg) return pd.DataFrame(confmat, columns=classes).set_index(np.array(classes)) diff --git a/src/holisticai/explainability/__init__.py b/src/holisticai/explainability/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/holisticai/explainability/metrics/_classification.py b/src/holisticai/explainability/metrics/_classification.py index 25a6bcc8..b6447bbc 100644 --- a/src/holisticai/explainability/metrics/_classification.py +++ b/src/holisticai/explainability/metrics/_classification.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING import pandas as pd + from holisticai.explainability.metrics.global_feature_importance import ( AlphaScore, PositionParity, @@ -27,9 +28,9 @@ def classification_explainability_metrics( importances: Importances, partial_dependencies: PartialDependence, conditional_importances: ConditionalImportances, - X: Union[pd.DataFrame, None] = None, - y_pred: Union[pd.Series, None] = None, - local_importances: Union[LocalImportances, None] = None, + X: pd.DataFrame | None = None, + y_pred: pd.Series | None = None, + local_importances: LocalImportances | None = None, ) -> pd.DataFrame: ranked_importances = importances.top_alpha(0.8) results = [] diff --git a/src/holisticai/explainability/metrics/_multiclass.py b/src/holisticai/explainability/metrics/_multiclass.py index 71c1408a..6ac39fc6 100644 --- a/src/holisticai/explainability/metrics/_multiclass.py +++ b/src/holisticai/explainability/metrics/_multiclass.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING import pandas as pd + from holisticai.explainability.metrics.global_feature_importance import ( AlphaScore, PositionParity, @@ -27,9 +28,9 @@ def multiclass_explainability_metrics( importances: Importances, partial_dependencies: PartialDependence, conditional_importances: ConditionalImportances, - X: Union[pd.DataFrame, None] = None, - y_pred: Union[pd.Series, None] = None, - local_importances: Union[LocalImportances, None] = None, + X: pd.DataFrame | None = None, + y_pred: pd.Series | None = None, + local_importances: LocalImportances | None = None, ) -> pd.DataFrame: ranked_importances = importances.top_alpha(0.8) results = [] diff --git a/src/holisticai/explainability/metrics/_regression.py b/src/holisticai/explainability/metrics/_regression.py index a4bb0da8..b7c5664e 100644 --- a/src/holisticai/explainability/metrics/_regression.py +++ b/src/holisticai/explainability/metrics/_regression.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING import pandas as pd + from holisticai.explainability.metrics.global_feature_importance import ( AlphaScore, PositionParity, @@ -22,9 +23,9 @@ def regression_explainability_metrics( importances: Importances, partial_dependencies: PartialDependence, conditional_importances: ConditionalImportances, - X: Union[pd.DataFrame, None] = None, - y_pred: Union[pd.Series, None] = None, - local_importances: Union[LocalImportances, None] = None, + X: pd.DataFrame | None = None, + y_pred: pd.Series | None = None, + local_importances: LocalImportances | None = None, ) -> pd.DataFrame: ranked_importances = importances.top_alpha(0.8) results = [] diff --git a/src/holisticai/explainability/metrics/_tree.py b/src/holisticai/explainability/metrics/_tree.py index 191b7b0c..1d09d58a 100644 --- a/src/holisticai/explainability/metrics/_tree.py +++ b/src/holisticai/explainability/metrics/_tree.py @@ -1,6 +1,7 @@ from __future__ import annotations import pandas as pd + from holisticai.explainability.metrics.tree import ( TreeDepthVariance, TreeNumberOfFeatures, diff --git a/src/holisticai/explainability/metrics/global_feature_importance/__init__.py b/src/holisticai/explainability/metrics/global_feature_importance/__init__.py index 6bc640d9..fdefdfe2 100644 --- a/src/holisticai/explainability/metrics/global_feature_importance/__init__.py +++ b/src/holisticai/explainability/metrics/global_feature_importance/__init__.py @@ -1,4 +1,5 @@ import pandas as pd + from holisticai.explainability.metrics.global_feature_importance._alpha_score import ( AlphaScore, alpha_score, diff --git a/src/holisticai/explainability/metrics/global_feature_importance/_fluctuation_ratio.py b/src/holisticai/explainability/metrics/global_feature_importance/_fluctuation_ratio.py index 40a912d4..edf78cd0 100644 --- a/src/holisticai/explainability/metrics/global_feature_importance/_fluctuation_ratio.py +++ b/src/holisticai/explainability/metrics/global_feature_importance/_fluctuation_ratio.py @@ -1,14 +1,16 @@ from __future__ import annotations import logging -from typing import Optional +from typing import TYPE_CHECKING import numpy as np import pandas as pd -from holisticai.typing import ArrayLike, MatrixLike -from holisticai.utils import Importances, PartialDependence from scipy.interpolate import interp1d +if TYPE_CHECKING: + from holisticai.typing import ArrayLike, MatrixLike + from holisticai.utils import Importances, PartialDependence + logger = logging.getLogger(__name__) @@ -27,7 +29,7 @@ def get_fluctuations_from_individuals(grid_values: ArrayLike, individuals: Matri def fluctuation_ratio( partial_dependencies: PartialDependence, - importances: Optional[Importances] = None, + importances: Importances | None = None, top_n=-1, label=0, weighted=False, diff --git a/src/holisticai/explainability/metrics/global_feature_importance/_importance_spread.py b/src/holisticai/explainability/metrics/global_feature_importance/_importance_spread.py index a0a7acf1..9ab48f3d 100644 --- a/src/holisticai/explainability/metrics/global_feature_importance/_importance_spread.py +++ b/src/holisticai/explainability/metrics/global_feature_importance/_importance_spread.py @@ -1,8 +1,9 @@ import numpy as np -from holisticai.typing import ArrayLike from scipy.spatial.distance import jensenshannon from scipy.stats import entropy +from holisticai.typing import ArrayLike + class FeatureImportanceSpread: """ diff --git a/src/holisticai/explainability/metrics/global_feature_importance/_surrogate.py b/src/holisticai/explainability/metrics/global_feature_importance/_surrogate.py index 81cf1907..2d568169 100644 --- a/src/holisticai/explainability/metrics/global_feature_importance/_surrogate.py +++ b/src/holisticai/explainability/metrics/global_feature_importance/_surrogate.py @@ -27,8 +27,7 @@ def smape(y_true, y_pred): numerator = np.abs(y_true - y_pred) # Avoid division by zero by adding a small constant (epsilon) to the denominator epsilon = 1e-10 - smape_value = np.mean(numerator / (denominator + epsilon)) - return smape_value + return np.mean(numerator / (denominator + epsilon)) def surrogate_fidelity(y_pred, y_surrogate): diff --git a/src/holisticai/explainability/metrics/local_feature_importance/__init__.py b/src/holisticai/explainability/metrics/local_feature_importance/__init__.py index 81324073..f27ea447 100644 --- a/src/holisticai/explainability/metrics/local_feature_importance/__init__.py +++ b/src/holisticai/explainability/metrics/local_feature_importance/__init__.py @@ -1,4 +1,5 @@ import pandas as pd + from holisticai.explainability.metrics.local_feature_importance._importance_stability import importance_stability from holisticai.explainability.metrics.local_feature_importance._rank_consistency import ( local_normalized_desviation, diff --git a/src/holisticai/explainability/metrics/local_feature_importance/_rank_consistency.py b/src/holisticai/explainability/metrics/local_feature_importance/_rank_consistency.py index 72eda654..5eb6808d 100644 --- a/src/holisticai/explainability/metrics/local_feature_importance/_rank_consistency.py +++ b/src/holisticai/explainability/metrics/local_feature_importance/_rank_consistency.py @@ -7,17 +7,15 @@ def get_average_local_importances(local_importances_values: np.ndarray): local_importances_values = np.abs(local_importances_values) local_importances_values = local_importances_values / local_importances_values.sum(axis=1, keepdims=True) - average_local_importances_values = local_importances_values.mean(axis=0) - return average_local_importances_values + return local_importances_values.mean(axis=0) def local_normalized_desviation(local_importances_values: np.ndarray): ranked_features = np.argsort(np.argsort(-local_importances_values, axis=1), axis=1) mode_result = stats.mode(ranked_features, axis=0) - normalized_desviation = (np.abs(ranked_features - np.array(mode_result.mode)[None, :])) / ( + return (np.abs(ranked_features - np.array(mode_result.mode)[None, :])) / ( np.max(ranked_features, axis=0) - np.min(ranked_features, axis=0) + EPSILON )[None, :] - return normalized_desviation def rank_consistency(local_importances_values: np.ndarray, weighted=False, aggregate=True): diff --git a/src/holisticai/explainability/metrics/local_feature_importance/_stability.py b/src/holisticai/explainability/metrics/local_feature_importance/_stability.py index 44d89e72..5370fe42 100644 --- a/src/holisticai/explainability/metrics/local_feature_importance/_stability.py +++ b/src/holisticai/explainability/metrics/local_feature_importance/_stability.py @@ -47,10 +47,10 @@ def __call__(self, local_feature_importance, strategy="variance", **kargs): if strategy == "variance": jsd_std = np.std(densities) jsd_max = np.max(densities) - stability = 1 - (jsd_std / jsd_max) - return stability + return 1 - (jsd_std / jsd_max) if strategy == "entropy": num_samples = len(densities) feature_equal_weight = np.array([1.0 / num_samples] * num_samples) return 1 - jensenshannon(densities, feature_equal_weight, base=2) - raise ValueError(f"Invalid strategy: {strategy}") + msg = f"Invalid strategy: {strategy}" + raise ValueError(msg) diff --git a/src/holisticai/explainability/metrics/surrogate/_classification.py b/src/holisticai/explainability/metrics/surrogate/_classification.py index b484e235..b1a207f3 100644 --- a/src/holisticai/explainability/metrics/surrogate/_classification.py +++ b/src/holisticai/explainability/metrics/surrogate/_classification.py @@ -2,6 +2,8 @@ import numpy as np import pandas as pd +from sklearn.metrics import accuracy_score + from holisticai.explainability.metrics.global_feature_importance._importance_spread import FeatureImportanceSpread from holisticai.explainability.metrics.global_feature_importance._surrogate import ( surrogate_accuracy_score, @@ -20,7 +22,6 @@ ) from holisticai.typing import ArrayLike from holisticai.utils.surrogate_models import BinaryClassificationSurrogate, MultiClassificationSurrogate -from sklearn.metrics import accuracy_score class AccuracyDegradation: @@ -30,8 +31,7 @@ class AccuracyDegradation: def __call__(self, y, y_pred, y_surrogate): Pb = accuracy_score(y, y_pred) Ps = accuracy_score(y, y_surrogate) - D = 2 * (Pb - Ps) / (Pb + Ps) # Normalized difference between the two SMAPE values - return D + return 2 * (Pb - Ps) / (Pb + Ps) # Normalized difference between the two SMAPE values def surrogate_accuracy_degradation(y: ArrayLike, y_pred: ArrayLike, y_surrogate: ArrayLike): @@ -124,7 +124,8 @@ def classification_surrogate_explainability_metrics( elif len(np.unique(y_pred)) > 2: surrogate = MultiClassificationSurrogate(X, y_pred=y_pred, model_type=surrogate_type) else: - raise ValueError("y_pred must have at least two unique values") + msg = "y_pred must have at least two unique values" + raise ValueError(msg) y_surrogate = surrogate.predict(X) results = {} diff --git a/src/holisticai/explainability/metrics/surrogate/_clustering.py b/src/holisticai/explainability/metrics/surrogate/_clustering.py index 16838079..c2532555 100644 --- a/src/holisticai/explainability/metrics/surrogate/_clustering.py +++ b/src/holisticai/explainability/metrics/surrogate/_clustering.py @@ -1,6 +1,8 @@ from typing import Any, Literal import pandas as pd +from sklearn.metrics import accuracy_score + from holisticai.explainability.metrics.global_feature_importance._importance_spread import ( FeatureImportanceSpread, ) @@ -20,7 +22,6 @@ WeightedTreeGini, ) from holisticai.utils.surrogate_models import ClusteringSurrogate -from sklearn.metrics import accuracy_score class AccuracyDegradation: @@ -32,8 +33,7 @@ def __call__(self, X, y, proxy, surrogate): Pb = accuracy_score(y, y_pred) y_pred_surrogate = surrogate.predict(X) Pt = accuracy_score(y, y_pred_surrogate) - D = Pb - Pt - return D + return Pb - Pt def accuracy_difference(X, y, proxy, surrogate): diff --git a/src/holisticai/explainability/metrics/surrogate/_regression.py b/src/holisticai/explainability/metrics/surrogate/_regression.py index 8beddd70..5cd53f87 100644 --- a/src/holisticai/explainability/metrics/surrogate/_regression.py +++ b/src/holisticai/explainability/metrics/surrogate/_regression.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd + from holisticai.explainability.metrics.global_feature_importance._importance_spread import ( FeatureImportanceSpread, ) @@ -31,8 +32,7 @@ class MSEDegradation: def __call__(self, y, y_pred, y_surrogate): Pb = surrogate_mean_squared_error(y, y_pred) Ps = surrogate_mean_squared_error(y, y_surrogate) - D = max(0, 2 * (Ps - Pb) / (Pb + Ps)) - return D + return max(0, 2 * (Ps - Pb) / (Pb + Ps)) def surrogate_mean_squared_error_degradation(y: ArrayLike, y_pred: ArrayLike, y_surrogate: ArrayLike): diff --git a/src/holisticai/explainability/metrics/surrogate/_stability.py b/src/holisticai/explainability/metrics/surrogate/_stability.py index 6b3dd770..106dc821 100644 --- a/src/holisticai/explainability/metrics/surrogate/_stability.py +++ b/src/holisticai/explainability/metrics/surrogate/_stability.py @@ -2,9 +2,10 @@ from typing import Any import numpy as np -from holisticai.utils.surrogate_models import Surrogate, get_features from sklearn.utils import resample +from holisticai.utils.surrogate_models import Surrogate, get_features + class FeaturesStability: """ diff --git a/src/holisticai/explainability/metrics/tree/_tree.py b/src/holisticai/explainability/metrics/tree/_tree.py index a4eb6371..ea900326 100644 --- a/src/holisticai/explainability/metrics/tree/_tree.py +++ b/src/holisticai/explainability/metrics/tree/_tree.py @@ -1,4 +1,5 @@ import numpy as np + from holisticai.utils.surrogate_models import get_features, get_number_of_rules @@ -338,8 +339,7 @@ def __call__(self, tree): """ depths, _ = get_depths_counts(0, tree, [], []) mean_depth = np.mean(depths) - variance = np.mean((depths - mean_depth) ** 2) - return variance + return np.mean((depths - mean_depth) ** 2) def tree_depth_variance(tree): diff --git a/src/holisticai/explainability/plots/_feature_importance.py b/src/holisticai/explainability/plots/_feature_importance.py index 70a465dd..52a592f7 100644 --- a/src/holisticai/explainability/plots/_feature_importance.py +++ b/src/holisticai/explainability/plots/_feature_importance.py @@ -1,6 +1,11 @@ import numpy as np import pandas as pd import seaborn as sns +from matplotlib import patches +from matplotlib import pyplot as plt +from matplotlib.colors import LinearSegmentedColormap +from scipy.spatial.distance import jensenshannon + from holisticai.explainability.metrics.global_feature_importance import fluctuation_ratio from holisticai.explainability.metrics.local_feature_importance import ( compute_importance_distribution, @@ -9,10 +14,6 @@ rank_consistency, ) from holisticai.utils import Importances -from matplotlib import patches -from matplotlib import pyplot as plt -from matplotlib.colors import LinearSegmentedColormap -from scipy.spatial.distance import jensenshannon def plot_feature_importance(feature_importance: Importances, alpha=0.8, top_n=20, ax=None): diff --git a/src/holisticai/explainability/plots/_partial_dependence.py b/src/holisticai/explainability/plots/_partial_dependence.py index 62102ed9..4b1fae3b 100644 --- a/src/holisticai/explainability/plots/_partial_dependence.py +++ b/src/holisticai/explainability/plots/_partial_dependence.py @@ -1,11 +1,12 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +from matplotlib import cm +from scipy.interpolate import interp1d + from holisticai.explainability.metrics.global_feature_importance._fluctuation_ratio import fluctuation_ratio from holisticai.explainability.metrics.global_feature_importance._xai_ease_score import XAIEaseAnnotator from holisticai.utils import Importances, PartialDependence -from matplotlib import cm -from scipy.interpolate import interp1d def plot_partial_dependence( diff --git a/src/holisticai/explainability/plots/_partial_dependencies.py b/src/holisticai/explainability/plots/_partial_dependencies.py index 62102ed9..4b1fae3b 100644 --- a/src/holisticai/explainability/plots/_partial_dependencies.py +++ b/src/holisticai/explainability/plots/_partial_dependencies.py @@ -1,11 +1,12 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd +from matplotlib import cm +from scipy.interpolate import interp1d + from holisticai.explainability.metrics.global_feature_importance._fluctuation_ratio import fluctuation_ratio from holisticai.explainability.metrics.global_feature_importance._xai_ease_score import XAIEaseAnnotator from holisticai.utils import Importances, PartialDependence -from matplotlib import cm -from scipy.interpolate import interp1d def plot_partial_dependence( diff --git a/src/holisticai/explainability/plots/_tree.py b/src/holisticai/explainability/plots/_tree.py index 96f6b7ee..69814cd7 100644 --- a/src/holisticai/explainability/plots/_tree.py +++ b/src/holisticai/explainability/plots/_tree.py @@ -2,9 +2,10 @@ import matplotlib.pyplot as plt import numpy as np -from holisticai.utils import Importances from sklearn.tree._export import _MPLTreeExporter +from holisticai.utils import Importances + def _color_brew(n): """Generate n colors using the 'viridis' colormap. @@ -25,9 +26,11 @@ def _color_brew(n): for i in range(n): color = cmap(0.075 + 0.875 * i / n)[:3] # Get RGB values from cmap if color[0] > 1 or color[1] > 1 or color[2] > 1: - raise ValueError("Color values must be in the range [0, 1] 1") + msg = "Color values must be in the range [0, 1] 1" + raise ValueError(msg) if color[0] < 0 or color[1] < 0 or color[2] < 0: - raise ValueError("Color values must be in the range [0, 1] 0") + msg = "Color values must be in the range [0, 1] 0" + raise ValueError(msg) rgb = (int(color[0] * 255), int(color[1] * 255), int(color[2] * 255)) color_list.append(rgb) @@ -119,7 +122,8 @@ def plot_surrogate(feature_importance: Importances, ax=None, **kargs): """ if "surrogate" not in feature_importance.extra_attrs: - raise ValueError("Surrogate key does not exist in feature_importance.extra_attrs") + msg = "Surrogate key does not exist in feature_importance.extra_attrs" + raise ValueError(msg) if ax is None: _, ax = plt.subplots(1, 1, figsize=(30, 10)) diff --git a/src/holisticai/inspection/_lime.py b/src/holisticai/inspection/_lime.py index 0792ebce..b4103e12 100644 --- a/src/holisticai/inspection/_lime.py +++ b/src/holisticai/inspection/_lime.py @@ -1,7 +1,6 @@ from __future__ import annotations import warnings -from typing import Union import numpy as np import pandas as pd @@ -18,8 +17,8 @@ def compute_lime_feature_importance( X: pd.DataFrame, y: pd.Series, proxy: ModelProxy, - max_samples: Union[int, None] = None, - random_state: Union[RandomState, None] = None, + max_samples: int | None = None, + random_state: RandomState | None = None, ) -> LocalImportances: if random_state is None: random_state = RandomState(42) @@ -36,8 +35,7 @@ def compute_lime_feature_importance( # y_: pd.Series = pd.Series(ds["y"]) y_ = pd.Series(proxy.predict(ds["X"])) condition = group_mask_samples_by_learning_task(y_, proxy.learning_task) - local_importances = LocalImportances(data=data, cond=condition) - return local_importances + return LocalImportances(data=data, cond=condition) class LIMEImportanceCalculator: @@ -48,7 +46,8 @@ def initialize_explainer(self, X: pd.DataFrame, proxy: ModelProxy): import lime # type: ignore import lime.lime_tabular # type: ignore except ImportError: - raise ImportError("LIME is not installed. Please install it using 'pip install lime'") from None + msg = "LIME is not installed. Please install it using 'pip install lime'" + raise ImportError(msg) from None if proxy.learning_task == "regression": self.explainer = lime.lime_tabular.LimeTabularExplainer( @@ -72,7 +71,8 @@ def initialize_explainer(self, X: pd.DataFrame, proxy: ModelProxy): self.predict_function = proxy.predict_proba self.feature_names = list(X.columns) else: - raise ValueError("Learning task must be regression or classification") + msg = "Learning task must be regression or classification" + raise ValueError(msg) def compute_importances(self, ds: Dataset, proxy: ModelProxy) -> pd.DataFrame: X = pd.DataFrame(ds["X"].astype(np.float64)) diff --git a/src/holisticai/inspection/_partial_dependence.py b/src/holisticai/inspection/_partial_dependence.py index e768c058..46b102ea 100644 --- a/src/holisticai/inspection/_partial_dependence.py +++ b/src/holisticai/inspection/_partial_dependence.py @@ -1,9 +1,9 @@ from __future__ import annotations import numbers +from typing import TYPE_CHECKING import numpy as np -import pandas as pd from joblib import Parallel, delayed from sklearn.inspection import partial_dependence from sklearn.metrics import accuracy_score, r2_score @@ -11,6 +11,9 @@ from holisticai.utils._commons import get_columns from holisticai.utils._definitions import ModelProxy, PartialDependence +if TYPE_CHECKING: + import pandas as pd + def get_partial_dependence( estimator, @@ -180,7 +183,8 @@ def wrap_sklearn_model(proxy: ModelProxy): def compute_partial_dependence(X: pd.DataFrame, features: list[str], proxy: ModelProxy) -> PartialDependence: supported_learning_tasks = ["binary_classification", "regression", "multi_classification"] if proxy.learning_task not in supported_learning_tasks: - raise ValueError(f"Learning task {proxy.learning_task} is not supported for partial dependence computation") + msg = f"Learning task {proxy.learning_task} is not supported for partial dependence computation" + raise ValueError(msg) model = wrap_sklearn_model(proxy) feature_names = np.array(get_columns(X)) diff --git a/src/holisticai/inspection/_permutation_importance.py b/src/holisticai/inspection/_permutation_importance.py index 6a6c193c..4bb69896 100644 --- a/src/holisticai/inspection/_permutation_importance.py +++ b/src/holisticai/inspection/_permutation_importance.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Literal, Union, overload +from typing import Any, Literal, overload import numpy as np import pandas as pd @@ -29,7 +29,7 @@ def compute_permutation_importance( y: pd.Series, n_repeats: int = 5, n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, + random_state: RandomState | int | None = None, importance_type: Literal["conditional"] = "conditional", ) -> ConditionalImportances: ... @@ -41,7 +41,7 @@ def compute_permutation_importance( y: pd.Series, n_repeats: int = 5, n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, + random_state: RandomState | int | None = None, ) -> Importances: ... @@ -51,9 +51,9 @@ def compute_permutation_importance( y: pd.Series, n_repeats: int = 5, n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, + random_state: RandomState | int | None = None, importance_type: Literal["standard", "conditional"] = "standard", -) -> Union[Importances, ConditionalImportances]: +) -> Importances | ConditionalImportances: pfi = PermutationFeatureImportanceCalculator(n_repeats=n_repeats, n_jobs=n_jobs, random_state=random_state) if importance_type == "conditional": return compute_conditional_permutation_importance( @@ -68,8 +68,8 @@ def compute_conditional_permutation_importance( y: pd.Series, n_repeats: int = 5, n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, -) -> Union[Importances, ConditionalImportances]: + random_state: RandomState | int | None = None, +) -> Importances | ConditionalImportances: pfi = PermutationFeatureImportanceCalculator(n_repeats=n_repeats, n_jobs=n_jobs, random_state=random_state) sample_groups = group_index_samples_by_learning_task(y, proxy.learning_task) values = {} @@ -108,7 +108,7 @@ def __init__( self, n_repeats: int = 5, n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, + random_state: RandomState | int | None = None, importance_type: str = "global", ): if random_state is None: diff --git a/src/holisticai/inspection/_shap.py b/src/holisticai/inspection/_shap.py index 1ee90841..43997e5f 100644 --- a/src/holisticai/inspection/_shap.py +++ b/src/holisticai/inspection/_shap.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Union - import numpy as np import pandas as pd from numpy.random import RandomState @@ -14,8 +12,8 @@ def compute_shap_feature_importance( proxy: ModelProxy, X: pd.DataFrame, - max_samples: Union[int, None] = None, - random_state: Union[RandomState, None] = None, + max_samples: int | None = None, + random_state: RandomState | None = None, ) -> LocalImportances: if random_state is None: random_state = RandomState(42) @@ -57,7 +55,8 @@ def initialize_explainer(self, X: pd.DataFrame, proxy: ModelProxy): try: import shap # type: ignore except ImportError: - raise ImportError("SHAP is not installed. Please install it using 'pip install shap'") from None + msg = "SHAP is not installed. Please install it using 'pip install shap'" + raise ImportError(msg) from None self.explainer = shap.Explainer(proxy.predict, X[:100]) diff --git a/src/holisticai/inspection/_surrogate_importance.py b/src/holisticai/inspection/_surrogate_importance.py index 330068b3..a0494dd2 100644 --- a/src/holisticai/inspection/_surrogate_importance.py +++ b/src/holisticai/inspection/_surrogate_importance.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, Optional, Union, overload +from typing import Literal, overload import numpy as np import pandas as pd @@ -22,8 +22,8 @@ def compute_surrogate_feature_importance( proxy: ModelProxy, X: pd.DataFrame, - y: Optional[pd.Series] = None, - random_state: Optional[Union[RandomState, int]] = None, + y: pd.Series | None = None, + random_state: RandomState | int | None = None, ) -> Importances: ... @@ -31,8 +31,8 @@ def compute_surrogate_feature_importance( def compute_surrogate_feature_importance( proxy: ModelProxy, X: pd.DataFrame, - y: Optional[pd.Series] = None, - random_state: Union[RandomState, int, None] = None, + y: pd.Series | None = None, + random_state: RandomState | int | None = None, importance_type: Literal["conditional"] = "conditional", ) -> Importances: ... @@ -40,10 +40,10 @@ def compute_surrogate_feature_importance( def compute_surrogate_feature_importance( proxy: ModelProxy, X: pd.DataFrame, - y: Optional[pd.Series] = None, - random_state: Optional[Union[RandomState, int]] = None, + y: pd.Series | None = None, + random_state: RandomState | int | None = None, importance_type: Literal["conditional", "standard"] = "standard", -) -> Union[Importances, ConditionalImportances]: +) -> Importances | ConditionalImportances: pfi = SurrogateFeatureImportanceCalculator(random_state=random_state) if importance_type == "conditional": y = pd.Series(proxy.predict(X)) if y is None else y @@ -59,7 +59,7 @@ def compute_surrogate_feature_importance( class SurrogateFeatureImportanceCalculator: def __init__( self, - random_state: Union[RandomState, int, None] = None, + random_state: RandomState | int | None = None, importance_type: str = "global", ): if random_state is None: @@ -92,7 +92,8 @@ def create_surrogate_model(self, X: pd.DataFrame, y: pd.Series, learning_task: s best_tree = tree if best_tree is None: - raise ValueError(f"Surrogate model could not be created for learning task {learning_task}") + msg = f"Surrogate model could not be created for learning task {learning_task}" + raise ValueError(msg) return best_tree diff --git a/src/holisticai/inspection/_utils.py b/src/holisticai/inspection/_utils.py index e7d1243d..6221b4bf 100644 --- a/src/holisticai/inspection/_utils.py +++ b/src/holisticai/inspection/_utils.py @@ -1,7 +1,11 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np -import pandas as pd + +if TYPE_CHECKING: + import pandas as pd def quantil_classify(q1, q2, q3, labels, x): @@ -33,4 +37,5 @@ def group_mask_samples_by_learning_task( v = np.array(y.quantile(labels_values)).squeeze() return y.map(lambda x: quantil_classify(v[1], v[2], v[3], labels, x)) - raise ValueError(f"Learning task {learning_task} not supported") + msg = f"Learning task {learning_task} not supported" + raise ValueError(msg) diff --git a/src/holisticai/robustness/__init__.py b/src/holisticai/robustness/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/holisticai/robustness/attackers/classification/commons.py b/src/holisticai/robustness/attackers/classification/commons.py index d9385ca0..895851f2 100644 --- a/src/holisticai/robustness/attackers/classification/commons.py +++ b/src/holisticai/robustness/attackers/classification/commons.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Optional, Union - import numpy as np import pandas as pd @@ -14,7 +12,7 @@ def x_to_nd_array(x: pd.DataFrame): return np.array(x) -def to_categorical(labels: Union[np.ndarray, list[float]], nb_classes: Optional[int] = None) -> np.ndarray: +def to_categorical(labels: np.ndarray | list[float], nb_classes: int | None = None) -> np.ndarray: """ Convert an array of labels to binary class matrix. diff --git a/src/holisticai/robustness/attackers/classification/hop_skip_jump.py b/src/holisticai/robustness/attackers/classification/hop_skip_jump.py index 317fe971..8e4c244f 100644 --- a/src/holisticai/robustness/attackers/classification/hop_skip_jump.py +++ b/src/holisticai/robustness/attackers/classification/hop_skip_jump.py @@ -7,12 +7,15 @@ from __future__ import annotations -from typing import Optional, Union +from typing import TYPE_CHECKING import numpy as np -import pandas as pd + from holisticai.robustness.attackers.classification.commons import x_array_to_df, x_to_nd_array +if TYPE_CHECKING: + import pandas as pd + class HopSkipJump: """ @@ -102,7 +105,7 @@ def predict(self, x: np.ndarray): return np.array(self.predictor(x_df)) def generate( - self, x_df: pd.DataFrame, y: Optional[np.ndarray] = None, mask: Optional[np.ndarray] = None, x_adv_init=None + self, x_df: pd.DataFrame, y: np.ndarray | None = None, mask: np.ndarray | None = None, x_adv_init=None ) -> pd.DataFrame: """ Generate adversarial samples and return them in an array. @@ -137,7 +140,8 @@ def generate( if y is None: # Throw error if attack is targeted, but no targets are provided if self.targeted: # pragma: no cover - raise ValueError("Target labels `y` need to be provided for a targeted attack.") + msg = "Target labels `y` need to be provided for a targeted attack." + raise ValueError(msg) # Use model predictions as correct outputs y = self.predict(x) @@ -210,7 +214,7 @@ def _perturb( y_p: int, init_pred: int, adv_init: np.ndarray, - mask: Optional[np.ndarray], + mask: np.ndarray | None, ) -> np.ndarray: """ Internal attack function for one example. @@ -244,9 +248,7 @@ def _perturb( return x # If an initial adversarial example found, then go with HopSkipJump attack - x_adv = self._attack(initial_sample[0], x, initial_sample[1], mask) - - return x_adv + return self._attack(initial_sample[0], x, initial_sample[1], mask) def _init_sample( self, @@ -255,8 +257,8 @@ def _init_sample( y_p: int, init_pred: int, adv_init: np.ndarray, - mask: Optional[np.ndarray], - ) -> Optional[Union[np.ndarray, tuple[np.ndarray, int]]]: + mask: np.ndarray | None, + ) -> np.ndarray | tuple[np.ndarray, int] | None: """ Find initial adversarial example for the attack. @@ -349,7 +351,7 @@ def _attack( initial_sample: np.ndarray, original_sample: np.ndarray, target: int, - mask: Optional[np.ndarray], + mask: np.ndarray | None, ) -> np.ndarray: """ Main function for the boundary attack. @@ -435,8 +437,8 @@ def _binary_search( current_sample: np.ndarray, original_sample: np.ndarray, target: int, - norm: Union[int, float, str], # noqa: PYI041 - threshold: Optional[float] = None, + norm: int | float | str, # noqa: PYI041 + threshold: float | None = None, ) -> np.ndarray: """ Binary search to approach the boundary. @@ -494,15 +496,13 @@ def _binary_search( lower_bound = np.where(satisfied == 0, alpha, lower_bound) upper_bound = np.where(satisfied == 1, alpha, upper_bound) - result = self._interpolate( + return self._interpolate( current_sample=current_sample, original_sample=original_sample, alpha=float(upper_bound), norm=norm, ) - return result - def _compute_delta( self, current_sample: np.ndarray, @@ -543,7 +543,7 @@ def _compute_update( num_eval: int, delta: float, target: int, - mask: Optional[np.ndarray], + mask: np.ndarray | None, ) -> np.ndarray: """ Compute the update in Eq.(14). @@ -603,9 +603,7 @@ def _compute_update( grad = np.mean(f_val * rnd_noise, axis=0) # Compute update - result = grad / np.linalg.norm(grad) if self.norm == 2 else np.sign(grad) - - return result + return grad / np.linalg.norm(grad) if self.norm == 2 else np.sign(grad) def _adversarial_satisfactory(self, samples: np.ndarray, target: int) -> np.ndarray: """ @@ -626,16 +624,14 @@ def _adversarial_satisfactory(self, samples: np.ndarray, target: int) -> np.ndar samples = np.clip(samples, self._clip_min, self._clip_max) preds = self.predict(samples) - result = preds == target if self.targeted else preds != target - - return result + return preds == target if self.targeted else preds != target @staticmethod def _interpolate( current_sample: np.ndarray, original_sample: np.ndarray, alpha: float, - norm: Union[int, float, str], # noqa: PYI041 + norm: int | float | str, # noqa: PYI041 ) -> np.ndarray: """ Interpolate a new sample based on the original and the current samples. diff --git a/src/holisticai/robustness/attackers/classification/zeroth_order_optimization.py b/src/holisticai/robustness/attackers/classification/zeroth_order_optimization.py index 274c5795..cb67ba5f 100644 --- a/src/holisticai/robustness/attackers/classification/zeroth_order_optimization.py +++ b/src/holisticai/robustness/attackers/classification/zeroth_order_optimization.py @@ -8,17 +8,20 @@ from __future__ import annotations -from typing import Any, Optional +from typing import TYPE_CHECKING, Any import numpy as np -import pandas as pd +from scipy.ndimage import zoom + from holisticai.robustness.attackers.classification.commons import ( format_function_predict_proba, to_categorical, x_array_to_df, x_to_nd_array, ) -from scipy.ndimage import zoom + +if TYPE_CHECKING: + import pandas as pd BATCH_SIZE = 1 @@ -132,10 +135,11 @@ def _initialize_vars(self, x: np.ndarray) -> None: if len(self.input_shape) == 1: self.input_is_feature_vector = True if self.batch_size != 1: - raise ValueError( + msg = ( "The current implementation of Zeroth-Order Optimisation attack only supports " "`batch_size=1` with feature vectors as input." ) + raise ValueError(msg) else: self.input_is_feature_vector = False @@ -196,7 +200,7 @@ def _loss( return preds, l2dist, c_weight * loss + l2dist - def generate(self, x_df: pd.DataFrame, y: Optional[np.ndarray] = None) -> pd.DataFrame: + def generate(self, x_df: pd.DataFrame, y: np.ndarray | None = None) -> pd.DataFrame: """ Generate adversarial samples and return them in an array. @@ -225,16 +229,16 @@ def generate(self, x_df: pd.DataFrame, y: Optional[np.ndarray] = None) -> pd.Dat # Check that `y` is provided for targeted attacks if self.targeted and y is None: # pragma: no cover - raise ValueError("Target labels `y` need to be provided for a targeted attack.") + msg = "Target labels `y` need to be provided for a targeted attack." + raise ValueError(msg) # No labels provided, use model prediction as correct class if y is None: y = self.predict_proba(x) if self.nb_classes == 2 and y.shape[1] == 1: # pragma: no cover - raise ValueError( - "This attack has not yet been tested for binary classification with a single output classifier." - ) + msg = "This attack has not yet been tested for binary classification with a single output classifier." + raise ValueError(msg) # Compute adversarial examples with implicit batching nb_batches = int(np.ceil(x.shape[0] / float(self.batch_size))) @@ -526,10 +530,11 @@ def _optimizer(self, x: np.ndarray, targets: np.ndarray, c_batch: np.ndarray) -> ) except ValueError as error: # pragma: no cover if "Cannot take a larger sample than population when 'replace=False'" in str(error): - raise ValueError( + msg = ( "Too many samples are requested for the random indices. Try to reduce the number of parallel" "coordinate updates `nb_parallel`." - ) from error + ) + raise ValueError(msg) from error raise @@ -560,7 +565,8 @@ def _optimizer(self, x: np.ndarray, targets: np.ndarray, c_batch: np.ndarray) -> True, ) else: - raise ValueError("Unexpected `None` in `adam_mean`, `adam_var` or `adam_epochs` detected.") + msg = "Unexpected `None` in `adam_mean`, `adam_var` or `adam_epochs` detected." + raise ValueError(msg) if self.use_importance and self._current_noise.shape[2] > self._init_size: self._sample_prob = self._get_prob(self._current_noise).flatten() @@ -626,7 +632,7 @@ def _optimizer_adam_coordinate( return current_noise.reshape(orig_shape) - def _reset_adam(self, nb_vars: int, indices: Optional[np.ndarray] = None) -> None: + def _reset_adam(self, nb_vars: int, indices: np.ndarray | None = None) -> None: """ Reset the ADAM optimizer. diff --git a/src/holisticai/robustness/attackers/regression/gb_attackers.py b/src/holisticai/robustness/attackers/regression/gb_attackers.py index 3d661f81..09bbb59b 100644 --- a/src/holisticai/robustness/attackers/regression/gb_attackers.py +++ b/src/holisticai/robustness/attackers/regression/gb_attackers.py @@ -1,8 +1,9 @@ import numpy as np import numpy.linalg as la -from holisticai.robustness.attackers.regression.gb_base import GDPoisoner from sklearn import linear_model +from holisticai.robustness.attackers.regression.gb_base import GDPoisoner + class LinRegGDPoisoner(GDPoisoner): """ @@ -121,8 +122,7 @@ def _compute_sigma(self): The covariance matrix. """ sigma = np.dot(np.transpose(self.trnx), self.trnx) - sigma = sigma / self.trnx.shape[0] - return sigma + return sigma / self.trnx.shape[0] def _compute_mu(self): """ @@ -133,8 +133,7 @@ def _compute_mu(self): array-like, shape (n_features,) The mean of the input samples. """ - mu = np.mean(np.matrix(self.trnx), axis=0) - return mu + return np.mean(np.matrix(self.trnx), axis=0) def _compute_m(self, clf, poisxelem, poisyelem): """ @@ -159,8 +158,7 @@ def _compute_m(self, clf, poisxelem, poisyelem): wtransp = np.reshape(w, (1, self.feanum)) errterm = (np.dot(w, poisxelemtransp) + b - poisyelem).reshape((1, 1)) first = np.dot(poisxelemtransp, wtransp) - m = first + errterm[0, 0] * np.identity(self.feanum) - return m + return first + errterm[0, 0] * np.identity(self.feanum) def _compute_wb_zc(self, eq7lhs, mu, w, m, n, poisxelem): # noqa: ARG002 """ @@ -220,8 +218,7 @@ def _compute_r(self, clf, lam): # noqa: ARG002 array-like, shape (1, n_features) The regularization term. """ - r = np.zeros((1, self.feanum)) - return r + return np.zeros((1, self.feanum)) def _comp_obj_trn(self, clf, lam, otherargs): # noqa: ARG002 """ @@ -242,9 +239,7 @@ def _comp_obj_trn(self, clf, lam, otherargs): # noqa: ARG002 The objective value. """ errs = clf.predict(np.asarray(self.trnx)) - self.trny - mse = np.linalg.norm(errs) ** 2 / self.samplenum - - return mse + return np.linalg.norm(errs) ** 2 / self.samplenum def _comp_attack_trn(self, clf, wxc, bxc, wyc, byc, otherargs): # noqa: ARG002 """ @@ -458,8 +453,7 @@ def _compute_sigma(self): """ basesigma = np.dot(np.transpose(self.trnx), self.trnx) basesigma = basesigma / self.trnx.shape[0] - sigma = basesigma + self.initlam * np.eye(self.feanum) - return sigma + return basesigma + self.initlam * np.eye(self.feanum) def learn_model(self, x, y, clf, lam=None): """ @@ -495,8 +489,7 @@ def _compute_mu(self): array-like, shape (n_features,) The mean of the input samples. """ - mu = np.mean(np.matrix(self.trnx), axis=0) - return mu + return np.mean(np.matrix(self.trnx), axis=0) def _compute_m(self, clf, poisxelem, poisyelem): """ @@ -521,8 +514,7 @@ def _compute_m(self, clf, poisxelem, poisyelem): wtransp = np.reshape(w, (1, self.feanum)) errterm = (np.dot(w, poisxelemtransp) + b - poisyelem).reshape((1, 1)) first = np.dot(poisxelemtransp, wtransp) - m = first + errterm[0, 0] * np.identity(self.feanum) - return m + return first + errterm[0, 0] * np.identity(self.feanum) def _compute_wb_zc(self, eq7lhs, mu, w, m, n, poisxelem): # noqa: ARG002 """ diff --git a/src/holisticai/robustness/attackers/regression/gb_base.py b/src/holisticai/robustness/attackers/regression/gb_base.py index 093f1256..ac937300 100644 --- a/src/holisticai/robustness/attackers/regression/gb_base.py +++ b/src/holisticai/robustness/attackers/regression/gb_base.py @@ -2,11 +2,12 @@ import numpy as np import pandas as pd -from holisticai.robustness.attackers.regression.initializers import adaptive, inf_flip, randflip, randflipnobd -from holisticai.robustness.attackers.regression.utils import one_hot_encode_columns, revert_one_hot_encoding from sklearn.linear_model import Ridge from sklearn.metrics import mean_squared_error +from holisticai.robustness.attackers.regression.initializers import adaptive, inf_flip, randflip, randflipnobd +from holisticai.robustness.attackers.regression.utils import one_hot_encode_columns, revert_one_hot_encoding + logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger() diff --git a/src/holisticai/robustness/metrics/classification/_binary_classification.py b/src/holisticai/robustness/metrics/classification/_binary_classification.py index 8a9a215b..45a83d81 100644 --- a/src/holisticai/robustness/metrics/classification/_binary_classification.py +++ b/src/holisticai/robustness/metrics/classification/_binary_classification.py @@ -36,7 +36,8 @@ def adversarial_accuracy(y, y_pred, y_adv_pred): y_pred = np.array(y_pred) y_adv_pred = np.array(y_adv_pred) except Exception as e: - raise ValueError("One or more inputs could not be converted to a numpy array.") from e + msg = "One or more inputs could not be converted to a numpy array." + raise ValueError(msg) from e if y is None: idxs = y_pred == y_adv_pred @@ -91,11 +92,13 @@ def empirical_robustness( y_pred = np.array(y_pred) y_adv_pred = np.array(y_adv_pred) except Exception as e: - raise ValueError("One or more inputs could not be converted to a numpy array.") from e + msg = "One or more inputs could not be converted to a numpy array." + raise ValueError(msg) from e # Verify that `norm` has the correct type if not isinstance(norm, int): - raise TypeError("norm must be of type int") + msg = "norm must be of type int" + raise TypeError(msg) idxs = y_adv_pred != y_pred if np.sum(idxs) == 0.0: diff --git a/src/holisticai/robustness/metrics/dataset_shift/_accuracy_degradation_profile.py b/src/holisticai/robustness/metrics/dataset_shift/_accuracy_degradation_profile.py index 04df48b8..5981fae0 100644 --- a/src/holisticai/robustness/metrics/dataset_shift/_accuracy_degradation_profile.py +++ b/src/holisticai/robustness/metrics/dataset_shift/_accuracy_degradation_profile.py @@ -27,7 +27,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any import numpy as np import pandas as pd @@ -91,8 +91,8 @@ def accuracy_degradation_profile( y_test: pd.Series, y_pred: pd.Series, n_neighbors: int = 5, - neighbor_estimator: Optional[Any] = None, - baseline_accuracy: Optional[float] = None, + neighbor_estimator: Any | None = None, + baseline_accuracy: float | None = None, threshold_percentual: float = 0.95, above_percentual: float = 0.90, step_size: float = STEP_SIZE, @@ -184,7 +184,8 @@ def accuracy_degradation_profile( # Check if the step size is too small if step_size < (1 / X_test.shape[0]): - raise ValueError("'step_size' is too small (less than 1 divided by the number of samples).") + msg = "'step_size' is too small (less than 1 divided by the number of samples)." + raise ValueError(msg) # Validate inputs if isinstance(X_test, pd.DataFrame): @@ -212,9 +213,7 @@ def accuracy_degradation_profile( results_summary_df = _summarize_results(results_df, baseline_accuracy, threshold_percentual, above_percentual) # Apply styling - styled_df = _styled_results(results_summary_df) - - return styled_df + return _styled_results(results_summary_df) def _validate_inputs( @@ -287,13 +286,17 @@ def _validate_inputs( """ if len(X_test) != len(y_test) or len(y_test) != len(y_pred): - raise ValueError("X_test, y_test, and y_pred must have the same length.") + msg = "X_test, y_test, and y_pred must have the same length." + raise ValueError(msg) if not (0 < threshold_percentual <= 1): - raise ValueError("threshold_percentual must be between 0 and 1.") + msg = "threshold_percentual must be between 0 and 1." + raise ValueError(msg) if not (0 < above_percentual <= 1): - raise ValueError("above_percentual must be between 0 and 1.") + msg = "above_percentual must be between 0 and 1." + raise ValueError(msg) if not (0 < baseline_accuracy <= 1): - raise ValueError("baseline_accuracy must be between 0 and 1.") + msg = "baseline_accuracy must be between 0 and 1." + raise ValueError(msg) def batched(X, batch_size=100): @@ -368,7 +371,8 @@ def _calculate_accuracies( # Validate step_size input if not (0 < step_size <= 1): - raise ValueError("step_size must be between 0 and 1.") + msg = "step_size must be between 0 and 1." + raise ValueError(msg) # Auxiliary data structures full_set_size = X_test.shape[0] @@ -565,5 +569,4 @@ def _styled_results(results_summary_df: pd.DataFrame, decision_column: str = DEC pd.DataFrame DataFrame with color-coded decisions. """ - styled_df = results_summary_df.style.map(_color_cells, subset=[decision_column]) - return styled_df + return results_summary_df.style.map(_color_cells, subset=[decision_column]) diff --git a/src/holisticai/robustness/plots/_dataset_shift.py b/src/holisticai/robustness/plots/_dataset_shift.py index 037819c3..c9820381 100644 --- a/src/holisticai/robustness/plots/_dataset_shift.py +++ b/src/holisticai/robustness/plots/_dataset_shift.py @@ -71,7 +71,8 @@ def _validate_and_extract_data(data): # Check if data is a pandas DataFrame if isinstance(data, pd.DataFrame): if data.shape[1] != 2: - raise ValueError("Data should have exactly two columns.") + msg = "Data should have exactly two columns." + raise ValueError(msg) data_vals = data.values # Convert DataFrame to NumPy array # Check if data is a pandas Series elif isinstance(data, pd.Series): @@ -82,7 +83,8 @@ def _validate_and_extract_data(data): # Raise error if the data is not a DataFrame or NumPy array else: - raise TypeError("Data must be either a pandas DataFrame or a NumPy array.") + msg = "Data must be either a pandas DataFrame or a NumPy array." + raise TypeError(msg) return data_vals @@ -212,7 +214,8 @@ def plot_2d(X, y, highlight_group=None, show_just_group=None, features_to_plot=N # If highlight_group is provided, outline the selected points if highlight_group is not None: if not isinstance(highlight_group, (np.ndarray, list)): - raise TypeError("highlight_group must be either a list or a NumPy array.") + msg = "highlight_group must be either a list or a NumPy array." + raise TypeError(msg) plt.scatter( X_vals[highlight_group, 0], @@ -485,7 +488,8 @@ def plot_neighborhood( for sample_index in points_of_interest: if sample_index not in indices_show: - raise ValueError(f"The point {sample_index} is not a point in 'indices_show'.") + msg = f"The point {sample_index} is not a point in 'indices_show'." + raise ValueError(msg) # Find the position of sample_index in indices_show position = np.where(indices_show == sample_index)[0] diff --git a/src/holisticai/robustness/plots/_utils.py b/src/holisticai/robustness/plots/_utils.py index 79213bf4..930d0212 100644 --- a/src/holisticai/robustness/plots/_utils.py +++ b/src/holisticai/robustness/plots/_utils.py @@ -55,18 +55,22 @@ def prepare_to_plot( # Check if X is a DataFrame and y is a NumPy array if not isinstance(X, pd.DataFrame): - raise TypeError("X must be a pandas DataFrame.") + msg = "X must be a pandas DataFrame." + raise TypeError(msg) if not isinstance(y, np.ndarray): - raise TypeError("y must be a NumPy array.") + msg = "y must be a NumPy array." + raise TypeError(msg) # Check if all the features are present in X missing_features = [feature for feature in features_to_plot if feature not in X.columns] if missing_features: - raise KeyError(f"The following features are missing from X: {', '.join(missing_features)}") + msg = f"The following features are missing from X: {', '.join(missing_features)}" + raise KeyError(msg) # Check if the length of X and y match if len(X) != len(y): - raise ValueError("The length of X and y must match.") + msg = "The length of X and y must match." + raise ValueError(msg) # Select only the columns for the chosen features to plot X_selected = X[features_to_plot] diff --git a/src/holisticai/security/__init__.py b/src/holisticai/security/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/holisticai/security/attackers/__init__.py b/src/holisticai/security/attackers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/holisticai/security/attackers/attribute_inference/__init__.py b/src/holisticai/security/attackers/attribute_inference/__init__.py new file mode 100644 index 00000000..9212499f --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/__init__.py @@ -0,0 +1,9 @@ +from holisticai.security.attackers.attribute_inference.baseline import AttributeInferenceBaseline +from holisticai.security.attackers.attribute_inference.black_box import AttributeInferenceBlackBox +from holisticai.security.attackers.attribute_inference.white_box_lifestyle import AttributeInferenceWhiteBoxLifestyleDecisionTree +from holisticai.security.attackers.attribute_inference.white_box import AttributeInferenceWhiteBoxDecisionTree + +__all__ = ["AttributeInferenceBaseline", + "AttributeInferenceBlackBox", + "AttributeInferenceWhiteBoxDecisionTree", + "AttributeInferenceWhiteBoxLifestyleDecisionTree"] diff --git a/src/holisticai/security/attackers/attribute_inference/baseline.py b/src/holisticai/security/attackers/attribute_inference/baseline.py new file mode 100644 index 00000000..35d9fd23 --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/baseline.py @@ -0,0 +1,88 @@ +import numpy as np +from holisticai.security.attackers.attribute_inference.dataset_utils import AttributeInferenceDataPreprocessor +from holisticai.security.attackers.attribute_inference.utils import get_attack_model, get_feature_index + + +class AttributeInferenceBaseline(): + """ + Attribute Inference Attack using a baseline model. + + This attack uses a baseline model to infer the value of a specific feature in the dataset. + The attack model is trained on the remaining features in the dataset. + + Parameters + ---------- + attack_model_type : str, default="nn" + The type of model to use for the attack. Options are "nn" for neural network or "tree" for decision tree. + attack_feature : int or slice, default=0 + The index or slice of the feature to attack. If a slice is provided, it should be of size 1. + """ + def __init__( + self, + attack_model_type="nn", + attack_feature=0, + ): + self._values = None + self._nb_classes = None + + self.attack_model = get_attack_model(attack_model_type) + self.attack_feature = attack_feature + + self._check_params() + self.attack_feature = get_feature_index(self.attack_feature) + self.ai_preprocessor = AttributeInferenceDataPreprocessor(attack_feature=attack_feature) + + def _check_params(self) -> None: + if not isinstance(self.attack_feature, int) and not isinstance(self.attack_feature, slice): + raise TypeError("Attack feature must be either an integer or a slice object.") + + if isinstance(self.attack_feature, int) and self.attack_feature < 0: + raise ValueError("Attack feature index must be non-negative.") + + def fit(self, x: np.ndarray) -> None: + """ + Train the attack model. + + Parameters + ---------- + x : np.ndarray + Input to training process. Includes all features used to train the original model. + """ + # train attack model + attack_x, attack_y = self.ai_preprocessor.fit_transform(x) + self._values = self.ai_preprocessor._values # noqa: SLF001 + self.attack_model.fit(attack_x, attack_y) + + def infer(self, x: np.ndarray, **kwargs) -> np.ndarray: + """ + Infer the attacked feature. + + Parameters + ---------- + x : np.ndarray + Input to attack. Includes all features except the attacked feature. + values : list + Possible values for attacked feature. For a single column feature this should be a simple list containing + all possible values, in increasing order (the smallest value in the 0 index and so on). For a multi-column + feature (for example 1-hot encoded and then scaled), this should be a list of lists, where each internal + list represents a column (in increasing order) and the values represent the possible values for that column + (in increasing order). + + Returns + ------- + np.ndarray + The inferred feature values. + """ + # if values are provided, override the values computed in fit() + values = kwargs.get("values", self._values) + attack_x = self.ai_preprocessor.transform(x) + predictions = self.attack_model.predict_proba(attack_x).astype(np.float32) + + if values is not None: + if isinstance(self.attack_feature, int): + predictions = np.array([values[np.argmax(arr)] for arr in predictions]) + else: + for value, column in zip(values, predictions.T): + for index in range(len(value)): + np.place(column, [column == index], value[index]) + return np.array(predictions) \ No newline at end of file diff --git a/src/holisticai/security/attackers/attribute_inference/black_box.py b/src/holisticai/security/attackers/attribute_inference/black_box.py new file mode 100644 index 00000000..9b775738 --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/black_box.py @@ -0,0 +1,146 @@ +""" +This module implements attribute inference attacks. +""" + +from __future__ import annotations + +import logging +from typing import Optional, Union + +import numpy as np +from holisticai.security.attackers.attribute_inference.dataset_utils import AttributeInferenceDataPreprocessor +from holisticai.security.attackers.attribute_inference.utils import get_attack_model, get_feature_index + +logger = logging.getLogger(__name__) + + +class AttributeInferenceBlackBox(): + """ + Implementation of a simple black-box attribute inference attack. + + The idea is to train a simple neural network to learn the attacked feature from the rest of the features and the + model's predictions. Assumes the availability of the attacked model's predictions for the samples under attack, + in addition to the rest of the feature values. If this is not available, the true class label of the samples may be + used as a proxy. + + Parameters + ---------- + estimator : object + Target estimator. + attack_model_type : str + The type of default attack model to train, optional. Should be one of `nn` (for neural network, default) or `rf` + (for random forest). If `attack_model` is supplied, this option will be ignored. + attack_model : object + The attack model to train, optional. If none is provided, a default model will be created. + attack_feature : int or slice + The index of the feature to be attacked or a slice representing multiple indexes in case of a one-hot encoded + feature. + scale_range : tuple + If supplied, the class labels (both true and predicted) will be scaled to the given range. Only applicable when + `estimator` is a regressor. + prediction_normal_factor : float + If supplied, the class labels (both true and predicted) are multiplied by the factor when used as inputs to the + attack-model. Only applicable when `estimator` is a regressor and if `scale_range` is not supplied. + """ + + def __init__( + self, + estimator, + attack_model_type: str = "nn", + attack_model=None, + attack_feature: Union[int, slice] = 0, + scale_range: Optional[tuple[float, float]] = None, + prediction_normal_factor: Optional[float] = 1, + ): + + self._values: Optional[list] = None + self._nb_classes: Optional[int] = None + self._attack_model_type = attack_model_type + self._attack_model = attack_model + self.estimator = estimator + self.attack_feature = attack_feature + self.attack_model = get_attack_model(attack_model_type) + + self.prediction_normal_factor = prediction_normal_factor + self.scale_range = scale_range + self._check_params() + self.attack_feature = get_feature_index(self.attack_feature) + self.ai_preprocessor = AttributeInferenceDataPreprocessor( + scale_range=scale_range, + prediction_normal_factor=prediction_normal_factor, + attack_feature=attack_feature, + ) + + def fit(self, x: np.ndarray, y: Optional[np.ndarray] = None, pred: Optional[np.ndarray] = None) -> None: + """ + Train the attack model. + + Parameters + ---------- + x : np.ndarray + Input to training process. Includes all features used to train the original model. + y : np.ndarray + True labels for x. + pred : np.ndarray + Predictions of the original model for x. + """ + + attack_x, attack_y = self.ai_preprocessor.fit_transform(x, y, pred) + self._values = self.ai_preprocessor._values # noqa: SLF001 + self.attack_model.fit(attack_x, attack_y) + + def infer(self, x: np.ndarray, y: np.ndarray, pred: np.ndarray, **kwargs) -> np.ndarray: + """ + Infer the attacked feature. + + Parameters + ---------- + x : np.ndarray + Input to attack. Includes all features except the attacked feature. + y : np.ndarray + True labels for x. + pred : np.ndarray + Original model's predictions for x. + values : list + Possible values for attacked feature. For a single column feature this should be a simple list + containing all possible values, in increasing order (the smallest value in the 0 index and so + on). For a multi-column feature (for example 1-hot encoded and then scaled), this should be a + list of lists, where each internal list represents a column (in increasing order) and the values + represent the possible values for that column (in increasing order). If not provided, is + computed from the training data when calling `fit`. + + Returns + ------- + np.ndarray + The inferred feature values. + """ + + values: Optional[list] = kwargs.get("values") + + # if provided, override the values computed in fit() + if values is not None: + self._values = values + + attack_x = self.ai_preprocessor.transform(x, y, pred) + predictions = self.attack_model.predict_proba(attack_x).astype(np.float32) + + if self._values is not None: + if isinstance(self.attack_feature, int): + predictions = np.array([self._values[np.argmax(arr)] for arr in predictions]) + else: + i = 0 + for column in predictions.T: + for index in range(len(self._values[i])): + np.place(column, [column == index], self._values[i][index]) + i += 1 + return np.array(predictions) + + def _check_params(self) -> None: + if not isinstance(self.attack_feature, int) and not isinstance(self.attack_feature, slice): + raise TypeError("Attack feature must be either an integer or a slice object.") + + if isinstance(self.attack_feature, int) and self.attack_feature < 0: + raise ValueError("Attack feature index must be positive.") + + if self._attack_model_type not in ["nn", "rf"]: + raise ValueError("Illegal value for parameter `attack_model_type`.") diff --git a/src/holisticai/security/attackers/attribute_inference/dataset_utils.py b/src/holisticai/security/attackers/attribute_inference/dataset_utils.py new file mode 100644 index 00000000..fec1696f --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/dataset_utils.py @@ -0,0 +1,379 @@ +from __future__ import annotations + +from typing import Optional + +import numpy as np +from sklearn.preprocessing import minmax_scale + + +def to_categorical(labels, nb_classes) -> np.ndarray: + """ + Convert an array of labels to binary class matrix. + + Parameters + ---------- + labels : np.ndarray + An array of integer labels of shape `(nb_samples,)`. + nb_classes : int + The number of classes (possible labels). + + Returns + ------- + np.ndarray + A binary matrix representation of `y` in the shape `(nb_samples, nb_classes)`. + """ + labels = np.array(labels, dtype=int) + if nb_classes is None: + nb_classes = np.max(labels) + 1 + categorical = np.zeros((labels.shape[0], nb_classes), dtype=np.float32) + categorical[np.arange(labels.shape[0]), np.squeeze(labels)] = 1 + return categorical + + +def check_and_transform_label_format( + labels: np.ndarray, nb_classes: Optional[int], return_one_hot: bool = True +) -> np.ndarray: + """ + Check label format and transform to one-hot-encoded labels if necessary + + Parameters + ---------- + labels : np.ndarray + An array of integer labels of shape `(nb_samples,)`, `(nb_samples, 1)` or `(nb_samples, nb_classes)`. + nb_classes : int + The number of classes. If None the number of classes is determined automatically. + return_one_hot : bool + True if returning one-hot encoded labels, False if returning index labels. + + Returns + ------- + np.ndarray + Labels with shape `(nb_samples, nb_classes)` (one-hot) or `(nb_samples,)` (index). + """ + labels_return = labels + + if len(labels.shape) == 2 and labels.shape[1] > 1: # multi-class, one-hot encoded + if not return_one_hot: + labels_return = np.argmax(labels, axis=1) + labels_return = np.expand_dims(labels_return, axis=1) + elif len(labels.shape) == 2 and labels.shape[1] == 1: + if nb_classes is None: + nb_classes = np.max(labels) + 1 + if nb_classes > 2: # multi-class, index labels + labels_return = to_categorical(labels, nb_classes) if return_one_hot else np.expand_dims(labels, axis=1) + elif nb_classes == 2: # binary, index labels + if return_one_hot: + labels_return = to_categorical(labels, nb_classes) + else: + raise ValueError( + "Shape of labels not recognised." + "Please provide labels in shape (nb_samples,) or (nb_samples, " + "nb_classes)" + ) + elif len(labels.shape) == 1: # index labels + labels_return = to_categorical(labels, nb_classes) if return_one_hot else np.expand_dims(labels, axis=1) + else: + raise ValueError( + "Shape of labels not recognised." + "Please provide labels in shape (nb_samples,) or (nb_samples, " + "nb_classes)" + ) + + return labels_return + + +def get_feature_values(x: np.ndarray, single_index_feature: bool) -> list: + """ + Returns a list of unique values of a given feature. + + Parameters + ---------- + x : np.ndarray + The feature column(s). + single_index_feature : bool + Bool representing whether this is a single-column or multiple-column feature (for + example 1-hot encoded and then scaled). + + Returns + ------- + list + For a single-column feature, a simple list containing all possible values, in increasing order. + For a multi-column feature, a list of lists, where each internal list represents a column and the values + represent the possible values for that column (in increasing order). + """ + values = None + if single_index_feature: + values = np.unique(x).tolist() + else: + for column in x.T: + column_values = np.unique(column) + values = column_values if values is None else np.vstack((values, column_values)) + if values is not None: + values = values.tolist() + return values + + +def floats_to_one_hot(labels: np.ndarray): + """ + Convert a 2D-array of floating point labels to binary class matrix. + + Parameters + ---------- + labels : np.ndarray + A 2D-array of floating point labels of shape `(nb_samples, nb_classes)`. + + Returns + ------- + np.ndarray + A binary matrix representation of `labels` in the shape `(nb_samples, nb_classes)`. + """ + labels = np.array(labels) + for feature in labels.T: # pylint: disable=E1133 + unique = np.unique(feature) + unique.sort() + for index, value in enumerate(unique): + feature[feature == value] = index + return labels.astype(np.float32) + + +def float_to_categorical(labels: np.ndarray, nb_classes: Optional[int] = None): + """ + Convert an array of floating point labels to binary class matrix. + + Parameters + ---------- + labels : np.ndarray + An array of floating point labels of shape `(nb_samples,)`. + nb_classes : int + The number of classes (possible labels). + + Returns + ------- + np.ndarray + A binary matrix representation of `labels` in the shape `(nb_samples, nb_classes)`. + """ + labels = np.array(labels) + unique = np.unique(labels) + unique.sort() + indexes = [np.where(unique == value)[0] for value in labels] + if nb_classes is None: + nb_classes = len(unique) + categorical = np.zeros((labels.shape[0], nb_classes), dtype=np.float32) + categorical[np.arange(labels.shape[0]), np.squeeze(indexes)] = 1 + return categorical + + +class AttackDataset: + """ + Class for splitting data into training and test sets for membership and attribute inference attacks. + + Parameters + ---------- + x : np.ndarray or tuple + Input data. If tuple, the first element is the training data and the second element is the test data. + y : np.ndarray or tuple, optional + Labels. If tuple, the first element is the training labels and the second element is the test labels. + attack_train_ratio : float, optional + The ratio of training data to use for the attack. Default is 0.5. + """ + + def __init__(self, x, y=None, attack_train_ratio: Optional[float] = 0.5): + if type(x) is tuple: + self.x_train, self.x_test = x + self.attack_train_size = int(len(self.x_train) * attack_train_ratio) + self.attack_test_size = int(len(self.x_test) * attack_train_ratio) + else: + self.x_train = x + self.attack_train_size = int(len(self.x_train) * attack_train_ratio) + + self.y_output = y is not None + + if self.y_output: + if type(y) is tuple: + self.y_train, self.y_test = y + else: + self.y_train = y + + def membership_inference_train(self): + """ + Get the training set for the membership inference attack. + + Returns + ------- + tuple + Tuple containing the training data and the membership + """ + x = np.concatenate([self.x_train[: self.attack_train_size :], self.x_test[: self.attack_test_size]]) + train_membership = np.ones(self.attack_train_size) + test_membership = np.zeros(self.attack_test_size) + membership = np.concatenate([train_membership, test_membership]) + + if not self.y_output: + return x, membership + + y = np.concatenate([self.y_train[: self.attack_train_size :], self.y_test[: self.attack_test_size]]) + return x, y, membership + + def membership_inference_test(self): + """ + Get the test set for the membership inference attack. + + Returns + ------- + tuple + Tuple containing the test data and the membership + """ + x = np.concatenate([self.x_train[self.attack_train_size :], self.x_test[self.attack_test_size :]]) + train_membership = np.ones(self.attack_train_size) + test_membership = np.zeros(self.attack_test_size) + membership = np.concatenate([train_membership, test_membership]) + + if not self.y_output: + return x, membership + + y = np.concatenate([self.y_train[self.attack_train_size :], self.y_test[self.attack_test_size :]]) + return x, y, membership + + def attribute_inference_train(self): + """ + Get the training set for the attribute inference attack. + + Returns + ------- + tuple + Tuple containing the training data and the attribute. + """ + attack_x_train = self.x_train[: self.attack_train_size] + + if not self.y_output: + return attack_x_train + + attack_y_train = self.y_train[: self.attack_train_size] + return attack_x_train, attack_y_train + + def attribute_inference_test(self): + """ + Get the test set for the attribute inference attack. + + Returns + ------- + tuple + Tuple containing the test data and the attribute. + """ + attack_x_test = self.x_train[self.attack_train_size :] + + if not self.y_output: + return attack_x_test + + attack_y_test = self.y_train[self.attack_train_size :] + return attack_x_test, attack_y_test + + +class AttributeInferenceDataPreprocessor: + """ + Class for preprocessing data for attribute inference attacks. + + Parameters + ---------- + attack_feature : int or slice + The feature to be attacked. + is_regression : bool, optional + Whether the attack is a regression attack. Default is False. + scale_range : tuple, optional + The range to scale the labels to. Default is None. + prediction_normal_factor : float, optional + The factor to normalize the predictions by. Default is 1. + """ + + def __init__(self, attack_feature, is_regression=None, scale_range=None, prediction_normal_factor=None): + self.is_regression = is_regression # if RegressorMixin in type(self.estimator).__mro__: + self.scale_range = scale_range + self.prediction_normal_factor = prediction_normal_factor + self.attack_feature = attack_feature + + def fit_transform(self, x, y=None, pred=None): + """ + Prepare the data for training the attack model. + + Parameters + ---------- + x : np.ndarray + The input data. + y : np.ndarray, optional + The true labels. + + Returns + ------- + np.ndarray or tuple + The training data for the attack model. If `y` is provided, a tuple is returned with the training data and + the labels. + """ + y_ready = self._get_feature_labels(x) + x_ready = np.delete(x, self.attack_feature, 1) + + # create training set for attack model + if y is not None: + normalized_labels = self._normalized_labels(y) + x_ready = np.c_[x_ready, normalized_labels].astype(np.float32) + + if pred is not None: + normalized_labels = self._normalized_labels(pred) + x_ready = np.c_[x_ready, normalized_labels].astype(np.float32) + + if y_ready is None: + return x_ready + return x_ready, y_ready + + def transform(self, x, y=None, pred=None): + """ + Prepare the data for inference with the attack model. + + Parameters + ---------- + x : np.ndarray + The input data. + y : np.ndarray, optional + The true labels. + + Returns + ------- + np.ndarray or tuple + The data for the attack model. If `y` is provided, a tuple is returned with the data and the labels. + """ + x_ready = x # np.delete(x, self.attack_feature, 1) + + # create training set for attack model + if y is not None: + normalized_labels = self._normalized_labels(y) + x_ready = np.c_[x_ready, normalized_labels].astype(np.float32) + + if pred is not None: + normalized_labels = self._normalized_labels(pred) + x_ready = np.c_[x_ready, normalized_labels].astype(np.float32) + + return x_ready + + def _get_feature_labels(self, x): + attacked_feature = x[:, self.attack_feature] + + self._values = get_feature_values(attacked_feature, isinstance(self.attack_feature, int)) + self._nb_classes = len(self._values) + + if isinstance(self.attack_feature, int): + y_one_hot = float_to_categorical(attacked_feature) + else: + y_one_hot = floats_to_one_hot(attacked_feature) + + y_ready = check_and_transform_label_format(y_one_hot, nb_classes=self._nb_classes, return_one_hot=True) + return y_ready + + def _normalized_labels(self, y): + if self.is_regression: + if self.scale_range is not None: + normalized_labels = minmax_scale(y, feature_range=self.scale_range) + else: + normalized_labels = y * self.prediction_normal_factor + normalized_labels = normalized_labels.reshape(-1, 1) + else: + normalized_labels = check_and_transform_label_format(y, nb_classes=None, return_one_hot=True) + return normalized_labels diff --git a/src/holisticai/security/attackers/attribute_inference/true_label_baseline.py b/src/holisticai/security/attackers/attribute_inference/true_label_baseline.py new file mode 100644 index 00000000..8a48e513 --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/true_label_baseline.py @@ -0,0 +1,107 @@ +""" +This module implements attribute inference attacks. +""" + +from __future__ import annotations + +import logging +from typing import Optional, Union + +import numpy as np +from holisticai.security.attackers.attribute_inference.dataset_utils import AttributeInferenceDataPreprocessor +from holisticai.security.attackers.attribute_inference.utils import get_attack_model, get_feature_index + +logger = logging.getLogger(__name__) + + +class AttributeInferenceBaselineTrueLabel(): + def __init__( + self, + attack_model_type: str = "nn", + attack_feature: Union[int, slice] = 0, + is_regression: Optional[bool] = False, + scale_range: Optional[tuple[float, float]] = None, + prediction_normal_factor: float = 1, + ): + + self._values: Optional[list] = None + self._nb_classes: Optional[int] = None + self.attack_model = get_attack_model(attack_model_type) + self.prediction_normal_factor = prediction_normal_factor + self.scale_range = scale_range + self.is_regression = is_regression + self.attack_feature = attack_feature + self._check_params() + self.attack_feature = get_feature_index(self.attack_feature) + self.ai_preprocessor = AttributeInferenceDataPreprocessor( + is_regression=is_regression, + scale_range=scale_range, + prediction_normal_factor=prediction_normal_factor, + attack_feature=attack_feature, + ) + + def _check_params(self) -> None: + if not isinstance(self.attack_feature, int) and not isinstance(self.attack_feature, slice): + raise TypeError("Attack feature must be either an integer or a slice object.") + + if isinstance(self.attack_feature, int) and self.attack_feature < 0: + raise ValueError("Attack feature index must be positive.") + + def fit(self, x: np.ndarray, y: np.ndarray) -> None: + """ + Train the attack model. + + Parameters + ---------- + x : np.ndarray + Input to training process. Includes all features used to train the original model. + y : np.ndarray + True labels of the features. + """ + + # train attack model + attack_x, attack_y = self.ai_preprocessor.fit_transform(x, y) + self._values = self.ai_preprocessor._values # noqa: SLF001 + self.attack_model.fit(attack_x, attack_y) + + def infer(self, x: np.ndarray, y: np.ndarray, **kwargs) -> np.ndarray: + """ + Infer the attacked feature. + + Parameters + ---------- + x : np.ndarray + Input to attack. Includes all features except the attacked feature. + y : np.ndarray + True labels of the features. + values : list + Possible values for attacked feature. For a single column feature this should be a simple list containing + all possible values, in increasing order (the smallest value in the 0 index and so on). For a multi-column + feature (for example 1-hot encoded and then scaled), this should be a list of lists, where each internal + list represents a column (in increasing order) and the values represent the possible values for that column + (in increasing order). + + Returns + ------- + np.ndarray + The inferred feature values. + """ + values: Optional[list] = kwargs.get("values") + + # if provided, override the values computed in fit() + if values is not None: + self._values = values + + attack_x = self.ai_preprocessor.transform(x, y) + predictions = self.attack_model.predict_proba(attack_x).astype(np.float32) + + if self._values is not None: + if isinstance(self.attack_feature, int): + predictions = np.array([self._values[np.argmax(arr)] for arr in predictions]) + else: + i = 0 + for column in predictions.T: + for index in range(len(self._values[i])): + np.place(column, [column == index], self._values[i][index]) + i += 1 + return np.array(predictions) \ No newline at end of file diff --git a/src/holisticai/security/attackers/attribute_inference/utils.py b/src/holisticai/security/attackers/attribute_inference/utils.py new file mode 100644 index 00000000..8bc7e6d7 --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/utils.py @@ -0,0 +1,150 @@ +def get_attack_model(attack_model_type): + """ + Returns an attack model of the specified type. + + Parameters + ---------- + attack_model_type : str + The type of the attack model. Possible values are 'nn' for neural network and 'rf' for random forest. + + Returns + ------- + object + The attack model. + """ + if attack_model_type == "nn": + from sklearn.neural_network import MLPClassifier + + attack_model = MLPClassifier( + hidden_layer_sizes=(100,), + activation="relu", + solver="adam", + alpha=0.0001, + batch_size="auto", + learning_rate="constant", + learning_rate_init=0.001, + power_t=0.5, + max_iter=2000, + shuffle=True, + random_state=None, + tol=0.0001, + verbose=False, + warm_start=False, + momentum=0.9, + nesterovs_momentum=True, + early_stopping=False, + validation_fraction=0.1, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-08, + n_iter_no_change=10, + max_fun=15000, + ) + elif attack_model_type == "rf": + from sklearn.ensemble import RandomForestClassifier + + attack_model = RandomForestClassifier() + else: + raise ValueError("Illegal value for parameter `attack_model_type`.") + return attack_model + + +def is_pipeline(estimator): + """ + Checks if the given estimator is a pipeline. + + Parameters + ---------- + estimator : object + The estimator to check. + + Returns + ------- + bool + True if the estimator is a pipeline. + """ + from holisticai.pipeline import Pipeline + + if type(estimator) is Pipeline: + return True + return None + + +def model_in_pipeline(estimator): + """ + Returns the model in a pipeline. + + Parameters + ---------- + estimator : object + The estimator to check. + + Returns + ------- + object + The model in the pipeline. + """ + return estimator.estimator_hdl.estimator.obj + + +def is_estimator_valid(estimator, estimator_requirements) -> bool: + """ + Checks if the given estimator satisfies the requirements for this attack. + + Parameters + ---------- + estimator : object + The estimator to check. + estimator_requirements : list + The requirements for the estimator. + + Returns + ------- + bool + True if the estimator is valid for the attack. + """ + + if is_pipeline(estimator): + model = model_in_pipeline(estimator) + return is_estimator_valid(model, estimator_requirements) + + for req in estimator_requirements: + # A requirement is either a class which the estimator must inherit from, or a tuple of classes and the + # estimator is required to inherit from at least one of the classes + if isinstance(req, tuple): + if all(p not in type(estimator).__mro__ for p in req): + return False + elif req not in type(estimator).__mro__: + return False + return True + + +def get_feature_index(feature): + """ + Returns a modified feature index: in case of a slice of size 1, returns the corresponding integer. Otherwise, + returns the same value (integer or slice) as passed. + + Parameters + ---------- + feature : int or slice + The index or slice representing a feature to attack (0-based). + + Returns + ------- + int or slice + An integer representing a single column index or a slice representing a multi-column index. + """ + if isinstance(feature, int): + return feature + + start = feature.start + stop = feature.stop + step = feature.step + if start is None: + start = 0 + if step is None: + step = 1 + if feature.stop is not None and ((stop - start) // step) == 1: + return start + + return feature diff --git a/src/holisticai/security/attackers/attribute_inference/white_box.py b/src/holisticai/security/attackers/attribute_inference/white_box.py new file mode 100644 index 00000000..17c450a4 --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/white_box.py @@ -0,0 +1,169 @@ +""" +This module implements attribute inference attacks. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +import numpy as np + + +logger = logging.getLogger(__name__) + + +class AttributeInferenceWhiteBoxDecisionTree(): + """ + A variation of the method proposed by of Fredrikson et al. in: + https://dl.acm.org/doi/10.1145/2810103.2813677 + + Assumes the availability of the attacked model's predictions for the samples under attack, in addition to access to + the model itself and the rest of the feature values. If this is not available, the true class label of the samples + may be used as a proxy. Also assumes that the attacked feature is discrete or categorical, with limited number of + possible values. For example: a boolean feature. + + Parameters + ---------- + classifier : ScikitlearnDecisionTreeClassifier + Target classifier. + attack_feature : int + The index of the feature to be attacked. + + References + ---------- + .. [1] Fredrikson, M., Jha, S., & Ristenpart, T. (2015, August). Model inversion attacks that exploit confidence + information and basic countermeasures. In Proceedings of the 22nd ACM SIGSAC Conference on Computer and + Communications Security (pp. 1322-1333). + """ + def __init__(self, classifier, attack_feature: int = 0): + self.attack_feature: int + self.attack_feature = attack_feature + self._check_params() + self.estimator = classifier + + def _check_params(self) -> None: + if self.attack_feature < 0: + raise ValueError("Attack feature must be positive.") + + def infer(self, x: np.ndarray, y: np.ndarray, **kwargs) -> np.ndarray: + """ + Infer the attacked feature. + + If the model's prediction coincides with the real prediction for the sample for a single value, choose it as the + predicted value. If not, fall back to the Fredrikson method (without phi) + + Parameters + ---------- + x : np.ndarray + Input to attack. Includes all features except the attacked feature. + y : np.ndarray + Original model's predictions for x. + values : list + Possible values for attacked feature. + priors : list + Prior distributions of attacked feature values. Same size array as `values`. + + Returns + ------- + np.ndarray + The inferred feature values. + """ + if "priors" not in kwargs: # pragma: no cover + raise ValueError("Missing parameter `priors`.") + if "values" not in kwargs: # pragma: no cover + raise ValueError("Missing parameter `values`.") + priors: Optional[list] = kwargs.get("priors") + values: Optional[list] = kwargs.get("values") + + if priors is None or values is None: # pragma: no cover + raise ValueError("`priors` and `values` are required as inputs.") + if len(priors) != len(values): # pragma: no cover + raise ValueError("Number of priors does not match number of values") + + n_values = len(values) + n_samples = x.shape[0] + + # Will contain the model's predictions for each value + pred_values = [] + # Will contain the probability of each value + prob_values = [] + + for i, value in enumerate(values): + # prepare data with the given value in the attacked feature + v_full = np.full((n_samples, 1), value).astype(x.dtype) + x_value = np.concatenate((x[:, : self.attack_feature], v_full), axis=1) + x_value = np.concatenate((x_value, x[:, self.attack_feature :]), axis=1) + + # Obtain the model's prediction for each possible value of the attacked feature + pred_value = [np.argmax(arr) for arr in self.estimator.predict_proba(x_value)] + pred_values.append(pred_value) + + # find the relative probability of this value for all samples being attacked + prob_value = [ + ( + (self.get_samples_at_node(self.get_decision_path([row])[-1]) / n_samples) + * priors[i] + ) + for row in x_value + ] + prob_values.append(prob_value) + + # Find the single value that coincides with the real prediction for the sample (if it exists) + pred_rows = zip(*pred_values) + predicted_pred = [] + for row_index, row in enumerate(pred_rows): + if y is not None: + matches = [1 if row[value_index] == y[row_index] else 0 for value_index in range(n_values)] + match_values = [ + values[value_index] if row[value_index] == y[row_index] else 0 for value_index in range(n_values) + ] + else: + matches = [0 for _ in range(n_values)] + match_values = [0 for _ in range(n_values)] + predicted_pred.append(sum(match_values) if sum(matches) == 1 else None) + + # Choose the value with highest probability for each sample + predicted_prob = [np.argmax(list(prob)) for prob in zip(*prob_values)] + + return np.array( + [ + value if value is not None else values[predicted_prob[index]] + for index, value in enumerate(predicted_pred) + ] + ) + + def get_decision_path(self, x: np.ndarray) -> np.ndarray: + """ + Returns the path through nodes in the tree when classifying x. Last one is leaf, first one root node. + + Parameters + ---------- + x : np.ndarray + Input sample. + + Returns + ------- + np.ndarray + The indices of the nodes in the array structure of the tree. + """ + if len(np.shape(x)) == 1: + return self.estimator.decision_path(x.reshape(1, -1)).indices + + return self.estimator.decision_path(x).indices + + def get_samples_at_node(self, node_id: int) -> int: + """ + Returns the number of training samples mapped to a node. + + Parameters + ---------- + node_id : int + Node id. + + Returns + ------- + int + Number of samples mapped this node. + """ + return self.estimator.tree_.n_node_samples[node_id] \ No newline at end of file diff --git a/src/holisticai/security/attackers/attribute_inference/white_box_lifestyle.py b/src/holisticai/security/attackers/attribute_inference/white_box_lifestyle.py new file mode 100644 index 00000000..fd551db9 --- /dev/null +++ b/src/holisticai/security/attackers/attribute_inference/white_box_lifestyle.py @@ -0,0 +1,171 @@ +""" +This module implements attribute inference attacks. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +import numpy as np + +logger = logging.getLogger(__name__) + + +class AttributeInferenceWhiteBoxLifestyleDecisionTree(): + """ + Implementation of Fredrikson et al. white box inference attack for decision trees. + + Assumes that the attacked feature is discrete or categorical, with limited number of possible values. For example: + a boolean feature. + + Parameters + ---------- + estimator : object + Target estimator. + attack_feature : int + The index of the feature to be attacked. + + References + ---------- + .. [1] Fredrikson, M., Jha, S., & Ristenpart, T. (2015, August). Model inversion attacks that exploit confidence\\ + information and basic countermeasures. In Proceedings of the 22nd ACM SIGSAC Conference on Computer and\\ + Communications Security (pp. 1322-1333). + + | Paper link: https://dl.acm.org/doi/10.1145/2810103.2813677 + """ + def __init__(self, estimator, attack_feature: int = 0): + self.attack_feature: int + self.attack_feature = attack_feature + self._check_params() + self.estimator = estimator + + def _check_params(self) -> None: + if self.attack_feature < 0: + raise ValueError("Attack feature must be positive.") + + def infer(self, x: np.ndarray, **kwargs) -> np.ndarray: + """ + Infer the attacked feature. + + Parameters + ---------- + x : np.ndarray + Input to attack. Includes all features except the attacked feature. + kwargs : dict + Possible values for attacked feature and prior distributions of attacked feature values. + + Returns + ------- + np.ndarray + The inferred feature values. + """ + priors: Optional[list] = kwargs.get("priors") + values: Optional[list] = kwargs.get("values") + + # Checks: + if priors is None or values is None: # pragma: no cover + raise ValueError("`priors` and `values` are required as inputs.") + if len(priors) != len(values): # pragma: no cover + raise ValueError("Number of priors does not match number of values") + + n_samples = x.shape[0] + + # Calculate phi for each possible value of the attacked feature + # phi is the total number of samples in all tree leaves corresponding to this value + phi = self._calculate_phi(x, values, n_samples) + + # Will contain the probability of each value + prob_values = [] + + for i, value in enumerate(values): + # prepare data with the given value in the attacked feature + v_full = np.full((n_samples, 1), value).astype(x.dtype) + x_value = np.concatenate((x[:, : self.attack_feature], v_full), axis=1) + x_value = np.concatenate((x_value, x[:, self.attack_feature :]), axis=1) + + # find the relative probability of this value for all samples being attacked + prob_value = [ + ( + (self.get_samples_at_node(self.get_decision_path([row])[-1]) / n_samples) + * priors[i] + / phi[i] + ) + for row in x_value + ] + prob_values.append(prob_value) + + # Choose the value with highest probability for each sample + return np.array([values[np.argmax(list(prob))] for prob in zip(*prob_values)]) + + def _calculate_phi(self, x, values, n_samples): + phi = [] + for value in values: + v_full = np.full((n_samples, 1), value).astype(x.dtype) + x_value = np.concatenate((x[:, : self.attack_feature], v_full), axis=1) + x_value = np.concatenate((x_value, x[:, self.attack_feature :]), axis=1) + nodes_value = {} + + for row in x_value: + # get leaf ids (no duplicates) + node_id = self.get_decision_path([row])[-1] + nodes_value[node_id] = self.get_samples_at_node(node_id) + # sum sample numbers + num_value = sum(nodes_value.values()) / n_samples + phi.append(num_value) + + return phi + + def get_decision_path(self, x: np.ndarray) -> np.ndarray: + """ + Returns the path through nodes in the tree when classifying x. Last one is leaf, first one root node. + + Parameters + ---------- + x : np.ndarray + Input sample. + + Returns + ------- + np.ndarray + The indices of the nodes in the array structure of the tree. + """ + if len(np.shape(x)) == 1: + return self.estimator.decision_path(x.reshape(1, -1)).indices + + return self.estimator.decision_path(x).indices + + def get_samples_at_node(self, node_id: int) -> int: + """ + Returns the number of training samples mapped to a node. + + Parameters + ---------- + node_id : int + Node id. + + Returns + ------- + int + Number of samples mapped this node. + """ + return self.estimator.tree_.n_node_samples[node_id] + + def _get_input_shape(self, model) -> Optional[tuple[int, ...]]: + _input_shape: Optional[tuple[int, ...]] + if hasattr(model, "n_features_"): + _input_shape = (model.n_features_,) + elif hasattr(model, "n_features_in_"): + _input_shape = (model.n_features_in_,) + elif hasattr(model, "feature_importances_"): + _input_shape = (len(model.feature_importances_),) + elif hasattr(model, "coef_"): + _input_shape = (model.coef_.shape[0],) if len(model.coef_.shape) == 1 else (model.coef_.shape[1],) + elif hasattr(model, "support_vectors_"): + _input_shape = (model.support_vectors_.shape[1],) + elif hasattr(model, "steps"): + _input_shape = self._get_input_shape(model.steps[-1][1].obj) + else: + logger.warning("Input shape not recognised. The model might not have been fitted.") + _input_shape = None + return _input_shape \ No newline at end of file diff --git a/src/holisticai/security/commons/_black_box_attack.py b/src/holisticai/security/commons/_black_box_attack.py index 4df03cb0..c2824469 100644 --- a/src/holisticai/security/commons/_black_box_attack.py +++ b/src/holisticai/security/commons/_black_box_attack.py @@ -88,4 +88,5 @@ def classification_security_features(X, y, attacker, attack_feature): attacker = BlackBoxAttack(learning_task="classification", attack_feature=attack_feature, attack_train_ratio=0.5) attacker.fit(X, y) return attacker - raise ValueError("Invalid attacker type. Please choose from 'black_box'") + msg = "Invalid attacker type. Please choose from 'black_box'" + raise ValueError(msg) diff --git a/src/holisticai/security/commons/data_minimization/_core.py b/src/holisticai/security/commons/data_minimization/_core.py index bb5c3262..e797494c 100644 --- a/src/holisticai/security/commons/data_minimization/_core.py +++ b/src/holisticai/security/commons/data_minimization/_core.py @@ -1,12 +1,15 @@ from __future__ import annotations import logging -from typing import Optional +from typing import TYPE_CHECKING -import pandas as pd from holisticai.security.commons.data_minimization._modificators import ModifierHandler, ModifierType from holisticai.security.commons.data_minimization._selectors import SelectorsHandler, SelectorType -from holisticai.utils import ModelProxy + +if TYPE_CHECKING: + import pandas as pd + + from holisticai.utils import ModelProxy logger = logging.getLogger(__name__) @@ -15,8 +18,8 @@ class DataMinimizer: def __init__( self, proxy: ModelProxy, - selector_types: Optional[list[SelectorType]] = None, - modifier_types: Optional[list[ModifierType]] = None, + selector_types: list[SelectorType] | None = None, + modifier_types: list[ModifierType] | None = None, ): if modifier_types is None: modifier_types = ["Average", "Permutation"] diff --git a/src/holisticai/security/commons/data_minimization/_modificators.py b/src/holisticai/security/commons/data_minimization/_modificators.py index e0390d12..7a4e00c3 100644 --- a/src/holisticai/security/commons/data_minimization/_modificators.py +++ b/src/holisticai/security/commons/data_minimization/_modificators.py @@ -22,7 +22,8 @@ def apply_method(self, method_name: ModifierType, x, indexes) -> pd.DataFrame: return replace_data_with_average(x, indexes) if method_name == "Permutation": return replace_data_with_permutation(x, indexes) - raise NotImplementedError(f"Method {method_name} not implemented") + msg = f"Method {method_name} not implemented" + raise NotImplementedError(msg) def replace_data_with_average(x, important_feature_names): diff --git a/src/holisticai/security/commons/data_minimization/_selectors.py b/src/holisticai/security/commons/data_minimization/_selectors.py index c9b7518b..a2433e68 100644 --- a/src/holisticai/security/commons/data_minimization/_selectors.py +++ b/src/holisticai/security/commons/data_minimization/_selectors.py @@ -1,12 +1,9 @@ from __future__ import annotations import logging -from typing import Literal, get_args +from typing import TYPE_CHECKING, Literal, get_args import numpy as np -import pandas as pd -from holisticai.inspection import compute_permutation_importance -from holisticai.utils import Importances, ModelProxy from sklearn.feature_selection import ( SelectPercentile, VarianceThreshold, @@ -14,6 +11,13 @@ f_regression, ) +from holisticai.inspection import compute_permutation_importance + +if TYPE_CHECKING: + import pandas as pd + + from holisticai.utils import Importances, ModelProxy + logger = logging.getLogger(__name__) SelectorbyData = Literal["Percentile", "Variance"] SelectorbyImportance = Literal["FImportance"] @@ -25,7 +29,8 @@ def get_score_fn(learning_task: str): return f_classif if learning_task == "regression": return f_regression - raise ValueError(f"Learning task {learning_task} not supported") + msg = f"Learning task {learning_task} not supported" + raise ValueError(msg) class SelectorsHandler: @@ -53,7 +58,8 @@ def _get_selectors_from_data(self, selector_type: SelectorbyData): for p in variance_thresholds: methods[f"Variance >{p}"] = VarianceThreshold(threshold=p / 100) return methods - raise ValueError(f"Selector type {selector_type} not supported") + msg = f"Selector type {selector_type} not supported" + raise ValueError(msg) def _get_selectors_from_importances(self, selector_type: SelectorbyImportance): if selector_type == "FImportance": @@ -61,7 +67,8 @@ def _get_selectors_from_importances(self, selector_type: SelectorbyImportance): for p in self.importance_thresholds: methods[f"FImportance >{p}"] = SelectFromFeatureImportance(threshold=p / 100) return methods - raise ValueError(f"Selector type {selector_type} not supported") + msg = f"Selector type {selector_type} not supported" + raise ValueError(msg) def fit(self, X: pd.DataFrame, y: pd.Series): selectors_by_data = {} diff --git a/src/holisticai/security/metrics/__init__.py b/src/holisticai/security/metrics/__init__.py index 75f8c764..e96df277 100644 --- a/src/holisticai/security/metrics/__init__.py +++ b/src/holisticai/security/metrics/__init__.py @@ -4,9 +4,8 @@ from __future__ import annotations -from typing import Union - import pandas as pd + from holisticai.security.metrics._anonymization import k_anonymity, l_diversity from holisticai.security.metrics._attribute_attack import AttributeAttackScore, attribute_attack_score from holisticai.security.metrics._data_minimization import ( @@ -26,7 +25,7 @@ def classification_privacy_metrics( y_pred_train: pd.Series, y_pred_test: pd.Series, y_pred_test_dm: dict[str, : pd.Series], - attribute_attack: Union[str, list[str]], + attribute_attack: str | list[str], ): shapr_score = ShaprScore() dm_accuracy_ratio = DataMinimizationAccuracyRatio() @@ -52,7 +51,7 @@ def regression_privacy_metrics( y_test: pd.Series, y_pred_test: pd.Series, y_pred_test_dm: dict[str, : pd.Series], - attribute_attack: Union[str, list[str]], + attribute_attack: str | list[str], ): attr_attack_score = AttributeAttackScore() dm_mse_ratio = DataMinimizationMSERatio() diff --git a/src/holisticai/security/metrics/_attribute_attack.py b/src/holisticai/security/metrics/_attribute_attack.py index 1a76a8d1..5fc79d2d 100644 --- a/src/holisticai/security/metrics/_attribute_attack.py +++ b/src/holisticai/security/metrics/_attribute_attack.py @@ -1,13 +1,16 @@ from __future__ import annotations -from typing import Any, Callable, Union +from typing import TYPE_CHECKING, Any, Callable -import pandas as pd -from holisticai.security.commons import BlackBoxAttack -from holisticai.security.metrics._utils import check_valid_output_type from sklearn.linear_model import LinearRegression, LogisticRegression from sklearn.metrics import accuracy_score, f1_score, mean_absolute_error, mean_squared_error +from holisticai.security.commons import BlackBoxAttack +from holisticai.security.metrics._utils import check_valid_output_type + +if TYPE_CHECKING: + import pandas as pd + def to_numerical_or_categorical(y: pd.Series): if y.dtype.kind in ["f"]: @@ -17,7 +20,8 @@ def to_numerical_or_categorical(y: pd.Series): return y.astype("category") if len(y.unique()) < 2: - raise ValueError("The target variable must have more than 1 unique value") + msg = "The target variable must have more than 1 unique value" + raise ValueError(msg) return y.astype("category") @@ -33,7 +37,7 @@ def __call__( y_test: pd.Series, attribute_attack: str, attack_train_ratio: float = 0.5, - metric_fn: Union[str, Callable, None] = None, + metric_fn: str | Callable | None = None, attacker_estimator: Any = None, ) -> float: check_valid_output_type(y_train) diff --git a/src/holisticai/security/metrics/_data_minimization.py b/src/holisticai/security/metrics/_data_minimization.py index dd075cb8..32daea51 100644 --- a/src/holisticai/security/metrics/_data_minimization.py +++ b/src/holisticai/security/metrics/_data_minimization.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Union - import numpy as np import pandas as pd from sklearn.metrics import accuracy_score, mean_squared_error @@ -86,7 +84,8 @@ def get_learning_task(y_true: pd.Series): return "classification" if y_true.dtype.kind in ["f"]: return "regression" - raise ValueError(f"Unknown learning task. dtype: {y_true.dtype.kind}") + msg = f"Unknown learning task. dtype: {y_true.dtype.kind}" + raise ValueError(msg) def data_minimization_score( @@ -94,7 +93,7 @@ def data_minimization_score( y_pred: pd.Series, y_pred_dm: dict[str, pd.Series], return_results=False, - learning_task: Union[str, None] = None, + learning_task: str | None = None, ): """ Calculate the accuracy ratio for data minimization. The accuracy ratio is the ratio of the accuracy of the data minimization model to the accuracy of the original model. diff --git a/src/holisticai/security/metrics/_privacy_risk_score.py b/src/holisticai/security/metrics/_privacy_risk_score.py index 5c52e965..6393beb4 100644 --- a/src/holisticai/security/metrics/_privacy_risk_score.py +++ b/src/holisticai/security/metrics/_privacy_risk_score.py @@ -283,5 +283,4 @@ def privacy_risk_score(shadow_train, shadow_test, target_train): tr_distrs, te_distrs, all_bins = distrs_compute( shadow_train_m_entr, shadow_test_m_entr, shadow_train_labels, shadow_test_labels, num_bins=5, log_bins=True ) - risk_score = _risk_score_compute(tr_distrs, te_distrs, all_bins, target_train_m_entr, target_train_labels) - return risk_score + return _risk_score_compute(tr_distrs, te_distrs, all_bins, target_train_m_entr, target_train_labels) diff --git a/src/holisticai/security/metrics/_shapr.py b/src/holisticai/security/metrics/_shapr.py index 42a07277..1dc4a7c0 100644 --- a/src/holisticai/security/metrics/_shapr.py +++ b/src/holisticai/security/metrics/_shapr.py @@ -1,16 +1,23 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING -import jax.numpy as jnp import numpy as np -import pandas as pd +from sklearn.preprocessing import LabelEncoder + from holisticai.typing import ArrayLike from holisticai.utils.models.neighbors import KNeighborsClassifier -from jax.nn import one_hot -from sklearn.preprocessing import LabelEncoder + +try: + import jax.numpy as jnp + from jax.nn import one_hot +except ImportError: + jnp = None + one_hot = None + if TYPE_CHECKING: + import pandas as pd from numpy.typing import ArrayLike @@ -48,13 +55,16 @@ def transform_label_to_numerical_label(labels: np.ndarray, le: LabelEncoder = No return np.array(labels) -def check_and_transform_label_format(labels: np.ndarray) -> jnp.ndarray: +def check_and_transform_label_format(labels: np.ndarray): """ Check label format and transform to one-hot-encoded labels if necessary :param labels: An array of integer or string labels of shape `(nb_samples,)`, `(nb_samples, 1)` or `(nb_samples, nb_classes)`. :return: Labels with shape `(nb_samples, nb_classes)` (one-hot) or `(nb_samples,)` (index). """ + if one_hot is None or jnp is None: + msg = "jax or jax.nn is not installed. Please install it with `pip install jax jaxlib`." + raise ImportError(msg) labels = jnp.array(labels) return_one_hot = True @@ -102,7 +112,11 @@ def __call__( batch_size=500, train_size=1.0, aggregated=True, - ) -> Union[np.ndarray, float]: + ) -> np.ndarray | float: + if one_hot is None or jnp is None: + msg = "jax or jax.nn is not installed. Please install it with `pip install jax jaxlib`." + raise ImportError(msg) + y_train, le = transform_label_to_numerical_label(y_train) y_test = transform_label_to_numerical_label(y_test, le) y_pred_train = transform_label_to_numerical_label(y_pred_train, le) @@ -152,7 +166,7 @@ def shapr_score( y_pred_test: pd.Series, batch_size=500, train_size=1.0, -) -> jnp.ndarray: +): """ Compute the SHAPr membership privacy risk metric [1]_ for the given classifier and training set. diff --git a/src/holisticai/security/metrics/_utils.py b/src/holisticai/security/metrics/_utils.py index 223051d8..5a24c14f 100644 --- a/src/holisticai/security/metrics/_utils.py +++ b/src/holisticai/security/metrics/_utils.py @@ -6,4 +6,5 @@ def check_valid_output_type(y_true: pd.Series): return "classification" if y_true.dtype.kind in ["f"]: return "regression" - raise ValueError(f"The target variable must be either continuous or categorical, found: {y_true.dtype}") + msg = f"The target variable must be either continuous or categorical, found: {y_true.dtype}" + raise ValueError(msg) diff --git a/src/holisticai/security/mitigation/_anonymization.py b/src/holisticai/security/mitigation/_anonymization.py index ef49147e..7eff8fc1 100644 --- a/src/holisticai/security/mitigation/_anonymization.py +++ b/src/holisticai/security/mitigation/_anonymization.py @@ -1,7 +1,6 @@ from __future__ import annotations from collections import Counter -from typing import Optional, Union import numpy as np import pandas as pd @@ -55,18 +54,20 @@ class Anonymize: def __init__( self, k: int, - quasi_identifiers: Union[np.ndarray, list], - features: Optional[list] = None, - features_names: Optional[list] = None, - quasi_identifer_slices: Optional[list] = None, - categorical_features: Optional[list] = None, - is_regression: Optional[bool] = False, - train_only_QI: Optional[bool] = False, + quasi_identifiers: np.ndarray | list, + features: list | None = None, + features_names: list | None = None, + quasi_identifer_slices: list | None = None, + categorical_features: list | None = None, + is_regression: bool | None = False, + train_only_QI: bool | None = False, ): if k < 2: - raise ValueError("k should be a positive integer with a value of 2 or higher") + msg = "k should be a positive integer with a value of 2 or higher" + raise ValueError(msg) if quasi_identifiers is None or len(quasi_identifiers) < 1: - raise ValueError("The list of quasi-identifiers cannot be empty") + msg = "The list of quasi-identifiers cannot be empty" + raise ValueError(msg) self.k = k self.quasi_identifiers = quasi_identifiers @@ -99,22 +100,25 @@ def anonymize(self, X_train, y_train): if X_train.shape[1] != 0: self.features = list(range(X_train.shape[1])) else: - raise ValueError("No data provided") + msg = "No data provided" + raise ValueError(msg) if self.features_names is None: # if no names provided, use numbers instead self.features_names = self.features if not set(self.quasi_identifiers).issubset(set(self.features_names)): - raise ValueError( + msg = ( "Quasi identifiers should bs a subset of the supplied features or indexes in range of " "the data columns" ) + raise ValueError(msg) if self.categorical_features and not set(self.categorical_features).issubset(set(self.features_names)): - raise ValueError( + msg = ( "Categorical features should bs a subset of the supplied features or indexes in range of " "the data columns" ) + raise ValueError(msg) # transform quasi identifiers to indexes self.quasi_identifiers = [i for i, v in enumerate(self.features_names) if v in self.quasi_identifiers] @@ -133,10 +137,12 @@ def anonymize(self, X_train, y_train): def _anonymize(self, x, y): if x.shape[0] != y.shape[0]: - raise ValueError("x and y should have same number of rows") + msg = "x and y should have same number of rows" + raise ValueError(msg) if any(x.select_dtypes(include="category").columns): if not self.categorical_features: - raise ValueError("when supplying an array with non-numeric data, categorical_features must be defined") + msg = "when supplying an array with non-numeric data, categorical_features must be defined" + raise ValueError(msg) x_prepared = self._modify_categorical_features(x) else: x_prepared = x @@ -252,5 +258,4 @@ def _modify_categorical_features(self, x): ("cat", categorical_transformer, categorical_features), ] ) - encoded = preprocessor.fit_transform(x) - return encoded + return preprocessor.fit_transform(x) diff --git a/src/holisticai/utils/_commons.py b/src/holisticai/utils/_commons.py index 2a90c4ce..f76f574e 100644 --- a/src/holisticai/utils/_commons.py +++ b/src/holisticai/utils/_commons.py @@ -30,7 +30,8 @@ def get_item(obj, key): return obj.loc[key] if isinstance(obj, np.ndarray): return obj[key] - raise ValueError(f"Type {type(obj)} not supported") + msg = f"Type {type(obj)} not supported" + raise ValueError(msg) def get_columns(obj): @@ -38,4 +39,5 @@ def get_columns(obj): return obj.columns if isinstance(obj, np.ndarray): return [f"feature_{i}" for i in range(obj.shape[1])] - raise ValueError(f"Type {type(obj)} not supported") + msg = f"Type {type(obj)} not supported" + raise ValueError(msg) diff --git a/src/holisticai/utils/_definitions.py b/src/holisticai/utils/_definitions.py index 3f7599f2..ac07e8e7 100644 --- a/src/holisticai/utils/_definitions.py +++ b/src/holisticai/utils/_definitions.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Literal, Optional, Union +from typing import TYPE_CHECKING, Callable, Literal, Union import numpy as np import pandas as pd @@ -19,8 +19,8 @@ class BinaryClassificationProxy(ReprObj): def __init__( self, predict: Callable, - predict_proba: Optional[Callable] = None, - classes: Union[list, None] = None, + predict_proba: Callable | None = None, + classes: list | None = None, ): if classes is None: classes = [0, 1] @@ -100,15 +100,14 @@ def create_proxy(**kargs) -> ModelProxy: return RegressionProxy(**kargs) if task == "clustering": return ClusteringProxy(**kargs) - raise ValueError("Unknown learning task type") + msg = "Unknown learning task type" + raise ValueError(msg) class Importances(ReprObj): _theme = "blue" - def __init__( - self, values: ArrayLike, feature_names: list[str], extra_attrs: Union[dict, None] = None, normalize=True - ): + def __init__(self, values: ArrayLike, feature_names: list[str], extra_attrs: dict | None = None, normalize=True): if extra_attrs is None: extra_attrs = {} values_ = np.abs(_array_like_to_numpy(values)) @@ -125,7 +124,8 @@ def __getitem__(self, idx: int | str) -> float: return self.values[idx] if isinstance(idx, str): return self.values[self.feature_names.index(idx)] - raise ValueError(f"Invalid index type: {type(idx)}") + msg = f"Invalid index type: {type(idx)}" + raise ValueError(msg) def select(self, idx: list[int]): data = pd.DataFrame({"feature_names": self.feature_names, "values": self.values}) @@ -224,7 +224,8 @@ def conditional(self): def __add__(self, other): if not isinstance(other, LocalImportances): - raise TypeError("Both operands must be instances of LocalImportances") + msg = "Both operands must be instances of LocalImportances" + raise TypeError(msg) data = self.data.droplevel(0, axis=1) serie = data["condition"] @@ -281,7 +282,7 @@ def __len__(self): class PartialDependence(ReprObj): - def __init__(self, values: list[list[dict]], feature_names: Optional[list[str]] = None): + def __init__(self, values: list[list[dict]], feature_names: list[str] | None = None): self.values = values if feature_names is None: feature_names = [f"Feature {i}" for i in range(len(values))] diff --git a/src/holisticai/utils/_validation.py b/src/holisticai/utils/_validation.py index 285f96d9..1bf61195 100644 --- a/src/holisticai/utils/_validation.py +++ b/src/holisticai/utils/_validation.py @@ -226,7 +226,8 @@ def _array_like_to_numpy(arr, name=""): return np.array([out]) if len(out.shape) == 1: return out - raise ValueError(f"input is not 1d array-like.: {out}") + msg = f"input is not 1d array-like.: {out}" + raise ValueError(msg) except TypeError as e: msg = f"input {name} is not array-like. This includes numpy 1d arrays, lists,\ pandas Series or pandas 1d DataFrame" @@ -272,14 +273,16 @@ def _matrix_like_to_dataframe(arr, name=""): # noqa: ARG001 return pd.DataFrame(out, columns=columns) ValueError("input is not matrix-like.") except TypeError as e: - raise TypeError("input is not matrix-like.") from e + msg = "input is not matrix-like." + raise TypeError(msg) from e def _array_like_to_series(arr, name=""): # noqa: ARG001 num_dimensions = 1 def raise_value_error(): - raise ValueError("input is not array-like.") + msg = "input is not array-like." + raise ValueError(msg) try: out = np.squeeze(np.asarray(arr)) @@ -287,7 +290,8 @@ def raise_value_error(): return pd.Series(out) raise_value_error() except TypeError as e: - raise TypeError("input is not array-like.") from e + msg = "input is not array-like." + raise TypeError(msg) from e def _coerce_and_check_arr(arr, name="input"): @@ -607,7 +611,8 @@ def _check_numerical_dataframe(df: pd.DataFrame): try: return df.astype(float) except ValueError as e: - raise ValueError("DataFrame cannot be converted to numerical values") from e + msg = "DataFrame cannot be converted to numerical values" + raise ValueError(msg) from e def _check_columns(df: pd.DataFrame, column: str): diff --git a/src/holisticai/utils/feature_importances/__init__.py b/src/holisticai/utils/feature_importances/__init__.py deleted file mode 100644 index 5d7f1ef1..00000000 --- a/src/holisticai/utils/feature_importances/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -from holisticai.utils.feature_importances._conditional import ( - group_index_samples_by_learning_task, - group_mask_samples_by_learning_task, -) -from holisticai.utils.feature_importances._lime import ( - LIMEImportanceCalculator, - compute_lime_feature_importance, -) -from holisticai.utils.feature_importances._permutation_feature_importance import ( - PermutationFeatureImportanceCalculator, - compute_conditional_permutation_feature_importance, - compute_permutation_feature_importance, -) -from holisticai.utils.feature_importances._shap import ( - SHAPImportanceCalculator, - compute_shap_feature_importance, -) -from holisticai.utils.feature_importances._surrogate_feature_importance import ( - SurrogateFeatureImportanceCalculator, - compute_surrogate_feature_importance, -) - -__all__ = [ - "PermutationFeatureImportanceCalculator", - "compute_permutation_feature_importance", - "group_index_samples_by_learning_task", - "group_mask_samples_by_learning_task", - "SurrogateFeatureImportanceCalculator", - "compute_surrogate_feature_importance", - "LIMEImportanceCalculator", - "compute_lime_feature_importance", - "SHAPImportanceCalculator", - "compute_shap_feature_importance", - "compute_conditional_permutation_feature_importance", -] diff --git a/src/holisticai/utils/feature_importances/_permutation_feature_importance.py b/src/holisticai/utils/feature_importances/_permutation_feature_importance.py deleted file mode 100644 index 7632c556..00000000 --- a/src/holisticai/utils/feature_importances/_permutation_feature_importance.py +++ /dev/null @@ -1,188 +0,0 @@ -from __future__ import annotations - -import os -from typing import Any, Literal, Union, overload - -import numpy as np -import pandas as pd -from joblib import Parallel, delayed -from numpy.random import RandomState -from sklearn.metrics import accuracy_score, mean_squared_error - -from holisticai.datasets import Dataset -from holisticai.utils._commons import get_columns, get_item -from holisticai.utils._definitions import ( - ConditionalImportances, - Importances, - ModelProxy, -) -from holisticai.utils.feature_importances import group_index_samples_by_learning_task - -metric_scores = { - "binary_classification": accuracy_score, - "regression": mean_squared_error, - "multi_classification": accuracy_score, -} - - -@overload -def compute_permutation_feature_importance( - proxy: ModelProxy, - X: pd.DataFrame, - y: pd.Series, - n_repeats: int = 5, - n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, - importance_type: Literal["conditional"] = "conditional", -) -> ConditionalImportances: ... - - -@overload -def compute_permutation_feature_importance( - proxy: ModelProxy, - X: pd.DataFrame, - y: pd.Series, - n_repeats: int = 5, - n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, -) -> Importances: ... - - -def compute_permutation_feature_importance( - proxy: ModelProxy, - X: pd.DataFrame, - y: pd.Series, - n_repeats: int = 5, - n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, - importance_type: Literal["standard", "conditional"] = "standard", -) -> Union[Importances, ConditionalImportances]: - pfi = PermutationFeatureImportanceCalculator(n_repeats=n_repeats, n_jobs=n_jobs, random_state=random_state) - if importance_type == "conditional": - return compute_conditional_permutation_feature_importance( - proxy=proxy, X=X, y=y, n_repeats=n_repeats, n_jobs=n_jobs, random_state=random_state - ) - return pfi.compute_importances(X=X, y=y, proxy=proxy) - - -def compute_conditional_permutation_feature_importance( - proxy: ModelProxy, - X: pd.DataFrame, - y: pd.Series, - n_repeats: int = 5, - n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, -) -> Union[Importances, ConditionalImportances]: - pfi = PermutationFeatureImportanceCalculator(n_repeats=n_repeats, n_jobs=n_jobs, random_state=random_state) - sample_groups = group_index_samples_by_learning_task(y, proxy.learning_task) - values = {} - for group_name, indexes in sample_groups.items(): - values[group_name] = pfi.compute_importances(X=get_item(X, indexes), y=get_item(y, indexes), proxy=proxy) - - return ConditionalImportances(values=values) - - -class SklearnClassifier: - @staticmethod - def from_proxy(proxy): - class Wrapper: - predict = proxy.predict - predict_proba = proxy.predict_proba - classes = proxy.classes - fit = lambda x, y: None # noqa: ARG005, E731 - score = lambda x, y: accuracy_score(y, proxy.predict(x)) # noqa: E731 - - return Wrapper - - -class SklearnRegressor: - @staticmethod - def from_proxy(proxy): - class Wrapper: - predict = proxy.predict - fit = lambda x, y: None # noqa: ARG005, E731 - score = lambda x, y: mean_squared_error(y, proxy.predict(x)) # noqa: E731 - - return Wrapper - - -class PermutationFeatureImportanceCalculator: - def __init__( - self, - n_repeats: int = 5, - n_jobs: int = -1, - random_state: Union[RandomState, int, None] = None, - importance_type: str = "global", - ): - if random_state is None: - random_state = RandomState(42) - if isinstance(random_state, int): - random_state = RandomState(random_state) - self.n_repeats = n_repeats - self.n_jobs = n_jobs - self.random_state = random_state - self.importance_type = importance_type - - def compute_importances(self, X: Any, y: Any, proxy: ModelProxy) -> Importances: - if proxy.learning_task == "regression": - sklearn_model = SklearnRegressor.from_proxy(proxy) - - elif proxy.learning_task in ("binary_classification", "multi_classification"): - sklearn_model = SklearnClassifier.from_proxy(proxy) - - from sklearn.inspection import permutation_importance - - r = permutation_importance(sklearn_model, X, y, n_repeats=self.n_repeats, random_state=0, n_jobs=self.n_jobs) - feature_importance_values = np.abs(r["importances_mean"]) - - features = list(get_columns(X)) - feature_importances = pd.DataFrame.from_dict( - {"Variable": features, "Importance": feature_importance_values} - ).sort_values("Importance", ascending=False) - feature_importances["Importance"] = feature_importances["Importance"] / feature_importances["Importance"].sum() - feature_names = list(feature_importances["Variable"].values) - importances = np.array(feature_importances["Importance"].values) - return Importances(values=importances, feature_names=feature_names) - - def compute_importances_(self, dataset: Dataset, proxy: ModelProxy) -> Importances: - X = dataset["X"] - y = dataset["y"] - metric = metric_scores[proxy.learning_task] - baseline_score = metric(y, proxy.predict(X)) - n_features = X.shape[1] - - def _calculate_permutation_scores(predictor, X, y, col_idx, n_repeats): - scores = [] - X_permuted = X.copy() - shuffling_idx = np.arange(X_permuted.shape[0]) - for _ in range(n_repeats): - self.random_state.shuffle(shuffling_idx) - col = X_permuted.iloc[shuffling_idx, col_idx] - col.index = X_permuted.index - X_permuted[X_permuted.columns[col_idx]] = col - scores.append(metric(y, predictor(X_permuted))) - return np.array(scores) - - n_jobs = os.cpu_count() if self.n_jobs == -1 else self.n_jobs - - scores = list( - Parallel(n_jobs=n_jobs)( - delayed(_calculate_permutation_scores)( - proxy.predict, - X, - y, - col_idx, - self.n_repeats, - ) - for col_idx in range(n_features) - ) - ) - feature_importances = [np.mean(np.abs(baseline_score - scores[col])) for col in range(n_features)] - features = list(X.columns) - feature_importances = pd.DataFrame.from_dict( - {"Variable": features, "Importance": feature_importances} - ).sort_values("Importance", ascending=False) - feature_importances["Importance"] = feature_importances["Importance"] / feature_importances["Importance"].sum() - feature_names = list(feature_importances["Variable"].values) - importances = np.array(feature_importances["Importance"].values) - return Importances(values=importances, feature_names=feature_names) diff --git a/src/holisticai/utils/models/cluster/_kcenters.py b/src/holisticai/utils/models/cluster/_kcenters.py index aa565ce4..cc14f056 100644 --- a/src/holisticai/utils/models/cluster/_kcenters.py +++ b/src/holisticai/utils/models/cluster/_kcenters.py @@ -1,13 +1,11 @@ from __future__ import annotations -from typing import Union - from holisticai.utils.models.cluster._utils import distance from numpy.random import RandomState class KCenters: - def __init__(self, n_clusters=5, random_state: Union[RandomState, int] = 42): + def __init__(self, n_clusters=5, random_state: RandomState | int = 42): """ k (int) : Number of centers to be identified """ diff --git a/src/holisticai/utils/models/cluster/_kmedoids.py b/src/holisticai/utils/models/cluster/_kmedoids.py index ddfaab95..50dec952 100644 --- a/src/holisticai/utils/models/cluster/_kmedoids.py +++ b/src/holisticai/utils/models/cluster/_kmedoids.py @@ -88,7 +88,8 @@ def _check_nonnegative_int(self, value, desc, strict=True): """Validates if value is a valid integer > 0""" negative = value is None or value <= 0 if strict else value is None or value < 0 if negative or not isinstance(value, (int, np.integer)): - raise ValueError(f"{desc} should be a nonnegative integer. " f"{value} was given") + msg = f"{desc} should be a nonnegative integer. " f"{value} was given" + raise ValueError(msg) def _check_init_args(self): """Validates the input arguments.""" @@ -289,7 +290,8 @@ def _initialize_medoids(self, D, n_clusters, random_state_): # to every other point. These are the initial medoids. medoids = np.argpartition(np.sum(D, axis=1), n_clusters - 1)[:n_clusters] else: - raise ValueError(f"init value '{self.init}' not recognized") + msg = f"init value '{self.init}' not recognized" + raise ValueError(msg) return medoids diff --git a/src/holisticai/utils/models/neighbors/_classification.py b/src/holisticai/utils/models/neighbors/_classification.py index d663ed9f..787b4208 100644 --- a/src/holisticai/utils/models/neighbors/_classification.py +++ b/src/holisticai/utils/models/neighbors/_classification.py @@ -1,8 +1,17 @@ -import jax.numpy as jnp -from jax import vmap +try: + import jax.numpy as jnp + from jax import vmap +except ImportError: + jnp = None + vmap = None class KNeighborsClassifier: + def __init__(self): + if jnp is None or vmap is None: + msg = "jax or jax.nn is not installed. Please install it with `pip install jax jaxlib`." + raise ImportError(msg) + def fit(self, X_train, y_train): self.X_train = jnp.array(X_train) self.y_train = jnp.array(y_train) @@ -49,5 +58,4 @@ def _predict_single(self, x): def predict(self, X): # Vectorize prediction for all input points - predictions = vmap(self._predict_single)(jnp.array(X)) - return predictions + return vmap(self._predict_single)(jnp.array(X)) diff --git a/src/holisticai/utils/obj_rep/object_repr.py b/src/holisticai/utils/obj_rep/object_repr.py index ccf751e7..48560a4f 100644 --- a/src/holisticai/utils/obj_rep/object_repr.py +++ b/src/holisticai/utils/obj_rep/object_repr.py @@ -109,12 +109,10 @@ def generate_html_for_generic_object(obj, feature_columns=5, theme="blue"): nested_objects_html += generate_html_for_generic_object(nested_obj, feature_columns, theme) header = f"[{obj_type}]" if name in ("N/A", "") else f"{name} [{obj_type}]" - html_output = html_template.format( + return html_template.format( header=header, attributes=attributes_html, nested_objects=nested_objects_html, css_template=css_template, theme=theme, ) - - return html_output diff --git a/src/holisticai/utils/optimizers/_genetic_algorithm.py b/src/holisticai/utils/optimizers/_genetic_algorithm.py index d7c73cf8..afa9642f 100644 --- a/src/holisticai/utils/optimizers/_genetic_algorithm.py +++ b/src/holisticai/utils/optimizers/_genetic_algorithm.py @@ -96,9 +96,11 @@ class containing GA algoirhtm hiperparameters self.var_type = np.array([["int"]] * self.dim) else: if type(variable_type_mixed).__module__ != "numpy": - raise ValueError("variable_type must be numpy array") + msg = "variable_type must be numpy array" + raise ValueError(msg) if len(variable_type_mixed) != self.dim: - raise ValueError("variable_type must have a length equal to dimension.") + msg = "variable_type must have a length equal to dimension." + raise ValueError(msg) for i in variable_type_mixed: assert i in ("real", "int"), ( @@ -112,10 +114,12 @@ class containing GA algoirhtm hiperparameters # input variables' boundaries if variable_type != "bool" or type(variable_type_mixed).__module__ == "numpy": if type(variable_boundaries).__module__ != "numpy": - raise ValueError("variable_boundaries must be numpy array") + msg = "variable_boundaries must be numpy array" + raise ValueError(msg) if len(variable_boundaries) != self.dim: - raise ValueError("variable_boundaries must have a length equal to dimension.") + msg = "variable_boundaries must have a length equal to dimension." + raise ValueError(msg) for i in variable_boundaries: assert len(i) == 2, "\n boundary for each variable must be a tuple of length two." @@ -130,7 +134,8 @@ class containing GA algoirhtm hiperparameters self.pop_s = int(self.param["population_size"]) if not (0 <= self.param["parents_portion"] <= 1): - raise ValueError("parents_portion must be in range [0,1]") + msg = "parents_portion must be in range [0,1]" + raise ValueError(msg) self.par_s = int(self.param["parents_portion"] * self.pop_s) trl = self.pop_s - self.par_s @@ -140,7 +145,8 @@ class containing GA algoirhtm hiperparameters self.prob_mut = self.param["mutation_probability"] if not (0 <= self.prob_mut <= 1): - raise ValueError("mutation_probability must be in range [0,1]") + msg = "mutation_probability must be in range [0,1]" + raise ValueError(msg) self.prob_cross = self.param["crossover_probability"] assert self.prob_cross <= 1, "mutation_probability must be less than or equal to 1" diff --git a/src/holisticai/utils/plots/_plots.py b/src/holisticai/utils/plots/_plots.py index 4f6c511b..c348ba3b 100644 --- a/src/holisticai/utils/plots/_plots.py +++ b/src/holisticai/utils/plots/_plots.py @@ -47,7 +47,8 @@ def _validate_and_extract_data(data): # Check if data is a pandas DataFrame if isinstance(data, pd.DataFrame): if data.shape[1] != 2: - raise ValueError("Data should have exactly two columns.") + msg = "Data should have exactly two columns." + raise ValueError(msg) data_vals = data.values # Convert DataFrame to NumPy array # Check if data is a pandas Series elif isinstance(data, pd.Series): @@ -57,7 +58,8 @@ def _validate_and_extract_data(data): data_vals = data # Raise error if the data is not a DataFrame or NumPy array else: - raise TypeError("Data must be either a pandas DataFrame or a NumPy array.") + msg = "Data must be either a pandas DataFrame or a NumPy array." + raise TypeError(msg) return data_vals @@ -92,7 +94,8 @@ def plot_graph(X, y, test_indices=None): # If test_indices are provided, outline the selected points if test_indices is not None: if not isinstance(test_indices, (np.ndarray, list)): - raise TypeError("test_indices must be either a list or a NumPy array.") + msg = "test_indices must be either a list or a NumPy array." + raise TypeError(msg) plt.scatter( X_vals[test_indices, 0], X_vals[test_indices, 1], facecolors="none", edgecolors="red", linewidths=2, s=150 diff --git a/src/holisticai/utils/plots/_utils.py b/src/holisticai/utils/plots/_utils.py index 79213bf4..930d0212 100644 --- a/src/holisticai/utils/plots/_utils.py +++ b/src/holisticai/utils/plots/_utils.py @@ -55,18 +55,22 @@ def prepare_to_plot( # Check if X is a DataFrame and y is a NumPy array if not isinstance(X, pd.DataFrame): - raise TypeError("X must be a pandas DataFrame.") + msg = "X must be a pandas DataFrame." + raise TypeError(msg) if not isinstance(y, np.ndarray): - raise TypeError("y must be a NumPy array.") + msg = "y must be a NumPy array." + raise TypeError(msg) # Check if all the features are present in X missing_features = [feature for feature in features_to_plot if feature not in X.columns] if missing_features: - raise KeyError(f"The following features are missing from X: {', '.join(missing_features)}") + msg = f"The following features are missing from X: {', '.join(missing_features)}" + raise KeyError(msg) # Check if the length of X and y match if len(X) != len(y): - raise ValueError("The length of X and y must match.") + msg = "The length of X and y must match." + raise ValueError(msg) # Select only the columns for the chosen features to plot X_selected = X[features_to_plot] diff --git a/src/holisticai/utils/surrogate_models/__init__.py b/src/holisticai/utils/surrogate_models/__init__.py index 52259d0b..764a3c2f 100644 --- a/src/holisticai/utils/surrogate_models/__init__.py +++ b/src/holisticai/utils/surrogate_models/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, Optional, Union +from typing import Literal, Union import numpy as np from numpy.random import RandomState @@ -22,16 +22,18 @@ def create_surrogate_model(X, y_pred, surrogate_type, learning_task="classificat elif len(np.unique(y_pred)) > 2: surrogate = MultiClassificationSurrogate(X, y_pred=y_pred, model_type=surrogate_type) else: - raise ValueError("y_pred must have at least two unique values") + msg = "y_pred must have at least two unique values" + raise ValueError(msg) elif learning_task == "regression": surrogate = RegressionSurrogate(X, y_pred=y_pred, model_type=surrogate_type) else: - raise ValueError(f"Learning task {learning_task} not supported") + msg = f"Learning task {learning_task} not supported" + raise ValueError(msg) return surrogate class BinaryClassificationSurrogate: - def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: Optional[RandomState] = None): + def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: RandomState | None = None): if random_state is None: random_state = RandomState(42) @@ -40,11 +42,12 @@ def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_s learning_task="binary_classification", model_type=model_type, random_state=random_state ) return surrogate.fit(X=X, y=y_pred) - raise ValueError(f"Surrogate type {model_type} not supported") + msg = f"Surrogate type {model_type} not supported" + raise ValueError(msg) class MultiClassificationSurrogate: - def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: Optional[RandomState] = None): + def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: RandomState | None = None): if random_state is None: random_state = RandomState(42) @@ -53,11 +56,12 @@ def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_s learning_task="multi_classification", model_type=model_type, random_state=random_state ) return surrogate.fit(X=X, y=y_pred) - raise ValueError(f"Surrogate type {model_type} not supported") + msg = f"Surrogate type {model_type} not supported" + raise ValueError(msg) class RegressionSurrogate: - def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: Optional[RandomState] = None): + def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: RandomState | None = None): if random_state is None: random_state = RandomState(42) if model_type in ["shallow_tree", "tree"]: @@ -65,11 +69,12 @@ def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_s learning_task="regression", model_type=model_type, random_state=random_state ) return surrogate.fit(X=X, y=y_pred) - raise ValueError(f"Surrogate type {model_type} not supported") + msg = f"Surrogate type {model_type} not supported" + raise ValueError(msg) class ClusteringSurrogate: - def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: Optional[RandomState] = None): + def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_state: RandomState | None = None): if random_state is None: random_state = RandomState(42) @@ -78,7 +83,8 @@ def __new__(cls, X, y_pred, model_type: SurrogateType = "shallow_tree", random_s learning_task="clustering", model_type=model_type, random_state=random_state ) return surrogate.fit(X=X, y=y_pred) - raise ValueError(f"Surrogate type {model_type} not supported") + msg = f"Surrogate type {model_type} not supported" + raise ValueError(msg) def get_features(surrogate): @@ -88,7 +94,8 @@ def get_features(surrogate): if isinstance(surrogate, (DecisionTreeClassifier, DecisionTreeRegressor)): assert surrogate.__is_fitted__, "Model not fitted" return surrogate._surrogate.tree_.feature # noqa: SLF001 - raise ValueError(f"Surrogate type {type(surrogate)} not supported") + msg = f"Surrogate type {type(surrogate)} not supported" + raise ValueError(msg) def get_number_of_rules(surrogate): @@ -98,4 +105,5 @@ def get_number_of_rules(surrogate): if isinstance(surrogate, (DecisionTreeClassifier, DecisionTreeRegressor)): assert surrogate.__is_fitted__, "Model not fitted" return surrogate._surrogate.get_n_leaves() # noqa: SLF001 - raise ValueError(f"Surrogate type {type(surrogate)} not supported") + msg = f"Surrogate type {type(surrogate)} not supported" + raise ValueError(msg) diff --git a/src/holisticai/utils/surrogate_models/_trees.py b/src/holisticai/utils/surrogate_models/_trees.py index adc9b68b..fa2239d2 100644 --- a/src/holisticai/utils/surrogate_models/_trees.py +++ b/src/holisticai/utils/surrogate_models/_trees.py @@ -1,15 +1,17 @@ from __future__ import annotations -from typing import Literal, Optional +from typing import TYPE_CHECKING, Literal import numpy as np import pandas as pd -from numpy.random import RandomState from sklearn.metrics import accuracy_score, mean_squared_error from holisticai.utils.obj_rep.object_repr import ReprObj from holisticai.utils.surrogate_models._base import SurrogateBase +if TYPE_CHECKING: + from numpy.random import RandomState + def validate_input(X, y=None): if isinstance(X, pd.DataFrame): @@ -72,7 +74,7 @@ def __init__( self, learning_task: Literal["binary_classification", "multi_classification", "clustering"], model_type: Literal["shallow_tree", "tree"] = "shallow_tree", - random_state: Optional[RandomState] = None, + random_state: RandomState | None = None, ): self.random_state = random_state self.model_type = model_type @@ -89,7 +91,8 @@ def build(self, X, y): elif self.model_type == "tree": rf = RandomForestClassifier(n_estimators=100, random_state=self.random_state) else: - raise ValueError(f"Model type {self.model_type} not supported") + msg = f"Model type {self.model_type} not supported" + raise ValueError(msg) rf.fit(X, y) best_score = -np.inf for tree in rf.estimators_: @@ -123,7 +126,7 @@ def __init__( self, learning_task: Literal["regression"], model_type: Literal["shallow_tree", "tree"] = "shallow_tree", - random_state: Optional[RandomState] = None, + random_state: RandomState | None = None, ): self.model_type = model_type self.random_state = random_state @@ -140,7 +143,8 @@ def build(self, X, y): elif self.model_type == "tree": rf = RandomForestRegressor(n_estimators=100, random_state=self.random_state) else: - raise ValueError(f"Model type {self.model_type} not supported") + msg = f"Model type {self.model_type} not supported" + raise ValueError(msg) rf.fit(X, y) best_score = np.inf for tree in rf.estimators_: diff --git a/tests/commons/test_feature_importance_strategies.py b/tests/commons/test_feature_importance_strategies.py index 95b1384b..0c5ad193 100644 --- a/tests/commons/test_feature_importance_strategies.py +++ b/tests/commons/test_feature_importance_strategies.py @@ -32,5 +32,5 @@ def test_permutation_feature_importance_call(input_data): fi_strategy = PermutationFeatureImportanceCalculator(random_state=RandomState(42)) importance = fi_strategy.compute_importances(test['X'], test['y'], proxy=proxy) assert isinstance(importance, Importances) - assert np.isclose(importance['capital-gain'], 0.38461538461538486, atol=5e-2) + assert np.isclose(importance['capital-gain'], 0.44444444444444453, atol=5e-2) \ No newline at end of file diff --git a/tests/robustness/dataset_shift/test_plot_neighborhood.py b/tests/robustness/dataset_shift/test_plot_neighborhood.py deleted file mode 100644 index 4a090028..00000000 --- a/tests/robustness/dataset_shift/test_plot_neighborhood.py +++ /dev/null @@ -1,270 +0,0 @@ -import pytest -import numpy as np -from unittest import mock -from matplotlib import pyplot as plt - -# Import the functions to be tested -from holisticai.robustness.plots._dataset_shift import ( - plot_neighborhood, -) - -@pytest.fixture -def sample_data(): - """ - Fixture to generate sample data for testing. - Returns: - X (np.ndarray): Feature matrix of shape (100, 2). - y (np.ndarray): True labels of shape (100,). - y_pred (np.ndarray): Predicted labels of shape (100,). - """ - np.random.seed(42) # For reproducibility - X = np.random.rand(100, 2) - y = np.random.randint(0, 2, size=100) - y_pred = np.random.randint(0, 2, size=100) - return X, y, y_pred - -@pytest.fixture -def indices_show_full(): - """ - Fixture to provide full indices for display. - Returns: - indices_show (np.ndarray): Indices from 0 to 99. - """ - return np.arange(100) - -@pytest.mark.parametrize( - "n_neighbors, points_of_interest, vertical_offset, features_to_plot, indices_show", - [ - (3, [0], 0.1, None, None), - (5, [10, 20], 0.2, ['Feature1', 'Feature2'], np.arange(100)), - (7, [50, 60, 70], 0.05, None, np.arange(100)), - ] -) -def test_plot_neighborhood_various_inputs( - sample_data, - n_neighbors, - points_of_interest, - vertical_offset, - features_to_plot, - indices_show -): - """ - Test plot_neighborhood with various input parameters. - """ - X, y, y_pred = sample_data - if indices_show is None: - indices_show = np.arange(len(X)) - else: - # Ensure indices_show is valid - assert np.all(np.isin(points_of_interest, indices_show)), \ - "All points_of_interest must be in indices_show." - - # Mock plt.show to prevent actual plotting during tests - with mock.patch("matplotlib.pyplot.show"): - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - vertical_offset=vertical_offset, - features_to_plot=features_to_plot, - indices_show=indices_show - ) - # If no exceptions occur, test passes - -def test_plot_neighborhood_with_invalid_points(sample_data, indices_show_full): - """ - Test plot_neighborhood raises ValueError when points_of_interest not in indices_show. - """ - X, y, y_pred = sample_data - n_neighbors = 5 - points_of_interest = [1000] # Invalid index - indices_show = indices_show_full - - with pytest.raises(ValueError, match=r"The point \d+ is not a point in 'indices_show'\."): - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - indices_show=indices_show - ) - -def test_plot_neighborhood_with_insufficient_neighbors(sample_data, indices_show_full): - """ - Test plot_neighborhood handles cases with insufficient neighbors gracefully. - """ - X, y, y_pred = sample_data - n_neighbors = 150 # More neighbors than available points - points_of_interest = [0, 1] - indices_show = indices_show_full - - with pytest.raises(ValueError): - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - indices_show=indices_show - ) - -@pytest.mark.parametrize( - "features_to_plot", - [ - None, - ['Feature1', 'Feature2'] - ] -) -def test_plot_neighborhood_with_features(sample_data, features_to_plot): - """ - Test plot_neighborhood with and without custom feature names. - """ - X, y, y_pred = sample_data - n_neighbors = 3 - points_of_interest = [0, 1] - indices_show = np.arange(len(X)) - - # Mock plt.show to prevent actual plotting during tests - with mock.patch("matplotlib.pyplot.show"): - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - features_to_plot=features_to_plot, - indices_show=indices_show - ) - -@pytest.mark.skip(reason="no call to this test at the moment") -def test_plot_neighborhood_plotting(sample_data, indices_show_full): - """ - Test that plot_neighborhood calls the plotting functions correctly. - """ - X, y, y_pred = sample_data - n_neighbors = 3 - points_of_interest = [0, 1] - indices_show = indices_show_full - - with mock.patch.object(plt, 'show') as mock_show, \ - mock.patch.object(plt.Axes, 'scatter') as mock_scatter, \ - mock.patch.object(plt.Axes, 'plot') as mock_plot, \ - mock.patch.object(plt.Axes, 'set_xlabel') as mock_set_xlabel, \ - mock.patch.object(plt.Axes, 'set_ylabel') as mock_set_ylabel, \ - mock.patch('matplotlib.pyplot.title') as mock_title: - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - indices_show=indices_show - ) - - # Check that plotting functions were called - assert mock_scatter.call_count >= 1 - assert mock_plot.call_count >= 1 - mock_set_xlabel.assert_called() - mock_set_ylabel.assert_called() - mock_title.assert_called() - mock_show.assert_called_once() - -def test_plot_neighborhood_accuracy_calculation(sample_data, indices_show_full): - """ - Test that accuracy is calculated correctly and text is added to the plot. - """ - X, y, y_pred = sample_data - n_neighbors = 3 - points_of_interest = [0] - indices_show = indices_show_full - - with mock.patch("matplotlib.pyplot.show"), \ - mock.patch("matplotlib.pyplot.text") as mock_text: - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - indices_show=indices_show - ) - - # Check that text was added to the plot - mock_text.assert_called() - # Extract the accuracy value from the call arguments - args, kwargs = mock_text.call_args - acc_text = args[2] - assert "Acc = " in acc_text - -def test_plot_neighborhood_with_custom_axes(sample_data): - """ - Test plot_neighborhood when a custom matplotlib axis is provided. - """ - X, y, y_pred = sample_data - n_neighbors = 3 - points_of_interest = [0] - indices_show = np.arange(len(X)) - fig, ax = plt.subplots() - - with mock.patch("matplotlib.pyplot.show"): - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - ax=ax, - indices_show=indices_show - ) - # Verify that the plot was drawn on the provided axis - assert ax.has_data() - -def test_plot_neighborhood_with_vertical_offset(sample_data): - """ - Test plot_neighborhood with different vertical offsets. - """ - X, y, y_pred = sample_data - n_neighbors = 3 - points_of_interest = [0] - indices_show = np.arange(len(X)) - - vertical_offsets = [0, 0.1, 0.5] - - for offset in vertical_offsets: - with mock.patch("matplotlib.pyplot.show"): - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - vertical_offset=offset, - indices_show=indices_show - ) - # No exception means test passes - -@pytest.mark.xfail(reason="ConvexHull from scipy.spatial presents error when the hull is flat.") -def test_plot_neighborhood_edge_cases(): - """ - Test plot_neighborhood with edge case inputs. - """ - X = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) - y = np.array([0, 1, 1, 0]) - y_pred = np.array([0, 1, 1, 1]) - n_neighbors = 3 - points_of_interest = [1] - indices_show = np.array([0, 1]) - - with mock.patch("matplotlib.pyplot.show"): - plot_neighborhood( - X, - y, - y_pred, - n_neighbors, - points_of_interest, - indices_show=indices_show - ) - # No exception means test passes diff --git a/tests/security/test_attribute_inference_classification.py b/tests/security/test_attribute_inference_classification.py new file mode 100644 index 00000000..fd254c9b --- /dev/null +++ b/tests/security/test_attribute_inference_classification.py @@ -0,0 +1,183 @@ +import numpy as np +from holisticai.datasets import load_dataset +from holisticai.security.attackers.attribute_inference.baseline import AttributeInferenceBaseline + +from holisticai.security.attackers.attribute_inference.wrappers.classification.scikitlearn import SklearnClassifier +from holisticai.security.attackers.attribute_inference.black_box import AttributeInferenceBlackBox +from holisticai.security.attackers.attribute_inference.true_label_baseline import AttributeInferenceBaselineTrueLabel +from holisticai.security.attackers.attribute_inference.wrappers.classification.scikitlearn import ScikitlearnDecisionTreeClassifier +from holisticai.security.attackers.attribute_inference.mitigation.attacks.white_box_lifestyle_decision_tree import ( + AttributeInferenceWhiteBoxLifestyleDecisionTree +) +from holisticai.security.attackers.attribute_inference.mitigation.scikitlearn import ( + AttributeInferenceWhiteBoxDecisionTree, +) +from sklearn.tree import DecisionTreeClassifier +from holisticai.efficacy.metrics import classification_efficacy_metrics +import pytest + +np.random.seed(100) + +SHARD_SIZE=60 + +@pytest.fixture +def categorical_dataset(): + dataset = load_dataset("adult", protected_attribute='race') # x,y,group_a,group_b + dataset = dataset.groupby(['y','group_a']).sample(SHARD_SIZE, random_state=42) # 0-ga | 0-gb | 1-ga | 1-gb + train_test = dataset.train_test_split(test_size=0.5, stratify=dataset['y'], random_state=0) + train = train_test['train'] + test = train_test['test'] + correlations = train['X'].corrwith(train['y']).sort_values(ascending=False) + top_10_features = correlations.head(10).index.tolist() + train['X'] = train['X'][top_10_features] + test['X'] = test['X'][top_10_features] + return train, test + + +def test_att_inf_baseline(categorical_dataset): + + train, test = categorical_dataset + + attack_feature = 0 # marital status + + x_train = train['X'].copy().values + y_train = train['y'].copy().values + x_test = test['X'].copy().values + y_test = test['y'].copy().values + + attack = AttributeInferenceBaseline(attack_feature=attack_feature) + attack.fit(x_train) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + feat_true = x_test[:, attack_feature] + + values = [0, 1] + feat_pred = attack.infer(attack_x_test, values=values) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 + + +def test_att_inf_black_box(categorical_dataset): + train, test = categorical_dataset + + attack_feature = 0 # marital status + + x_train = train['X'].copy().values + y_train = train['y'].copy().values + x_test = test['X'].copy().values + y_test = test['y'].copy().values + + classifier = DecisionTreeClassifier() + classifier.fit(x_train, y_train) + classifier = SklearnClassifier(classifier) + + attack = AttributeInferenceBlackBox(estimator=classifier, attack_feature=attack_feature) + + pred = classifier.predict_proba(x_train) + attack.fit(x_train, y_train, pred) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + pred = classifier.predict_proba(x_test) + + feat_true = x_test[:, attack_feature] + + values = [0, 1] + feat_pred = attack.infer(attack_x_test, y_test, pred, values=values) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 + + +def test_att_inf_truelabel(categorical_dataset): + train, test = categorical_dataset + + attack_feature = 0 # marital status + + x_train = train['X'].copy().values + y_train = train['y'].copy().values + x_test = test['X'].copy().values + y_test = test['y'].copy().values + + attack = AttributeInferenceBaselineTrueLabel(attack_feature=attack_feature) + attack.fit(x_train, y_train) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + feat_true = x_test[:, attack_feature] + + values = [0, 1] + feat_pred = attack.infer(attack_x_test, y_test, values=values) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 + + +def test_att_inf_white_box_lifestyle(categorical_dataset): + train, test = categorical_dataset + + attack_feature = 0 # marital status + + x_train = train['X'].copy().values + y_train = train['y'].copy().values + x_test = test['X'].copy().values + y_test = test['y'].copy().values + + classifier = DecisionTreeClassifier() + classifier.fit(x_train, y_train) + classifier = ScikitlearnDecisionTreeClassifier(classifier) + + attack = AttributeInferenceWhiteBoxLifestyleDecisionTree(attack_feature=attack_feature, estimator=classifier) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + feat_true = x_test[:, attack_feature] + + values = [0, 1] + priors = [53/120, 67/120] + + feat_pred = attack.infer(attack_x_test, values=values, priors=priors) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 + + +def test_att_inf_white_box(categorical_dataset): + train, test = categorical_dataset + + attack_feature = 0 # marital status + + x_train = train['X'].copy().values + y_train = train['y'].copy().values + x_test = test['X'].copy().values + y_test = test['y'].copy().values + + classifier = DecisionTreeClassifier() + classifier.fit(x_train, y_train) + classifier = ScikitlearnDecisionTreeClassifier(classifier) + + attack = AttributeInferenceWhiteBoxDecisionTree(attack_feature=attack_feature, classifier=classifier) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + feat_true = x_test[:, attack_feature] + + values = [0, 1] + priors = [53/120, 67/120] + + feat_pred = attack.infer(attack_x_test, y_test, values=values, priors=priors) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 \ No newline at end of file diff --git a/tests/security/test_attribute_inference_regression.py b/tests/security/test_attribute_inference_regression.py new file mode 100644 index 00000000..104c47e4 --- /dev/null +++ b/tests/security/test_attribute_inference_regression.py @@ -0,0 +1,171 @@ +import numpy as np +from holisticai.datasets import load_dataset +from holisticai.security.attackers.attribute_inference.baseline import AttributeInferenceBaseline +from holisticai.security.attackers.attribute_inference.wrappers.regression.scikitlearn import ScikitlearnRegressor +from holisticai.security.attackers.attribute_inference.black_box import AttributeInferenceBlackBox +from holisticai.security.attackers.attribute_inference.true_label_baseline import AttributeInferenceBaselineTrueLabel +from holisticai.security.attackers.attribute_inference.wrappers.regression.scikitlearn import ScikitlearnDecisionTreeRegressor +from holisticai.security.attackers.attribute_inference.mitigation.attacks.white_box_lifestyle_decision_tree import ( + AttributeInferenceWhiteBoxLifestyleDecisionTree +) +from holisticai.efficacy.metrics import classification_efficacy_metrics +from holisticai.pipeline import Pipeline +from sklearn.tree import DecisionTreeRegressor +import pytest + +np.random.seed(100) + +SHARD_SIZE = 60 + +@pytest.fixture +def regression_dataset(): + dataset = load_dataset('us_crime', preprocessed=True, protected_attribute='race') + dataset = dataset.sample(SHARD_SIZE, random_state=42) + train_test = dataset.train_test_split(test_size=0.5, random_state=42) + train = train_test['train'] + test = train_test['test'] + return train, test + +def test_att_inf_baseline_regression(regression_dataset): + train, test = regression_dataset + train_data = train['X'].copy() + train_data['group_a'] = train['group_a'].astype(int) + + test_data = test['X'].copy() + test_data['group_a'] = test['group_a'].astype(int) + + x_train = train_data.values + x_test = test_data.values + + y_train = train['y'].values + y_test = test['y'].values + + attack_feature = 101 # last column represents the sensitive attribute + + attack = AttributeInferenceBaseline(attack_feature=attack_feature) + attack.fit(x_train) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + feat_true = x_test[:, attack_feature] + + values = [0, 1] + feat_pred = attack.infer(attack_x_test, values=values) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 + +def test_att_inf_blackbox_regression(regression_dataset): + train, test = regression_dataset + train_data = train['X'].copy() + train_data['group_a'] = train['group_a'].astype(int) + + test_data = test['X'].copy() + test_data['group_a'] = test['group_a'].astype(int) + + x_train = train_data.values + x_test = test_data.values + + y_train = train['y'].values + y_test = test['y'].values + + attack_feature = 101 # last column represents the sensitive attribute + + regressor = Pipeline(steps=[ + ('model', DecisionTreeRegressor()) + ]) + + regressor.fit(x_train, y_train) + + # regressor = train_holisticai_regressor(x_train, y_train) + regressor = ScikitlearnRegressor(regressor) + pred = regressor.predict(x_train) + + attack = AttributeInferenceBlackBox(estimator=regressor, attack_feature=attack_feature, scale_range=(0,1)) + attack.fit(x_train, y_train, pred) + + pred = regressor.predict(x_test) + attack_x_test = np.delete(x_test, attack_feature, axis=1) + + feat_true = x_test[:, attack_feature] + + values = [False, True] + feat_pred = attack.infer(attack_x_test, y_test, pred, values=values) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 + + +def test_att_inf_baseline_true_label_regression(regression_dataset): + train, test = regression_dataset + train_data = train['X'].copy() + train_data['group_a'] = train['group_a'].astype(int) + + test_data = test['X'].copy() + test_data['group_a'] = test['group_a'].astype(int) + + x_train = train_data.values + x_test = test_data.values + + y_train = train['y'].values + y_test = test['y'].values + + attack_feature = 101 # last column represents the sensitive attribute + + attack = AttributeInferenceBaselineTrueLabel(attack_feature=attack_feature, is_regression=True) + attack.fit(x_train, y_train) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + feat_true = x_test[:, attack_feature] + + values = [0, 1] + feat_pred = attack.infer(attack_x_test, y_test, values=values) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 + + +def test_att_inf_white_box_lifestyle_decision_tree_regression(regression_dataset): + train, test = regression_dataset + train_data = train['X'].copy() + train_data['group_a'] = train['group_a'].astype(int) + + test_data = test['X'].copy() + test_data['group_a'] = test['group_a'].astype(int) + + x_train = train_data.values + x_test = test_data.values + + y_train = train['y'].values + y_test = test['y'].values + + attack_feature = 101 # last column represents the sensitive attribute + + regressor = DecisionTreeRegressor() + regressor.fit(x_train, y_train) + + regressor = ScikitlearnDecisionTreeRegressor(regressor) + attack = AttributeInferenceWhiteBoxLifestyleDecisionTree(estimator=regressor, attack_feature=attack_feature) + + attack_x_test = np.delete(x_test, attack_feature, axis=1) + + feat_true = x_test[:, attack_feature] + + values = [0, 1] + priors = [2 / 30, 28 / 30] + + feat_pred = attack.infer(attack_x_test, values=values, priors=priors) + + assert len(feat_true) == len(feat_pred) + + df = classification_efficacy_metrics(feat_true, feat_pred) + + assert df.loc['Accuracy']['Value'] > 0.5 \ No newline at end of file diff --git a/tutorials/security/attribute_inference_attacks/classification/baseline.ipynb b/tutorials/security/attribute_inference_attacks/classification/baseline.ipynb new file mode 100644 index 00000000..9c10a107 --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/classification/baseline.ipynb @@ -0,0 +1,688 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the baseline module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the baseline module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will consider the 'Job' attribute of the German credit dataset as the attribute to be inferred. \n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.baseline import AttributeInferenceBaseline\n", + "\n", + "np.random.seed(100)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 800
Features: X , y , p_attrs
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":800,\"Features\":[\"X , y , p_attrs\"]},\"metadata\":null}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('german_credit', preprocessed=True)\n", + "train_test = dataset.train_test_split(test_size=0.2, stratify=dataset['y'], random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
JobCredit amountDurationHousing_freeHousing_ownHousing_rentSaving accounts_quite richSaving accounts_littleSaving accounts_moderateSaving accounts_rich...Checking account_moderateChecking account_richPurpose_'domestic appliances'Purpose_businessPurpose_carPurpose_educationPurpose_furniture/equipmentPurpose_radio/TVPurpose_repairsPurpose_vacation/others
022181300.01.00.00.01.00.00.0...1.00.00.00.01.00.00.00.00.00.0
126148200.01.00.00.00.01.00.0...1.00.00.00.01.00.00.00.00.00.0
222058240.01.00.00.01.00.00.0...0.00.00.00.00.00.00.00.01.00.0
322613360.01.00.00.01.00.00.0...0.00.00.00.00.00.00.00.01.00.0
4173180.01.00.00.01.00.00.0...0.00.00.00.01.00.00.00.00.00.0
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " Job Credit amount Duration Housing_free Housing_own Housing_rent \\\n", + "0 2 2181 30 0.0 1.0 0.0 \n", + "1 2 6148 20 0.0 1.0 0.0 \n", + "2 2 2058 24 0.0 1.0 0.0 \n", + "3 2 2613 36 0.0 1.0 0.0 \n", + "4 1 731 8 0.0 1.0 0.0 \n", + "\n", + " Saving accounts_quite rich Saving accounts_little \\\n", + "0 0.0 1.0 \n", + "1 0.0 0.0 \n", + "2 0.0 1.0 \n", + "3 0.0 1.0 \n", + "4 0.0 1.0 \n", + "\n", + " Saving accounts_moderate Saving accounts_rich ... \\\n", + "0 0.0 0.0 ... \n", + "1 1.0 0.0 ... \n", + "2 0.0 0.0 ... \n", + "3 0.0 0.0 ... \n", + "4 0.0 0.0 ... \n", + "\n", + " Checking account_moderate Checking account_rich \\\n", + "0 1.0 0.0 \n", + "1 1.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "\n", + " Purpose_'domestic appliances' Purpose_business Purpose_car \\\n", + "0 0.0 0.0 1.0 \n", + "1 0.0 0.0 1.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 1.0 \n", + "\n", + " Purpose_education Purpose_furniture/equipment Purpose_radio/TV \\\n", + "0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 \n", + "\n", + " Purpose_repairs Purpose_vacation/others \n", + "0 0.0 0.0 \n", + "1 0.0 0.0 \n", + "2 1.0 0.0 \n", + "3 1.0 0.0 \n", + "4 0.0 0.0 \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train['X'].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The German credit dataset is a dataset that contains information about 1000 loan applicants. The dataset has 20 attributes, including the 'Job' attribute, which will be the attribute to be inferred. The 'Job' attribute has four possible values: 'Unskilled', 'Unskilled Resident', 'Skilled', and 'Highly Skilled'. The dataset also contains a binary target attribute, 'Credit', which indicates whether the loan application was approved or not.\n", + "\n", + "For this demonstration, we will transform the 'Job' attribute into a binary attribute by considering only two values: 'Highly Skilled', represented by 1, and 'Not Highly Skilled', represented by 0. The goal of the attack will be to infer the value of the 'Job' attribute of a data record by querying a model trained on the data." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "job_feat_train = train['X']['Job'].values\n", + "job_feat_train[job_feat_train < 2] = 0\n", + "job_feat_train[job_feat_train >= 2] = 1\n", + "\n", + "job_feat_test = test['X']['Job'].values\n", + "job_feat_test[job_feat_test < 2] = 0\n", + "job_feat_test[job_feat_test >= 2] = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make this demonstration more simple, we will also remove the 'Credit amount' attribute and the 'Duration' attribute from the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "x_test = test['X'].drop(columns=['Credit amount', 'Duration']).values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also replace the transformed 'Job' attribute into the original training data and test data." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 0 # job feature\n", + "\n", + "x_train[:, attack_feature] = job_feat_train\n", + "x_test[:, attack_feature] = job_feat_test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - baseline**\n", + "\n", + "Now, we will perform a baseline attack to infer the selected attribute using the `AttributeInferenceBaseline` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. The attack is performed by querying the internal model with the target model's predictions and the target model's input data." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "attack = AttributeInferenceBaseline(attack_feature=attack_feature)\n", + "\n", + "attack.fit(x_train)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "feat_pred = attack.infer(attack_x_test, values=values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.7500001
Balanced Accuracy0.5022591
Precision0.7708331
Recall0.9610391
F1-Score0.8554911
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.750000 1\n", + "Balanced Accuracy 0.502259 1\n", + "Precision 0.770833 1\n", + "Recall 0.961039 1\n", + "F1-Score 0.855491 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.75, which means that the attack was able to infer the 'Job' attribute with an accuracy of 75%." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/classification/black_box.ipynb b/tutorials/security/attribute_inference_attacks/classification/black_box.ipynb new file mode 100644 index 00000000..09775eb7 --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/classification/black_box.ipynb @@ -0,0 +1,470 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the blackbox module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the blackbox module on a classification model. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will consider the 'Job' attribute of the German credit dataset as the attribute to be inferred. \n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.black_box import AttributeInferenceBlackBox" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 800
Features: X , y , p_attrs
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":800,\"Features\":[\"X , y , p_attrs\"]},\"metadata\":null}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('german_credit', preprocessed=True)\n", + "train_test = dataset.train_test_split(test_size=0.2, stratify=dataset['y'], random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The German credit dataset is a dataset that contains information about 1000 loan applicants. The dataset has 20 attributes, including the 'Job' attribute, which will be the attribute to be inferred. The 'Job' attribute has four possible values: 'Unskilled', 'Unskilled Resident', 'Skilled', and 'Highly Skilled'. The dataset also contains a binary target attribute, 'Credit', which indicates whether the loan application was approved or not.\n", + "\n", + "For this demonstration, we will transform the 'Job' attribute into a binary attribute by considering only two values: 'Highly Skilled', represented by 1, and 'Not Highly Skilled', represented by 0. The goal of the attack will be to infer the value of the 'Job' attribute of a data record by querying a model trained on the data." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "job_feat_train = train['X']['Job'].values\n", + "job_feat_train[job_feat_train < 2] = 0\n", + "job_feat_train[job_feat_train >= 2] = 1\n", + "\n", + "job_feat_test = test['X']['Job'].values\n", + "job_feat_test[job_feat_test < 2] = 0\n", + "job_feat_test[job_feat_test >= 2] = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make this demonstration more simple, we will also remove the 'Credit amount' attribute and the 'Duration' attribute from the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "x_test = test['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also replace the transformed 'Job' attribute into the original training data and test data." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 0\n", + "x_train[:, attack_feature] = job_feat_train\n", + "x_test[:, attack_feature] = job_feat_test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - blackbox**\n", + "\n", + "Now, we will perform an attack to infer the selected attribute using the `AttributeInferenceBlackBox` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. This module assumes the availability of the attacked model's predictions for the samples under attack, in addition to the rest of the feature values. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/franklin/.local/share/hatch/env/virtual/holisticai/Rsz3flyg/testing/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:690: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (2000) reached and the optimization hasn't converged yet.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from sklearn.tree import DecisionTreeClassifier\n", + "\n", + "classifier = DecisionTreeClassifier()\n", + "classifier.fit(x_train, y_train)\n", + "\n", + "attack = AttributeInferenceBlackBox(estimator=classifier, attack_feature=attack_feature)\n", + "\n", + "pred = classifier.predict_proba(x_train)\n", + "attack.fit(x_train, y_train, pred)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "pred = classifier.predict_proba(x_test)\n", + "\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "feat_pred = attack.infer(attack_x_test, y_test, pred, values=values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.9300001
Balanced Accuracy0.8935631
Precision0.9487181
Recall0.9610391
F1-Score0.9548391
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.930000 1\n", + "Balanced Accuracy 0.893563 1\n", + "Precision 0.948718 1\n", + "Recall 0.961039 1\n", + "F1-Score 0.954839 1" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.915, which means that the attack was able to infer the 'Job' attribute with an accuracy of 91.5%, which is a high accuracy. This demonstrates the vulnerability of the model to attribute inference attacks and the importance of protecting sensitive attributes in the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/classification/true_label.ipynb b/tutorials/security/attribute_inference_attacks/classification/true_label.ipynb new file mode 100644 index 00000000..8fc221a5 --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/classification/true_label.ipynb @@ -0,0 +1,691 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the TrueLabel module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the TrueLabel module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will consider the 'Job' attribute of the German credit dataset as the attribute to be inferred. \n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.true_label_baseline import AttributeInferenceBaselineTrueLabel\n", + "\n", + "np.random.seed(100)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 800
Features: X , y , p_attrs
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":800,\"Features\":[\"X , y , p_attrs\"]},\"metadata\":null}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('german_credit', preprocessed=True)\n", + "train_test = dataset.train_test_split(test_size=0.2, stratify=dataset['y'], random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
JobCredit amountDurationHousing_freeHousing_ownHousing_rentSaving accounts_quite richSaving accounts_littleSaving accounts_moderateSaving accounts_rich...Checking account_moderateChecking account_richPurpose_'domestic appliances'Purpose_businessPurpose_carPurpose_educationPurpose_furniture/equipmentPurpose_radio/TVPurpose_repairsPurpose_vacation/others
022181300.01.00.00.01.00.00.0...1.00.00.00.01.00.00.00.00.00.0
126148200.01.00.00.00.01.00.0...1.00.00.00.01.00.00.00.00.00.0
222058240.01.00.00.01.00.00.0...0.00.00.00.00.00.00.00.01.00.0
322613360.01.00.00.01.00.00.0...0.00.00.00.00.00.00.00.01.00.0
4173180.01.00.00.01.00.00.0...0.00.00.00.01.00.00.00.00.00.0
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " Job Credit amount Duration Housing_free Housing_own Housing_rent \\\n", + "0 2 2181 30 0.0 1.0 0.0 \n", + "1 2 6148 20 0.0 1.0 0.0 \n", + "2 2 2058 24 0.0 1.0 0.0 \n", + "3 2 2613 36 0.0 1.0 0.0 \n", + "4 1 731 8 0.0 1.0 0.0 \n", + "\n", + " Saving accounts_quite rich Saving accounts_little \\\n", + "0 0.0 1.0 \n", + "1 0.0 0.0 \n", + "2 0.0 1.0 \n", + "3 0.0 1.0 \n", + "4 0.0 1.0 \n", + "\n", + " Saving accounts_moderate Saving accounts_rich ... \\\n", + "0 0.0 0.0 ... \n", + "1 1.0 0.0 ... \n", + "2 0.0 0.0 ... \n", + "3 0.0 0.0 ... \n", + "4 0.0 0.0 ... \n", + "\n", + " Checking account_moderate Checking account_rich \\\n", + "0 1.0 0.0 \n", + "1 1.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "\n", + " Purpose_'domestic appliances' Purpose_business Purpose_car \\\n", + "0 0.0 0.0 1.0 \n", + "1 0.0 0.0 1.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 1.0 \n", + "\n", + " Purpose_education Purpose_furniture/equipment Purpose_radio/TV \\\n", + "0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 \n", + "\n", + " Purpose_repairs Purpose_vacation/others \n", + "0 0.0 0.0 \n", + "1 0.0 0.0 \n", + "2 1.0 0.0 \n", + "3 1.0 0.0 \n", + "4 0.0 0.0 \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train['X'].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The German credit dataset is a dataset that contains information about 1000 loan applicants. The dataset has 20 attributes, including the 'Job' attribute, which will be the attribute to be inferred. The 'Job' attribute has four possible values: 'Unskilled', 'Unskilled Resident', 'Skilled', and 'Highly Skilled'. The dataset also contains a binary target attribute, 'Credit', which indicates whether the loan application was approved or not.\n", + "\n", + "For this demonstration, we will transform the 'Job' attribute into a binary attribute by considering only two values: 'Highly Skilled', represented by 1, and 'Not Highly Skilled', represented by 0. The goal of the attack will be to infer the value of the 'Job' attribute of a data record by querying a model trained on the data." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "job_feat_train = train['X']['Job'].values\n", + "job_feat_train[job_feat_train < 2] = 0\n", + "job_feat_train[job_feat_train >= 2] = 1\n", + "\n", + "job_feat_test = test['X']['Job'].values\n", + "job_feat_test[job_feat_test < 2] = 0\n", + "job_feat_test[job_feat_test >= 2] = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make this demonstration more simple, we will also remove the 'Credit amount' attribute and the 'Duration' attribute from the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "x_test = test['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also replace the transformed 'Job' attribute into the original training data and test data." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 0 # job feature\n", + "\n", + "x_train[:, attack_feature] = job_feat_train\n", + "x_test[:, attack_feature] = job_feat_test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - TrueLabel**\n", + "\n", + "Now, we will perform an attack to infer the selected attribute using the `AttributeInferenceBaselineTrueLabel` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. The attack is performed by querying the internal model with the target model's predictions and the target model's input data. This module works similar to the baseline module, but requires the target model's input data to be provided." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "attack = AttributeInferenceBaselineTrueLabel(attack_feature=attack_feature)\n", + "\n", + "attack.fit(x_train, y_train)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "feat_pred = attack.infer(attack_x_test, y_test, values=values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.7300001
Balanced Accuracy0.4892721
Precision0.7659571
Recall0.9350651
F1-Score0.8421051
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.730000 1\n", + "Balanced Accuracy 0.489272 1\n", + "Precision 0.765957 1\n", + "Recall 0.935065 1\n", + "F1-Score 0.842105 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.73, which means that the attack was able to infer the 'Job' attribute with an accuracy of 73%." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/classification/white_box.ipynb b/tutorials/security/attribute_inference_attacks/classification/white_box.ipynb new file mode 100644 index 00000000..0f7ceadf --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/classification/white_box.ipynb @@ -0,0 +1,480 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the WhiteBox module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the WhiteBox module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will consider the 'Job' attribute of the German credit dataset as the attribute to be inferred. \n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.white_box import AttributeInferenceWhiteBoxDecisionTree" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 800
Features: X , y , p_attrs
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":800,\"Features\":[\"X , y , p_attrs\"]},\"metadata\":null}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('german_credit', preprocessed=True)\n", + "train_test = dataset.train_test_split(test_size=0.2, stratify=dataset['y'], random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The German credit dataset is a dataset that contains information about 1000 loan applicants. The dataset has 20 attributes, including the 'Job' attribute, which will be the attribute to be inferred. The 'Job' attribute has four possible values: 'Unskilled', 'Unskilled Resident', 'Skilled', and 'Highly Skilled'. The dataset also contains a binary target attribute, 'Credit', which indicates whether the loan application was approved or not.\n", + "\n", + "For this demonstration, we will transform the 'Job' attribute into a binary attribute by considering only two values: 'Highly Skilled', represented by 1, and 'Not Highly Skilled', represented by 0. The goal of the attack will be to infer the value of the 'Job' attribute of a data record by querying a model trained on the data." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "job_feat_train = train['X']['Job'].values\n", + "job_feat_train[job_feat_train < 2] = 0\n", + "job_feat_train[job_feat_train >= 2] = 1\n", + "\n", + "job_feat_test = test['X']['Job'].values\n", + "job_feat_test[job_feat_test < 2] = 0\n", + "job_feat_test[job_feat_test >= 2] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "((array([0, 1]), array([ 46, 154])), 200)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unique_values = np.unique(job_feat_test, return_counts=True)\n", + "unique_values, len(job_feat_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make this demonstration more simple, we will also remove the 'Credit amount' attribute and the 'Duration' attribute from the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "x_test = test['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also replace the transformed 'Job' attribute into the original training data and test data." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 0\n", + "x_train[:, attack_feature] = job_feat_train\n", + "x_test[:, attack_feature] = job_feat_test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - WhiteBox**\n", + "\n", + "Now, we will perform an attack to infer the selected attribute using the `AttributeInferenceWhiteBoxDecisionTree` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. The attack is performed by querying the internal model with the target model's predictions and the target model's input data. This is a variation of the method proposed by [Fredrikson et al.](https://dl.acm.org/doi/10.1145/2810103.2813677), since it assumes the availability of the attacked model's predictions for the samples under attack, in addition to access to the model itself and the rest of the feature values. Unlike the WhiteBoxLifeStyle module, this module requires the target values to pefrom the attack." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.tree import DecisionTreeClassifier\n", + "\n", + "classifier = DecisionTreeClassifier()\n", + "classifier.fit(x_train, y_train)\n", + "\n", + "attack = AttributeInferenceWhiteBoxDecisionTree(classifier=classifier, attack_feature=attack_feature)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "priors = [46/200, 154/200]\n", + "\n", + "feat_pred = attack.infer(attack_x_test, y_test, values=values, priors=priors)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.7150001
Balanced Accuracy0.4795311
Precision0.7621621
Recall0.9155841
F1-Score0.8318581
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.715000 1\n", + "Balanced Accuracy 0.479531 1\n", + "Precision 0.762162 1\n", + "Recall 0.915584 1\n", + "F1-Score 0.831858 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.71, which means that the attack was able to infer the 'Job' attribute with an accuracy of 71%." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/classification/white_box_lifestyle.ipynb b/tutorials/security/attribute_inference_attacks/classification/white_box_lifestyle.ipynb new file mode 100644 index 00000000..089f0a8c --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/classification/white_box_lifestyle.ipynb @@ -0,0 +1,480 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the WhiteBoxLifeStyle module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the WhiteBoxLifeStyle module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will consider the 'Job' attribute of the German credit dataset as the attribute to be inferred. \n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.white_box_lifestyle import AttributeInferenceWhiteBoxLifestyleDecisionTree" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 800
Features: X , y , p_attrs
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":800,\"Features\":[\"X , y , p_attrs\"]},\"metadata\":null}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('german_credit', preprocessed=True)\n", + "train_test = dataset.train_test_split(test_size=0.2, stratify=dataset['y'], random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The German credit dataset is a dataset that contains information about 1000 loan applicants. The dataset has 20 attributes, including the 'Job' attribute, which will be the attribute to be inferred. The 'Job' attribute has four possible values: 'Unskilled', 'Unskilled Resident', 'Skilled', and 'Highly Skilled'. The dataset also contains a binary target attribute, 'Credit', which indicates whether the loan application was approved or not.\n", + "\n", + "For this demonstration, we will transform the 'Job' attribute into a binary attribute by considering only two values: 'Highly Skilled', represented by 1, and 'Not Highly Skilled', represented by 0. The goal of the attack will be to infer the value of the 'Job' attribute of a data record by querying a model trained on the data." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "job_feat_train = train['X']['Job'].values\n", + "job_feat_train[job_feat_train < 2] = 0\n", + "job_feat_train[job_feat_train >= 2] = 1\n", + "\n", + "job_feat_test = test['X']['Job'].values\n", + "job_feat_test[job_feat_test < 2] = 0\n", + "job_feat_test[job_feat_test >= 2] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "((array([0, 1]), array([ 46, 154])), 200)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unique_values = np.unique(job_feat_test, return_counts=True)\n", + "unique_values, len(job_feat_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make this demonstration more simple, we will also remove the 'Credit amount' attribute and the 'Duration' attribute from the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "x_test = test['X'].drop(columns=['Credit amount', 'Duration']).values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also replace the transformed 'Job' attribute into the original training data and test data." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 0\n", + "x_train[:, attack_feature] = job_feat_train\n", + "x_test[:, attack_feature] = job_feat_test" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - WhiteBoxLifestyle**\n", + "\n", + "Now, we will perform an attack to infer the selected attribute using the `AttributeInferenceWhiteBoxLifestyleDecisionTree` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. The attack is performed by querying the internal model with the target model's predictions and the target model's input data. For our case we will use a implementation of [Fredrikson et al.](https://dl.acm.org/doi/10.1145/2810103.2813677) to perform the attack, specifically for decision trees." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.tree import DecisionTreeClassifier\n", + "\n", + "classifier = DecisionTreeClassifier()\n", + "classifier.fit(x_train, y_train)\n", + "\n", + "attack = AttributeInferenceWhiteBoxLifestyleDecisionTree(estimator=classifier, attack_feature=attack_feature)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "priors = [46/200, 154/200]\n", + "\n", + "feat_pred = attack.infer(attack_x_test, values=values, priors=priors)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.7300001
Balanced Accuracy0.4968941
Precision0.7688171
Recall0.9285711
F1-Score0.8411761
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.730000 1\n", + "Balanced Accuracy 0.496894 1\n", + "Precision 0.768817 1\n", + "Recall 0.928571 1\n", + "F1-Score 0.841176 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.735, which means that the attack was able to infer the 'Job' attribute with an accuracy of 73.5%." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/regression/baseline.ipynb b/tutorials/security/attribute_inference_attacks/regression/baseline.ipynb new file mode 100644 index 00000000..d3b96047 --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/regression/baseline.ipynb @@ -0,0 +1,649 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the baseline module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the baseline module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will work with the 'us_crime' dataset, which contains information about crime rates in the United States, and we will consider the 'race' attribute to perform the attack.\n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.baseline import AttributeInferenceBaseline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 1594
Features: X , y , p_attrs , group_a , group_b
Metadata: race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":1594,\"Features\":[\"X , y , p_attrs , group_a , group_b\"]},\"metadata\":\"race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}\"}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('us_crime', preprocessed=True, protected_attribute='race')\n", + "train_test = dataset.train_test_split(test_size=0.2, random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
statefoldpopulationhouseholdsizeracepctblackracePctAsianracePctHispagePct12t21agePct12t29agePct16t24...NumStreetPctForeignBornPctBornSameStatePctSameHouse85PctSameCity85PctSameState85LandAreaPopDensPctUsePubTransLemasPctOfficDrugUn
04580.140.560.850.090.030.760.830.77...0.090.090.620.330.360.360.340.070.271.0
1620.010.400.020.110.220.390.420.25...0.010.190.660.300.570.780.010.260.020.0
25550.000.330.000.010.010.360.400.24...0.000.030.670.760.770.710.020.150.020.0
33430.010.710.360.210.060.770.650.66...0.000.350.480.660.530.570.010.480.820.0
41380.010.650.080.200.040.480.360.20...0.000.130.220.200.000.000.070.070.010.0
\n", + "

5 rows × 101 columns

\n", + "
" + ], + "text/plain": [ + " state fold population householdsize racepctblack racePctAsian \\\n", + "0 45 8 0.14 0.56 0.85 0.09 \n", + "1 6 2 0.01 0.40 0.02 0.11 \n", + "2 55 5 0.00 0.33 0.00 0.01 \n", + "3 34 3 0.01 0.71 0.36 0.21 \n", + "4 13 8 0.01 0.65 0.08 0.20 \n", + "\n", + " racePctHisp agePct12t21 agePct12t29 agePct16t24 ... NumStreet \\\n", + "0 0.03 0.76 0.83 0.77 ... 0.09 \n", + "1 0.22 0.39 0.42 0.25 ... 0.01 \n", + "2 0.01 0.36 0.40 0.24 ... 0.00 \n", + "3 0.06 0.77 0.65 0.66 ... 0.00 \n", + "4 0.04 0.48 0.36 0.20 ... 0.00 \n", + "\n", + " PctForeignBorn PctBornSameState PctSameHouse85 PctSameCity85 \\\n", + "0 0.09 0.62 0.33 0.36 \n", + "1 0.19 0.66 0.30 0.57 \n", + "2 0.03 0.67 0.76 0.77 \n", + "3 0.35 0.48 0.66 0.53 \n", + "4 0.13 0.22 0.20 0.00 \n", + "\n", + " PctSameState85 LandArea PopDens PctUsePubTrans LemasPctOfficDrugUn \n", + "0 0.36 0.34 0.07 0.27 1.0 \n", + "1 0.78 0.01 0.26 0.02 0.0 \n", + "2 0.71 0.02 0.15 0.02 0.0 \n", + "3 0.57 0.01 0.48 0.82 0.0 \n", + "4 0.00 0.07 0.07 0.01 0.0 \n", + "\n", + "[5 rows x 101 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train['X'].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The 'us_crime' dataset that we will use in this notebook is a processed version that contains 1594 records and 102 attributes, including the protected attribute that we will use in the attack. This protected attribute is a binary attribute that indicates whether the individual is white or non-white." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = train['X'].copy()\n", + "train_data['group_a'] = train['group_a'].astype(int)\n", + "\n", + "test_data = test['X'].copy()\n", + "test_data['group_a'] = test['group_a'].astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train_data.values\n", + "x_test = test_data.values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 101 # last column represents the sensitive attribute" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - baseline**\n", + "\n", + "Now, we will perform a baseline attack to infer the selected attribute using the `AttributeInferenceBaseline` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. The attack is performed by querying the internal model with the target model's predictions and the target model's input data." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "attack = AttributeInferenceBaseline(attack_feature=attack_feature)\n", + "\n", + "attack.fit(x_train)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "feat_pred = attack.infer(attack_x_test, values=values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.9749371
Balanced Accuracy0.9509591
Precision0.9852511
Recall0.9852511
F1-Score0.9852511
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.974937 1\n", + "Balanced Accuracy 0.950959 1\n", + "Precision 0.985251 1\n", + "Recall 0.985251 1\n", + "F1-Score 0.985251 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy around of 0.97, which means that the attack was able to infer the 'group_a' attribute with an accuracy of 97%. This high accuracy indicates that the target model is leaking information about the 'group_a' attribute, which can be a privacy concern." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/regression/black_box.ipynb b/tutorials/security/attribute_inference_attacks/regression/black_box.ipynb new file mode 100644 index 00000000..58b90e4b --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/regression/black_box.ipynb @@ -0,0 +1,663 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the BlackBox module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the BlackBox module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will work with the 'us_crime' dataset, which contains information about crime rates in the United States, and we will consider the 'race' attribute to perform the attack.\n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.black_box import AttributeInferenceBlackBox" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 1594
Features: X , y , p_attrs , group_a , group_b
Metadata: race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":1594,\"Features\":[\"X , y , p_attrs , group_a , group_b\"]},\"metadata\":\"race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}\"}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('us_crime', preprocessed=True, protected_attribute='race')\n", + "train_test = dataset.train_test_split(test_size=0.2, random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
statefoldpopulationhouseholdsizeracepctblackracePctAsianracePctHispagePct12t21agePct12t29agePct16t24...NumStreetPctForeignBornPctBornSameStatePctSameHouse85PctSameCity85PctSameState85LandAreaPopDensPctUsePubTransLemasPctOfficDrugUn
04580.140.560.850.090.030.760.830.77...0.090.090.620.330.360.360.340.070.271.0
1620.010.400.020.110.220.390.420.25...0.010.190.660.300.570.780.010.260.020.0
25550.000.330.000.010.010.360.400.24...0.000.030.670.760.770.710.020.150.020.0
33430.010.710.360.210.060.770.650.66...0.000.350.480.660.530.570.010.480.820.0
41380.010.650.080.200.040.480.360.20...0.000.130.220.200.000.000.070.070.010.0
\n", + "

5 rows × 101 columns

\n", + "
" + ], + "text/plain": [ + " state fold population householdsize racepctblack racePctAsian \\\n", + "0 45 8 0.14 0.56 0.85 0.09 \n", + "1 6 2 0.01 0.40 0.02 0.11 \n", + "2 55 5 0.00 0.33 0.00 0.01 \n", + "3 34 3 0.01 0.71 0.36 0.21 \n", + "4 13 8 0.01 0.65 0.08 0.20 \n", + "\n", + " racePctHisp agePct12t21 agePct12t29 agePct16t24 ... NumStreet \\\n", + "0 0.03 0.76 0.83 0.77 ... 0.09 \n", + "1 0.22 0.39 0.42 0.25 ... 0.01 \n", + "2 0.01 0.36 0.40 0.24 ... 0.00 \n", + "3 0.06 0.77 0.65 0.66 ... 0.00 \n", + "4 0.04 0.48 0.36 0.20 ... 0.00 \n", + "\n", + " PctForeignBorn PctBornSameState PctSameHouse85 PctSameCity85 \\\n", + "0 0.09 0.62 0.33 0.36 \n", + "1 0.19 0.66 0.30 0.57 \n", + "2 0.03 0.67 0.76 0.77 \n", + "3 0.35 0.48 0.66 0.53 \n", + "4 0.13 0.22 0.20 0.00 \n", + "\n", + " PctSameState85 LandArea PopDens PctUsePubTrans LemasPctOfficDrugUn \n", + "0 0.36 0.34 0.07 0.27 1.0 \n", + "1 0.78 0.01 0.26 0.02 0.0 \n", + "2 0.71 0.02 0.15 0.02 0.0 \n", + "3 0.57 0.01 0.48 0.82 0.0 \n", + "4 0.00 0.07 0.07 0.01 0.0 \n", + "\n", + "[5 rows x 101 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train['X'].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The 'us_crime' dataset that we will use in this notebook is a processed version that contains 1594 records and 102 attributes, including the protected attribute that we will use in the attack. This protected attribute is a binary attribute that indicates whether the individual is white or non-white." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = train['X'].copy()\n", + "train_data['group_a'] = train['group_a'].astype(int)\n", + "\n", + "test_data = test['X'].copy()\n", + "test_data['group_a'] = test['group_a'].astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train_data.values\n", + "x_test = test_data.values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 101 # last column represents the sensitive attribute" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - blackbox**\n", + "\n", + "Now, we will perform a attack to infer the selected attribute using the `AttributeInferenceBlackBox` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. This module assumes the availability of the attacked model's predictions for the samples under attack, in addition to the rest of the feature values. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from holisticai.pipeline import Pipeline\n", + "from sklearn.tree import DecisionTreeRegressor\n", + "\n", + "regressor = Pipeline(steps=[\n", + " ('model', DecisionTreeRegressor())\n", + "])\n", + "\n", + "regressor.fit(x_train, y_train)\n", + "\n", + "attack = AttributeInferenceBlackBox(estimator=regressor, attack_feature=attack_feature, scale_range=(0,1))\n", + "\n", + "pred = regressor.predict(x_train)\n", + "\n", + "attack.fit(x_train, y_train, pred)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "\n", + "pred = regressor.predict(x_test)\n", + "\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [False, True]\n", + "feat_pred = attack.infer(attack_x_test, y_test, pred, values=values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.9699251
Balanced Accuracy0.9137171
Precision0.9711821
Recall0.9941001
F1-Score0.9825071
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.969925 1\n", + "Balanced Accuracy 0.913717 1\n", + "Precision 0.971182 1\n", + "Recall 0.994100 1\n", + "F1-Score 0.982507 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.969, which means that the attack was able to infer the 'group_a' attribute with an accuracy of 96.9%, which is a high accuracy. This demonstrates the vulnerability of the model to attribute inference attacks and the importance of protecting sensitive attributes in the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/regression/true_label.ipynb b/tutorials/security/attribute_inference_attacks/regression/true_label.ipynb new file mode 100644 index 00000000..ec588feb --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/regression/true_label.ipynb @@ -0,0 +1,650 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the TrueLabel module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the TrueLabel module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will work with the 'us_crime' dataset, which contains information about crime rates in the United States, and we will consider the 'race' attribute to perform the attack.\n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.true_label_baseline import AttributeInferenceBaselineTrueLabel" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 1594
Features: X , y , p_attrs , group_a , group_b
Metadata: race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":1594,\"Features\":[\"X , y , p_attrs , group_a , group_b\"]},\"metadata\":\"race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}\"}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('us_crime', preprocessed=True, protected_attribute='race')\n", + "train_test = dataset.train_test_split(test_size=0.2, random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
statefoldpopulationhouseholdsizeracepctblackracePctAsianracePctHispagePct12t21agePct12t29agePct16t24...NumStreetPctForeignBornPctBornSameStatePctSameHouse85PctSameCity85PctSameState85LandAreaPopDensPctUsePubTransLemasPctOfficDrugUn
04580.140.560.850.090.030.760.830.77...0.090.090.620.330.360.360.340.070.271.0
1620.010.400.020.110.220.390.420.25...0.010.190.660.300.570.780.010.260.020.0
25550.000.330.000.010.010.360.400.24...0.000.030.670.760.770.710.020.150.020.0
33430.010.710.360.210.060.770.650.66...0.000.350.480.660.530.570.010.480.820.0
41380.010.650.080.200.040.480.360.20...0.000.130.220.200.000.000.070.070.010.0
\n", + "

5 rows × 101 columns

\n", + "
" + ], + "text/plain": [ + " state fold population householdsize racepctblack racePctAsian \\\n", + "0 45 8 0.14 0.56 0.85 0.09 \n", + "1 6 2 0.01 0.40 0.02 0.11 \n", + "2 55 5 0.00 0.33 0.00 0.01 \n", + "3 34 3 0.01 0.71 0.36 0.21 \n", + "4 13 8 0.01 0.65 0.08 0.20 \n", + "\n", + " racePctHisp agePct12t21 agePct12t29 agePct16t24 ... NumStreet \\\n", + "0 0.03 0.76 0.83 0.77 ... 0.09 \n", + "1 0.22 0.39 0.42 0.25 ... 0.01 \n", + "2 0.01 0.36 0.40 0.24 ... 0.00 \n", + "3 0.06 0.77 0.65 0.66 ... 0.00 \n", + "4 0.04 0.48 0.36 0.20 ... 0.00 \n", + "\n", + " PctForeignBorn PctBornSameState PctSameHouse85 PctSameCity85 \\\n", + "0 0.09 0.62 0.33 0.36 \n", + "1 0.19 0.66 0.30 0.57 \n", + "2 0.03 0.67 0.76 0.77 \n", + "3 0.35 0.48 0.66 0.53 \n", + "4 0.13 0.22 0.20 0.00 \n", + "\n", + " PctSameState85 LandArea PopDens PctUsePubTrans LemasPctOfficDrugUn \n", + "0 0.36 0.34 0.07 0.27 1.0 \n", + "1 0.78 0.01 0.26 0.02 0.0 \n", + "2 0.71 0.02 0.15 0.02 0.0 \n", + "3 0.57 0.01 0.48 0.82 0.0 \n", + "4 0.00 0.07 0.07 0.01 0.0 \n", + "\n", + "[5 rows x 101 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train['X'].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The 'us_crime' dataset that we will use in this notebook is a processed version that contains 1594 records and 102 attributes, including the protected attribute that we will use in the attack. This protected attribute is a binary attribute that indicates whether the individual is white or non-white." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = train['X'].copy()\n", + "train_data['group_a'] = train['group_a'].astype(int)\n", + "\n", + "test_data = test['X'].copy()\n", + "test_data['group_a'] = test['group_a'].astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train_data.values\n", + "x_test = test_data.values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 101 # last column represents the sensitive attribute" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - TrueLabel**\n", + "\n", + "Now, we will perform an attack to infer the selected attribute using the `AttributeInferenceBaselineTrueLabel` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. The attack is performed by querying the internal model with the target model's predictions and the target model's input data. This module works similar to the baseline module, but requires the target model's input data to be provided." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "attack = AttributeInferenceBaselineTrueLabel(attack_feature=attack_feature, is_regression=True)\n", + "\n", + "attack.fit(x_train, y_train)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "feat_pred = attack.infer(attack_x_test, y_test, values=values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.9674191
Balanced Accuracy0.9396761
Precision0.9822491
Recall0.9793511
F1-Score0.9807981
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.967419 1\n", + "Balanced Accuracy 0.939676 1\n", + "Precision 0.982249 1\n", + "Recall 0.979351 1\n", + "F1-Score 0.980798 1" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.967, which means that the attack was able to infer the 'group_a' attribute with an accuracy of 96.7%. This high accuracy indicates that the target model is leaking information about the 'group_a' attribute, which can be a privacy concern." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/security/attribute_inference_attacks/regression/white_box.ipynb b/tutorials/security/attribute_inference_attacks/regression/white_box.ipynb new file mode 100644 index 00000000..e52f89dd --- /dev/null +++ b/tutorials/security/attribute_inference_attacks/regression/white_box.ipynb @@ -0,0 +1,677 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# **Attribute inference attack with the WhiteBox module**\n", + "\n", + "In this notebook, we will demonstrate how to perform an attribute inference attack with the WhiteBox module. The goal of an attribute inference attack is to infer the value of a sensitive attribute of a data record by querying a model trained on the data. In this case, we will work with the 'us_crime' dataset, which contains information about crime rates in the United States, and we will consider the 'race' attribute to perform the attack.\n", + "\n", + "### **Importing the necessary libraries and loading the data** " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from holisticai.datasets import load_dataset\n", + "from holisticai.security.attackers.attribute_inference.white_box_lifestyle import AttributeInferenceWhiteBoxLifestyleDecisionTree" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " [Dataset]\n", + "
\n", + "
\n", + "
Instances: 1594
Features: X , y , p_attrs , group_a , group_b
Metadata: race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + " " + ], + "text/plain": [ + "{\"dtype\":\"Dataset\",\"attributes\":{\"Instances\":1594,\"Features\":[\"X , y , p_attrs , group_a , group_b\"]},\"metadata\":\"race: {'group_a': 'racePctWhite>0.5', 'group_b': 'racePctWhite<=0.5'}\"}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset = load_dataset('us_crime', preprocessed=True, protected_attribute='race')\n", + "train_test = dataset.train_test_split(test_size=0.2, random_state=0)\n", + "train = train_test['train']\n", + "test = train_test['test']\n", + "train" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
statefoldpopulationhouseholdsizeracepctblackracePctAsianracePctHispagePct12t21agePct12t29agePct16t24...NumStreetPctForeignBornPctBornSameStatePctSameHouse85PctSameCity85PctSameState85LandAreaPopDensPctUsePubTransLemasPctOfficDrugUn
04580.140.560.850.090.030.760.830.77...0.090.090.620.330.360.360.340.070.271.0
1620.010.400.020.110.220.390.420.25...0.010.190.660.300.570.780.010.260.020.0
25550.000.330.000.010.010.360.400.24...0.000.030.670.760.770.710.020.150.020.0
33430.010.710.360.210.060.770.650.66...0.000.350.480.660.530.570.010.480.820.0
41380.010.650.080.200.040.480.360.20...0.000.130.220.200.000.000.070.070.010.0
\n", + "

5 rows × 101 columns

\n", + "
" + ], + "text/plain": [ + " state fold population householdsize racepctblack racePctAsian \\\n", + "0 45 8 0.14 0.56 0.85 0.09 \n", + "1 6 2 0.01 0.40 0.02 0.11 \n", + "2 55 5 0.00 0.33 0.00 0.01 \n", + "3 34 3 0.01 0.71 0.36 0.21 \n", + "4 13 8 0.01 0.65 0.08 0.20 \n", + "\n", + " racePctHisp agePct12t21 agePct12t29 agePct16t24 ... NumStreet \\\n", + "0 0.03 0.76 0.83 0.77 ... 0.09 \n", + "1 0.22 0.39 0.42 0.25 ... 0.01 \n", + "2 0.01 0.36 0.40 0.24 ... 0.00 \n", + "3 0.06 0.77 0.65 0.66 ... 0.00 \n", + "4 0.04 0.48 0.36 0.20 ... 0.00 \n", + "\n", + " PctForeignBorn PctBornSameState PctSameHouse85 PctSameCity85 \\\n", + "0 0.09 0.62 0.33 0.36 \n", + "1 0.19 0.66 0.30 0.57 \n", + "2 0.03 0.67 0.76 0.77 \n", + "3 0.35 0.48 0.66 0.53 \n", + "4 0.13 0.22 0.20 0.00 \n", + "\n", + " PctSameState85 LandArea PopDens PctUsePubTrans LemasPctOfficDrugUn \n", + "0 0.36 0.34 0.07 0.27 1.0 \n", + "1 0.78 0.01 0.26 0.02 0.0 \n", + "2 0.71 0.02 0.15 0.02 0.0 \n", + "3 0.57 0.01 0.48 0.82 0.0 \n", + "4 0.00 0.07 0.07 0.01 0.0 \n", + "\n", + "[5 rows x 101 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train['X'].head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Dataset preprocessing**\n", + "\n", + "The 'us_crime' dataset that we will use in this notebook is a processed version that contains 1594 records and 102 attributes, including the protected attribute that we will use in the attack. This protected attribute is a binary attribute that indicates whether the individual is white or non-white." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = train['X'].copy()\n", + "train_data['group_a'] = train['group_a'].astype(int)\n", + "\n", + "test_data = test['X'].copy()\n", + "test_data['group_a'] = test['group_a'].astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([0, 1]), array([ 60, 339]))" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.unique(test['group_a'].astype(int), return_counts=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train_data.values\n", + "x_test = test_data.values\n", + "\n", + "y_train = train['y'].values\n", + "y_test = test['y'].values" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "attack_feature = 101 # last column represents the sensitive attribute" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Attribute inference attack - WhiteBox**\n", + "\n", + "Now, we will perform an attack to infer the selected attribute using the `AttributeInferenceWhiteBoxDecisionTree` class from the `holisticai` library. This class, creates an object that uses an internal model to perform the attack. The internal model is trained on the same dataset used to train the target model to learn the attacked feature from the remaining features. The attack is performed by querying the internal model with the target model's predictions and the target model's input data. This is a variation of the method proposed by [Fredrikson et al.](https://dl.acm.org/doi/10.1145/2810103.2813677), since it assumes the availability of the attacked model's predictions for the samples under attack, in addition to access to the model itself and the rest of the feature values. Unlike the WhiteBoxLifeStyle module, this module requires the target values to pefrom the attack.\n", + "\n", + "To do this, we will first train a decision tree regressor model on the 'us_crime' dataset and then we will pass this model to the `AttributeInferenceWhiteBoxDecisionTree` class to perform the attack." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.tree import DecisionTreeRegressor\n", + "\n", + "regressor = DecisionTreeRegressor()\n", + "regressor.fit(x_train, y_train)\n", + "\n", + "attack = AttributeInferenceWhiteBoxLifestyleDecisionTree(estimator=regressor, attack_feature=attack_feature)\n", + "\n", + "attack_x_test = np.delete(x_test, attack_feature, axis=1)\n", + "\n", + "feat_true = x_test[:, attack_feature]\n", + "\n", + "values = [0, 1]\n", + "priors = [60 / 399, 339 / 399]\n", + "\n", + "feat_pred = attack.infer(attack_x_test, values=values, priors=priors)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Measuring the attack success**\n", + "\n", + "The success of the attack is measured by the accuracy of the inferred attribute. The accuracy is calculated as the ratio of the correctly inferred attributes to the total number of data records. This can be done by using traditional classification metrics such as accuracy, precision, recall, and F1-score. For our case, we will use the `classification_efficacy_metrics` function from the `holisticai` library to calculate these metrics." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ValueReference
Metric
Accuracy0.8496241
Balanced Accuracy0.5000001
Precision0.8496241
Recall1.0000001
F1-Score0.9186991
\n", + "
" + ], + "text/plain": [ + " Value Reference\n", + "Metric \n", + "Accuracy 0.849624 1\n", + "Balanced Accuracy 0.500000 1\n", + "Precision 0.849624 1\n", + "Recall 1.000000 1\n", + "F1-Score 0.918699 1" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from holisticai.efficacy.metrics import classification_efficacy_metrics\n", + "\n", + "classification_efficacy_metrics(feat_true, feat_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can see, our attack achieved an accuracy of 0.849, which means that the attack was able to infer the 'Job' attribute with an accuracy of 84.9%. This is a high accuracy, which indicates that the attack was successful in inferring the sensitive attribute." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "testing", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}