Merge pull request #19 from automl/dev

Version 0.0.3

Author: mwever

CHANGELOG.md

@@ -1,3 +1,9 @@
+# v0.0.3
+- Added multi-baseline ablation game. This game computes ablation paths with respect to multiple baseline configurations and aggregates values for different paths via mean, min, max or variance.
+- Added waterfall plots to the HyperSHAP interface.
+- Added support for multi-data settings
+- Enabled approximation of Shapley values and interactions for settings exceeding a certain number of hyperparameters.
+
 # v0.0.2
 - Added parallelization for faster analysis
 

README.md

@@ -74,14 +74,14 @@ The example demonstrates how to:
 
 ## API Overview
 
-| Method | Purpose | Key Arguments |
-|--------|---------|---------------|
-| `HyperSHAP(explanation_task)` | Initialise the explainer with a generic `ExplanationTask`. |
+| Method | Purpose                                                                                                                           | Key Arguments |
+|--------|-----------------------------------------------------------------------------------------------------------------------------------|---------------|
+| `HyperSHAP(explanation_task)` | Initialize the explainer with a generic `ExplanationTask`.                                                                        |
 | `ablation(config_of_interest, baseline_config, index="FSII", order=2)` | Explain the contribution of each hyperparameter value (and interactions) when moving from a baseline to a specific configuration. |
-| `tunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be gained by tuning subsets of hyper‑parameters. |
-| `optimizer_bias(optimizer_of_interest, optimizer_ensemble, index="FSII", order=2)` | Attribute performance differences to a particular optimizer vs. an ensemble of optimizers. |
-| `plot_si_graph(interaction_values=None, save_path=None)` | Plot the Shapley Interaction (SI) graph; uses the most recent interaction values if none are supplied. |
-| `ExplanationTask.get_hyperparameter_names()` | Helper to retrieve ordered hyper‑parameter names (used for visualisation). |
+| `tunability(baseline_config=None, index="FSII", order=2, n_samples=10_000)` | Quantify how much performance can be gained by tuning subsets of hyper‑parameters.                                                |
+| `optimizer_bias(optimizer_of_interest, optimizer_ensemble, index="FSII", order=2)` | Attribute performance differences to a particular optimizer vs. an ensemble of optimizers.                                        |
+| `plot_si_graph(interaction_values=None, save_path=None)` | Plot the Shapley Interaction (SI) graph; uses the most recent interaction values if none are supplied.                            |
+| `ExplanationTask.get_hyperparameter_names()` | Helper to retrieve ordered hyper‑parameter names (used for visualisation).                                                        |
 
 All methods return an `InteractionValues` object (from **shapiq**) that can be inspected, saved, or passed to the visualisation routine.
 
@@ -121,11 +121,11 @@ The paper introduces the underlying game-theoretic framework and demonstrates it
 
 Contributions are welcome! Please follow these steps:
 
-     Fork the repo and create a feature branch (git checkout -b feat/your-feature).
-     Write tests (the project uses pytest).
-     Ensure all tests pass (pytest).
-     Update documentation if you add new functionality.
-     Submit a Pull Request with a clear description of the changes.
+1. Fork the repo and create a feature branch (git checkout -b feat/your-feature).
+2. Write tests (the project uses pytest).
+3. Ensure all tests pass (pytest).
+4. Update documentation if you add new functionality.
+5. Submit a Pull Request with a clear description of the changes.
 
 
 See CONTRIBUTING.md for detailed guidelines.

docs/modules.md

@@ -21,7 +21,7 @@ dependencies = [
     "numpy>=2.2.6",
     "scikit-learn>=1.7.1",
     "matplotlib>=3.10.5",
-    "networkx>=3.4.2",
+    "networkx>=3.4.2"
 ]
 
 [project.urls]
@@ -60,6 +60,10 @@ dev = [
     {include-group = "docs"}
 ]
 
+carps = [
+    "carps>=1.0.4"
+]
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

examples/carps-benchmark-example.ipynb

@@ -2,15 +2,18 @@
 
 from __future__ import annotations
 
-from .ablation import AblationGame
+from .ablation import AblationGame, MultiBaselineAblationGame
 from .abstract import AbstractHPIGame
+from .multi_data import MultiDataHPIGame
 from .optimizerbias import OptimizerBiasGame
 from .tunability import MistunabilityGame, SearchBasedGame, SensitivityGame, TunabilityGame
 
 __all__ = [
     "AblationGame",
     "AbstractHPIGame",
     "MistunabilityGame",
+    "MultiBaselineAblationGame",
+    "MultiDataHPIGame",
     "OptimizerBiasGame",
     "SearchBasedGame",
     "SensitivityGame",

pyproject.toml

@@ -27,7 +27,8 @@
 import numpy as np
 
 from hypershap.games.abstract import AbstractHPIGame
-from hypershap.task import AblationExplanationTask
+from hypershap.task import AblationExplanationTask, MultiBaselineAblationExplanationTask
+from hypershap.utils import Aggregation, evaluate_aggregation
 
 
 class AblationGame(AbstractHPIGame):
@@ -78,7 +79,8 @@ def evaluate_single_coalition(self, coalition: np.ndarray) -> float:
         baseline_cfg = self._get_explanation_task().baseline_config.get_array()
         cfg_of_interest = self._get_explanation_task().config_of_interest.get_array()
         blend = np.where(coalition == 0, baseline_cfg, cfg_of_interest)
-        res = self._get_explanation_task().surrogate_model.evaluate(blend)
+
+        res = self._get_explanation_task().get_single_surrogate_model().evaluate(blend)
 
         # validate that we do not get a list of floats by accident
         if isinstance(res, list):  # pragma: no cover
@@ -100,3 +102,48 @@ def _get_explanation_task(self) -> AblationExplanationTask:
             return self.explanation_task
 
         raise ValueError  # pragma: no cover
+
+
+class MultiBaselineAblationGame(AbstractHPIGame):
+    """The multi-baseline ablation game generates local explanations for hyperparameter configurations.
+
+    It does so by considering multiple baseline configurations as starting points.
+    """
+
+    def __init__(
+        self,
+        explanation_task: MultiBaselineAblationExplanationTask,
+        aggregation: Aggregation = Aggregation.AVG,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize a MultiBaselineAblationGame.
+
+        n_workers (int | None): The number of worker threads to use for parallel
+            evaluation of coalitions. Defaults to None, which disables parallelization.
+            Using more workers can significantly speed up the computation of Shapley values.
+            The maximum number of workers is capped by the number of coalitions.
+        verbose (bool | None): A boolean indicating whether to print verbose messages
+            during computation. Defaults to None. When set to True, the method prints
+            debugging information and progress updates.
+        """
+        self.aggregation = aggregation
+        self.ablation_games = [
+            AblationGame(
+                AblationExplanationTask(
+                    config_space=explanation_task.config_space,
+                    surrogate_model=explanation_task.surrogate_model,
+                    baseline_config=baseline_config,
+                    config_of_interest=explanation_task.config_of_interest,
+                ),
+                n_workers=n_workers,
+                verbose=verbose,
+            )
+            for baseline_config in explanation_task.baseline_configs
+        ]
+        super().__init__(explanation_task, n_workers, verbose)
+
+    def evaluate_single_coalition(self, coalition: np.ndarray) -> float:
+        """Evaluate a single coalition (combination of baseline and optimized hyperparameters)."""
+        vals = np.array([ag.evaluate_single_coalition(coalition) for ag in self.ablation_games])
+        return evaluate_aggregation(self.aggregation, vals)

src/hypershap/games/__init__.py

@@ -0,0 +1,63 @@
+"""The multi-data module provides a wrapper for extending explanation games to a multi-data setting."""
+
+from __future__ import annotations
+
+import copy
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from hypershap.games.ablation import MultiBaselineAblationGame
+from hypershap.games.abstract import AbstractHPIGame
+from hypershap.games.tunability import SearchBasedGame
+
+if TYPE_CHECKING:
+    from hypershap.task import ExplanationTask
+
+from hypershap.utils import Aggregation, evaluate_aggregation
+
+
+class MultiDataHPIGame(AbstractHPIGame):
+    """The multi-data game generalizes an explanation game to multiple datasets."""
+
+    def __init__(
+        self,
+        explanation_task: ExplanationTask,
+        base_game: AbstractHPIGame,
+        aggregation: Aggregation,
+    ) -> None:
+        """Initialize the multi-data game wrapper.
+
+        Args:
+            explanation_task: The explanation task containing the configuration space and surrogate model.
+            base_game: The base game instance.
+            aggregation: The aggregation method to use.
+
+        """
+        self.aggregation = aggregation
+        self.base_game = base_game
+
+        self.sub_games = []
+        for surrogate_model in explanation_task.get_surrogate_model_list():
+            game_copy = copy.deepcopy(base_game)
+            game_copy.explanation_task.surrogate_model = surrogate_model
+            if isinstance(game_copy, SearchBasedGame):
+                game_copy.cs_searcher.explanation_task.surrogate_model = surrogate_model
+            if isinstance(game_copy, MultiBaselineAblationGame):
+                for ag in game_copy.ablation_games:
+                    ag.explanation_task.surrogate_model = surrogate_model
+            self.sub_games.append(game_copy)
+
+        super().__init__(explanation_task)
+
+    def evaluate_single_coalition(self, coalition: np.ndarray) -> float:
+        """Evaluate the multi-data game on the coalition.
+
+        Args:
+            coalition: The coalition to evaluate.
+
+        Returns: The value of the multi-data game on the coalition.
+
+        """
+        vals = np.array([game.evaluate_single_coalition(coalition) for game in self.sub_games])
+        return evaluate_aggregation(self.aggregation, vals)

src/hypershap/games/ablation.py

@@ -32,7 +32,7 @@
     )
 
 from hypershap.games.abstract import AbstractHPIGame
-from hypershap.utils import ConfigSpaceSearcher, RandomConfigSpaceSearcher
+from hypershap.utils import Aggregation, ConfigSpaceSearcher, RandomConfigSpaceSearcher
 
 logger = logging.getLogger(__name__)
 
@@ -106,10 +106,10 @@ def __init__(
         """
         # set cs searcher if not given by default to a random config space searcher.
         if cs_searcher is None:
-            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode="max")
-        elif cs_searcher.mode != "max":  # ensure that cs_searcher is maximizing
+            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode=Aggregation.MAX)
+        elif cs_searcher.mode != Aggregation.MAX:  # ensure that cs_searcher is maximizing
             logger.warning("WARN: Tunability game set mode of given ConfigSpaceSearcher to maximize.")
-            cs_searcher.mode = "max"
+            cs_searcher.mode = Aggregation.MAX
         super().__init__(explanation_task, cs_searcher, n_workers=n_workers, verbose=verbose)
 
 
@@ -140,10 +140,10 @@ def __init__(
         """
         # set cs searcher if not given by default to a random config space searcher.
         if cs_searcher is None:
-            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode="var")
-        elif cs_searcher.mode != "var":  # ensure that cs_searcher is maximizing
+            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode=Aggregation.VAR)
+        elif cs_searcher.mode != Aggregation.VAR:  # ensure that cs_searcher is maximizing
             logger.warning("WARN: Sensitivity game set mode of given ConfigSpaceSearcher to variance.")
-            cs_searcher.mode = "var"
+            cs_searcher.mode = Aggregation.VAR
 
         super().__init__(explanation_task, cs_searcher, n_workers=n_workers, verbose=verbose)
 
@@ -175,9 +175,9 @@ def __init__(
         """
         # set cs searcher if not given by default to a random config space searcher.
         if cs_searcher is None:
-            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode="min")
-        elif cs_searcher.mode != "min":  # ensure that cs_searcher is maximizing
+            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode=Aggregation.MIN)
+        elif cs_searcher.mode != Aggregation.MIN:  # ensure that cs_searcher is maximizing
             logger.warning("WARN: Mistunability game set mode of given ConfigSpaceSearcher to minimize.")
-            cs_searcher.mode = "min"
+            cs_searcher.mode = Aggregation.MIN
 
         super().__init__(explanation_task, cs_searcher, n_workers=n_workers, verbose=verbose)