Added a function to the random config space searcher that caches already evaluated coalitions and ensures monotonicity for min/max search settings.

Furthermore, a fallback solution was introduced to cater for cases where all samples configs become invalid according to conditions after blinding.

Author: mwever

CHANGELOG.md

@@ -1,3 +1,9 @@
+# v0.0.6
+
+## Improvements
+- Added fallback for configuration spaces with conditions resulting in all configurations being filtered out.
+- Added caching and a function in RandomConfigSpaceSearcher to ensure monotonicity of the value function.
+
 # v0.0.5
 
 ## Features

src/hypershap/utils.py

@@ -154,15 +154,77 @@ def search(self, coalition: np.ndarray) -> float:
             validity = np.apply_along_axis(self._is_valid, axis=1, arr=temp_random_sample)
             filtered_samples = temp_random_sample[validity]
 
+            if coalition.any():
+                filtered_samples = []
+
             if len(filtered_samples) < 0.05 * len(temp_random_sample):  # pragma: no cover
                 logger.warning(
                     "WARNING: Due to blinding less than 5% of the samples in the random search remain valid. "
                     "Consider increasing the sampling budget of the random search.",
                 )
 
             # predict performance values with the help of the surrogate model for the filtered configurations
-            vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples))
+            if len(filtered_samples) > 0:
+                vals: np.ndarray = np.array(
+                    self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples),
+                )
+            else:
+                logger.warning(
+                    "WARNING: After filtering for conditions, no configurations were left, thus, using baseline value.",
+                )
+                vals = np.array([self.search(np.array([False] * len(coalition)))])
         else:
             vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(temp_random_sample))
 
-        return evaluate_aggregation(self.mode, vals)
+        # determine the final, aggregated value of the coalition
+        res = evaluate_aggregation(self.mode, vals)
+
+        # in case we are maximizing or minimizing, ensure that the value function is monotone
+        if self.mode in (Aggregation.MAX, Aggregation.MIN):
+            res = self._ensure_monotonicity(coalition, res)
+
+        # cache the coalition's value
+        self.coalition_cache[str(coalition.tolist())] = res
+
+        return res
+
+    def _ensure_monotonicity(self, coalition: np.ndarray, value: float) -> float:
+        """Ensure that the value function is monotonically increasing/decreasing depending on whether we want to maximize or minimize respectively.
+
+        Args:
+            coalition: The current coalition.
+            value: The value of the coalition as determined by searching.
+
+        Returns: The monotonicity-ensured value of the coalition.
+
+        """
+        monotone_value = value
+        checked_one = False
+
+        for i in range(len(coalition)):
+            if coalition[i]:  # check whether the entry is True and set it to False to check for a cached result
+                temp_coalition = coalition.copy()
+                temp_coalition[i] = False
+                if str(temp_coalition.tolist()) in self.coalition_cache:
+                    checked_one = True
+                    monotone_value = evaluate_aggregation(
+                        self.mode,
+                        np.array([
+                            monotone_value,
+                            self.coalition_cache[str(temp_coalition.tolist())],
+                        ]),
+                    )
+
+        if not checked_one and coalition.any():
+            logger.warning(
+                "Could not ensure monotonicity as none of the coalitions with one player less has been cached so far.",
+            )
+
+        if value < monotone_value:
+            logger.debug(
+                "Ensured monotonicity with a sub-coalition's value. Increased the value of the current coalition from %s to %s.",
+                value,
+                monotone_value,
+            )
+
+        return monotone_value

tests/fixtures/simple_setup.py

@@ -3,7 +3,13 @@
 from __future__ import annotations
 
 import pytest
-from ConfigSpace import Configuration, ConfigurationSpace, LessThanCondition, UniformFloatHyperparameter
+from ConfigSpace import (
+    Configuration,
+    ConfigurationSpace,
+    GreaterThanCondition,
+    LessThanCondition,
+    UniformFloatHyperparameter,
+)
 
 from hypershap import ExplanationTask
 
@@ -88,10 +94,34 @@ def simple_cond_config_space() -> ConfigurationSpace:
     return config_space
 
 
+@pytest.fixture(scope="session")
+def simple_act_config_space() -> ConfigurationSpace:
+    """Return a simple config space with activation structure for testing."""
+    config_space = ConfigurationSpace()
+    config_space.seed(42)
+
+    a = UniformFloatHyperparameter("a", 0, 1, 0)
+    b = UniformFloatHyperparameter("b", 0, 1, 0)
+    config_space.add(a)
+    config_space.add(b)
+
+    config_space.add(GreaterThanCondition(b, a, 0.3))
+    return config_space
+
+
 @pytest.fixture(scope="session")
 def simple_cond_base_et(
     simple_cond_config_space: ConfigurationSpace,
     simple_blackbox_function: SimpleBlackboxFunction,
 ) -> ExplanationTask:
     """Return a base explanation task for the simple setup with conditions."""
     return ExplanationTask.from_function(simple_cond_config_space, simple_blackbox_function.evaluate)
+
+
+@pytest.fixture(scope="session")
+def simple_act_base_et(
+    simple_act_config_space: ConfigurationSpace,
+    simple_blackbox_function: SimpleBlackboxFunction,
+) -> ExplanationTask:
+    """Return a base explanation task for the simple setup with conditions."""
+    return ExplanationTask.from_function(simple_act_config_space, simple_blackbox_function.evaluate)

tests/test_extended_settings.py

@@ -79,3 +79,10 @@ def test_tunability_with_conditions(simple_cond_base_et: ExplanationTask) -> Non
     hypershap = HyperSHAP(simple_cond_base_et)
     iv = hypershap.tunability(simple_cond_base_et.config_space.get_default_configuration())
     assert iv is not None, "Interaction values should not be none."
+
+
+def test_tunability_with_activation_structures(simple_act_base_et: ExplanationTask) -> None:
+    """Test the tunability task with a configuration space that has conditions."""
+    hypershap = HyperSHAP(simple_act_base_et)
+    iv = hypershap.tunability(simple_act_base_et.config_space.get_default_configuration())
+    assert iv is not None, "Interaction values should not be none."