Version 2.0.0a2 (#883)

renesass · Katharina Eggensperger · mfeurer · web-flow · commit dd249cdb90c2 · 2022-10-26T13:59:51.000+02:00
## Bugfixes
- Fixed random weight (re-)generalization of multi-objective algorithms: Before the weights were generated for each call to ``build_matrix``, now we only re-generate them for every iteration.
- Optimization may get stuck because of deep copying an iterator for callback: We removed the configuration call from ``on_next_configurations_end``.

## Minor
- Removed example badget in README.
- Added SMAC logo to README.

Co-authored-by: Katharina Eggensperger &lt;eggenspk@informatik.uni-freiburg.de&gt;
Co-authored-by: Matthias Feurer &lt;feurerm@informatik.uni-freiburg.de&gt;
Co-authored-by: dengdifan &lt;difandeng@gmail.com&gt;
Co-authored-by: Carolin Benjamins &lt;benjamins@tnt.uni-hannover.de&gt;
Co-authored-by: Eric Kalosa-Kenyon &lt;ekquasar@appdynamics.com&gt;
diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml
@@ -9,14 +9,12 @@ on:
     branches:
       - main
       - development
-      - development-2.0
 
   # Trigger on a open/push to a PR targeting one of these branches
   pull_request:
     branches:
       - main
       - development
-      - development-2.0
 
 jobs:
   dist:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -11,14 +11,12 @@ on:
     branches:
       - main
       - development
-      - development-2.0
 
   # Trigger on a open/push to a PR targeting one of these branches
   pull_request:
     branches:
       - main
       - development
-      - development-2.0
 
 env:
   name: SMAC3
@@ -36,9 +34,16 @@ jobs:
         python-version: "3.10"
 
     - name: Install dependencies
+      id: install
       run: |
         pip install ".[gpytorch,dev]"
 
+        # Getting the version
+        SMAC_VERSION=$(python -c "import smac; print('v' + str(smac.version));")
+
+        # Make it a global variable
+        echo "SMAC_VERSION=$SMAC_VERSION" >> $GITHUB_ENV
+
     - name: Make docs
       run: |
         make clean
@@ -58,13 +63,19 @@ jobs:
         rm -rf $branch_name
         cp -r ../${{ env.name }}/docs/build/html $branch_name
 
+        # we also copy the current SMAC_VERSION
+        rm -rf $SMAC_VERSION
+        cp -r ../${{ env.name }}/docs/build/html $SMAC_VERSION
+
+
     - name: Push to gh-pages
       if: (contains(github.ref, 'develop') || contains(github.ref, 'main')) && github.event_name == 'push'
       run: |
         last_commit=$(git log --pretty=format:"%an: %s")
         cd ../gh-pages
         branch_name=${GITHUB_REF##*/}
         git add $branch_name/
+        git add $SMAC_VERSION/
         git config --global user.name 'Github Actions'
         git config --global user.email 'not@mail.com'
         git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -9,14 +9,12 @@ on:
     branches:
       - main
       - development
-      - development-2.0
 
   # When a push occurs on a PR that targets these branches
   pull_request:
     branches:
       - main
       - development
-      - development-2.0
 
 jobs:
   run-all-files:
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -9,14 +9,12 @@ on:
     branches:
       - main
       - development
-      - development-2.0
 
   # Triggers with push to a pr aimed at main
   pull_request:
     branches:
       - main
       - development
-      - development-2.0
 
   schedule:
     # Every day at 7AM UTC
diff --git a/README.md b/README.md
@@ -2,11 +2,12 @@
 
 
 [![Tests](https://github.com/automl/SMAC3/actions/workflows/pytest.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/pytest.yml)
-[![Docs](https://github.com/automl/SMAC3/actions/workflows/docs.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/docs.yml)
-[![Examples](https://github.com/automl/SMAC3/actions/workflows/examples.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/examples.yml)
+[![Documentation](https://github.com/automl/SMAC3/actions/workflows/docs.yml/badge.svg?branch=main)](https://github.com/automl/SMAC3/actions/workflows/docs.yml)
 [![codecov
 Status](https://codecov.io/gh/automl/SMAC3/branch/master/graph/badge.svg)](https://codecov.io/gh/automl/SMAC3)
 
+<img src="docs/images/logo.png" style="width: 50%;" />
+
 SMAC is a tool for algorithm configuration to optimize the parameters of arbitrary algorithms, including hyperparameter 
 optimization of Machine Learning algorithms. The main core consists of Bayesian Optimization in combination with an 
 aggressive racing mechanism to efficiently decide which of two configurations performs better.
diff --git a/changelog.md b/changelog.md
@@ -1,3 +1,14 @@
+# 2.0.0a2
+
+## Bugfixes
+- Fixed random weight (re-)generalization of multi-objective algorithms: Before the weights were generated for each call to ``build_matrix``, now we only re-generate them for every iteration.
+- Optimization may get stuck because of deep copying an iterator for callback: We removed the configuration call from ``on_next_configurations_end``.
+
+## Minor
+- Removed example badget in README.
+- Added SMAC logo to README.
+
+
 # 2.0.0a1
 
 ## Big Changes
diff --git a/docs/conf.py b/docs/conf.py
@@ -10,6 +10,11 @@
     "copyright": copyright,
     "author": author,
     "version": version,
+    "versions": {
+        f"v{version} (unstable)": "#",
+        "v2.0.0a1": "https://automl.github.io/SMAC3/v2.0.0a1/",
+        "v1.4.0": "https://automl.github.io/SMAC3/v1.4.0/",
+    },
     "name": name,
     "html_theme_options": {
         "github_url": "https://github.com/automl/SMAC3",
diff --git a/setup.py b/setup.py
@@ -32,7 +32,7 @@ def read_file(filepath: str) -> str:
         "pytest-xdist",
         "pytest-timeout",
         # Docs
-        "automl-sphinx-theme>=0.1.18",
+        "automl-sphinx-theme>=0.2",
         # Others
         "mypy",
         "isort",
diff --git a/smac/__init__.py b/smac/__init__.py
@@ -19,7 +19,7 @@
     Copyright {datetime.date.today().strftime('%Y')}, Marius Lindauer, Katharina Eggensperger,
     Matthias Feurer, André Biedenkapp, Difan Deng, Carolin Benjamins, Tim Ruhkopf, René Sass
     and Frank Hutter"""
-version = "2.0.0a1"
+version = "2.0.0a2"
 
 
 try:
diff --git a/smac/callback.py b/smac/callback.py
@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-from ConfigSpace import Configuration
-
 import smac
 from smac.runhistory import TrialInfo, TrialInfoIntent, TrialValue
 
@@ -37,7 +35,7 @@ def on_next_configurations_start(self, smbo: smac.main.BaseSMBO) -> None:
         """
         pass
 
-    def on_next_configurations_end(self, smbo: smac.main.BaseSMBO, configurations: list[Configuration]) -> None:
+    def on_next_configurations_end(self, smbo: smac.main.BaseSMBO) -> None:
         """Called after the intensification asks for new configurations. Essentially, this callback is called
         before the surrogate model is trained and before the acquisition function is called.
         """
diff --git a/smac/main/smbo.py b/smac/main/smbo.py
@@ -2,8 +2,6 @@
 
 from typing import Any, Iterator
 
-import copy
-
 import numpy as np
 from ConfigSpace import Configuration
 
@@ -35,6 +33,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         self._considered_budgets: list[float | None] = [None]
 
     def get_next_configurations(self, n: int | None = None) -> Iterator[Configuration]:  # noqa: D102
+        # TODO: Let's return the initial configurations from this method too
+
         for callback in self._callbacks:
             callback.on_next_configurations_start(self)
 
@@ -53,6 +53,7 @@ def get_next_configurations(self, n: int | None = None) -> Iterator[Configuratio
             # the configspace.
             return iter([self._scenario.configspace.sample_configuration(1)])
 
+        # TODO: Check if X/Y differs from the last run, otherwise use cached results
         self._model.train(X, Y)
 
         x_best_array: np.ndarray | None = None
@@ -79,15 +80,17 @@ def get_next_configurations(self, n: int | None = None) -> Iterator[Configuratio
         )
 
         for callback in self._callbacks:
-            challenger_list = list(copy.deepcopy(challengers))
-            callback.on_next_configurations_end(self, challenger_list)
+            callback.on_next_configurations_end(self)
 
         return challengers
 
     def ask(self) -> tuple[TrialInfoIntent, TrialInfo]:  # noqa: D102
         for callback in self._callbacks:
             callback.on_ask_start(self)
 
+        if self._runhistory_encoder.multi_objective_algorithm is not None:
+            self._runhistory_encoder.multi_objective_algorithm.update_on_iteration_start()
+
         intent, trial_info = self._intensifier.get_next_trial(
             challengers=self._initial_design_configs,
             incumbent=self._incumbent,
diff --git a/smac/model/gaussian_process/priors/tophat_prior.py b/smac/model/gaussian_process/priors/tophat_prior.py
@@ -35,6 +35,8 @@ def __init__(
         self._log_min = np.log(lower_bound)
         self._max = upper_bound
         self._log_max = np.log(upper_bound)
+        self._prob = 1 / (self._max - self._min)
+        self._log_prob = np.log(self._prob)
 
         if not (self._max > self._min):
             raise Exception("Upper bound of Tophat prior must be greater than the lower bound.")
@@ -50,7 +52,7 @@ def _get_log_probability(self, theta: float) -> float:
         if theta < self._min or theta > self._max:
             return -np.inf
         else:
-            return 0
+            return self._log_prob
 
     def _sample_from_prior(self, n_samples: int) -> np.ndarray:
         if np.ndim(n_samples) != 0:
diff --git a/smac/multi_objective/parego.py b/smac/multi_objective/parego.py
@@ -37,8 +37,8 @@ def __init__(
         self._rng = np.random.RandomState(seed)
 
         self._rho = rho
-        self._theta = self._rng.rand(self._n_objectives)
-        self.update_on_iteration_start()
+        # Will be set on starting an SMBO iteration
+        self._theta: np.ndarray | None = None
 
     @property
     def meta(self) -> dict[str, Any]:  # noqa: D102
@@ -61,5 +61,8 @@ def update_on_iteration_start(self) -> None:  # noqa: D102
 
     def __call__(self, values: list[float]) -> float:  # noqa: D102
         # Weight the values
+        if self._theta is None:
+            raise ValueError("Iteration not yet initalized; Call `update_on_iteration_start()` first")
+
         theta_f = self._theta * values
         return float(np.max(theta_f, axis=0) + self._rho * np.sum(theta_f, axis=0))
diff --git a/smac/runhistory/encoder/encoder.py b/smac/runhistory/encoder/encoder.py
@@ -33,9 +33,6 @@ def _build_matrix(
         # TODO: Extend for native multi-objective
         y = np.ones([n_rows, 1])
 
-        if self._multi_objective_algorithm is not None:
-            self._multi_objective_algorithm.update_on_iteration_start()
-
         # Then populate matrix
         for row, (key, run) in enumerate(trials.items()):
             # Scaling is automatically done in configSpace
diff --git a/smac/utils/multi_objective.py b/smac/utils/multi_objective.py
@@ -38,6 +38,6 @@ def normalize_costs(
             cost = 1.0
         else:
             cost = p / q
-        costs += [cost]
+        costs.append(cost)
 
     return costs
diff --git a/tests/test_callback.py b/tests/test_callback.py
@@ -39,7 +39,7 @@ def on_iteration_end(self, smbo: smac.main.BaseSMBO) -> None:
     def on_next_configurations_start(self, smbo: smac.main.BaseSMBO) -> None:
         self.next_configurations_start_counter += 1
 
-    def on_next_configurations_end(self, smbo: smac.main.BaseSMBO, configurations: list[Configuration]) -> None:
+    def on_next_configurations_end(self, smbo: smac.main.BaseSMBO) -> None:
         self.next_configurations_end_counter += 1
 
     def on_ask_start(self, smbo: smac.main.BaseSMBO) -> None:
diff --git a/tests/test_multi_objective/test_schaffer.py b/tests/test_multi_objective/test_schaffer.py
@@ -1,13 +1,17 @@
+from __future__ import annotations
+
 import numpy as np
 import pytest
 from ConfigSpace import ConfigurationSpace, Float
 
 from smac import (
     AlgorithmConfigurationFacade,
     BlackBoxFacade,
+    Callback,
     HyperparameterOptimizationFacade,
     RandomFacade,
 )
+from smac.multi_objective import AbstractMultiObjectiveAlgorithm
 from smac.multi_objective.aggregation_strategy import MeanAggregationStrategy
 from smac.multi_objective.parego import ParEGO
 
@@ -52,16 +56,33 @@ def configspace():
     return cs
 
 
+class WrapStrategy(AbstractMultiObjectiveAlgorithm):
+    def __init__(self, strategy: AbstractMultiObjectiveAlgorithm, *args, **kwargs):
+        self.strategy = strategy(*args, **kwargs)
+        self.n_calls_update_on_iteration_start = 0
+        self.n_calls___call__ = 0
+
+    def update_on_iteration_start(self) -> None:  # noqa: D102
+        self.n_calls_update_on_iteration_start += 1
+        return self.strategy.update_on_iteration_start()
+
+    def __call__(self, values: list[float]) -> float:  # noqa: D102
+        self.n_calls___call__ += 1
+        return self.strategy(values)
+
+
 @pytest.mark.parametrize(
     "facade", [BlackBoxFacade, HyperparameterOptimizationFacade, AlgorithmConfigurationFacade, RandomFacade]
 )
 def test_mean_aggregation(facade, make_scenario, configspace):
     scenario = make_scenario(configspace, use_multi_objective=True)
 
+    multi_objective_algorithm = WrapStrategy(MeanAggregationStrategy, scenario=scenario)
+
     smac = facade(
         scenario=scenario,
         target_function=tae,
-        multi_objective_algorithm=MeanAggregationStrategy(scenario=scenario),
+        multi_objective_algorithm=multi_objective_algorithm,
         overwrite=True,
     )
     incumbent = smac.optimize()
@@ -75,32 +96,41 @@ def test_mean_aggregation(facade, make_scenario, configspace):
 
     assert diff < 0.06
 
+    assert multi_objective_algorithm.n_calls_update_on_iteration_start >= 100
+    assert multi_objective_algorithm.n_calls_update_on_iteration_start <= 130
+    assert multi_objective_algorithm.n_calls___call__ >= 100
+
 
 @pytest.mark.parametrize(
     "facade", [BlackBoxFacade, HyperparameterOptimizationFacade, AlgorithmConfigurationFacade, RandomFacade]
 )
 def test_parego(facade, make_scenario, configspace):
     scenario = make_scenario(configspace, use_multi_objective=True)
 
+    multi_objective_algorithm = WrapStrategy(ParEGO, scenario=scenario)
+
     smac = facade(
         scenario=scenario,
         target_function=tae,
-        multi_objective_algorithm=ParEGO(scenario=scenario),
+        multi_objective_algorithm=multi_objective_algorithm,
         overwrite=True,
     )
-    # The incumbent is not ambiguously because we have a Pareto front
+
     smac.optimize()
 
-    # We use the mean aggregation strategy to get the same weights
-    multi_objective_algorithm = MeanAggregationStrategy(scenario=scenario)
-    smac.runhistory.multi_objective_algorithm = multi_objective_algorithm
+    # The incumbent is not ambiguous because we have a Pareto front
+    confs, vals = smac.runhistory.get_pareto_front()
 
-    incumbent, _ = smac.runhistory.get_incumbent()
+    min_ = np.inf
+    for x, y in zip(confs, vals):
+        tr = schaffer(x["x"])
+        assert np.allclose(tr, y)
+        if np.sum(y) < min_:
+            min_ = np.sum(y)
 
-    f1_inc, f2_inc = schaffer(incumbent["x"])
-    f1_opt, f2_opt = get_optimum()
-    inc = f1_inc + f2_inc
-    opt = f1_opt + f2_opt
-    diff = abs(inc - opt)
+    opt = np.sum(get_optimum())
+    assert abs(np.sum(min_) - opt) <= 0.06
 
-    assert diff < 0.06
+    assert multi_objective_algorithm.n_calls_update_on_iteration_start >= 100
+    assert multi_objective_algorithm.n_calls_update_on_iteration_start <= 120
+    assert multi_objective_algorithm.n_calls___call__ >= 100
diff --git a/tests/test_multi_objective/test_schaffer_upscaled.py b/tests/test_multi_objective/test_schaffer_upscaled.py
