Update SyneTune API, add CQR

.gitignore

@@ -169,6 +169,7 @@ runs*
 *.parquet
 run-data*
 *.txt
+subselection/data*
 slurm-*
 tmp*
 *.aux

CHANGELOG.md

@@ -1,6 +1,9 @@
+# 1.0.5
+- Update SyneTune API, add CQR as optimizer (#211).
+
 # 1.0.4
-- Fix yahpo error (#201)
-- Add missing requirement files (#202)
+- Fix yahpo error (#201).
+- Add missing requirement files (#202).
 
 # v1.0.3
 - Fix the order of optimizer names in the critical difference plots (#196).

carps/build/Makefile

@@ -73,8 +73,9 @@ OS := $(shell uname)
 
 uvenv:
 	pip install uv
-	uv venv --python=3.12 carpsenv
-	. carpsenv/bin/activate
+	export PATH="$HOME/.local/bin:$PATH"
+	uv venv --python=3.12 .venv
+	. .venv/bin/activate
 	python -m ensurepip --upgrade
 	python -m pip install uv --upgrade
 	uv pip install setuptools wheel
@@ -94,8 +95,11 @@ optimizer_skopt:
 	$(PIP) install -r ../container/recipes/optimizers/Scikit_Optimize/Scikit_Optimize_requirements.txt
 
 optimizer_synetune:
-	$(PIP) install -r ../container/recipes/optimizers/SyneTune/SyneTune_requirements.txt
+# 	$(PIP) install -r ../container/recipes/optimizers/SyneTune/SyneTune_requirements.txt
+	git clone https://github.com/awslabs/syne-tune.git lib/syne-tune || true
+	$(PIP) install -e 'lib/syne-tune[basic,blackbox-repository]'
 	$(PIP) install numpy --upgrade
+	$(PIP) install xgboost --upgrade
 
 optimizer_ax:
 	$(PIP) install -r ../container/recipes/optimizers/Ax/Ax_requirements.txt

carps/configs/optimizer/synetune/ASHA.yaml

@@ -0,0 +1,11 @@
+# @package _global_
+defaults:
+  - base
+optimizer_id: SyneTune-ASHA
+optimizer:
+  optimizer_name: ASHA
+  optimizer_kwargs:
+    random_seed: ${seed}
+    max_t: ${task.input_space.fidelity_space.max_fidelity}
+  expects_multiple_objectives: false
+  expects_fidelities: true

carps/configs/optimizer/synetune/BOHB.yaml

@@ -0,0 +1,11 @@
+# @package _global_
+defaults:
+  - base
+optimizer_id: SyneTune-BOHB
+optimizer:
+  optimizer_name: BOHB
+  optimizer_kwargs:
+    random_seed: ${seed}
+    max_t: ${task.input_space.fidelity_space.max_fidelity}
+  expects_multiple_objectives: false
+  expects_fidelities: true

carps/configs/optimizer/synetune/BO.yaml → carps/configs/optimizer/synetune/BOTorch.yaml

@@ -1,8 +1,8 @@
 # @package _global_
 defaults:
   - base
-optimizer_id: SyneTune-BO
+optimizer_id: SyneTune-BOTorch
 optimizer:
-  optimizer_name: 'BayesianOptimization'
+  optimizer_name: BOTorch
   optimizer_kwargs:
     random_seed: ${seed}

carps/configs/optimizer/synetune/KDE.yaml → carps/configs/optimizer/synetune/CQR.yaml

@@ -1,8 +1,8 @@
 # @package _global_
 defaults:
   - base
-optimizer_id: SyneTune-KDE
+optimizer_id: SyneTune-CQR
 optimizer:
-  optimizer_name: 'KDE'
+  optimizer_name: CQR
   optimizer_kwargs:
     random_seed: ${seed}

carps/configs/optimizer/synetune/TPE.yaml

@@ -0,0 +1,8 @@
+# @package _global_
+defaults:
+  - base
+optimizer_id: SyneTune-TPE
+optimizer:
+  optimizer_name: TPE
+  optimizer_kwargs:
+    random_seed: ${seed}

carps/container/recipes/optimizers/SyneTune/SyneTune_requirements.txt

@@ -1,4 +1,2 @@
-syne-tune[basic]  #==0.13.0
-# for bore
-xgboost  #==2.0.3
-scikit-learn  #==1.4.0
+syne-tune[basic,blackbox-repository]  >= 0.14.2
+

carps/optimizers/synetune.py

@@ -33,20 +33,7 @@
     randint,
     uniform,
 )
-from syne_tune.optimizer.baselines import (  # type: ignore
-    ASHA,
-    BOHB,
-    BORE,
-    DEHB,
-    KDE,
-    MOASHA,
-    MOBSTER,
-    MOREA,
-    BayesianOptimization,
-    MOLinearScalarizationBayesOpt,
-    MORandomScalarizationBayesOpt,
-    SyncMOBSTER,
-)
+from syne_tune.optimizer.baselines import ASHA, BOHB, BORE, CQR, TPE, BOTorch  # type: ignore
 
 from carps.optimizers.optimizer import Optimizer
 from carps.utils.pareto_front import pareto
@@ -62,39 +49,27 @@
 
 # This is a subset from the syne-tune baselines
 optimizers_dict = {
-    "BayesianOptimization": BayesianOptimization,
-    "BO-MO-RS": MORandomScalarizationBayesOpt,
-    "BO-MO-LS": MOLinearScalarizationBayesOpt,
-    "MOREA": MOREA,
     "ASHA": ASHA,
-    "MOBSTER": MOBSTER,
     "BOHB": BOHB,
-    "KDE": KDE,
+    "CQR": CQR,
     "BORE": BORE,
-    "DEHB": DEHB,
-    "MOASHA": MOASHA,
-    "SyncMOBSTER": SyncMOBSTER,
+    "BOTorch": BOTorch,
+    "TPE": TPE,
 }
 
 
 metric_type_dict = {
-    "BayesianOptimization": str,
-    "BO-MO-RS": list,
-    "BO-MO-LS": list,
-    "MOREA": list,
     "ASHA": str,
-    "MOBSTER": str,
     "BOHB": str,
-    "KDE": str,
+    "CQR": str,
     "BORE": str,
-    "DEHB": str,
-    "MOASHA": list,
-    "SyncMOBSTER": str,
+    "BOTorch": str,
+    "TPE": str,
 }
 
 mf_optimizer_dicts = {
-    "with_mf": {"ASHA", "MOASHA", "DEHB", "MOBSTER", "BOHB", "SyncMOBSTER"},
-    "without_mf": {"BORE", "BayesianOptimization", "KDE"},
+    "with_mf": {"ASHA", "BOHB"},
+    "without_mf": {"BORE", "CQR", "BOTorch", "TPE"},
 }
 
 
@@ -274,7 +249,7 @@ def ask(self) -> TrialInfo:
             trial info (config, seed, instance, budget)
         """
         assert self._solver is not None
-        trial_suggestion = self._solver.suggest(self.trial_counter)
+        trial_suggestion = self._solver.suggest()
         trial = SyneTrial(
             trial_id=self.trial_counter,
             config=trial_suggestion.config,
@@ -350,10 +325,7 @@ def tell(self, trial_info: TrialInfo, trial_value: TrialValue) -> None:
 
     def best_trial(self, metric: str) -> TrialResult:
         """Return the best trial according to the provided metric."""
-        if self.optimizer_name == "MOASHA":
-            self.solver.mode = "min"
-
-        sign = 1.0 if self.solver.mode == "max" else -1.0
+        sign = -1.0 if self.solver.do_minimize else 1.0
 
         return max(
             [value for key, value in self.completed_experiments.items()],
@@ -397,13 +369,11 @@ def _setup_optimizer(self) -> SyneTrialScheduler:
 
         _optimizer_kwargs: dict[str, Any] = {
             "metric": self.metric,
-            "mode": "min"
-            if self.task.output_space.n_objectives == 1
-            else list(np.repeat("min", self.task.output_space.n_objectives)),
         }
 
         if self.optimizer_name in mf_optimizer_dicts["with_mf"]:
-            _optimizer_kwargs["resource_attr"] = self.fidelity_type
+            key_fidelity_type = "time_attr"
+            _optimizer_kwargs[key_fidelity_type] = self.fidelity_type
             # _optimizer_kwargs["max_t"] = self.max_fidelity  # TODO check how to set n trials / wallclock limit
             # for synetune
 
@@ -426,10 +396,6 @@ def _setup_optimizer(self) -> SyneTrialScheduler:
 
         self.optimizer_kwargs.update(_optimizer_kwargs)
 
-        if self.optimizer_name == "MOASHA":
-            del self.optimizer_kwargs["metric"]
-            del self.optimizer_kwargs["resource_attr"]
-
         if self.optimizer_name in ["SyncMOBSTER"] and "time_attr" in self.optimizer_kwargs:
             del self.optimizer_kwargs["time_attr"]
 

tests/test_optimizers.py

@@ -32,6 +32,7 @@ def filenames():
     # Replace with your desired pattern, e.g., "*.txt" to match all .txt files
     _filenames = list(Path("carps/configs/optimizer").glob("**/*.yaml"))
     _filenames = [fn for fn in _filenames if "base" not in str(fn)]
+    _filenames = [fn for fn in _filenames if not str(fn.name).startswith("_")]
     _filenames = [fn for fn in _filenames if "hebo" not in str(fn)]
     _filenames.sort()
     return _filenames