Merge pull request #21 from Sohambasu07/main

Update to ConfigSpace>=1.0

Author: Sohambasu07

.github/workflows/pre-commit.yml

@@ -19,10 +19,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Setup Python 3.8
+      - name: Setup Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - run: pip install pre-commit
       - run: pre-commit install
       - run: pre-commit run --all-files

.github/workflows/pytest.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     steps:
       - uses: actions/checkout@v4

pyproject.toml

@@ -4,8 +4,8 @@
 name = "mf-prior-bench"
 dependencies = [
   "pyyaml",
-  "numpy",
-  "configspace<=0.7",
+  "numpy<2.0",
+  "configspace>=1.0",
   "pandas",
   "more_itertools",
   "pyarrow"
@@ -15,7 +15,7 @@ description = "A wrapper for multi-fidelity benchmarks with priors"
 authors = [{name = "Eddie Bergman", email="[email protected]"}]
 readme = "README.md"
 license = { file = "LICENSE.txt" }
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 classifiers = [
   'Intended Audience :: Science/Research',
   'Intended Audience :: Developers',
@@ -32,11 +32,13 @@ classifiers = [
 [project.optional-dependencies]
 yahpo = ["yahpo-gym==1.0.1"]
 jahs-bench = [
-  "jahs_bench==1.1",
-  "pandas<1.4",
+  "jahs_bench==1.2.0",
+  "pandas<2.0",
+  "numpy<2.0.0"
 ]
 tabular = ["pandas>2", "pyarrow"]
-pd1 = ["xgboost>=1.7"]
+pd1 = ["xgboost[scikit-learn]>=1.7"]
+taskset_tabular = ["tensorflow<=2.18.0"]
 surrogates = ["dehb"]
 docs = [
   "mkdocs",
@@ -82,10 +84,10 @@ exclude_lines = [
 ] # These are lines to exclude from coverage
 
 [tool.black]
-target-version = ['py38']
+target-version = ['py310']
 
 [tool.ruff]
-target-version = "py38"
+target-version = "py310"
 line-length = 88
 src = ["src", "tests"]
 
@@ -224,7 +226,7 @@ convention = "google"
 max-args = 10 # Changed from default of 5
 
 [tool.mypy]
-python_version = "3.8"
+python_version = "3.10"
 packages = ["src/mfpbench", "tests"]
 
 show_error_codes = true

src/mfpbench/config.py

@@ -146,7 +146,9 @@ def set_as_default_prior(self, configspace: ConfigurationSpace) -> None:
                     )
                 # No need to do anything here
             else:
-                hp.default_value = hp.check_default(v)
+                if not hp.legal_value(v):
+                    raise ValueError(f"Value {v} is not legal for {k}")
+                hp.default_value = v
 
     @classmethod
     def from_file(cls, path: str | Path) -> Self:

src/mfpbench/jahs/benchmark.py

@@ -264,7 +264,7 @@ def _jahs_configspace(
             ) from e
 
         space = ConfigurationSpace(name=name, seed=seed)
-        space.add_hyperparameters(
+        space.add(
             [
                 Constant(
                     "N",
@@ -342,5 +342,5 @@ def _jahs_configspace(
             log=True,
         )
 
-        space.add_hyperparameters([optimizers, lr, weight_decay])
+        space.add([optimizers, lr, weight_decay])
         return space

src/mfpbench/lcbench_tabular/benchmark.py

@@ -43,7 +43,7 @@ def _get_raw_lcbench_space(
     """
     # obtained from https://github.com/automl/lcbench#dataset-overview
     cs = ConfigurationSpace(name=name, seed=seed)
-    cs.add_hyperparameters(
+    cs.add(
         [
             UniformIntegerHyperparameter(
                 "batch_size",
@@ -98,7 +98,7 @@ def _get_raw_lcbench_space(
     )
 
     if with_constants:
-        cs.add_hyperparameters(
+        cs.add(
             [
                 Constant("cosine_annealing_T_max", 50),
                 Constant("cosine_annealing_eta_min", 0.0),

src/mfpbench/nb201_tabular/benchmark.py

@@ -5,9 +5,10 @@
 """
 from __future__ import annotations
 
+from collections.abc import Mapping
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, ClassVar, Literal, Mapping
+from typing import Any, ClassVar, Literal
 
 import numpy as np
 import pandas as pd
@@ -28,7 +29,7 @@ def _raw_space(name: str, *, seed: int | None = None) -> ConfigurationSpace:
         "avg_pool_3x3",
     ]
     cs = ConfigurationSpace(name=name, seed=seed)
-    cs.add_hyperparameters(
+    cs.add(
         [
             CategoricalHyperparameter("edge_0_1", choices=choices.copy()),
             CategoricalHyperparameter("edge_0_2", choices=choices.copy()),

src/mfpbench/pd1/benchmarks/cifar100.py

@@ -13,7 +13,7 @@ class PD1cifar100_wideresnet_2048(PD1Benchmark):
     @classmethod
     def _create_space(cls, seed: int | None = None) -> ConfigurationSpace:
         cs = ConfigurationSpace(seed=seed)
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "lr_decay_factor",

src/mfpbench/pd1/benchmarks/imagenet.py

@@ -13,7 +13,7 @@ class PD1imagenet_resnet_512(PD1Benchmark):
     @classmethod
     def _create_space(cls, seed: int | None = None) -> ConfigurationSpace:
         cs = ConfigurationSpace(seed=seed)
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "lr_decay_factor",

src/mfpbench/pd1/benchmarks/lm1b.py

@@ -13,7 +13,7 @@ class PD1lm1b_transformer_2048(PD1Benchmark):
     @classmethod
     def _create_space(cls, seed: int | None = None) -> ConfigurationSpace:
         cs = ConfigurationSpace(seed=seed)
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "lr_decay_factor",

src/mfpbench/pd1/benchmarks/translate_wmt.py

@@ -13,7 +13,7 @@ class PD1translatewmt_xformer_64(PD1Benchmark):
     @classmethod
     def _create_space(cls, seed: int | None = None) -> ConfigurationSpace:
         cs = ConfigurationSpace(seed=seed)
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "lr_decay_factor",

src/mfpbench/pd1/benchmarks/uniref50.py

@@ -13,7 +13,7 @@ class PD1uniref50_transformer_128(PD1Benchmark):
     @classmethod
     def _create_space(cls, seed: int | None = None) -> ConfigurationSpace:
         cs = ConfigurationSpace(seed=seed)
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "lr_decay_factor",

src/mfpbench/pd1/processing/process_script.py

@@ -235,7 +235,7 @@ def process_pd1(  # noqa: C901, PLR0912, PLR0915
             for r in dataset["train_cost"]  # type: ignore
         ]
 
-        # Explode out the lists in the entries of the datamframe to be a single long
+        # Explode out the lists in the entries of the dataframe to be a single long
         # dataframe with each element of that list on its own row
         dataset = dataset.explode(explode_columns, ignore_index=True)
         logger.info(f"{len(dataset)} rows")

src/mfpbench/pd1/surrogate/train_xgboost.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import pandas as pd
 
@@ -17,7 +17,7 @@
 
 
 def train_xgboost(
-    config: Configuration,
+    config: Configuration | dict[str, Any],
     budget: int,
     X: pd.DataFrame,
     y: pd.Series,

src/mfpbench/pd1/surrogate/training.py

@@ -131,7 +131,7 @@ def find_xgboost_surrogate(
     dehb = DEHB(
         f=dehb_target_function,
         cs=cs,
-        dimensions=len(cs.get_hyperparameters()),
+        dimensions=len(list(cs.values())),
         min_budget=MIN_ESTIMATORS,
         max_budget=MAX_ESTIMATORS,
         n_workers=n_workers,

src/mfpbench/pd1/surrogate/xgboost_space.py

@@ -15,7 +15,7 @@ def space(seed: int | None) -> ConfigurationSpace:
     """Space for the xgboost surrogate."""
     cs = ConfigurationSpace(seed=seed)
 
-    cs.add_hyperparameters(
+    cs.add(
         [
             UniformFloatHyperparameter(
                 "learning_rate",

src/mfpbench/pd1_tabular/benchmark.py

@@ -23,10 +23,10 @@ def _get_raw_pd1_space(
     name: str,
     seed: int | None = None,
     *,
-    with_constants: bool | None = None,
+    with_constants: bool | None = None, # noqa: ARG001
 ) -> ConfigurationSpace:
     cs = ConfigurationSpace(name=name, seed=seed)
-    cs.add_hyperparameters(
+    cs.add(
         [
             UniformFloatHyperparameter(
                 "lr_decay_factor",
@@ -117,7 +117,7 @@ class PD1TabularBenchmark(TabularBenchmark):
         "translate_wmt-xformer_translate-64_tabular",
     )
 
-    def __init__(
+    def __init__(  # noqa: D107, PLR0913
         self,
         dataset: str,
         model: str,

src/mfpbench/result.py

@@ -68,17 +68,18 @@ def from_dict(
         renames: Mapping[str, str] | None = None,
     ) -> Self:
         """Create from a dict or mapping object."""
+        if renames is not None:
+            values = {renames.get(k, k): v for k, v in result.items()}
+        else:
+            values = result
         values = {
             k: (
                 metric.as_value(v)
                 if (metric := cls.metric_defs.get(k)) is not None
                 else v
             )
-            for k, v in result.items()
+            for k, v in values.items()
         }
-        if renames is not None:
-            values = {renames.get(k, k): v for k, v in values.items()}
-
         if value_metric is None:
             value_metric = cls.default_value_metric
 

src/mfpbench/synthetic/hartmann/benchmark.py

@@ -151,7 +151,7 @@ def __init__(
             else f"mfh{cls.mfh_dims}"
         )
         space = ConfigurationSpace(name=name, seed=seed)
-        space.add_hyperparameters(
+        space.add(
             [
                 UniformFloatHyperparameter(f"X_{i}", lower=0.0, upper=1.0)
                 for i in range(cls.mfh_dims)

src/mfpbench/taskset_tabular/benchmark.py

@@ -24,7 +24,7 @@ def _get_raw_taskset_space(
     optimizer: str,
 ) -> ConfigurationSpace:
     cs = ConfigurationSpace(name=name, seed=seed)
-    cs.add_hyperparameters(
+    cs.add(
         [
             UniformFloatHyperparameter(
                 "learning_rate",
@@ -35,7 +35,7 @@ def _get_raw_taskset_space(
         ],
     )
     if optimizer.split("_")[0] in ["adam4p", "adam6p", "adam8p"]:
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "beta1",
@@ -58,7 +58,7 @@ def _get_raw_taskset_space(
             ],
         )
     if optimizer.split("_")[0] in ["adam6p", "adam8p"]:
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "l1",
@@ -75,7 +75,7 @@ def _get_raw_taskset_space(
             ],
         )
     if optimizer.split("_")[0] in ["adam8p"]:
-        cs.add_hyperparameters(
+        cs.add(
             [
                 UniformFloatHyperparameter(
                     "linear_decay",
@@ -1128,7 +1128,7 @@ def _normalize_metrics(config_frame_column: pd.DataFrame) -> pd.DataFrame:
         return df.groupby("id").transform(_normalize_metrics)
 
     def _remove_zero_step(self, df: pd.DataFrame) -> pd.DataFrame:
-        """Drops the loss curve at step 0, that is, at initialization"""
+        """Drops the loss curve at step 0, that is, at initialization."""
         unique_ids = df.index.get_level_values(0).unique()
         # check if step=0 exists for all unique IDs
         step_zero_exists_for_all = sum(df["step"] == 0) == len(unique_ids)