refactor

Author: ltiao

bore/decorators.py

@@ -41,7 +41,7 @@ def value_and_gradient_fn(x):
 
         # Equivalent to `tfp.math.value_and_gradient(value_fn, x)`, with the
         # only difference that the gradients preserve their `dtype` rather than
-        # casting to tf.float32, which is problematic for scipy optimize
+        # casting to `tf.float32`, which is problematic for scipy.optimize
         with tf.GradientTape(watch_accessed_variables=False) as tape:
             tape.watch(x)
             val = value_fn(x)

bore/engine.py

@@ -4,18 +4,19 @@
 from scipy.optimize import minimize
 
 from tensorflow.keras.losses import BinaryCrossentropy
-# from tensorflow.keras.initializers import GlorotUniform
 
 from .types import DenseConfigurationSpace, DenseConfiguration
 from .models import DenseSequential
 from .decorators import unbatch, value_and_gradient, numpy_io
 from .optimizers import multi_start
 
-# from hpbandster.core.master import Master
 from hpbandster.optimizers.hyperband import HyperBand
 from hpbandster.core.base_config_generator import base_config_generator
 
 
+minimize_multi_start = multi_start(minimizer_fn=minimize)
+
+
 def is_duplicate(x, xs, rtol=1e-5, atol=1e-8):
     # Clever ways of doing this would involve data structs. like KD-trees
     # or locality sensitive hashing (LSH), but these are premature
@@ -97,14 +98,15 @@ def __init__(self, config_space, gamma=1/3, num_random_init=10,
         self.logit = self._build_compile_network(num_layers, num_units,
                                                  activation, optimizer)
         self.loss = self._build_loss(self.logit, normalize=normalize)
-        self.minimizer = self._build_minimizer(num_restarts=num_restarts,
-                                               method=method, ftol=ftol,
-                                               max_iter=max_iter)
 
         self.gamma = gamma
         self.num_random_init = num_random_init
         self.random_rate = random_rate
+
         self.num_restarts = num_restarts
+        self.method = method
+        self.ftol = ftol
+        self.max_iter = max_iter
 
         self.batch_size = batch_size
         self.num_steps_per_iter = num_steps_per_iter
@@ -115,6 +117,35 @@ def __init__(self, config_space, gamma=1/3, num_random_init=10,
         self.seed = seed
         self.random_state = np.random.RandomState(seed)
 
+    def _array_from_dict(self, dct):
+        config = DenseConfiguration(self.config_space, values=dct)
+        return config.to_array()
+
+    def _dict_from_array(self, array):
+        config = DenseConfiguration.from_array(self.config_space,
+                                               array_dense=array)
+        return config.get_dictionary()
+
+    def _get_dataset_size(self):
+        return len(self.config_arrs)
+
+    def _load_data(self):
+        X = np.vstack(self.config_arrs)
+        y = np.hstack(self.losses)
+        return X, y
+
+    def _load_labels(self, y):
+        # TODO(LT): we can use clever data structures like heaps to make this
+        #   labelling constant-time, but this is probably a premature
+        #   optimization at this time...
+        tau = np.quantile(y, q=self.gamma)
+        return np.less(y, tau)
+
+    def _get_steps_per_epoch(self, dataset_size):
+        steps_per_epoch = int(np.ceil(np.true_divide(dataset_size,
+                                                     self.batch_size)))
+        return steps_per_epoch
+
     @staticmethod
     def _build_compile_network(num_layers, num_units, activation, optimizer):
 
@@ -142,52 +173,12 @@ def loss(x):
 
         return loss
 
-    @staticmethod
-    def _build_minimizer(num_restarts, method="L-BFGS-B", max_iter=100,
-                         ftol=1e-2):
-
-        @multi_start(num_restarts=num_restarts)
-        def multi_start_minimizer(fn, x0, bounds):
-            return minimize(fn, x0=x0, method=method, jac=True, bounds=bounds,
-                            options=dict(maxiter=max_iter, ftol=ftol))
-
-        return multi_start_minimizer
-
-    def _load_data(self):
-        X = np.vstack(self.config_arrs)
-        y = np.hstack(self.losses)
-        return X, y
-
-    def _load_labels(self, y):
-        tau = np.quantile(y, q=self.gamma)
-        return np.less(y, tau)
-
-    def _get_steps_per_epoch(self, dataset_size):
-        steps_per_epoch = int(np.ceil(np.true_divide(dataset_size,
-                                                     self.batch_size)))
-        return steps_per_epoch
-
-    def get_config(self, budget):
-
-        dataset_size = len(self.config_arrs)
-
-        config_random = self.config_space.sample_configuration()
-        config_random_dict = config_random.get_dictionary()
-
-        if dataset_size < self.num_random_init:
-            self.logger.debug(f"Completed {dataset_size}/{self.num_random_init}"
-                              " initial runs. Returning random candidate...")
-            return (config_random_dict, {})
-
-        if self.random_state.binomial(p=self.random_rate, n=1):
-            self.logger.info("[Glob. maximum: skipped "
-                             f"(prob={self.random_rate:.2f})] "
-                             "Returning random candidate ...")
-            return (config_random_dict, {})
+    def _update_model(self):
 
         X, y = self._load_data()
         z = self._load_labels(y)
 
+        dataset_size = self._get_dataset_size()
         steps_per_epoch = self._get_steps_per_epoch(dataset_size)
         num_epochs = self.num_steps_per_iter // steps_per_epoch
 
@@ -203,15 +194,21 @@ def get_config(self, budget):
                          f"num steps per iter: {self.num_steps_per_iter}, "
                          f"num epochs: {num_epochs}")
 
-        # Maximize acquisition function
+    def _get_maximum(self):
+
         self.logger.debug("Beginning multi-start maximization with "
                           f"{self.num_restarts} starts...")
 
-        results = self.minimizer(self.loss, self.bounds,
-                                 random_state=self.random_state)
+        results = minimize_multi_start(self.loss, self.bounds,
+                                       num_restarts=self.num_restarts,
+                                       method=self.method, jac=True,
+                                       options=dict(maxiter=self.max_iter,
+                                                    ftol=self.ftol),
+                                       random_state=self.random_state)
 
         res_best = None
         for i, res in enumerate(results):
+            # TODO(LT): This currently assumes `normalize=False`
             self.logger.debug(f"[Maximum {i+1:02d}/{self.num_restarts:02d}: "
                               f"logit={-res.fun:.3f}] success: {res.success}, "
                               f"iterations: {res.nit:02d}, status: {res.status}"
@@ -225,7 +222,32 @@ def get_config(self, budget):
                 if res_best is None or res.fun < res_best.fun:
                     res_best = res
 
-        if res_best is None:
+        return res_best
+
+    def get_config(self, budget):
+
+        dataset_size = self._get_dataset_size()
+
+        config_random = self.config_space.sample_configuration()
+        config_random_dict = config_random.get_dictionary()
+
+        if dataset_size < self.num_random_init:
+            self.logger.debug(f"Completed {dataset_size}/{self.num_random_init}"
+                              " initial runs. Returning random candidate...")
+            return (config_random_dict, {})
+
+        if self.random_state.binomial(p=self.random_rate, n=1):
+            self.logger.info("[Glob. maximum: skipped "
+                             f"(prob={self.random_rate:.2f})] "
+                             "Returning random candidate ...")
+            return (config_random_dict, {})
+
+        # Update model
+        self._update_model()
+
+        # Maximize acquisition function
+        opt = self._get_maximum()
+        if opt is None:
             # TODO(LT): It's actually important to report what one of these
             # occurred...
             self.logger.warn("[Glob. maximum: not found!] Either optimization "
@@ -234,29 +256,28 @@ def get_config(self, budget):
                              " Returning random candidate...")
             return (config_random_dict, {})
 
-        self.logger.info(f"[Glob. maximum: logit={-res_best.fun:.3f}, "
-                         f"prob={tf.sigmoid(-res_best.fun):.3f}, "
-                         f"rel. ratio={tf.sigmoid(-res_best.fun)/self.gamma:.3f}] "
-                         f"x={res_best.x}")
+        # TODO(LT): This currently assumes `normalize=False`
+        self.logger.info(f"[Glob. maximum: logit={-opt.fun:.3f}, "
+                         f"prob={tf.sigmoid(-opt.fun):.3f}, "
+                         f"rel. ratio={tf.sigmoid(-opt.fun)/self.gamma:.3f}] "
+                         f"x={opt.x}")
 
-        config_opt_arr = res_best.x
-        config_opt = DenseConfiguration.from_array(self.config_space,
-                                                   array_dense=config_opt_arr)
-        config_opt_dict = config_opt.get_dictionary()
+        config_opt_arr = opt.x
+        config_opt_dict = self._dict_from_array(config_opt_arr)
 
         return (config_opt_dict, {})
 
     def new_result(self, job, update_model=True):
 
         super(DRE, self).new_result(job)
 
-        # TODO: ignoring this right now
+        # TODO(LT): support multi-fidelity
         budget = job.kwargs["budget"]
 
-        loss = job.result["loss"]
         config_dict = job.kwargs["config"]
-        config = DenseConfiguration(self.config_space, values=config_dict)
-        config_arr = config.to_array()
+        config_arr = self._array_from_dict(config_dict)
+
+        loss = job.result["loss"]
 
-        self.losses.append(loss)
         self.config_arrs.append(config_arr)
+        self.losses.append(loss)

bore/optimizers.py

@@ -1,7 +1,6 @@
 import numpy as np
 
-from scipy.optimize import minimize
-from scipy.optimize import Bounds
+from scipy.optimize import minimize, Bounds
 from sklearn.utils import check_random_state
 
 
@@ -19,56 +18,35 @@ def deduplicate(results, atol=1e-6):
     return results_unique
 
 
-def multi_start(num_restarts):
+def multi_start(minimizer_fn=minimize):
 
-    def decorator(minimizer_fn):
+    def new_minimizer(fn, bounds, num_restarts, random_state=None, *args, **kwargs):
 
-        def new_minimizer(fn, bounds, random_state=None):
-            # We deliberately don't use args/kwargs here which would increase
-            # flexibility but also complexity. The aim here to to expose a
-            # simplied interface so users can't accidently pass conflicting
-            # arguments, e.g. `x0` which is the whole point of this decorator.
+        assert "x0" not in kwargs, "`x0` should not be specified"
 
-            # TODO(LT): Allow alternative arbitary generator function callbacks
-            # to support e.g. Gaussian sampling, low-discrepancy sequences, etc
-            random_state = check_random_state(random_state)
+        if not (num_restarts > 0):
+            return []
 
-            if isinstance(bounds, Bounds):
-                low = bounds.lb
-                high = bounds.ub
-                dims = len(low)
-                assert dims == len(high), "lower and upper bounds sizes do not match"
-            else:
-                # assumes `bounds` is a list of tuples
-                low, high = zip(*bounds)
-                dims = len(bounds)
+        # TODO(LT): Allow alternative arbitary generator function callbacks
+        # to support e.g. Gaussian sampling, low-discrepancy sequences, etc
+        random_state = check_random_state(random_state)
 
-            x_inits = random_state.uniform(low=low, high=high,
-                                           size=(num_restarts, dims))
+        if isinstance(bounds, Bounds):
+            low = bounds.lb
+            high = bounds.ub
+            dims = len(low)
+            assert dims == len(high), "lower and upper bounds sizes do not match"
+        else:
+            # assumes `bounds` is a list of tuples
+            low, high = zip(*bounds)
+            dims = len(bounds)
 
-            results = []
-            for x_init in x_inits:
-                res = minimizer_fn(fn, x0=x_init, bounds=bounds)
-                results.append(res)
+        results = new_minimizer(fn, bounds, num_restarts-1, random_state,
+                                *args, **kwargs)
+        x0 = random_state.uniform(low=low, high=high, size=(dims,))
+        result = minimizer_fn(fn, x0=x0, bounds=bounds, *args, **kwargs)
+        results.append(result)
 
-            # TODO(LT): support reduction function callback? e.g. argmin which
-            #   is what one ultimately cares about. But perhaps suboptimal
-            #   points can be useful as well, e.g. to be queued up for
-            #   evaluation by idle workers.
-            return results
+        return results
 
-        return new_minimizer
-
-    return decorator
-
-
-@multi_start(num_restarts=10)
-def multi_start_lbfgs_minimizer(fn, x0, bounds):
-    """
-    Wrapper around SciPy L-BFGS-B minimizer with a simplified interface and
-    sensible defaults specified.
-    """
-    # TODO(LT): L-BFGS-B has its own set of `tol` options so I suspect the
-    #   following `tol=1e-8` is completely ignored.
-    return minimize(fn, x0=x0, method="L-BFGS-B", jac=True, bounds=bounds,
-                    tol=1e-8, options=dict(maxiter=10000))
+    return new_minimizer

bore/types.py

@@ -42,13 +42,9 @@ def get_bounds(self):
         #             lowers.append(0.)
         #             uppers.append(1.)
         #     elif isinstance(hp, CS.UniformFloatHyperparameter):
-        #         # TODO(LT): These should never not be 0. and 1. respectively,
-        #         #   so I am really overcomplicating things here...
         #         lowers.append(hp._inverse_transform(hp.lower))
         #         uppers.append(hp._inverse_transform(hp.upper))
         #     elif isinstance(hp, CS.UniformIntegerHyperparameter):
-        #         # TODO(LT): These should never not be 0. and 1. respectively,
-        #         #   so I am really overcomplicating things here...
         #         lowers.append(hp._inverse_transform(hp.lower - 1))
         #         uppers.append(hp._inverse_transform(hp.upper + 1))
         #     else:
@@ -60,6 +56,8 @@ def get_bounds(self):
         # assert len(lowers) == self.size_dense
         # assert len(uppers) == self.size_dense
 
+        # All of the above commented code is equivalent to the following two
+        # lines...
         lowers = np.zeros(self.size_dense)
         uppers = np.ones(self.size_dense)