added distortion

Author: ltiao

bore/plugins/hpbandster.py

@@ -2,6 +2,7 @@
 import tensorflow as tf
 
 from tensorflow.keras.losses import BinaryCrossentropy
+import scipy.stats as sps
 
 from ..engine import Ledger, minimize_multi_start, is_duplicate
 from ..types import DenseConfigurationSpace, DenseConfiguration
@@ -19,7 +20,7 @@ def __init__(self, config_space, eta=3, min_budget=0.01, max_budget=1,
                  num_restarts=10, batch_size=64, num_steps_per_iter=1000,
                  optimizer="adam", num_layers=2, num_units=32,
                  activation="relu", normalize=True, method="L-BFGS-B",
-                 max_iter=100, ftol=1e-2, seed=None, **kwargs):
+                 max_iter=100, ftol=1e-2, distortion=1e-3, seed=None, **kwargs):
 
         if gamma is None:
             gamma = 1/eta
@@ -31,7 +32,7 @@ def __init__(self, config_space, eta=3, min_budget=0.01, max_budget=1,
                             optimizer=optimizer, num_layers=num_layers,
                             num_units=num_units, activation=activation,
                             normalize=normalize, method=method,
-                            max_iter=max_iter, ftol=ftol, seed=seed)
+                            max_iter=max_iter, ftol=ftol, distortion=distortion, seed=seed)
         # (LT): Note this is using the *grandparent* class initializer to
         # replace the config_generator!
         super(HyperBand, self).__init__(config_generator=cg, **kwargs)
@@ -70,8 +71,8 @@ def __init__(self, config_space, gamma=1/3, num_random_init=10,
                  random_rate=0.25, num_restarts=3, batch_size=64,
                  num_steps_per_iter=1000, optimizer="adam", num_layers=2,
                  num_units=32, activation="relu", normalize=True,
-                 method="L-BFGS-B", max_iter=100, ftol=1e-2, seed=None,
-                 **kwargs):
+                 method="L-BFGS-B", max_iter=100, ftol=1e-2, distortion=1e-3,
+                 seed=None, **kwargs):
 
         super(RatioEstimator, self).__init__(**kwargs)
 
@@ -100,6 +101,7 @@ def __init__(self, config_space, gamma=1/3, num_random_init=10,
         self.method = method
         self.ftol = ftol
         self.max_iter = max_iter
+        self.distortion = distortion
 
         self.batch_size = batch_size
         self.num_steps_per_iter = num_steps_per_iter
@@ -230,7 +232,20 @@ def get_config(self, budget):
 
         self.logger.info(f"[Glob. maximum: value={-opt.fun:.3f}, x={opt.x}")
 
-        config_opt_arr = opt.x
+        loc = opt.x
+
+        if self.distortion is None:
+
+            config_opt_arr = loc
+        else:
+
+            # dist = multivariate_normal(mean=opt.x, cov=opt.hess_inv.todense())
+            a = (self.bounds.lb - loc) / self.distortion
+            b = (self.bounds.ub - loc) / self.distortion
+            dist = sps.truncnorm(a=a, b=b, loc=loc, scale=self.distortion)
+
+            config_opt_arr = dist.rvs(random_state=self.random_state)
+
         config_opt_dict = self._dict_from_array(config_opt_arr)
 
         return (config_opt_dict, {})

scripts/plotting/plot_result.py

@@ -20,6 +20,7 @@
 @click.option('--num-runs', '-n', default=20)
 @click.option('--methods', '-m', multiple=True)
 @click.option('--ci')
+@click.option('--duration-key', default="info")
 @click.option('--context', default="paper")
 @click.option('--style', default="ticks")
 @click.option('--palette', default="muted")
@@ -29,8 +30,8 @@
 @click.option("--output-dir", default=OUTPUT_DIR,
               type=click.Path(file_okay=False, dir_okay=True),
               help="Output directory.")
-def main(benchmark_name, input_dir, num_runs, methods, ci, context, style,
-         palette, width, aspect, extension, output_dir):
+def main(benchmark_name, input_dir, num_runs, methods, ci, duration_key,
+         context, style, palette, width, aspect, extension, output_dir):
 
     figsize = size(width, aspect)
     height = width / aspect
@@ -53,8 +54,8 @@ def main(benchmark_name, input_dir, num_runs, methods, ci, context, style,
         "bore": "BORE",
         "bore-steps-50": "BORE (50 steps)",
         "bore-steps": "BORE (250 steps)",
-        # "boredom": "BORE II",
-        # "boredom-real": "BORE III",
+        "boredom": "BOREDOM",
+        "boredom-real": "BORE",
         "bore-sigmoid-elu-ftol-1e-2-gamma-0.33333333333333333333": "BORE",
         "boredom-real": "BORE",
         "bore-logit-elu-ftol-1e-2-gamma-0.33333333333333333333": "BORE",
@@ -74,7 +75,8 @@ def main(benchmark_name, input_dir, num_runs, methods, ci, context, style,
 
             path = input_path.joinpath(benchmark_name, method, f"{run:03d}.csv")
 
-            frame = load_frame(path, run, loss_min=loss_min)
+            frame = load_frame(path, run, loss_min=loss_min,
+                               duration_key=duration_key)
             frames.append(frame.assign(method=method))
 
             series[run] = extract_series(frame, index="elapsed", column="regret")
@@ -124,9 +126,9 @@ def main(benchmark_name, input_dir, num_runs, methods, ci, context, style,
                  data=data_merged, ax=ax)
 
     ax.set_xlabel("wall-clock time elapsed (s)")
-    ax.set_ylabel("evaluations")
+    ax.set_ylabel("simple regret")
 
-    # ax.set_yscale("log")
+    ax.set_yscale("log")
 
     for ext in extension:
         fig.savefig(output_path.joinpath(f"regret_elapsed_{context}_{suffix}.{ext}"),

scripts/run_bore.py

@@ -38,8 +38,9 @@
 @click.option("--activation", default="elu")
 @click.option('--normalize/--no-normalize', default=True)
 @click.option("--method", default="L-BFGS-B")
-@click.option("--max-iter", default=100)
-@click.option("--ftol", default=1e-3)
+@click.option("--max-iter", default=1000)
+@click.option("--ftol", default=1e-9)
+@click.option("--distortion", default=1e-3)
 @click.option("--input-dir", default="datasets/fcnet_tabular_benchmarks",
               type=click.Path(file_okay=False, dir_okay=True),
               help="Input data directory.")
@@ -50,7 +51,7 @@ def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
          num_iterations, eta, min_budget, max_budget, gamma, num_random_init,
          random_rate, num_restarts, batch_size, num_steps_per_iter, optimizer,
          num_layers, num_units, activation, normalize, method, max_iter, ftol,
-         input_dir, output_dir):
+         distortion, input_dir, output_dir):
 
     Worker, worker_kws = get_worker(benchmark_name, dimensions=dimensions,
                                     dataset_name=dataset_name,
@@ -68,7 +69,7 @@ def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
                    optimizer=optimizer, num_layers=num_layers,
                    num_units=num_units, activation=activation,
                    normalize=normalize, method=method, max_iter=max_iter,
-                   ftol=ftol)
+                   ftol=ftol, distortion=distortion)
     with open(output_path.joinpath("options.json"), 'w') as f:
         json.dump(options, f, sort_keys=True, indent=2)
 
@@ -105,6 +106,7 @@ def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
                   method=method,
                   max_iter=max_iter,
                   ftol=ftol,
+                  distortion=distortion,
                   seed=run_id,
                   nameserver=ns_host,
                   nameserver_port=ns_port,