Merge pull request #232 from automl/development

Release 1.1.5

Author: shukon

cave/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.4"
+__version__ = "1.1.5"

cave/analyzer/bohb_learning_curves.py

@@ -116,9 +116,10 @@ def _plot(self, result_object, learning_curves, hyperparameter_names, reset_time
             data['colors_iteration'].append(c_id[0])
 
         # Tooltips
-        tooltips=[(key, '@' + key) for key in data.keys() if not key in ['times', 'duration', 'colors',
-                                                                         'colors_performance', 'colors_iteration']]
+        tooltips = [(key, '@' + key) for key in data.keys() if not key in ['times', 'duration', 'colors',
+                                                                           'colors_performance', 'colors_iteration']]
         tooltips.insert(4, ('duration (sec)', '@duration'))
+        tooltips.insert(5, ('Configuration', ' '))
         hover = HoverTool(tooltips=tooltips)
 
         # Create sources
@@ -154,14 +155,15 @@ def _plot(self, result_object, learning_curves, hyperparameter_names, reset_time
         p = figure(plot_height=500, plot_width=600,
                    y_axis_type=y_axis_type,
                    tools=[hover, 'save', 'pan', 'wheel_zoom', 'box_zoom', 'reset'],
-                   x_axis_label='Time', y_axis_label='Quality',
+                   x_axis_label='Time', y_axis_label='Cost',
                    x_range=Range1d(x_min, x_max, bounds='auto'),
                    y_range=Range1d(y_min, y_max, bounds='auto'),
                    )
 
         # Plot per HB_iteration, each config individually
         HB_iterations = sorted(set(data['HB_iteration']))
-        HB_handles = []
+        max_label_len = max([len(l) for l in HB_iterations])
+        HB_handles, HB_labels = [], []
         self.logger.debug("Assuming config_info to be either \"model_based_pick=True\" or \"model_based_pick=False\"")
         for it in HB_iterations:
             line_handles = []
@@ -197,12 +199,15 @@ def _plot(self, result_object, learning_curves, hyperparameter_names, reset_time
                                          size=20,
                                       ))
             HB_handles.append(line_handles)
+            HB_labels.append('warmstart data' if l in [-1, '-1']  else '{number:0{width}d}'.format(width=max_label_len,
+                                                                                                   number=int(it)))
 
-        iteration_labels = sorted(['warmstart data' if l in [-1, '-1'] else '{:02d}'.format(int(l)) for l in HB_iterations])
-        self.logger.debug("iteration_labels: %s", str(iteration_labels))
-        self.logger.debug("HB_iterations: %s", str(HB_iterations))
+        # Sort all lists according to label
+        HB_iterations, HB_handles, HB_labels = zip(*sorted(zip(HB_iterations, HB_handles, HB_labels), key=lambda tup: tup[2]))
+        HB_iterations, HB_handles, HB_labels = list(HB_iterations), list(HB_handles), list(HB_labels)
+        self.logger.debug("HB_iterations to labels: %s", str(list(zip(HB_iterations, HB_labels))))
 
-        checkbox, select_all, select_none = get_checkbox(HB_handles, iteration_labels)
+        checkbox, select_all, select_none = get_checkbox(HB_handles, HB_labels)
 
         callback_color = CustomJS(args=dict(source_multiline=source_multiline,
                                             source_scatter=source_scatter,
@@ -226,12 +231,17 @@ def _plot(self, result_object, learning_curves, hyperparameter_names, reset_time
             }}
             source.change.emit();
             """.format(min_perf, max_perf, min_iter, max_iter))
-        select_color = Select(title="Select colors", value="performance", options = ["performance", "iteration"],
-                              callback=callback_color)
 
+        select_color = Select(title="Select colors",
+                              value="performance",
+                              options=["performance", "iteration"],
+                              callback=callback_color)
 
-        # Put it all together
-        layout = column(p, row(widgetbox(select_all, select_none, width=100), widgetbox(checkbox, width=500),
+        # Put it all together in a layout (width of checkbox-field sizes with number of elements
+        width_of_checkbox = 650 if len(HB_labels) > 100 else 500 if len(HB_labels) > 70 else 400
+        layout = row(p, column(widgetbox(checkbox, width=width_of_checkbox),
+                               row(widgetbox(select_all, width=50),
+                                   widgetbox(select_none, width=50)),
                                widgetbox(select_color, width=200)))
         return layout
 

cave/analyzer/cave_fanova.py

@@ -95,7 +95,7 @@ def parse_pairwise(p):
         # Get plot-paths
         self.single_plots = {p : os.path.join(self.output_dir, "fanova", p + '.png') for p in single_imp.keys()}
         # Right now no way to access paths of the plots -> file issue
-        self.pairwise_plots = {" & ".join(parse_pairwise(k)) : os.path.join(self.output_dir, 'fanova', '_'.join(parse_pairwise(k)) + '.png') for p in pairwise_imp.keys()}
+        self.pairwise_plots = {" & ".join(parse_pairwise(p)) : os.path.join(self.output_dir, 'fanova', '_'.join(parse_pairwise(p)) + '.png') for p in pairwise_imp.keys()}
         self.pairwise_plots = {p : path for p, path in self.pairwise_plots.items() if os.path.exists(path)}
 
     def get_table(self):

cave/analyzer/compare_default_incumbent.py

@@ -21,7 +21,7 @@ def __init__(self, default, incumbent):
         self.logger = logging.getLogger(self.__module__ + '.' + self.__class__.__name__)
 
         # Remove unused parameters
-        keys = [k for k in default.keys() if default[k] or incumbent[k]]
+        keys = [k for k in default.configuration_space.get_hyperparameter_names() if default[k] or incumbent[k]]
         default = [default[k] if default[k] is not None else "inactive" for k in keys]
         incumbent = [incumbent[k] if incumbent[k] is not None else "inactive" for k in keys]
         zipped = list(zip(keys, default, incumbent))

cave/analyzer/cost_over_time.py

@@ -334,7 +334,6 @@ def plot(self):
         # Wrap renderers in nested lists for checkbox-code
         checkbox, select_all, select_none = get_checkbox(renderers, [l[0] for l in legend_it])
 
-
         # Tilt tick labels and configure axis labels
         p.xaxis.major_label_orientation = 3/4
 

cave/analyzer/overview_table.py

@@ -11,27 +11,30 @@
 from cave.utils.bokeh_routines import array_to_bokeh_table
 
 class OverviewTable(BaseAnalyzer):
-    def __init__(self, runs, output_dir):
+    def __init__(self, runs, bohb_parallel, output_dir):
         """ Create overview-table.
 
         Parameters
         ----------
         runs: List[ConfiguratorRun]
             list with all runs
+        bohb_parallel: False or int
+            number of parallel bohb-runs if present
         output_dir: str
             output-directory for CAVE
         """
         self.logger = logging.getLogger(self.__module__ + '.' + self.__class__.__name__)
         self.output_dir = output_dir
         self.runs = runs
+        self.bohb_parallel = bohb_parallel
 
         self.html_table_general, self.html_table_specific = self.run()
 
     def run(self):
         """ Generate tables. """
         scenario = self.runs[0].scenario
 
-        general_dict = self._general_dict(scenario)
+        general_dict = self._general_dict(scenario, self.bohb_parallel)
         html_table_general = DataFrame(data=OrderedDict([('General', general_dict)]))
         html_table_general = html_table_general.reindex(list(general_dict.keys()))
         html_table_general = html_table_general.to_html(escape=False, header=False, justify='left')
@@ -44,7 +47,7 @@ def run(self):
 
         return html_table_general, html_table_specific
 
-    def _general_dict(self, scenario):
+    def _general_dict(self, scenario, bohb_parallel=False):
         """ Generate the meta-information that holds for all runs (scenario info etc) """
         # general stores information that holds for all runs, runspec holds information on a run-basis
         general = OrderedDict()
@@ -54,6 +57,10 @@ def _general_dict(self, scenario):
         #if num_conf_runs != 1:
         #    overview['Number of configurator runs'] = num_conf_runs
 
+        self.logger.debug("bohb_parallel in overview: %s", bohb_parallel)
+        if bohb_parallel:
+            general['# aggregated parallel BOHB runs'] = bohb_parallel
+
         # Scenario related
         general['# parameters'] = len(scenario.cs.get_hyperparameters())
         general['Deterministic target algorithm'] = scenario.deterministic

cave/analyzer/parallel_coordinates.py

@@ -39,12 +39,13 @@ def __init__(self,
             param_imp: Union[None, Dict[str, float]],
             params: Union[int, List[str]],
             n_configs: int,
+            pc_sort_by: str,
             output_dir: str,
             cs: ConfigurationSpace,
             runtime: bool=False,
             max_runs_epm: int=3000000,
                  ):
-        """"This function prepares the data from a SMAC-related
+        """This function prepares the data from a SMAC-related
         format (using runhistories and parameters) to a more general format
         (using a dataframe). The resulting dataframe is passed to the
         parallel_coordinates-routine
@@ -69,6 +70,8 @@ def __init__(self,
             important ones
         n_configs: int
             number of configs to be plotted
+        pc_sort_by: str
+            defines the pimp-method by which to choose the plotted parameters
         max_runs_epm: int
             maximum number of runs to train the epm with. this should prevent MemoryErrors
         output_dir: str
@@ -86,13 +89,33 @@ def __init__(self,
         self.param_imp = param_imp
         self.cs = cs
 
-        # Sorting by importance, if possible (choose first executed parameter-importance
+        # Sorting by importance, if possible (choose first executed parameter-importance)
         self.method, self.importance = "", {}
-        for m, i in list(self.param_imp.items()):
-            if i:
-                self.method, self.importance = m, i
+        if pc_sort_by == 'all':
+            self.logger.debug("Sorting by average importance")
+            self.method = 'average'
+            for m, i in self.param_imp.items():
+                if i:
+                    for p, imp in i.items():
+                        if p in self.importance:
+                            self.importance[p].append(imp)
+                        else:
+                            self.importance[p] = [imp]
+            self.importance = {k : sum(v) / len(v) for k, v in self.importance.items()}
+        elif pc_sort_by in self.param_imp:
+            self.method, self.importance = pc_sort_by, self.param_imp[pc_sort_by]
+        else:
+            self.logger.debug("%s not evaluated.. choosing at random from: %s", pc_sort_by,
+                              str(list(self.param_imp.keys())))
+            for m, i in self.param_imp.items():
+                if i:
+                    self.method, self.importance = m, i
+                    break
+
         self.hp_names = sorted([hp for hp in self.cs.get_hyperparameter_names()],
-                               key=lambda x: self.importance.get(x, 0))
+                               key=lambda x: self.importance.get(x, 0),
+                               reverse=True)
+        self.logger.debug("Sorted hp's by method \'%s\': %s", self.method, str(self.hp_names))
 
         # To be set
         self.plots = []
@@ -146,6 +169,7 @@ def get_plots(self):
                 self.plots = [self.pcp.plot_n_configs(self.n_configs, self.params)]
             except ValueError as err:
                 self.error = str(err)
+        self.logger.debug("Paths to plot(s): %s", str(self.plots))
         return self.plots
 
     def get_html(self, d=None, tooltip=None):

cave/analyzer/plot_ecdf.py

@@ -70,7 +70,7 @@ def prepare_data(x_data):
         output_fns = []
 
         for insts, name in [(train, 'train'), (test, 'test')]:
-            if insts == [None]:
+            if len(insts) <= 1:
                 self.logger.debug("No %s instances, skipping cdf", name)
                 continue
             data = [prepare_data(np.array([v for k, v in costs if k in insts])) for costs in [def_costs, inc_costs]]
@@ -82,6 +82,8 @@ def prepare_data(x_data):
         self.output_fns = output_fns
 
     def get_html(self, d=None, tooltip=None):
+        if not self.output_fns:
+            return
         if d is not None and self.output_fns:
             d["figure"] = self.output_fns
             d["tooltip"] = tooltip

cave/analyzer/plot_scatter.py

@@ -56,7 +56,7 @@ def __init__(self,
 
         out_fns = []
         for insts, name in [(train, 'train'), (test, 'test')]:
-            if insts == [None]:
+            if len(insts) <= 1:
                 self.logger.debug("No %s instances, skipping scatter", name)
                 continue
             default = np.array([v for k, v in def_costs if k in insts])
@@ -65,9 +65,12 @@ def __init__(self,
             out_fn = out_fn_base + name + '.png'
             out_fns.append(plot_scatter_plot((default,), (incumbent,), labels, metric=metric,
                            min_val=min_val, max_val=timeout, out_fn=out_fn))
-        self.output_fns = out_fns
+        self.output_fns = out_fns if len(out_fns) > 0 else None
 
     def get_html(self, d=None, tooltip=None):
+        if not self.output_fns:
+            return
+
         if d is not None and self.output_fns:
             d["figure"] = self.output_fns
             d["tooltip"] = tooltip