diff --git a/examples/ohbm2023/config_selectors.py b/examples/ohbm2023/config_selectors.py index da168670..a4d0b7b6 100644 --- a/examples/ohbm2023/config_selectors.py +++ b/examples/ohbm2023/config_selectors.py @@ -47,3 +47,21 @@ def __call__(self, list_of_non_failed_configs, metric, fold_operation, maximize_ return best_config_outer_fold + +class PercentileConfigSelector(BaseConfigSelector): + + def __init__(self, percentile: int = 75): + self.percentile = percentile + + def __call__(self, list_of_non_failed_configs, metric, fold_operation, maximize_metric): + + all_metrics_mean, all_metrics_std = self.prepare_metrics(list_of_non_failed_configs, metric) + best_config_metric_values = [c.get_test_metric(metric, fold_operation) for c in list_of_non_failed_configs] + + position = (self.percentile / 100.) if maximize_metric else 1 - (self.percentile / 100.) + index = position * (len(best_config_metric_values) - 1) + index = int(index + 0.5) # index to integer conversion + # following line is a hack for speedup + best_config_outer_fold = list_of_non_failed_configs[np.argpartition(best_config_metric_values, index)[index]] + + return best_config_outer_fold diff --git a/examples/ohbm2023/run.py b/examples/ohbm2023/run.py index b35894e8..86013f40 100644 --- a/examples/ohbm2023/run.py +++ b/examples/ohbm2023/run.py @@ -1,19 +1,20 @@ import pandas as pd from run_elements import * -from config_selectors import DefaultConfigSelector, RandomConfigSelector +from config_selectors import DefaultConfigSelector, RandomConfigSelector, PercentileConfigSelector from collect_results import ResultCollector from multiprocessing import Process import os list_of_config_selectors = {'default': DefaultConfigSelector, + 'percentile': PercentileConfigSelector} 'random': RandomConfigSelector} config_selector_name = 'default' multiprocessing = False -calculate = False +calculate = True list_of_dataset_runners = {