Add KDD'26 camera-ready experiments and update BibTeX

README.md

@@ -162,10 +162,15 @@ The raw experiment outputs (pickled results) and the values used in the tables a
 If you find condPED-ANOVA useful in your research, please consider citing the following paper:
 
 ```bibtex
-@article{baba2026condpedanova,
-  title={Conditional {PED-ANOVA}: Hyperparameter Importance in Hierarchical \& Dynamic Search Spaces},
-  author={Baba, Kaito and Ozaki, Yoshihiko and Watanabe, Shuhei},
-  journal={arXiv preprint arXiv:2601.20800},
-  year={2026},
+@inproceedings{baba2026condpedanova,
+  title     = {Conditional {PED-ANOVA}: Hyperparameter Importance in Hierarchical \& Dynamic Search Spaces},
+  author    = {Baba, Kaito and Ozaki, Yoshihiko and Watanabe, Shuhei},
+  year      = {2026},
+  month     = {August},
+  booktitle = {Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2},
+  publisher = {Association for Computing Machinery},
+  address   = {Jeju Island, Republic of Korea},
+  doi       = {10.1145/3770855.3817758},
+  isbn      = {979-8-4007-2259-2/2026/08},
 }
 ```

experiments/_print.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from typing import Any, Literal
+
+
+def print_table(
+    filename: str | Path,
+    data: list[list[Any]],
+    mode: Literal["md", "csv"] = "md",
+    float_format: str = "{:.4f}",
+    tight: bool = False,
+) -> None:
+    data = [[float_format.format(x) if isinstance(x, float) else x for x in row] for row in data]
+    formatted = {"md": lambda d: _format_md(d, tight=tight), "csv": _format_csv}[mode](data)
+    Path(filename).write_text(formatted)
+
+
+def _format_md(data: list[list[str]], tight: bool) -> str:
+    bar_left = "|" if tight else "| "
+    bar_right = "|" if tight else " |"
+    bar_middle = "|" if tight else bar_right
+    header_line = bar_left + bar_middle.join(data[0]) + bar_right
+    separator_line = bar_left + bar_middle.join(["---"] * len(data[0])) + bar_right
+    data_lines = [bar_left + bar_middle.join(row) + bar_right for row in data[1:]]
+    return "\n".join([header_line, separator_line, *data_lines])
+
+
+def _format_csv(data: list[list[str]]) -> str:
+    data = [[x.replace('"', '""') for x in row] for row in data]
+    data = [[f'"{x}"' if any(c in x for c in '",\n\r') else x for x in row] for row in data]
+    return "\n".join([",".join(row) for row in data])

experiments/run_cond_ped_anova.py

@@ -43,7 +43,10 @@ def _run_experiment_once(
     args: Namespace, seed: int
 ) -> tuple[list[dict[str, list[float]]], list[dict[str, list[float]]], list[np.ndarray]]:
     objective = get_objective(args.objective_name)
-    sampler = optuna.samplers.RandomSampler(seed=seed)
+    sampler = {
+        "random": optuna.samplers.RandomSampler,
+        "tpe": optuna.samplers.TPESampler,
+    }[args.sampler](seed=seed)
     study = optuna.create_study(direction="minimize", sampler=sampler)
     study.optimize(objective, n_trials=args.n_trials)
 
@@ -74,8 +77,11 @@ def _run_experiment_once(
 
 
 def main(args: Namespace) -> None:
+    # For backward compatibility, add the sampler name only for non-random samplers.
     save_path = Path(args.output_dir) / (
-        f"{args.objective_name}_{args.evaluator_name.replace('/', '-')}_{args.n_trials}trials.pkl"
+        f"{args.objective_name}_{args.evaluator_name.replace('/', '-')}"
+        + ("" if args.sampler == "random" else f"_{args.sampler}")
+        + f"_{args.n_trials}trials.pkl"
     )
     if save_path.exists():
         print(f"Results already exist at {save_path}, skipping...")
@@ -126,6 +132,7 @@ def main(args: Namespace) -> None:
     )
     parser.add_argument("--n-trials", type=int, default=1000)
     parser.add_argument("--n-seeds", type=int, default=10)
+    parser.add_argument("--sampler", type=str, choices=["random", "tpe"], default="random")
     parser.add_argument("--region-quantiles", type=float, nargs="+", default=[1.0, 0.75, 0.5])
     parser.add_argument("--target-quantile-step", type=float, default=0.01)
     parser.add_argument("--output-dir", type=str, default="results")

experiments/run_yahpo_comparison.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import pickle
+from typing import TYPE_CHECKING
+
+import numpy as np
+from scipy.stats import pearsonr
+from yahpo_gym import list_scenarios
+
+from experiments._evaluator_registry import get_all_evaluator_names
+from experiments._print import print_table
+
+
+if TYPE_CHECKING:
+    from argparse import Namespace
+
+
+def _calc_corr(
+    evaluator_names: list[str],
+    scenario: str,
+    instance: int,
+    n_trials: int,
+    output_dir: str,
+) -> dict[str, float]:
+    with open(
+        Path(output_dir)
+        / f"yahpo_{scenario}_{instance}_{evaluator_names[0]}_{n_trials}trials.stats_raw.pkl",
+        "rb",
+    ) as f:
+        stats = pickle.load(f)
+    data = {}
+    for evaluator in evaluator_names:
+        with open(
+            Path(output_dir) / f"yahpo_{scenario}_{instance}_{evaluator}_{n_trials}trials.pkl",
+            "rb",
+        ) as f:
+            data[evaluator] = pickle.load(f)["results_raw"]
+    data = {e: {k: np.mean(v) for k, v in d.items()} for e, d in data.items()}
+    means = {k: np.mean(v, axis=0)[2] for k, v in stats.items()}
+
+    max_importances = {
+        evaluator: {
+            learner_id: max(
+                {k: v for k, v in importances.items() if k.startswith(learner_id)}.values()
+            )
+            for learner_id in means
+        }
+        for evaluator, importances in data.items()
+    }
+    corr = {
+        evaluator: pearsonr(list(val.values()), list(means.values()))[0]
+        for evaluator, val in max_importances.items()
+    }
+    return corr
+
+
+def main(args: Namespace) -> None:
+    corrs = [
+        _calc_corr(
+            args.evaluator_names,
+            args.scenario,
+            instance,
+            args.n_trials,
+            args.output_dir,
+        )
+        for instance in args.instances
+    ]
+    evaluator_names = {
+        "cond_ped_anova": "condPED-ANOVA (Ours)",
+        "ped_anova_filtering": "PED-ANOVA w/ Filtering",
+        "ped_anova_imputation": "PED-ANOVA w/ Imputation",
+        "fanova_filtering": "f-ANOVA w/ Filtering",
+        "fanova_imputation": "f-ANOVA w/ Imputation",
+        "mdi_filtering": "MDI w/ Filtering",
+        "mdi_imputation": "MDI w/ Imputation",
+        "shap_filtering": "SHAP w/ Filtering",
+        "shap_imputation": "SHAP w/ Imputation",
+    }
+    header = ["Instance ID"] + [evaluator_names[e] for e in args.evaluator_names]
+    data = [
+        [f"{instance}"] + [f"{corr[evaluator]:.2f}" for evaluator in args.evaluator_names]
+        for instance, corr in zip(args.instances, corrs, strict=True)
+    ]
+    stats = ["Mean $\\pm$ StdErr"] + [
+        f"{np.mean([corr[evaluator] for corr in corrs]):.2f} $\\pm$ "
+        f"{np.std([corr[evaluator] for corr in corrs]) / np.sqrt(len(corrs)):.2f}"
+        for evaluator in args.evaluator_names
+    ]
+    print_table(
+        Path(args.output_dir) / f"yahpo_{args.scenario}_correlations_{args.n_trials}trials.md",
+        [header, *data, stats],
+        mode="md",
+        tight=True,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--scenario",
+        type=str,
+        default="rbv2_super",
+        choices=list_scenarios(),
+    )
+    parser.add_argument(
+        "--instances",
+        type=str,
+        nargs="+",
+        default=[1053, 1457, 1063, 1479, 15, 1468],
+    )
+    parser.add_argument(
+        "--evaluator-names",
+        type=str,
+        choices=get_all_evaluator_names(),
+        nargs="+",
+        default=[
+            "cond_ped_anova",
+            "ped_anova_filtering",
+            "ped_anova_imputation",
+            "fanova_filtering",
+            "fanova_imputation",
+            "mdi_filtering",
+            "mdi_imputation",
+            "shap_filtering",
+            "shap_imputation",
+        ],
+    )
+    parser.add_argument("--n-trials", type=int, default=1000)
+    parser.add_argument("--output-dir", type=str, default="results")
+    args = parser.parse_args()
+    main(args)

experiments/run_yahpo_gym.py

@@ -12,10 +12,8 @@
 from optuna.distributions import CategoricalDistribution, FloatDistribution, IntDistribution
 from yahpo_gym import BenchmarkSet, list_scenarios, local_config
 
-from experiments._evaluator_registry import (
-    get_all_evaluator_names,
-    get_evaluator,
-)
+from experiments._evaluator_registry import get_all_evaluator_names, get_evaluator
+from experiments._print import print_table
 
 
 if TYPE_CHECKING:
@@ -114,17 +112,22 @@ def main(args: Namespace) -> None:
             },
             f,
         )
-    with open(save_path.with_suffix(".stats.txt"), "w") as f:
-        f.write("Learner ID\tMin\tMean\tMax\n")
-        for learner_id in means:
-            mean_min, mean_mean, mean_max = means[learner_id]
-            stderr_min, stderr_mean, stderr_max = stderrs[learner_id]
-            f.write(
-                f"{learner_id}\t"
-                f"{mean_min:.6f} ± {stderr_min:.6f}\t"
-                f"{mean_mean:.6f} ± {stderr_mean:.6f}\t"
-                f"{mean_max:.6f} ± {stderr_max:.6f}\n"
-            )
+    data = [["Learner ID", "Min", "Mean", "Max"]] + [
+        [
+            f"{learner_id}",
+            f"{mean_min:.6f} ± {stderr_min:.6f}",
+            f"{mean_mean:.6f} ± {stderr_mean:.6f}",
+            f"{mean_max:.6f} ± {stderr_max:.6f}",
+        ]
+        for (learner_id, (mean_min, mean_mean, mean_max)), (
+            stderr_min,
+            stderr_mean,
+            stderr_max,
+        ) in zip(means.items(), stderrs.values(), strict=True)
+    ]
+    print_table(save_path.with_suffix(".stats.md"), data, mode="md")
+    with open(save_path.with_suffix(".stats_raw.pkl"), "wb") as f:
+        pickle.dump(stats, f)
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -39,6 +39,7 @@ experiments = [
   "scikit-learn", # for ShapleyImportanceEvaluator and FanovaImportanceEvaluator
   "shap", # for ShapleyImportanceEvaluator
   "matplotlib", # for plotting results
+  "scipy", # for computing metrics such as spearmanr
   "yahpo_gym==1.0.2", # for YAHPO Gym experiments
   "ConfigSpace<1.0.0", # for YAHPO Gym experiments; to be compatible with yahpo_gym
   # uv may incorrectly resolve an old numba (e.g., 0.53.x) via shap's dependencies,

run_cond_ped_anova.sh

@@ -4,24 +4,35 @@ set -xeuo pipefail
 
 n_trials=1000
 objectives=("activation-disjoint" "activation-overlap" "regime-dependent-domain")
+samplers=("random" "tpe")
 
-results=()
-for objective in "${objectives[@]}"; do
-    python -m experiments.run_cond_ped_anova \
-        --objective-name "$objective" \
-        --n-trials "$n_trials"
-    results+=("results/${objective}_cond_ped_anova_${n_trials}trials.pkl")
-done
+for sampler in "${samplers[@]}"; do
+    # For backward compatibility, add the sampler name only for non-random samplers.
+    if [ "$sampler" != "random" ]; then
+        sampler_suffix="_${sampler}"
+    else
+        sampler_suffix=""
+    fi
+
+    results=()
+    for objective in "${objectives[@]}"; do
+        python -m experiments.run_cond_ped_anova \
+            --objective-name "$objective" \
+            --n-trials "$n_trials" \
+            --sampler "$sampler"
+        results+=("results/${objective}_cond_ped_anova${sampler_suffix}_${n_trials}trials.pkl")
+    done
 
-python -m experiments.plot_mixed \
-    --input-paths "${results[@]}" \
-    --titles \
-    --save-name "cond_ped_anova_${n_trials}trials" \
-    --legend-loc "upper right" \
-    --legend-column 2
+    python -m experiments.plot_mixed \
+        --input-paths "${results[@]}" \
+        --titles \
+        --save-name "cond_ped_anova${sampler_suffix}_${n_trials}trials" \
+        --legend-loc "upper right" \
+        --legend-column 2
 
-for result in "${results[@]}"; do
-    python -m experiments.plot_cond_ped_anova \
-        --input-path "$result" \
-        --data-indices 1 2
+    for result in "${results[@]}"; do
+        python -m experiments.plot_cond_ped_anova \
+            --input-path "$result" \
+            --data-indices 1 2
+    done
 done

run_yahpo_gym.sh

@@ -2,18 +2,50 @@
 
 set -xeuo pipefail
 
-scenario="rbv2_super"
-instances=(1053 1457 1063 1479 15 1468)
-evaluator="cond_ped_anova"
+evaluators=(
+    "cond_ped_anova"
+    "ped_anova_filtering"
+    "ped_anova_imputation"
+    "fanova_filtering"
+    "fanova_imputation"
+    "mdi_filtering"
+    "mdi_imputation"
+    "shap_filtering"
+    "shap_imputation"
+)
 n_trials=1000
 
-for instance in "${instances[@]}"; do
-    python -m experiments.run_yahpo_gym \
+run_yahpo_suite() {
+    local scenario="$1"
+    local instances_var="$2"
+    local metric="$3"
+    local stat_key="$4"
+    local -n instances="$instances_var"
+
+    for instance in "${instances[@]}"; do
+        for evaluator in "${evaluators[@]}"; do
+            python -m experiments.run_yahpo_gym \
+                --scenario "$scenario" \
+                --instance "$instance" \
+                --evaluator-name "$evaluator" \
+                --n-trials "$n_trials" \
+                --metric "$metric" \
+                --stat-key "$stat_key"
+
+            python -m experiments.plot_yahpo_gym \
+                --input-path "results/yahpo_${scenario}_${instance}_${evaluator}_${n_trials}trials.pkl"
+        done
+    done
+
+    python -m experiments.run_yahpo_comparison \
         --scenario "$scenario" \
-        --instance "$instance" \
-        --evaluator-name "$evaluator" \
+        --instances "${instances[@]}" \
+        --evaluator-names "${evaluators[@]}" \
         --n-trials "$n_trials"
+}
+
+rbv2_instances=(1053 1457 1063 1479 15 1468)
+iaml_instances=(40981 41146 1489 1067)
 
-    python -m experiments.plot_yahpo_gym \
-        --input-path "results/yahpo_${scenario}_${instance}_${evaluator}_${n_trials}trials.pkl"
-done
+run_yahpo_suite "rbv2_super" rbv2_instances "acc" "learner_id"
+run_yahpo_suite "iaml_super" iaml_instances "f1" "learner"