This repository was archived by the owner on Jul 13, 2026. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathruntime_data.py
More file actions
285 lines (284 loc) · 16.5 KB
/
Copy pathruntime_data.py
File metadata and controls
285 lines (284 loc) · 16.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
# Copyright (c) Meta Platforms, Inc. and affiliates.
PAST_AVERAGE_RUNTIME_BY_BENCHMARK = {
("Ax::fast+Center", "Ackley Mixed"): 544.1506170232285,
("Ax::fast+Center", "BBOB01_01_20d"): 649.6102951189503,
("Ax::fast+Center", "BBOB01_01_20d_noise_1"): 632.7653389191255,
("Ax::fast+Center", "BBOB02_01_20d"): 1038.4914609579369,
("Ax::fast+Center", "BBOB03_01_20d"): 786.7487048069946,
("Ax::fast+Center", "BBOB04_01_20d"): 830.3524326929822,
("Ax::fast+Center", "BBOB05_01_20d"): 515.9084882248193,
("Ax::fast+Center", "BBOB06_01_20d"): 717.766075802967,
("Ax::fast+Center", "BBOB07_01_20d"): 658.1017976914533,
("Ax::fast+Center", "BBOB08_01_20d"): 530.7998055386357,
("Ax::fast+Center", "BBOB09_01_20d"): 606.9413947268389,
("Ax::fast+Center", "BBOB10_01_20d"): 688.9502366289496,
("Ax::fast+Center", "BBOB11_01_20d"): 800.493925104849,
("Ax::fast+Center", "BBOB12_01_20d"): 639.5636447004042,
("Ax::fast+Center", "BBOB13_01_20d"): 541.9295049700886,
("Ax::fast+Center", "BBOB14_01_20d"): 899.9268841695041,
("Ax::fast+Center", "BBOB15_01_20d"): 550.2940703662113,
("Ax::fast+Center", "BBOB16_01_20d"): 649.0583270345815,
("Ax::fast+Center", "BBOB17_01_20d"): 504.9168007627595,
("Ax::fast+Center", "BBOB18_01_20d"): 713.1826220454648,
("Ax::fast+Center", "BBOB19_01_20d"): 546.7854141483083,
("Ax::fast+Center", "BBOB20_01_20d"): 493.1027928569354,
("Ax::fast+Center", "BBOB21_01_20d"): 491.7312921254896,
("Ax::fast+Center", "BBOB22_01_20d"): 522.0799982554279,
("Ax::fast+Center", "BBOB23_01_20d"): 473.4027391904965,
("Ax::fast+Center", "BBOB24_01_20d"): 500.65114158401263,
("Ax::fast+Center", "Branin"): 174.6476198810269,
("Ax::fast+Center", "Branin_noise_23"): 81.87652449118904,
("Ax::fast+Center", "Branin_variable_runtime"): 152.61061395381694,
("Ax::fast+Center", "DTLZ2_2m_6d"): 796.2597921892302,
("Ax::fast+Center", "Hartmann_30d"): 922.7533200738951,
("Ax::fast+Center", "Hartmann_6d"): 187.6371951564215,
("Ax::fast+Center", "PressureVessel"): 3906.3174313796685,
("Ax::fast+Center", "SixHumpCamel"): 81.38804853273322,
("Ax::fast+Center", "TensionCompressionString"): 2693.2643067160157,
("Ax::fast+Center", "VehicleSafety"): 1908.8520224839915,
("Ax::fast+Center", "WeldedBeamSO"): 8021.828493806906,
("Ax::fast+Center", "ZDT1_2m_5d"): 556.5999200265389,
("Ax::fast+Center", "ZDT2_2m_5d"): 1117.0446009150705,
("Ax::fast+Center", "ZDT3_2m_5d"): 735.1157971833484,
("Ax::fast::ES+Center", "LCBench_Surrogate_Fashion-MNIST:v1"): 582.7082356737926,
("Ax::fast::ES+Center", "LCBench_Surrogate_airlines:v1"): 778.6357114578598,
("Ax::fast::ES+Center", "LCBench_Surrogate_albert:v1"): 596.4262313634157,
("Ax::fast::ES+Center", "LCBench_Surrogate_christine:v1"): 1066.113397302106,
("Ax::fast::ES+Center", "LCBench_Surrogate_covertype:v1"): 1020.3563080691732,
("Ax::random_search+Center", "Ackley Mixed"): 8.046856630739057,
("Ax::random_search+Center", "BBOB01_01_20d"): 2.589231173618464,
("Ax::random_search+Center", "BBOB01_01_20d_noise_1"): 2.593421075750375,
("Ax::random_search+Center", "BBOB02_01_20d"): 2.6460589008312674,
("Ax::random_search+Center", "BBOB03_01_20d"): 2.658361605796963,
("Ax::random_search+Center", "BBOB04_01_20d"): 2.618407857952407,
("Ax::random_search+Center", "BBOB05_01_20d"): 2.625237969257869,
("Ax::random_search+Center", "BBOB06_01_20d"): 2.6557872264145406,
("Ax::random_search+Center", "BBOB07_01_20d"): 2.6509660253056793,
("Ax::random_search+Center", "BBOB08_01_20d"): 2.633236360367155,
("Ax::random_search+Center", "BBOB09_01_20d"): 2.6241747722565196,
("Ax::random_search+Center", "BBOB10_01_20d"): 2.626488171791425,
("Ax::random_search+Center", "BBOB11_01_20d"): 2.636808432242833,
("Ax::random_search+Center", "BBOB12_01_20d"): 2.6022150982171297,
("Ax::random_search+Center", "BBOB13_01_20d"): 2.5480548652634023,
("Ax::random_search+Center", "BBOB14_01_20d"): 2.5872377471392976,
("Ax::random_search+Center", "BBOB15_01_20d"): 2.613904303818708,
("Ax::random_search+Center", "BBOB16_01_20d"): 2.5784668344061354,
("Ax::random_search+Center", "BBOB17_01_20d"): 2.609974397844635,
("Ax::random_search+Center", "BBOB18_01_20d"): 2.6116656042286195,
("Ax::random_search+Center", "BBOB19_01_20d"): 2.605758932135068,
("Ax::random_search+Center", "BBOB20_01_20d"): 2.613184698612895,
("Ax::random_search+Center", "BBOB21_01_20d"): 2.6215256019181106,
("Ax::random_search+Center", "BBOB22_01_20d"): 2.608398549091071,
("Ax::random_search+Center", "BBOB23_01_20d"): 2.5980010123259856,
("Ax::random_search+Center", "BBOB24_01_20d"): 2.592377391937189,
("Ax::random_search+Center", "Branin"): 2.260835421751253,
("Ax::random_search+Center", "Branin_noise_23"): 2.288850175376283,
("Ax::random_search+Center", "Branin_variable_runtime"): 2.331353244371712,
("Ax::random_search+Center", "CarSideImpact"): 4.1423472837824376,
("Ax::random_search+Center", "DTLZ2_2m_6d"): 3.6843936552572996,
("Ax::random_search+Center", "Hartmann_30d"): 2.874173802806763,
("Ax::random_search+Center", "Hartmann_6d"): 2.3792201253748497,
("Ax::random_search+Center", "Penicillin"): 3.1964832364290485,
("Ax::random_search+Center", "PressureVessel"): 3.378283989218762,
("Ax::random_search+Center", "SixHumpCamel"): 2.270278179321904,
("Ax::random_search+Center", "TensionCompressionString"): 4.0741557060775815,
("Ax::random_search+Center", "VehicleSafety"): 3.9574600115418432,
("Ax::random_search+Center", "WeldedBeamSO"): 4.443850504283327,
("Ax::random_search+Center", "ZDT1_2m_5d"): 2.9938504402642137,
("Ax::random_search+Center", "ZDT2_2m_5d"): 2.8989457801543175,
("Ax::random_search+Center", "ZDT3_2m_5d"): 2.8605013970972504,
("HEBO+Center", "Ackley Mixed"): 355.25730349421644,
("HEBO+Center", "BBOB01_01_20d"): 136.9135827784543,
("HEBO+Center", "BBOB01_01_20d_noise_1"): 124.32140242493479,
("HEBO+Center", "BBOB02_01_20d"): 136.62514678337612,
("HEBO+Center", "BBOB03_01_20d"): 137.12839121309807,
("HEBO+Center", "BBOB04_01_20d"): 136.30365805710898,
("HEBO+Center", "BBOB05_01_20d"): 137.64709201011573,
("HEBO+Center", "BBOB06_01_20d"): 136.113837967359,
("HEBO+Center", "BBOB07_01_20d"): 136.88314572292148,
("HEBO+Center", "BBOB08_01_20d"): 137.39816947748187,
("HEBO+Center", "BBOB09_01_20d"): 137.97870820693205,
("HEBO+Center", "BBOB10_01_20d"): 138.16519923055313,
("HEBO+Center", "BBOB11_01_20d"): 136.24585361798526,
("HEBO+Center", "BBOB12_01_20d"): 137.7780203344999,
("HEBO+Center", "BBOB13_01_20d"): 136.45201182216405,
("HEBO+Center", "BBOB14_01_20d"): 135.66491386099952,
("HEBO+Center", "BBOB15_01_20d"): 136.37060296577403,
("HEBO+Center", "BBOB16_01_20d"): 137.26975369337015,
("HEBO+Center", "BBOB17_01_20d"): 138.3325617157505,
("HEBO+Center", "BBOB18_01_20d"): 137.28986893310213,
("HEBO+Center", "BBOB19_01_20d"): 139.04689924337436,
("HEBO+Center", "BBOB20_01_20d"): 139.209298092511,
("HEBO+Center", "BBOB21_01_20d"): 137.60538999816634,
("HEBO+Center", "BBOB22_01_20d"): 138.7430356221972,
("HEBO+Center", "BBOB23_01_20d"): 137.78925394392573,
("HEBO+Center", "BBOB24_01_20d"): 139.0222549816477,
("HEBO+Center", "Branin"): 191.8916861312231,
("HEBO+Center", "Branin_noise_23"): 194.39389735446895,
("HEBO+Center", "Branin_variable_runtime"): 182.8335562483873,
("HEBO+Center", "CarSideImpact"): 497.8893774916884,
("HEBO+Center", "DTLZ2_2m_6d"): 390.6071797545534,
("HEBO+Center", "Hartmann_30d"): 85.87852811976919,
("HEBO+Center", "Hartmann_6d"): 169.0682366175635,
("HEBO+Center", "Penicillin"): 439.91668403362854,
("HEBO+Center", "PressureVessel"): 511.3634185578732,
("HEBO+Center", "SixHumpCamel"): 188.1415301709494,
("HEBO+Center", "TensionCompressionString"): 591.7535278126364,
("HEBO+Center", "VehicleSafety"): 453.477885009395,
("HEBO+Center", "WeldedBeamSO"): 691.498881318036,
("HEBO+Center", "ZDT1_2m_5d"): 391.30816462753575,
("HEBO+Center", "ZDT2_2m_5d"): 357.0777983253123,
("HEBO+Center", "ZDT3_2m_5d"): 357.71818898563504,
("Optuna+Center", "Ackley Mixed"): 23.286435805085056,
("Optuna+Center", "BBOB01_01_20d"): 4.967987521062605,
("Optuna+Center", "BBOB01_01_20d_noise_1"): 5.0812789348606024,
("Optuna+Center", "BBOB02_01_20d"): 4.9488145976886155,
("Optuna+Center", "BBOB03_01_20d"): 5.023095728665571,
("Optuna+Center", "BBOB04_01_20d"): 5.052676760037429,
("Optuna+Center", "BBOB05_01_20d"): 4.969211882501841,
("Optuna+Center", "BBOB06_01_20d"): 5.029841006915085,
("Optuna+Center", "BBOB07_01_20d"): 5.09105036880821,
("Optuna+Center", "BBOB08_01_20d"): 5.038875578953885,
("Optuna+Center", "BBOB09_01_20d"): 5.0419192939344795,
("Optuna+Center", "BBOB10_01_20d"): 4.943070147638209,
("Optuna+Center", "BBOB11_01_20d"): 5.123035387517884,
("Optuna+Center", "BBOB12_01_20d"): 5.304912677928805,
("Optuna+Center", "BBOB13_01_20d"): 5.007154214251787,
("Optuna+Center", "BBOB14_01_20d"): 4.9583241400588305,
("Optuna+Center", "BBOB15_01_20d"): 4.994768361253664,
("Optuna+Center", "BBOB16_01_20d"): 5.116711516212672,
("Optuna+Center", "BBOB17_01_20d"): 5.116636860086583,
("Optuna+Center", "BBOB18_01_20d"): 5.200591629827395,
("Optuna+Center", "BBOB19_01_20d"): 5.110948944007978,
("Optuna+Center", "BBOB20_01_20d"): 5.100935896420851,
("Optuna+Center", "BBOB21_01_20d"): 4.987925362093375,
("Optuna+Center", "BBOB22_01_20d"): 4.942503640051,
("Optuna+Center", "BBOB23_01_20d"): 4.962931894632057,
("Optuna+Center", "BBOB24_01_20d"): 4.964082819665782,
("Optuna+Center", "Branin"): 1.1006031828466802,
("Optuna+Center", "Branin_noise_23"): 1.0506172667210922,
("Optuna+Center", "Branin_variable_runtime"): 1.1194748663681093,
("Optuna+Center", "CarSideImpact"): 3.7184605475421995,
("Optuna+Center", "DTLZ2_2m_6d"): 2.3020523418765517,
("Optuna+Center", "Hartmann_30d"): 7.257086506555789,
("Optuna+Center", "Hartmann_6d"): 1.8947090418636798,
("Optuna+Center", "Penicillin"): 3.3176295846607537,
("Optuna+Center", "SixHumpCamel"): 1.0508573641069234,
("Optuna+Center", "VehicleSafety"): 2.422655969611369,
("Optuna+Center", "ZDT1_2m_5d"): 1.9953931740950792,
("Optuna+Center", "ZDT2_2m_5d"): 1.9965447521349415,
("Optuna+Center", "ZDT3_2m_5d"): 2.0343329239450396,
("SMAC-BB+Center", "Ackley Mixed"): 2075.1529258572655,
("SMAC-BB+Center", "BBOB01_01_20d"): 1063.6181134269573,
("SMAC-BB+Center", "BBOB02_01_20d"): 1176.9530336075463,
("SMAC-BB+Center", "BBOB03_01_20d"): 1295.8487420764752,
("SMAC-BB+Center", "BBOB04_01_20d"): 1112.6037860198878,
("SMAC-BB+Center", "BBOB05_01_20d"): 839.5900130761787,
("SMAC-BB+Center", "BBOB06_01_20d"): 1028.878260383848,
("SMAC-BB+Center", "BBOB07_01_20d"): 1071.5905805127695,
("SMAC-BB+Center", "BBOB14_01_20d"): 1223.311489250278,
("SMAC-BB+Center", "BBOB15_01_20d"): 1261.1674780222006,
("SMAC-BB+Center", "BBOB16_01_20d"): 893.0401514552766,
("SMAC-BB+Center", "BBOB19_01_20d"): 1978.2307699652388,
("SMAC-BB+Center", "Branin"): 248.1948464940302,
("SMAC-BB+Center", "Branin_noise_23"): 262.8377652512863,
("SMAC-BB+Center", "Branin_variable_runtime"): 278.5651880057761,
("SMAC-BB+Center", "CarSideImpact"): 267.64197394195946,
("SMAC-BB+Center", "DTLZ2_2m_6d"): 242.46392807485535,
("SMAC-BB+Center", "Hartmann_6d"): 641.9660647088197,
("SMAC-BB+Center", "Penicillin"): 407.45022947243416,
("SMAC-BB+Center", "SixHumpCamel"): 396.5106906215427,
("SMAC-BB+Center", "VehicleSafety"): 326.5899607438594,
("SMAC-BB+Center", "ZDT1_2m_5d"): 226.29884493611752,
("SMAC-BB+Center", "ZDT2_2m_5d"): 216.70588705111294,
("SMAC-BB+Center", "ZDT3_2m_5d"): 244.3948271553032,
("SMAC-HPO+Center", "Ackley Mixed"): 33.093085178596084,
("SMAC-HPO+Center", "BBOB01_01_20d"): 13.803440073039383,
("SMAC-HPO+Center", "BBOB02_01_20d"): 13.602883766032756,
("SMAC-HPO+Center", "BBOB03_01_20d"): 13.888552371505648,
("SMAC-HPO+Center", "BBOB04_01_20d"): 13.588001734483987,
("SMAC-HPO+Center", "BBOB05_01_20d"): 13.57642125505954,
("SMAC-HPO+Center", "BBOB06_01_20d"): 13.220608349330723,
("SMAC-HPO+Center", "BBOB07_01_20d"): 13.651719933748245,
("SMAC-HPO+Center", "BBOB08_01_20d"): 13.93979202657938,
("SMAC-HPO+Center", "BBOB09_01_20d"): 12.388480084100738,
("SMAC-HPO+Center", "BBOB10_01_20d"): 11.953291720114649,
("SMAC-HPO+Center", "BBOB11_01_20d"): 12.313961058172863,
("SMAC-HPO+Center", "BBOB12_01_20d"): 12.407091403459198,
("SMAC-HPO+Center", "BBOB13_01_20d"): 12.851642867519987,
("SMAC-HPO+Center", "BBOB14_01_20d"): 12.498771702508675,
("SMAC-HPO+Center", "BBOB15_01_20d"): 12.46589947383385,
("SMAC-HPO+Center", "BBOB16_01_20d"): 12.44824261114234,
("SMAC-HPO+Center", "BBOB17_01_20d"): 12.390945335635916,
("SMAC-HPO+Center", "BBOB18_01_20d"): 12.260369105489225,
("SMAC-HPO+Center", "BBOB19_01_20d"): 12.412291102115995,
("SMAC-HPO+Center", "BBOB20_01_20d"): 12.262900901827962,
("SMAC-HPO+Center", "BBOB21_01_20d"): 12.28093829419464,
("SMAC-HPO+Center", "BBOB22_01_20d"): 11.745347658870742,
("SMAC-HPO+Center", "BBOB23_01_20d"): 11.623373140580952,
("SMAC-HPO+Center", "Branin"): 3.5190696310065688,
("SMAC-HPO+Center", "Branin_noise_23"): 3.5379238757956775,
("SMAC-HPO+Center", "Branin_variable_runtime"): 3.726370598997592,
("SMAC-HPO+Center", "CarSideImpact"): 6.056060686511919,
("SMAC-HPO+Center", "DTLZ2_2m_6d"): 5.560079685486853,
("SMAC-HPO+Center", "Hartmann_6d"): 5.5033728508465,
("SMAC-HPO+Center", "Penicillin"): 7.407741984538734,
("SMAC-HPO+Center", "SixHumpCamel"): 3.570112335132435,
("SMAC-HPO+Center", "VehicleSafety"): 5.14315292541869,
("SMAC-HPO+Center", "ZDT1_2m_5d"): 5.1331782553903755,
("SMAC-HPO+Center", "ZDT2_2m_5d"): 5.163017928516492,
("SMAC-HPO+Center", "ZDT3_2m_5d"): 5.11691466299817,
("Vizier", "Ackley Mixed"): 3111.7750602033775,
("Vizier", "BBOB01_01_20d"): 1388.1057877809042,
("Vizier", "BBOB01_01_20d_noise_1"): 3088.2256237462934,
("Vizier", "BBOB02_01_20d"): 1594.1976033631713,
("Vizier", "BBOB03_01_20d"): 1658.259861622774,
("Vizier", "BBOB04_01_20d"): 1583.7568749463885,
("Vizier", "BBOB05_01_20d"): 1196.8809362595202,
("Vizier", "BBOB06_01_20d"): 1491.6484978771537,
("Vizier", "BBOB07_01_20d"): 1652.7605198863428,
("Vizier", "BBOB08_01_20d"): 1695.5957367307972,
("Vizier", "BBOB09_01_20d"): 1716.1401858804281,
("Vizier", "BBOB10_01_20d"): 1708.137355636852,
("Vizier", "BBOB11_01_20d"): 1681.424214523798,
("Vizier", "BBOB12_01_20d"): 1692.8449678405655,
("Vizier", "BBOB13_01_20d"): 1680.930803057854,
("Vizier", "BBOB14_01_20d"): 1695.5308751825476,
("Vizier", "BBOB15_01_20d"): 1718.4326709089335,
("Vizier", "BBOB16_01_20d"): 1290.361077798577,
("Vizier", "BBOB17_01_20d"): 1374.7311208781787,
("Vizier", "BBOB18_01_20d"): 1446.2520969888196,
("Vizier", "BBOB19_01_20d"): 1465.3017042629422,
("Vizier", "BBOB20_01_20d"): 1409.965745873656,
("Vizier", "BBOB21_01_20d"): 1345.7366682451684,
("Vizier", "BBOB22_01_20d"): 1391.9293117268942,
("Vizier", "BBOB23_01_20d"): 1297.5176479740535,
("Vizier", "BBOB24_01_20d"): 1353.3727215859108,
("Vizier", "Branin"): 1120.9575905623265,
("Vizier", "Branin_noise_23"): 706.9293799998006,
("Vizier", "Branin_variable_runtime"): 936.5797979278839,
("Vizier", "CarSideImpact"): 1380.7215038021095,
("Vizier", "DTLZ2_2m_6d"): 1205.9157086708583,
("Vizier", "Discrete Labs"): 2080.97632496799,
("Vizier", "Hartmann_30d"): 1317.5518271096748,
("Vizier", "Hartmann_6d"): 1541.9186798716314,
("Vizier", "Penicillin"): 1733.2438224195503,
("Vizier", "PressureVessel"): 1047.110226731957,
("Vizier", "SixHumpCamel"): 699.2186566144344,
("Vizier", "TensionCompressionString"): 1161.3408887739758,
("Vizier", "VehicleSafety"): 929.2489898727042,
("Vizier", "WeldedBeamSO"): 1470.967353972816,
("Vizier", "ZDT1_2m_5d"): 710.6755482986919,
("Vizier", "ZDT2_2m_5d"): 905.1761489394528,
("Vizier", "ZDT3_2m_5d"): 716.3772842964215,
}
PAST_AVERAGE_RUNTIME_BY_METHOD = {
"Ax::fast+Center": 934.4863914025534,
"Ax::fast::ES+Center": 808.8479767732695,
"Ax::random_search+Center": 2.863206572517144,
"HEBO+Center": 224.81724724265342,
"Optuna+Center": 4.300884519331207,
"SMAC-BB+Center": 775.0705641796308,
"SMAC-HPO+Center": 9.279847992361555,
"Vizier": 1428.2346809913863,
}