Skip to content

Commit 3458c66

Browse files
motusbpkroth
motus
and
bpkroth
authored
Add support for ConfigSpace distributions (#663)
To be merged after #662 Closes #665 --------- Co-authored-by: Brian Kroth <[email protected]> Co-authored-by: Brian Kroth <[email protected]>
1 parent 9175d18 commit 3458c66

File tree

6 files changed

+496
-76
lines changed

6 files changed

+496
-76
lines changed

Diff for: ‎mlos_bench/mlos_bench/config/schemas/tunables/tunable-params-schema.json

+109-2
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,113 @@
1515
}
1616
}
1717
},
18+
"tunable_param_distribution": {
19+
"description": "Distribution config to use when sampling from numerical range.",
20+
"allOf": [
21+
{
22+
"description": "Distribution config to use when sampling from numerical range.",
23+
"type": "object",
24+
"properties": {
25+
"type": {
26+
"description": "Which distribution to use when sampling from the numerical range.",
27+
"$comment": "The possible distribution types. Full match must be below. This is just for IDE help.",
28+
"enum": ["uniform", "beta", "normal"]
29+
},
30+
"params": {
31+
"description": "Per distribution type configuration parameters",
32+
"$comment": "Stub. Full match should be below.",
33+
"type": "object",
34+
"minItems": 1
35+
}
36+
},
37+
"required": ["type"]
38+
},
39+
{
40+
"oneOf": [
41+
{
42+
"type": "object",
43+
"properties": {
44+
"type": {
45+
"description": "Use a uniform distribution for sampling from the numerical range.",
46+
"const": "uniform"
47+
},
48+
"params": {
49+
"description": "(null) parameters for the uniform distribution",
50+
"type": "object",
51+
"unevaluatedProperties": false
52+
}
53+
},
54+
"required": ["type"],
55+
"unevaluatedProperties": false
56+
},
57+
{
58+
"type": "object",
59+
"properties": {
60+
"type": {
61+
"description": "Use a beta distribution for sampling from the numerical range.",
62+
"const": "beta"
63+
},
64+
"params": {
65+
"description": "Parameters for the beta distribution",
66+
"type": "object",
67+
"properties": {
68+
"alpha": {
69+
"description": "The alpha shape parameter for the beta distribution.",
70+
"type": "number",
71+
"exclusiveMinimum": 0
72+
},
73+
"beta": {
74+
"description": "The beta shape parameter for the beta distribution.",
75+
"type": "number",
76+
"exclusiveMinimum": 0
77+
}
78+
},
79+
"required": ["alpha", "beta"],
80+
"unevaluatedProperties": false
81+
}
82+
},
83+
"required": ["type", "params"],
84+
"unevaluatedProperties": false
85+
},
86+
{
87+
"type": "object",
88+
"properties": {
89+
"type": {
90+
"description": "Use a normal distribution for sampling from the numerical range.",
91+
"const": "normal"
92+
},
93+
"params": {
94+
"description": "Parameters for the normal distribution",
95+
"type": "object",
96+
"properties": {
97+
"mu": {
98+
"description": "The mean of the normal distribution.",
99+
"type": "number"
100+
},
101+
"sigma": {
102+
"description": "The standard deviation of the normal distribution.",
103+
"type": "number",
104+
"exclusiveMinimum": 0
105+
}
106+
},
107+
"required": ["mu", "sigma"],
108+
"unevaluatedProperties": false
109+
}
110+
},
111+
"required": ["type", "params"],
112+
"unevaluatedProperties": false
113+
}
114+
]
115+
}
116+
]
117+
},
18118
"numeric_range": {
19119
"description": "Two element array representing the lower and upper bounds of the range.",
20120
"type": "array",
21121
"$comment": "items type left unspecified here",
22122
"minItems": 2,
23-
"maxItems": 2
123+
"maxItems": 2,
124+
"uniqueItems": true
24125
},
25126
"quantization": {
26127
"description": "The number of buckets to quantize the range into.\nSee Also:\nhttps://automl.github.io/ConfigSpace/main/api/hyperparameters.html#module-ConfigSpace.api.types.float,\nhttps://automl.github.io/ConfigSpace/main/api/hyperparameters.html#module-ConfigSpace.api.types.integer",
@@ -85,7 +186,7 @@
85186
},
86187
"required": ["type", "default", "values"],
87188
"not": {
88-
"required": ["range", "special", "special_weights", "range_weight"]
189+
"required": ["range", "special", "special_weights", "range_weight", "log", "quantization", "distribution"]
89190
},
90191
"$comment": "TODO: add check that default is in values",
91192
"unevaluatedProperties": false
@@ -112,6 +213,9 @@
112213
"type": "integer"
113214
}
114215
},
216+
"distribution": {
217+
"$ref": "#/$defs/tunable_param_distribution"
218+
},
115219
"quantization": {
116220
"$ref": "#/$defs/quantization",
117221
"type": "integer",
@@ -159,6 +263,9 @@
159263
"type": "number"
160264
}
161265
},
266+
"distribution": {
267+
"$ref": "#/$defs/tunable_param_distribution"
268+
},
162269
"quantization": {
163270
"$ref": "#/$defs/quantization",
164271
"type": "number",

Diff for: ‎mlos_bench/mlos_bench/optimizers/convert_configspace.py

+47-25
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,18 @@
88

99
import logging
1010

11-
from typing import Dict, List, Optional, Tuple
11+
from typing import Dict, List, Optional, Tuple, Union
1212

1313
from ConfigSpace import (
14+
Beta,
1415
CategoricalHyperparameter,
1516
Configuration,
1617
ConfigurationSpace,
1718
EqualsCondition,
18-
UniformFloatHyperparameter,
19-
UniformIntegerHyperparameter,
19+
Float,
20+
Integer,
21+
Normal,
22+
Uniform,
2023
)
2124
from mlos_bench.tunables.tunable import Tunable, TunableValue
2225
from mlos_bench.tunables.tunable_groups import TunableGroups
@@ -76,24 +79,51 @@ def _tunable_to_configspace(
7679
meta=meta)
7780
})
7881

82+
distribution: Union[Uniform, Normal, Beta, None] = None
83+
if tunable.distribution == "uniform":
84+
distribution = Uniform()
85+
elif tunable.distribution == "normal":
86+
distribution = Normal(
87+
mu=tunable.distribution_params["mu"],
88+
sigma=tunable.distribution_params["sigma"]
89+
)
90+
elif tunable.distribution == "beta":
91+
distribution = Beta(
92+
alpha=tunable.distribution_params["alpha"],
93+
beta=tunable.distribution_params["beta"]
94+
)
95+
elif tunable.distribution is not None:
96+
raise TypeError(f"Invalid Distribution Type: {tunable.distribution}")
97+
7998
if tunable.type == "int":
80-
hp_type = UniformIntegerHyperparameter
99+
range_hp = Integer(
100+
name=tunable.name,
101+
bounds=(int(tunable.range[0]), int(tunable.range[1])),
102+
log=bool(tunable.is_log),
103+
q=int(tunable.quantization) if tunable.quantization is not None else None,
104+
distribution=distribution,
105+
default=(int(tunable.default)
106+
if tunable.in_range(tunable.default) and tunable.default is not None
107+
else None),
108+
meta=meta
109+
)
81110
elif tunable.type == "float":
82-
hp_type = UniformFloatHyperparameter
111+
range_hp = Float(
112+
name=tunable.name,
113+
bounds=tunable.range,
114+
log=bool(tunable.is_log),
115+
q=tunable.quantization, # type: ignore[arg-type]
116+
distribution=distribution, # type: ignore[arg-type]
117+
default=(float(tunable.default)
118+
if tunable.in_range(tunable.default) and tunable.default is not None
119+
else None),
120+
meta=meta
121+
)
83122
else:
84-
raise TypeError(f"Undefined Parameter Type: {tunable.type}")
123+
raise TypeError(f"Invalid Parameter Type: {tunable.type}")
85124

86125
if not tunable.special:
87-
return ConfigurationSpace({
88-
tunable.name: hp_type(
89-
name=tunable.name,
90-
lower=tunable.range[0],
91-
upper=tunable.range[1],
92-
log=tunable.is_log,
93-
q=tunable.quantization,
94-
default_value=tunable.default if tunable.in_range(tunable.default) else None,
95-
meta=meta)
96-
})
126+
return ConfigurationSpace({tunable.name: range_hp})
97127

98128
# Compute the probabilities of switching between regular and special values.
99129
special_weights: Optional[List[float]] = None
@@ -106,15 +136,7 @@ def _tunable_to_configspace(
106136
# one for special values, and one to choose between the two.
107137
(special_name, type_name) = special_param_names(tunable.name)
108138
conf_space = ConfigurationSpace({
109-
tunable.name: hp_type(
110-
name=tunable.name,
111-
lower=tunable.range[0],
112-
upper=tunable.range[1],
113-
log=tunable.is_log,
114-
q=tunable.quantization,
115-
default_value=tunable.default if tunable.in_range(tunable.default) else None,
116-
meta=meta
117-
),
139+
tunable.name: range_hp,
118140
special_name: CategoricalHyperparameter(
119141
name=special_name,
120142
choices=tunable.special,

Diff for: ‎mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/good/full/full-tunable-params-test.jsonc

+31-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"cost": 1,
44
"description": "Covariant group description",
55
"params": {
6-
"int": {
6+
"beta-int": {
77
"description": "Int",
88
"type": "int",
99
"default": 10,
@@ -13,16 +13,45 @@
1313
"special_weights": [0.1],
1414
"range_weight": 0.9,
1515
"quantization": 50,
16+
"distribution": {
17+
"type": "beta",
18+
"params": {
19+
"alpha": 0.1,
20+
"beta": 0.1
21+
}
22+
},
1623
"log": true
1724
},
18-
"float": {
25+
"normal-int": {
26+
"description": "Int",
27+
"type": "int",
28+
"default": 10,
29+
"range": [1, 500],
30+
"meta": {"suffix": "MB"},
31+
"special": [-1],
32+
"special_weights": [0.1],
33+
"range_weight": 0.9,
34+
"quantization": 50,
35+
"distribution": {
36+
"type": "normal",
37+
"params": {
38+
"mu": 0,
39+
"sigma": 0.1
40+
}
41+
},
42+
"log": true
43+
},
44+
"uniform-float": {
1945
"description": "Float",
2046
"type": "float",
2147
"default": 10.1,
2248
"meta": {"scale": 1000, "prefix": "/proc/var/random/", "base": 2.71828},
2349
"range": [1.1, 111.1],
2450
"special": [-1.1],
2551
"quantization": 10,
52+
"distribution": {
53+
"type": "uniform"
54+
},
2655
"log": false
2756
},
2857
"cat": {

0 commit comments

Comments
 (0)