Skip to content

Commit ee1f2a4

Browse files
be-marcbe-marc
authored
feat: add one se rule callback (#464)
* feat: add one se rule callback * ... * ... * ... * ... * ... * ... * async * ... * ... * ... * ... * ...
1 parent 4dc1160 commit ee1f2a4

27 files changed

+1355
-162
lines changed

‎DESCRIPTION

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ Depends:
2929
paradox (>= 1.0.1),
3030
R (>= 3.1.0)
3131
Imports:
32-
bbotk (>= 1.2.0),
32+
bbotk (>= 1.3.0),
3333
checkmate (>= 2.0.0),
3434
data.table,
3535
lgr,
@@ -50,8 +50,6 @@ Suggests:
5050
rpart,
5151
testthat (>= 3.0.0),
5252
xgboost
53-
Remotes:
54-
mlr-org/bbotk
5553
VignetteBuilder:
5654
knitr
5755
Config/testthat/edition: 3

‎NAMESPACE

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ export(TuningInstanceSingleCrit)
5050
export(as_search_space)
5151
export(as_tuner)
5252
export(as_tuners)
53+
export(assert_async_tuning_callback)
54+
export(assert_async_tuning_callbacks)
55+
export(assert_batch_tuning_callback)
56+
export(assert_batch_tuning_callbacks)
5357
export(assert_tuner)
5458
export(assert_tuner_async)
5559
export(assert_tuner_batch)
@@ -88,5 +92,6 @@ importFrom(bbotk,trms)
8892
importFrom(mlr3misc,clbk)
8993
importFrom(mlr3misc,clbks)
9094
importFrom(mlr3misc,mlr_callbacks)
95+
importFrom(stats,sd)
9196
importFrom(utils,bibentry)
9297
importFrom(utils,tail)

‎NEWS.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# mlr3tuning (development version)
22

3+
* feat: Add new callback `clbk("mlr3tuning.one_se_rule")` that selects the the hyperparameter configuration with the smallest feature set within one standard error of the best.
4+
* feat: Add new stages `on_tuning_result_begin` and `on_result_begin` to `CallbackAsyncTuning` and `CallbackBatchTuning`.
5+
* refactor: Rename stage `on_result` to `on_result_end` in `CallbackAsyncTuning` and `CallbackBatchTuning`.
6+
* docs: Extend the `CallbackAsyncTuning` and `CallbackBatchTuning` documentation.
37
* compatibility: mlr3 0.22.0
48

59
# mlr3tuning 1.1.0

‎R/CallbackAsyncTuning.R

Lines changed: 109 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,24 @@ CallbackAsyncTuning = R6Class("CallbackAsyncTuning",
1313
public = list(
1414

1515
#' @field on_eval_after_xs (`function()`)\cr
16-
#' Stage called after xs is passed.
17-
#' Called in `ObjectiveTuning$eval()`.
16+
#' Stage called after xs is passed.
17+
#' Called in `ObjectiveTuningAsync$eval()`.
1818
on_eval_after_xs = NULL,
1919

2020
#' @field on_eval_after_resample (`function()`)\cr
21-
#' Stage called after hyperparameter configurations are evaluated.
22-
#' Called in `ObjectiveTuning$eval()`.
21+
#' Stage called after hyperparameter configurations are evaluated.
22+
#' Called in `ObjectiveTuningAsync$eval()`.
2323
on_eval_after_resample = NULL,
2424

2525
#' @field on_eval_before_archive (`function()`)\cr
26-
#' Stage called before performance values are written to the archive.
27-
#' Called in `ObjectiveTuning$eval()`.
28-
on_eval_before_archive = NULL
26+
#' Stage called before performance values are written to the archive.
27+
#' Called in `ObjectiveTuningAsync$eval()`.
28+
on_eval_before_archive = NULL,
29+
30+
#' @field on_tuning_result_begin (`function()`)\cr
31+
#' Stage called before the results are written.
32+
#' Called in `TuningInstance*$assign_result()`.
33+
on_tuning_result_begin = NULL
2934
)
3035
)
3136

@@ -54,7 +59,9 @@ CallbackAsyncTuning = R6Class("CallbackAsyncTuning",
5459
#' End Optimization on Worker
5560
#' - on_worker_end
5661
#' End Worker
57-
#' - on_result
62+
#' - on_tuning_result_begin
63+
#' - on_result_begin
64+
#' - on_result_end
5865
#' - on_optimization_end
5966
#' End Tuning
6067
#' ```
@@ -68,39 +75,70 @@ CallbackAsyncTuning = R6Class("CallbackAsyncTuning",
6875
#' Tuning callbacks access [ContextAsyncTuning].
6976
#'
7077
#' @param id (`character(1)`)\cr
71-
#' Identifier for the new instance.
78+
#' Identifier for the new instance.
7279
#' @param label (`character(1)`)\cr
73-
#' Label for the new instance.
80+
#' Label for the new instance.
7481
#' @param man (`character(1)`)\cr
75-
#' String in the format `[pkg]::[topic]` pointing to a manual page for this object.
76-
#' The referenced help package can be opened via method `$help()`.
82+
#' String in the format `[pkg]::[topic]` pointing to a manual page for this object.
83+
#' The referenced help package can be opened via method `$help()`.
84+
#'
7785
#' @param on_optimization_begin (`function()`)\cr
78-
#' Stage called at the beginning of the optimization.
79-
#' Called in `Optimizer$optimize()`.
86+
#' Stage called at the beginning of the optimization.
87+
#' Called in `Optimizer$optimize()`.
88+
#' The functions must have two arguments named `callback` and `context`.
8089
#' @param on_worker_begin (`function()`)\cr
81-
#' Stage called at the beginning of the optimization on the worker.
82-
#' Called in the worker loop.
90+
#' Stage called at the beginning of the optimization on the worker.
91+
#' Called in the worker loop.
92+
#' The functions must have two arguments named `callback` and `context`.
8393
#' @param on_optimizer_before_eval (`function()`)\cr
84-
#' Stage called after the optimizer proposes points.
85-
#' Called in `OptimInstance$.eval_point()`.
94+
#' Stage called after the optimizer proposes points.
95+
#' Called in `OptimInstance$.eval_point()`.
96+
#' The functions must have two arguments named `callback` and `context`.
97+
#' The argument of `instance$.eval_point(xs)` and `xs_trafoed` and `extra` are available in the `context`.
98+
#' Or `xs` and `xs_trafoed` of `instance$.eval_queue()` are available in the `context`.
8699
#' @param on_eval_after_xs (`function()`)\cr
87-
#' Stage called after xs is passed.
88-
#' Called in `ObjectiveTuning$eval()`.
100+
#' Stage called after xs is passed to the objective.
101+
#' Called in `ObjectiveTuningAsync$eval()`.
102+
#' The functions must have two arguments named `callback` and `context`.
103+
#' The argument of `$.eval(xs)` is available in the `context`.
89104
#' @param on_eval_after_resample (`function()`)\cr
90-
#' Stage called after a hyperparameter configuration is evaluated.
91-
#' Called in `ObjectiveTuning$eval()`.
105+
#' Stage called after a hyperparameter configuration is evaluated.
106+
#' Called in `ObjectiveTuningAsync$eval()`.
107+
#' The functions must have two arguments named `callback` and `context`.
108+
#' The `resample_result` is available in the `context
92109
#' @param on_eval_before_archive (`function()`)\cr
93-
#' Stage called before performance values are written to the archive.
94-
#' Called in `ObjectiveTuning$eval()`.
110+
#' Stage called before performance values are written to the archive.
111+
#' Called in `ObjectiveTuningAsync$eval()`.
112+
#' The functions must have two arguments named `callback` and `context`.
113+
#' The `aggregated_performance` is available in `context`.
95114
#' @param on_optimizer_after_eval (`function()`)\cr
96-
#' Stage called after points are evaluated.
97-
#' Called in `OptimInstance$.eval_point()`.
115+
#' Stage called after points are evaluated.
116+
#' Called in `OptimInstance$.eval_point()`.
117+
#' The functions must have two arguments named `callback` and `context`.
98118
#' @param on_worker_end (`function()`)\cr
99-
#' Stage called at the end of the optimization on the worker.
100-
#' Called in the worker loop.
119+
#' Stage called at the end of the optimization on the worker.
120+
#' Called in the worker loop.
121+
#' The functions must have two arguments named `callback` and `context`.
122+
#' @param on_tuning_result_begin (`function()`)\cr
123+
#' Stage called at the beginning of the result writing.
124+
#' Called in `TuningInstance*$assign_result()`.
125+
#' The functions must have two arguments named `callback` and `context`.
126+
#' The arguments of `$assign_result(xdt, y, learner_param_vals, extra)` are available in `context`.
127+
#' @param on_result_begin (`function()`)\cr
128+
#' Stage called at the beginning of the result writing.
129+
#' Called in `OptimInstance$assign_result()`.
130+
#' The functions must have two arguments named `callback` and `context`.
131+
#' The arguments of `$.assign_result(xdt, y, extra)` are available in the `context`.
132+
#' @param on_result_end (`function()`)\cr
133+
#' Stage called after the result is written.
134+
#' Called in `OptimInstance$assign_result()`.
135+
#' The functions must have two arguments named `callback` and `context`.
136+
#' The final result `instance$result` is available in the `context`.
101137
#' @param on_result (`function()`)\cr
102-
#' Stage called after the result is written.
103-
#' Called in `OptimInstance$assign_result()`.
138+
#' Deprecated.
139+
#' Use `on_result_end` instead.
140+
#' Stage called after the result is written.
141+
#' Called in `OptimInstance$assign_result()`.
104142
#' @param on_optimization_end (`function()`)\cr
105143
#' Stage called at the end of the optimization.
106144
#' Called in `Optimizer$optimize()`.
@@ -118,6 +156,9 @@ callback_async_tuning = function(
118156
on_eval_before_archive = NULL,
119157
on_optimizer_after_eval = NULL,
120158
on_worker_end = NULL,
159+
on_tuning_result_begin = NULL,
160+
on_result_begin = NULL,
161+
on_result_end = NULL,
121162
on_result = NULL,
122163
on_optimization_end = NULL
123164
) {
@@ -130,6 +171,9 @@ callback_async_tuning = function(
130171
on_eval_before_archive,
131172
on_optimizer_after_eval,
132173
on_worker_end,
174+
on_tuning_result_begin,
175+
on_result_begin,
176+
on_result_end,
133177
on_result,
134178
on_optimization_end),
135179
c(
@@ -141,10 +185,45 @@ callback_async_tuning = function(
141185
"on_eval_before_archive",
142186
"on_optimizer_after_eval",
143187
"on_worker_end",
188+
"on_tuning_result_begin",
189+
"on_result_begin",
190+
"on_result_end",
144191
"on_result",
145192
"on_optimization_end")), is.null)
193+
194+
if ("on_result" %in% names(stages)) {
195+
.Deprecated(old = "on_result", new = "on_result_end")
196+
stages$on_result_end = stages$on_result
197+
stages$on_result = NULL
198+
}
199+
146200
walk(stages, function(stage) assert_function(stage, args = c("callback", "context")))
147201
callback = CallbackAsyncTuning$new(id, label, man)
148202
iwalk(stages, function(stage, name) callback[[name]] = stage)
149203
callback
150204
}
205+
206+
207+
#' @title Assertions for Callbacks
208+
#'
209+
#' @description
210+
#' Assertions for [CallbackAsyncTuning] class.
211+
#'
212+
#' @param callback ([CallbackAsyncTuning]).
213+
#' @param null_ok (`logical(1)`)\cr
214+
#' If `TRUE`, `NULL` is allowed.
215+
#'
216+
#' @return [CallbackAsyncTuning | List of [CallbackAsyncTuning]s.
217+
#' @export
218+
assert_async_tuning_callback = function(callback, null_ok = FALSE) {
219+
if (null_ok && is.null(callback)) return(invisible(NULL))
220+
assert_class(callback, "CallbackAsyncTuning")
221+
invisible(callback)
222+
}
223+
224+
#' @export
225+
#' @param callbacks (list of [CallbackAsyncTuning]).
226+
#' @rdname assert_async_tuning_callback
227+
assert_async_tuning_callbacks = function(callbacks) {
228+
invisible(lapply(callbacks, assert_callback))
229+
}

0 commit comments

Comments
 (0)