From ee1f2a42c5c8c4b490e3a8157e56e4fd4e0f3ce5 Mon Sep 17 00:00:00 2001 From: Marc Becker <33069354+be-marc@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:13:13 +0100 Subject: [PATCH] feat: add one se rule callback (#464) * feat: add one se rule callback * ... * ... * ... * ... * ... * ... * async * ... * ... * ... * ... * ... --- DESCRIPTION | 4 +- NAMESPACE | 5 + NEWS.md | 4 + R/CallbackAsyncTuning.R | 139 +++++++-- R/CallbackBatchTuning.R | 138 +++++++-- R/ContextAsyncTuning.R | 22 +- R/ContextBatchTuning.R | 26 +- R/TuningInstanceAsyncMulticrit.R | 37 ++- R/TuningInstanceAsyncSingleCrit.R | 44 ++- R/TuningInstanceBatchMulticrit.R | 38 ++- R/TuningInstanceBatchSingleCrit.R | 40 ++- R/bibentries.R | 11 + R/mlr_callbacks.R | 133 +++++++++ R/zzz.R | 3 + man/CallbackAsyncTuning.Rd | 10 +- man/CallbackBatchTuning.Rd | 10 +- man/ContextAsyncTuning.Rd | 3 + man/ContextBatchTuning.Rd | 3 + man/assert_async_tuning_callback.Rd | 25 ++ man/assert_batch_tuning_callback.Rd | 25 ++ man/callback_async_tuning.Rd | 58 +++- man/callback_batch_tuning.Rd | 60 +++- man/mlr3tuning.one_se_rule.Rd | 35 +++ pkgdown/_pkgdown.yml | 4 + tests/testthat/test_CallbackAsyncTuning.R | 341 ++++++++++++++++++++++ tests/testthat/test_CallbackBatchTuning.R | 261 +++++++++++++++++ tests/testthat/test_mlr_callbacks.R | 38 +++ 27 files changed, 1355 insertions(+), 162 deletions(-) create mode 100644 man/assert_async_tuning_callback.Rd create mode 100644 man/assert_batch_tuning_callback.Rd create mode 100644 man/mlr3tuning.one_se_rule.Rd create mode 100644 tests/testthat/test_CallbackAsyncTuning.R create mode 100644 tests/testthat/test_CallbackBatchTuning.R diff --git a/DESCRIPTION b/DESCRIPTION index 4c3bc5173..d5816c9cf 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -29,7 +29,7 @@ Depends: paradox (>= 1.0.1), R (>= 3.1.0) Imports: - bbotk (>= 1.2.0), + bbotk (>= 1.3.0), checkmate (>= 2.0.0), data.table, lgr, @@ -50,8 +50,6 @@ Suggests: rpart, testthat (>= 3.0.0), xgboost -Remotes: - mlr-org/bbotk VignetteBuilder: knitr Config/testthat/edition: 3 diff --git a/NAMESPACE b/NAMESPACE index be76cdfa0..b886aaafd 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -50,6 +50,10 @@ export(TuningInstanceSingleCrit) export(as_search_space) export(as_tuner) export(as_tuners) +export(assert_async_tuning_callback) +export(assert_async_tuning_callbacks) +export(assert_batch_tuning_callback) +export(assert_batch_tuning_callbacks) export(assert_tuner) export(assert_tuner_async) export(assert_tuner_batch) @@ -88,5 +92,6 @@ importFrom(bbotk,trms) importFrom(mlr3misc,clbk) importFrom(mlr3misc,clbks) importFrom(mlr3misc,mlr_callbacks) +importFrom(stats,sd) importFrom(utils,bibentry) importFrom(utils,tail) diff --git a/NEWS.md b/NEWS.md index 6937d9ab6..d492b35b3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # mlr3tuning (development version) +* feat: Add new callback `clbk("mlr3tuning.one_se_rule")` that selects the the hyperparameter configuration with the smallest feature set within one standard error of the best. +* feat: Add new stages `on_tuning_result_begin` and `on_result_begin` to `CallbackAsyncTuning` and `CallbackBatchTuning`. +* refactor: Rename stage `on_result` to `on_result_end` in `CallbackAsyncTuning` and `CallbackBatchTuning`. +* docs: Extend the `CallbackAsyncTuning` and `CallbackBatchTuning` documentation. * compatibility: mlr3 0.22.0 # mlr3tuning 1.1.0 diff --git a/R/CallbackAsyncTuning.R b/R/CallbackAsyncTuning.R index b590184ac..616addad7 100644 --- a/R/CallbackAsyncTuning.R +++ b/R/CallbackAsyncTuning.R @@ -13,19 +13,24 @@ CallbackAsyncTuning = R6Class("CallbackAsyncTuning", public = list( #' @field on_eval_after_xs (`function()`)\cr - #' Stage called after xs is passed. - #' Called in `ObjectiveTuning$eval()`. + #' Stage called after xs is passed. + #' Called in `ObjectiveTuningAsync$eval()`. on_eval_after_xs = NULL, #' @field on_eval_after_resample (`function()`)\cr - #' Stage called after hyperparameter configurations are evaluated. - #' Called in `ObjectiveTuning$eval()`. + #' Stage called after hyperparameter configurations are evaluated. + #' Called in `ObjectiveTuningAsync$eval()`. on_eval_after_resample = NULL, #' @field on_eval_before_archive (`function()`)\cr - #' Stage called before performance values are written to the archive. - #' Called in `ObjectiveTuning$eval()`. - on_eval_before_archive = NULL + #' Stage called before performance values are written to the archive. + #' Called in `ObjectiveTuningAsync$eval()`. + on_eval_before_archive = NULL, + + #' @field on_tuning_result_begin (`function()`)\cr + #' Stage called before the results are written. + #' Called in `TuningInstance*$assign_result()`. + on_tuning_result_begin = NULL ) ) @@ -54,7 +59,9 @@ CallbackAsyncTuning = R6Class("CallbackAsyncTuning", #' End Optimization on Worker #' - on_worker_end #' End Worker -#' - on_result +#' - on_tuning_result_begin +#' - on_result_begin +#' - on_result_end #' - on_optimization_end #' End Tuning #' ``` @@ -68,39 +75,70 @@ CallbackAsyncTuning = R6Class("CallbackAsyncTuning", #' Tuning callbacks access [ContextAsyncTuning]. #' #' @param id (`character(1)`)\cr -#' Identifier for the new instance. +#' Identifier for the new instance. #' @param label (`character(1)`)\cr -#' Label for the new instance. +#' Label for the new instance. #' @param man (`character(1)`)\cr -#' String in the format `[pkg]::[topic]` pointing to a manual page for this object. -#' The referenced help package can be opened via method `$help()`. +#' String in the format `[pkg]::[topic]` pointing to a manual page for this object. +#' The referenced help package can be opened via method `$help()`. +#' #' @param on_optimization_begin (`function()`)\cr -#' Stage called at the beginning of the optimization. -#' Called in `Optimizer$optimize()`. +#' Stage called at the beginning of the optimization. +#' Called in `Optimizer$optimize()`. +#' The functions must have two arguments named `callback` and `context`. #' @param on_worker_begin (`function()`)\cr -#' Stage called at the beginning of the optimization on the worker. -#' Called in the worker loop. +#' Stage called at the beginning of the optimization on the worker. +#' Called in the worker loop. +#' The functions must have two arguments named `callback` and `context`. #' @param on_optimizer_before_eval (`function()`)\cr -#' Stage called after the optimizer proposes points. -#' Called in `OptimInstance$.eval_point()`. +#' Stage called after the optimizer proposes points. +#' Called in `OptimInstance$.eval_point()`. +#' The functions must have two arguments named `callback` and `context`. +#' The argument of `instance$.eval_point(xs)` and `xs_trafoed` and `extra` are available in the `context`. +#' Or `xs` and `xs_trafoed` of `instance$.eval_queue()` are available in the `context`. #' @param on_eval_after_xs (`function()`)\cr -#' Stage called after xs is passed. -#' Called in `ObjectiveTuning$eval()`. +#' Stage called after xs is passed to the objective. +#' Called in `ObjectiveTuningAsync$eval()`. +#' The functions must have two arguments named `callback` and `context`. +#' The argument of `$.eval(xs)` is available in the `context`. #' @param on_eval_after_resample (`function()`)\cr -#' Stage called after a hyperparameter configuration is evaluated. -#' Called in `ObjectiveTuning$eval()`. +#' Stage called after a hyperparameter configuration is evaluated. +#' Called in `ObjectiveTuningAsync$eval()`. +#' The functions must have two arguments named `callback` and `context`. +#' The `resample_result` is available in the `context #' @param on_eval_before_archive (`function()`)\cr -#' Stage called before performance values are written to the archive. -#' Called in `ObjectiveTuning$eval()`. +#' Stage called before performance values are written to the archive. +#' Called in `ObjectiveTuningAsync$eval()`. +#' The functions must have two arguments named `callback` and `context`. +#' The `aggregated_performance` is available in `context`. #' @param on_optimizer_after_eval (`function()`)\cr -#' Stage called after points are evaluated. -#' Called in `OptimInstance$.eval_point()`. +#' Stage called after points are evaluated. +#' Called in `OptimInstance$.eval_point()`. +#' The functions must have two arguments named `callback` and `context`. #' @param on_worker_end (`function()`)\cr -#' Stage called at the end of the optimization on the worker. -#' Called in the worker loop. +#' Stage called at the end of the optimization on the worker. +#' Called in the worker loop. +#' The functions must have two arguments named `callback` and `context`. +#' @param on_tuning_result_begin (`function()`)\cr +#' Stage called at the beginning of the result writing. +#' Called in `TuningInstance*$assign_result()`. +#' The functions must have two arguments named `callback` and `context`. +#' The arguments of `$assign_result(xdt, y, learner_param_vals, extra)` are available in `context`. +#' @param on_result_begin (`function()`)\cr +#' Stage called at the beginning of the result writing. +#' Called in `OptimInstance$assign_result()`. +#' The functions must have two arguments named `callback` and `context`. +#' The arguments of `$.assign_result(xdt, y, extra)` are available in the `context`. +#' @param on_result_end (`function()`)\cr +#' Stage called after the result is written. +#' Called in `OptimInstance$assign_result()`. +#' The functions must have two arguments named `callback` and `context`. +#' The final result `instance$result` is available in the `context`. #' @param on_result (`function()`)\cr -#' Stage called after the result is written. -#' Called in `OptimInstance$assign_result()`. +#' Deprecated. +#' Use `on_result_end` instead. +#' Stage called after the result is written. +#' Called in `OptimInstance$assign_result()`. #' @param on_optimization_end (`function()`)\cr #' Stage called at the end of the optimization. #' Called in `Optimizer$optimize()`. @@ -118,6 +156,9 @@ callback_async_tuning = function( on_eval_before_archive = NULL, on_optimizer_after_eval = NULL, on_worker_end = NULL, + on_tuning_result_begin = NULL, + on_result_begin = NULL, + on_result_end = NULL, on_result = NULL, on_optimization_end = NULL ) { @@ -130,6 +171,9 @@ callback_async_tuning = function( on_eval_before_archive, on_optimizer_after_eval, on_worker_end, + on_tuning_result_begin, + on_result_begin, + on_result_end, on_result, on_optimization_end), c( @@ -141,10 +185,45 @@ callback_async_tuning = function( "on_eval_before_archive", "on_optimizer_after_eval", "on_worker_end", + "on_tuning_result_begin", + "on_result_begin", + "on_result_end", "on_result", "on_optimization_end")), is.null) + + if ("on_result" %in% names(stages)) { + .Deprecated(old = "on_result", new = "on_result_end") + stages$on_result_end = stages$on_result + stages$on_result = NULL + } + walk(stages, function(stage) assert_function(stage, args = c("callback", "context"))) callback = CallbackAsyncTuning$new(id, label, man) iwalk(stages, function(stage, name) callback[[name]] = stage) callback } + + +#' @title Assertions for Callbacks +#' +#' @description +#' Assertions for [CallbackAsyncTuning] class. +#' +#' @param callback ([CallbackAsyncTuning]). +#' @param null_ok (`logical(1)`)\cr +#' If `TRUE`, `NULL` is allowed. +#' +#' @return [CallbackAsyncTuning | List of [CallbackAsyncTuning]s. +#' @export +assert_async_tuning_callback = function(callback, null_ok = FALSE) { + if (null_ok && is.null(callback)) return(invisible(NULL)) + assert_class(callback, "CallbackAsyncTuning") + invisible(callback) +} + +#' @export +#' @param callbacks (list of [CallbackAsyncTuning]). +#' @rdname assert_async_tuning_callback +assert_async_tuning_callbacks = function(callbacks) { + invisible(lapply(callbacks, assert_callback)) +} diff --git a/R/CallbackBatchTuning.R b/R/CallbackBatchTuning.R index 7f27e1430..2de95b4dd 100644 --- a/R/CallbackBatchTuning.R +++ b/R/CallbackBatchTuning.R @@ -20,19 +20,24 @@ CallbackBatchTuning= R6Class("CallbackBatchTuning", public = list( #' @field on_eval_after_design (`function()`)\cr - #' Stage called after design is created. - #' Called in `ObjectiveTuning$eval_many()`. + #' Stage called after design is created. + #' Called in `ObjectiveTuningBatch$eval_many()`. on_eval_after_design = NULL, #' @field on_eval_after_benchmark (`function()`)\cr - #' Stage called after hyperparameter configurations are evaluated. - #' Called in `ObjectiveTuning$eval_many()`. + #' Stage called after hyperparameter configurations are evaluated. + #' Called in `ObjectiveTuningBatch$eval_many()`. on_eval_after_benchmark = NULL, #' @field on_eval_before_archive (`function()`)\cr - #' Stage called before performance values are written to the archive. - #' Called in `ObjectiveTuning$eval_many()`. - on_eval_before_archive = NULL + #' Stage called before performance values are written to the archive. + #' Called in `ObjectiveTuningBatch$eval_many()`. + on_eval_before_archive = NULL, + + #' @field on_tuning_result_begin (`function()`)\cr + #' Stage called before the results are written. + #' Called in `TuningInstance*$assign_result()`. + on_tuning_result_begin = NULL ) ) @@ -57,7 +62,9 @@ CallbackBatchTuning= R6Class("CallbackBatchTuning", #' End Evaluation #' - on_optimizer_after_eval #' End Tuner Batch -#' - on_result +#' - on_tuning_result_begin +#' - on_result_begin +#' - on_result_end #' - on_optimization_end #' End Tuning #' ``` @@ -71,39 +78,68 @@ CallbackBatchTuning= R6Class("CallbackBatchTuning", #' Tuning callbacks access [ContextBatchTuning]. #' #' @param id (`character(1)`)\cr -#' Identifier for the new instance. +#' Identifier for the new instance. #' @param label (`character(1)`)\cr -#' Label for the new instance. +#' Label for the new instance. #' @param man (`character(1)`)\cr -#' String in the format `[pkg]::[topic]` pointing to a manual page for this object. -#' The referenced help package can be opened via method `$help()`. +#' String in the format `[pkg]::[topic]` pointing to a manual page for this object. +#' The referenced help package can be opened via method `$help()`. +#' #' @param on_optimization_begin (`function()`)\cr -#' Stage called at the beginning of the optimization. -#' Called in `Optimizer$optimize()`. +#' Stage called at the beginning of the optimization. +#' Called in `Optimizer$optimize()`. +#' The functions must have two arguments named `callback` and `context`. #' @param on_optimizer_before_eval (`function()`)\cr -#' Stage called after the optimizer proposes points. -#' Called in `OptimInstance$eval_batch()`. +#' Stage called after the optimizer proposes points. +#' Called in `OptimInstance$eval_batch()`. +#' The functions must have two arguments named `callback` and `context`. +#' The argument of `$eval_batch(xdt)` is available in `context`. #' @param on_eval_after_design (`function()`)\cr -#' Stage called after the design is created. -#' Called in `ObjectiveTuning$eval_many()`. -#' The context available is [ContextBatchTuning]. +#' Stage called after the design is created. +#' Called in `ObjectiveTuningBatch$eval_many()`. +#' The functions must have two arguments named `callback` and `context`. +#' The arguments of `$eval_many(xss, resampling)` are available in `context`. +#' Additionally, the `design` is available in `context`. #' @param on_eval_after_benchmark (`function()`)\cr -#' Stage called after hyperparameter configurations are evaluated. -#' Called in `ObjectiveTuning$eval_many()`. -#' The context available is [ContextBatchTuning]. +#' Stage called after hyperparameter configurations are evaluated. +#' Called in `ObjectiveTuningBatch$eval_many()`. +#' The functions must have two arguments named `callback` and `context`. +#' The `benchmark_result` is available in `context`. #' @param on_eval_before_archive (`function()`)\cr -#' Stage called before performance values are written to the archive. -#' Called in `ObjectiveTuning$eval_many()`. -#' The context available is [ContextBatchTuning]. +#' Stage called before performance values are written to the archive. +#' Called in `ObjectiveTuningBatch$eval_many()`. +#' The functions must have two arguments named `callback` and `context`. +#' The `aggregated_performance` is available in `context`. #' @param on_optimizer_after_eval (`function()`)\cr -#' Stage called after points are evaluated. -#' Called in `OptimInstance$eval_batch()`. +#' Stage called after points are evaluated. +#' Called in `OptimInstance$eval_batch()`. +#' The functions must have two arguments named `callback` and `context`. +#' The new configurations and performances in `instance$archive` are available in `context`. +#' @param on_tuning_result_begin (`function()`)\cr +#' Stage called at the beginning of the result writing. +#' Called in `TuningInstanceBatch$assign_result()`. +#' The functions must have two arguments named `callback` and `context`. +#' The arguments of `$assign_result(xdt, y, learner_param_vals, extra)` are available in `context`. +#' @param on_result_begin (`function()`)\cr +#' Stage called at the beginning of the result writing. +#' Called in `OptimInstance$assign_result()`. +#' The functions must have two arguments named `callback` and `context`. +#' The arguments of `$assign_result(xdt, y, extra)` are available in `context`. +#' @param on_result_end (`function()`)\cr +#' Stage called after the result is written. +#' Called in `OptimInstance$assign_result()`. +#' The functions must have two arguments named `callback` and `context`. +#' The final result `instance$result` is available in `context`. #' @param on_result (`function()`)\cr -#' Stage called after the result is written. -#' Called in `OptimInstance$assign_result()`. +#' Deprecated. +#' Use `on_result_end` instead. +#' Stage called after the result is written. +#' Called in `OptimInstance$assign_result()`. +#' The functions must have two arguments named `callback` and `context`. #' @param on_optimization_end (`function()`)\cr -#' Stage called at the end of the optimization. -#' Called in `Optimizer$optimize()`. +#' Stage called at the end of the optimization. +#' Called in `Optimizer$optimize()`. +#' The functions must have two arguments named `callback` and `context`. #' #' @export #' @inherit CallbackBatchTuning examples @@ -117,6 +153,9 @@ callback_batch_tuning = function( on_eval_after_benchmark = NULL, on_eval_before_archive = NULL, on_optimizer_after_eval = NULL, + on_tuning_result_begin = NULL, + on_result_begin = NULL, + on_result_end = NULL, on_result = NULL, on_optimization_end = NULL ) { @@ -127,6 +166,9 @@ callback_batch_tuning = function( on_eval_after_benchmark, on_eval_before_archive, on_optimizer_after_eval, + on_tuning_result_begin, + on_result_begin, + on_result_end, on_result, on_optimization_end), c( @@ -136,10 +178,44 @@ callback_batch_tuning = function( "on_eval_after_benchmark", "on_eval_before_archive", "on_optimizer_after_eval", + "on_tuning_result_begin", + "on_result_begin", + "on_result_end", "on_result", "on_optimization_end")), is.null) + + if ("on_result" %in% names(stages)) { + .Deprecated(old = "on_result", new = "on_result_end") + stages$on_result_end = stages$on_result + stages$on_result = NULL + } + walk(stages, function(stage) assert_function(stage, args = c("callback", "context"))) callback = CallbackBatchTuning$new(id, label, man) iwalk(stages, function(stage, name) callback[[name]] = stage) callback } + +#' @title Assertions for Callbacks +#' +#' @description +#' Assertions for [CallbackBatchTuning] class. +#' +#' @param callback ([CallbackBatchTuning]). +#' @param null_ok (`logical(1)`)\cr +#' If `TRUE`, `NULL` is allowed. +#' +#' @return [CallbackBatchTuning | List of [CallbackBatchTuning]s. +#' @export +assert_batch_tuning_callback = function(callback, null_ok = FALSE) { + if (null_ok && is.null(callback)) return(invisible(NULL)) + assert_class(callback, "CallbackBatchTuning") + invisible(callback) +} + +#' @export +#' @param callbacks (list of [CallbackBatchTuning]). +#' @rdname assert_batch_tuning_callback +assert_batch_tuning_callbacks = function(callbacks) { + invisible(lapply(callbacks, assert_callback)) +} diff --git a/R/ContextAsyncTuning.R b/R/ContextAsyncTuning.R index eb68289ed..3f9ef6a2a 100644 --- a/R/ContextAsyncTuning.R +++ b/R/ContextAsyncTuning.R @@ -17,8 +17,8 @@ ContextAsyncTuning = R6Class("ContextAsyncTuning", active = list( #' @field xs_learner (list())\cr - #' The hyperparameter configuration currently evaluated. - #' Contains the values on the learner scale i.e. transformations are applied. + #' The hyperparameter configuration currently evaluated. + #' Contains the values on the learner scale i.e. transformations are applied. xs_learner = function(rhs) { if (missing(rhs)) { return(get_private(self$instance$objective)$.xs) @@ -28,7 +28,7 @@ ContextAsyncTuning = R6Class("ContextAsyncTuning", }, #' @field resample_result ([mlr3::BenchmarkResult])\cr - #' The resample result of the hyperparameter configuration currently evaluated. + #' The resample result of the hyperparameter configuration currently evaluated. resample_result = function(rhs) { if (missing(rhs)) { return(get_private(self$instance$objective)$.resample_result) @@ -38,15 +38,25 @@ ContextAsyncTuning = R6Class("ContextAsyncTuning", }, #' @field aggregated_performance (`list()`)\cr - #' Aggregated performance scores and training time of the evaluated hyperparameter configuration. - #' This list is passed to the archive. - #' A callback can add additional elements which are also written to the archive. + #' Aggregated performance scores and training time of the evaluated hyperparameter configuration. + #' This list is passed to the archive. + #' A callback can add additional elements which are also written to the archive. aggregated_performance = function(rhs) { if (missing(rhs)) { return(get_private(self$instance$objective)$.aggregated_performance) } else { self$instance$objective$.__enclos_env__$private$.aggregated_performance = rhs } + }, + + #' @field result_learner_param_vals (list())\cr + #' The learner parameter values passed to `instance$assign_result()`. + result_learner_param_vals = function(rhs) { + if (missing(rhs)) { + return(get_private(self$instance)$.result_learner_param_vals) + } else { + self$instance$.__enclos_env__$private$.result_learner_param_vals = rhs + } } ) ) diff --git a/R/ContextBatchTuning.R b/R/ContextBatchTuning.R index 4297ec9b1..45cb7afa1 100644 --- a/R/ContextBatchTuning.R +++ b/R/ContextBatchTuning.R @@ -14,9 +14,9 @@ ContextBatchTuning = R6Class("ContextBatchTuning", active = list( #' @field xss (list())\cr - #' The hyperparameter configurations of the latest batch. - #' Contains the values on the learner scale i.e. transformations are applied. - #' See `$xdt` for the untransformed values. + #' The hyperparameter configurations of the latest batch. + #' Contains the values on the learner scale i.e. transformations are applied. + #' See `$xdt` for the untransformed values. xss = function(rhs) { if (missing(rhs)) { return(get_private(self$instance$objective)$.xss) @@ -26,7 +26,7 @@ ContextBatchTuning = R6Class("ContextBatchTuning", }, #' @field design ([data.table::data.table])\cr - #' The benchmark design of the latest batch. + #' The benchmark design of the latest batch. design = function(rhs) { if (missing(rhs)) { return(get_private(self$instance$objective)$.design) @@ -36,7 +36,7 @@ ContextBatchTuning = R6Class("ContextBatchTuning", }, #' @field benchmark_result ([mlr3::BenchmarkResult])\cr - #' The benchmark result of the latest batch. + #' The benchmark result of the latest batch. benchmark_result = function(rhs) { if (missing(rhs)) { return(get_private(self$instance$objective)$.benchmark_result) @@ -46,15 +46,25 @@ ContextBatchTuning = R6Class("ContextBatchTuning", }, #' @field aggregated_performance ([data.table::data.table])\cr - #' Aggregated performance scores and training time of the latest batch. - #' This data table is passed to the archive. - #' A callback can add additional columns which are also written to the archive. + #' Aggregated performance scores and training time of the latest batch. + #' This data table is passed to the archive. + #' A callback can add additional columns which are also written to the archive. aggregated_performance = function(rhs) { if (missing(rhs)) { return(get_private(self$instance$objective)$.aggregated_performance) } else { self$instance$objective$.__enclos_env__$private$.aggregated_performance = rhs } + }, + + #' @field result_learner_param_vals (list())\cr + #' The learner parameter values passed to `instance$assign_result()`. + result_learner_param_vals = function(rhs) { + if (missing(rhs)) { + return(get_private(self$instance)$.result_learner_param_vals) + } else { + self$instance$.__enclos_env__$private$.result_learner_param_vals = rhs + } } ) ) diff --git a/R/TuningInstanceAsyncMulticrit.R b/R/TuningInstanceAsyncMulticrit.R index 432b7bded..7fae8e8d7 100644 --- a/R/TuningInstanceAsyncMulticrit.R +++ b/R/TuningInstanceAsyncMulticrit.R @@ -55,6 +55,7 @@ TuningInstanceAsyncMultiCrit = R6Class("TuningInstanceAsyncMultiCrit", ) { require_namespaces("rush") learner = assert_learner(as_learner(learner, clone = TRUE)) + callbacks = assert_async_tuning_callbacks(as_callbacks(callbacks)) # tune token and search space if (!is.null(search_space) && length(learner$param_set$get_values(type = "only_token"))) { @@ -161,26 +162,34 @@ TuningInstanceAsyncMultiCrit = R6Class("TuningInstanceAsyncMultiCrit", # workaround extra = extra %??% xydt + # assign for callbacks + private$.result_xdt = xdt + private$.result_ydt = ydt + private$.result_learner_param_vals = learner_param_vals + private$.result_extra = extra + + call_back("on_tuning_result_begin", self$objective$callbacks, self$objective$context) + # extract internal tuned values - if ("internal_tuned_values" %in% names(extra)) { - set(xdt, j = "internal_tuned_values", value = list(extra[["internal_tuned_values"]])) + if ("internal_tuned_values" %in% names(private$.result_extra)) { + set(private$.result_xdt, j = "internal_tuned_values", value = list(private$.result_extra[["internal_tuned_values"]])) } # set the column with the learner param_vals that were not optimized over but set implicitly - if (is.null(learner_param_vals)) { - learner_param_vals = self$objective$learner$param_set$values - if (length(learner_param_vals) == 0) learner_param_vals = list() - learner_param_vals = replicate(nrow(ydt), list(learner_param_vals)) + if (is.null(private$.result_learner_param_vals)) { + private$.result_learner_param_vals = self$objective$learner$param_set$values + if (length(private$.result_learner_param_vals) == 0) private$.result_learner_param_vals = list() + private$.result_learner_param_vals = replicate(nrow(private$.result_ydt), list(private$.result_learner_param_vals)) } - opt_x = transform_xdt_to_xss(xdt, self$search_space) - if (length(opt_x) == 0) opt_x = replicate(length(ydt), list()) - learner_param_vals = Map(insert_named, learner_param_vals, opt_x) + opt_x = transform_xdt_to_xss(private$.result_xdt, self$search_space) + if (length(opt_x) == 0) opt_x = replicate(length(private$.result_ydt), list()) + private$.result_learner_param_vals = Map(insert_named, private$.result_learner_param_vals, opt_x) # disable internal tuning - if (!is.null(xdt$internal_tuned_values)) { + if (!is.null(private$.result_xdt$internal_tuned_values)) { learner = self$objective$learner$clone(deep = TRUE) - learner_param_vals = pmap(list(learner_param_vals, xdt$internal_tuned_values), function(lpv, itv) { + private$.result_learner_param_vals = pmap(list(private$.result_learner_param_vals, private$.result_xdt$internal_tuned_values), function(lpv, itv) { values = insert_named(lpv, itv) learner$param_set$set_values(.values = values, .insert = FALSE) learner$param_set$disable_internal_tuning(self$internal_search_space$ids()) @@ -188,9 +197,9 @@ TuningInstanceAsyncMultiCrit = R6Class("TuningInstanceAsyncMultiCrit", }) } - set(xdt, j = "learner_param_vals", value = list(learner_param_vals)) + set(private$.result_xdt, j = "learner_param_vals", value = list(private$.result_learner_param_vals)) - super$assign_result(xdt, ydt) + super$assign_result(private$.result_xdt, private$.result_ydt) } ), @@ -204,6 +213,8 @@ TuningInstanceAsyncMultiCrit = R6Class("TuningInstanceAsyncMultiCrit", ), private = list( + # intermediate objects + .result_learner_param_vals = NULL, # initialize context for optimization .initialize_context = function(optimizer) { diff --git a/R/TuningInstanceAsyncSingleCrit.R b/R/TuningInstanceAsyncSingleCrit.R index 6cc576284..8ae52451f 100644 --- a/R/TuningInstanceAsyncSingleCrit.R +++ b/R/TuningInstanceAsyncSingleCrit.R @@ -65,6 +65,7 @@ TuningInstanceAsyncSingleCrit = R6Class("TuningInstanceAsyncSingleCrit", ) { require_namespaces("rush") learner = assert_learner(as_learner(learner, clone = TRUE)) + callbacks = assert_async_tuning_callbacks(as_callbacks(callbacks)) # tune token and search space if (!is.null(search_space) && length(learner$param_set$get_values(type = "only_token"))) { @@ -171,34 +172,43 @@ TuningInstanceAsyncSingleCrit = R6Class("TuningInstanceAsyncSingleCrit", # workaround extra = extra %??% xydt + # assign for callbacks + private$.result_xdt = xdt + private$.result_y = y + private$.result_learner_param_vals = learner_param_vals + private$.result_extra = extra + + call_back("on_tuning_result_begin", self$objective$callbacks, self$objective$context) + # set the column with the learner param_vals that were not optimized over but set implicitly - assert_list(learner_param_vals, null.ok = TRUE, names = "named") + assert_list(private$.result_learner_param_vals, null.ok = TRUE, names = "named") # extract internal tuned values - if ("internal_tuned_values" %in% names(extra)) { - set(xdt, j = "internal_tuned_values", value = list(extra[["internal_tuned_values"]])) + if ("internal_tuned_values" %in% names(private$.result_extra)) { + set(private$.result_xdt, j = "internal_tuned_values", value = list(private$.result_extra[["internal_tuned_values"]])) } - if (is.null(learner_param_vals)) { - learner_param_vals = self$objective$learner$param_set$values + # learner param values + if (is.null(private$.result_learner_param_vals)) { + private$.result_learner_param_vals = self$objective$learner$param_set$values } - opt_x = unlist(transform_xdt_to_xss(xdt, self$search_space), recursive = FALSE) - learner_param_vals = insert_named(learner_param_vals, opt_x) - - # maintain list column - if (length(learner_param_vals) < 2 | !nrow(xdt)) learner_param_vals = list(learner_param_vals) + opt_x = unlist(transform_xdt_to_xss(private$.result_xdt, self$search_space), recursive = FALSE) + private$.result_learner_param_vals = insert_named(private$.result_learner_param_vals, opt_x) # disable internal tuning - if (!is.null(xdt$internal_tuned_values)) { + if (!is.null(private$.result_xdt$internal_tuned_values)) { learner = self$objective$learner$clone(deep = TRUE) - learner_param_vals = insert_named(learner_param_vals, xdt$internal_tuned_values[[1]]) - learner$param_set$set_values(.values = learner_param_vals) + private$.result_learner_param_vals = insert_named(private$.result_learner_param_vals, private$.result_xdt$internal_tuned_values[[1]]) + learner$param_set$set_values(.values = private$.result_learner_param_vals) learner$param_set$disable_internal_tuning(self$internal_search_space$ids()) - learner_param_vals = learner$param_set$values + private$.result_learner_param_vals = learner$param_set$values } - set(xdt, j = "learner_param_vals", value = list(learner_param_vals)) - super$assign_result(xdt, y) + # maintain list column + if (length(private$.result_learner_param_vals) < 2 | !nrow(private$.result_xdt)) private$.result_learner_param_vals = list(private$.result_learner_param_vals) + + set(private$.result_xdt, j = "learner_param_vals", value = list(private$.result_learner_param_vals)) + super$assign_result(private$.result_xdt, private$.result_y) } ), @@ -212,6 +222,8 @@ TuningInstanceAsyncSingleCrit = R6Class("TuningInstanceAsyncSingleCrit", ), private = list( + # intermediate objects + .result_learner_param_vals = NULL, # initialize context for optimization .initialize_context = function(optimizer) { diff --git a/R/TuningInstanceBatchMulticrit.R b/R/TuningInstanceBatchMulticrit.R index 0bcfd52c2..a8acf9e20 100644 --- a/R/TuningInstanceBatchMulticrit.R +++ b/R/TuningInstanceBatchMulticrit.R @@ -90,6 +90,7 @@ TuningInstanceBatchMultiCrit = R6Class("TuningInstanceBatchMultiCrit", callbacks = NULL ) { learner = assert_learner(as_learner(learner, clone = TRUE)) + callbacks = assert_batch_tuning_callbacks(as_callbacks(callbacks)) # tune token and search space if (!is.null(search_space) && length(learner$param_set$get_values(type = "only_token"))) { @@ -195,26 +196,34 @@ TuningInstanceBatchMultiCrit = R6Class("TuningInstanceBatchMultiCrit", # workaround extra = extra %??% xydt + # assign for callbacks + private$.result_xdt = xdt + private$.result_ydt = ydt + private$.result_learner_param_vals = learner_param_vals + private$.result_extra = extra + + call_back("on_tuning_result_begin", self$objective$callbacks, self$objective$context) + # extract internal tuned values - if ("internal_tuned_values" %in% names(extra)) { - set(xdt, j = "internal_tuned_values", value = list(extra[["internal_tuned_values"]])) + if ("internal_tuned_values" %in% names(private$.result_extra)) { + set(private$.result_xdt, j = "internal_tuned_values", value = list(private$.result_extra[["internal_tuned_values"]])) } # set the column with the learner param_vals that were not optimized over but set implicitly - if (is.null(learner_param_vals)) { - learner_param_vals = self$objective$learner$param_set$values - if (length(learner_param_vals) == 0) learner_param_vals = list() - learner_param_vals = replicate(nrow(ydt), list(learner_param_vals)) + if (is.null(private$.result_learner_param_vals)) { + private$.result_learner_param_vals = self$objective$learner$param_set$values + if (length(private$.result_learner_param_vals) == 0) private$.result_learner_param_vals = list() + private$.result_learner_param_vals = replicate(nrow(private$.result_ydt), list(private$.result_learner_param_vals)) } - opt_x = transform_xdt_to_xss(xdt, self$search_space) - if (length(opt_x) == 0) opt_x = replicate(length(ydt), list()) - learner_param_vals = Map(insert_named, learner_param_vals, opt_x) + opt_x = transform_xdt_to_xss(private$.result_xdt, self$search_space) + if (length(opt_x) == 0) opt_x = replicate(length(private$.result_ydt), list()) + private$.result_learner_param_vals = Map(insert_named, private$.result_learner_param_vals, opt_x) # disable internal tuning - if (!is.null(xdt$internal_tuned_values)) { + if (!is.null(private$.result_xdt$internal_tuned_values)) { learner = self$objective$learner$clone(deep = TRUE) - learner_param_vals = pmap(list(learner_param_vals, xdt$internal_tuned_values), function(lpv, itv) { + private$.result_learner_param_vals = pmap(list(private$.result_learner_param_vals, private$.result_xdt$internal_tuned_values), function(lpv, itv) { values = insert_named(lpv, itv) learner$param_set$set_values(.values = values, .insert = FALSE) learner$param_set$disable_internal_tuning(self$internal_search_space$ids()) @@ -222,9 +231,9 @@ TuningInstanceBatchMultiCrit = R6Class("TuningInstanceBatchMultiCrit", }) } - set(xdt, j = "learner_param_vals", value = list(learner_param_vals)) + set(private$.result_xdt, j = "learner_param_vals", value = list(private$.result_learner_param_vals)) - super$assign_result(xdt, ydt) + super$assign_result(private$.result_xdt, private$.result_ydt) } ), @@ -238,6 +247,9 @@ TuningInstanceBatchMultiCrit = R6Class("TuningInstanceBatchMultiCrit", ), private = list( + # intermediate objects + .result_learner_param_vals = NULL, + # initialize context for optimization .initialize_context = function(optimizer) { context = ContextBatchTuning$new(self, optimizer) diff --git a/R/TuningInstanceBatchSingleCrit.R b/R/TuningInstanceBatchSingleCrit.R index 27d5c3b13..25adb7453 100644 --- a/R/TuningInstanceBatchSingleCrit.R +++ b/R/TuningInstanceBatchSingleCrit.R @@ -128,6 +128,7 @@ TuningInstanceBatchSingleCrit = R6Class("TuningInstanceBatchSingleCrit", callbacks = NULL ) { learner = assert_learner(as_learner(learner, clone = TRUE)) + callbacks = assert_batch_tuning_callbacks(as_callbacks(callbacks)) # tune token and search space if (!is.null(search_space) && length(learner$param_set$get_values(type = "only_token"))) { @@ -233,35 +234,43 @@ TuningInstanceBatchSingleCrit = R6Class("TuningInstanceBatchSingleCrit", # workaround extra = extra %??% xydt + # assign for callbacks + private$.result_xdt = xdt + private$.result_y = y + private$.result_learner_param_vals = learner_param_vals + private$.result_extra = extra + + call_back("on_tuning_result_begin", self$objective$callbacks, self$objective$context) + # set the column with the learner param_vals that were not optimized over but set implicitly - assert_list(learner_param_vals, null.ok = TRUE, names = "named") + assert_list(private$.result_learner_param_vals, null.ok = TRUE, names = "named") # extract internal tuned values - if ("internal_tuned_values" %in% names(extra)) { - set(xdt, j = "internal_tuned_values", value = list(extra[["internal_tuned_values"]])) + if ("internal_tuned_values" %in% names(private$.result_extra)) { + set(private$.result_xdt, j = "internal_tuned_values", value = list(private$.result_extra[["internal_tuned_values"]])) } # learner param values - if (is.null(learner_param_vals)) { - learner_param_vals = self$objective$learner$param_set$values + if (is.null(private$.result_learner_param_vals)) { + private$.result_learner_param_vals = self$objective$learner$param_set$values } - opt_x = unlist(transform_xdt_to_xss(xdt, self$search_space), recursive = FALSE) - learner_param_vals = insert_named(learner_param_vals, opt_x) + opt_x = unlist(transform_xdt_to_xss(private$.result_xdt, self$search_space), recursive = FALSE) + private$.result_learner_param_vals = insert_named(private$.result_learner_param_vals, opt_x) # disable internal tuning - if (!is.null(xdt$internal_tuned_values)) { + if (!is.null(private$.result_xdt$internal_tuned_values)) { learner = self$objective$learner$clone(deep = TRUE) - learner_param_vals = insert_named(learner_param_vals, xdt$internal_tuned_values[[1]]) - learner$param_set$set_values(.values = learner_param_vals) + private$.result_learner_param_vals = insert_named(private$.result_learner_param_vals, private$.result_xdt$internal_tuned_values[[1]]) + learner$param_set$set_values(.values = private$.result_learner_param_vals) learner$param_set$disable_internal_tuning(self$internal_search_space$ids()) - learner_param_vals = learner$param_set$values + private$.result_learner_param_vals = learner$param_set$values } # maintain list column - if (length(learner_param_vals) < 2 | !nrow(xdt)) learner_param_vals = list(learner_param_vals) + if (length(private$.result_learner_param_vals) < 2 | !nrow(private$.result_xdt)) private$.result_learner_param_vals = list(private$.result_learner_param_vals) - set(xdt, j = "learner_param_vals", value = list(learner_param_vals)) - super$assign_result(xdt, y) + set(private$.result_xdt, j = "learner_param_vals", value = list(private$.result_learner_param_vals)) + super$assign_result(private$.result_xdt, private$.result_y) } ), @@ -274,6 +283,9 @@ TuningInstanceBatchSingleCrit = R6Class("TuningInstanceBatchSingleCrit", ), private = list( + # intermediate objects + .result_learner_param_vals = NULL, + # initialize context for optimization .initialize_context = function(optimizer) { context = ContextBatchTuning$new(self, optimizer) diff --git a/R/bibentries.R b/R/bibentries.R index 1ec2e0b84..ca840de0e 100644 --- a/R/bibentries.R +++ b/R/bibentries.R @@ -60,5 +60,16 @@ bibentries = c( eprint = "1604.00772", archivePrefix = "arXiv", primaryClass = "cs.LG" + ), + + kuhn2013 = bibentry("Inbook", + author = "Kuhn, Max and Johnson, Kjell", + chapter = "Over-Fitting and Model Tuning", + title = "Applied Predictive Modeling", + year = "2013", + publisher = "Springer New York", + address = "New York, NY", + pages = "61--92", + isbn = "978-1-4614-6849-3" ) ) diff --git a/R/mlr_callbacks.R b/R/mlr_callbacks.R index 680cf6743..95286b6b4 100644 --- a/R/mlr_callbacks.R +++ b/R/mlr_callbacks.R @@ -293,3 +293,136 @@ load_callback_async_save_logs = function() { } ) } + +#' @title One Standard Error Rule Callback +#' +#' @include CallbackBatchTuning.R +#' @name mlr3tuning.one_se_rule +#' +#' @description +#' The one standard error rule takes the number of features into account when selecting the best hyperparameter configuration. +#' Many learners support internal feature selection, which can be accessed via `$selected_features()`. +#' The callback selects the hyperparameter configuration with the smallest feature set within one standard error of the best performing configuration. +#' If there are multiple such hyperparameter configurations with the same number of features, the first one is selected. +#' +#' @source +#' `r format_bib("kuhn2013")` +#' +#' @examples +#' clbk("mlr3tuning.one_se_rule") +#' +#' # Run optimization on the pima data set with the callback +#' instance = tune( +#' tuner = tnr("random_search", batch_size = 15), +#' task = tsk("pima"), +#' learner = lrn("classif.rpart", cp = to_tune(1e-04, 1e-1, logscale = TRUE)), +#' resampling = rsmp("cv", folds = 3), +#' measures = msr("classif.ce"), +#' term_evals = 30, +#' callbacks = clbk("mlr3tuning.one_se_rule") +#' ) +#' +#' # Hyperparameter configuration with the smallest feature set within one standard error of the best +#' instance$result +NULL + +load_callback_async_one_se_rule = function() { + callback_async_tuning("mlr3tuning.async_one_se_rule", + label = "One Standard Error Rule Callback", + man = "mlr3tuning::mlr3tuning.one_se_rule", + + on_optimization_begin = function(callback, context) { + if ("selected_features" %nin% context$instance$objective$learner$properties) { + stopf("Learner '%s' does not support `$selected_features()`", context$instance$objective$learner$id) + } + callback$state$store_models = context$instance$objective$store_models + context$instance$objective$store_models = TRUE + }, + + on_eval_before_archive = function(callback, context) { + res = context$resample_result$aggregate(msr("selected_features")) + context$aggregated_performance$n_features = res + if (!callback$state$store_models) { + context$resample_result$discard(models = TRUE) + } + }, + + on_tuning_result_begin = function(callback, context) { + archive = context$instance$archive + data = as.data.table(archive) + + # standard error + y = data[[archive$cols_y]] + se = sd(y) / sqrt(length(y)) + + if (se == 0) { + # select smallest future set when all scores are the same + best = data[which.min(get("n_features"))] + } else { + # select smallest future set within one standard error of the best + best_y = context$result_y + best = data[y > best_y - se & y < best_y + se, ][which.min(get("n_features"))] + } + + cols_x = context$instance$archive$cols_x + cols_y = context$instance$archive$cols_y + + context$result_xdt = best[, c(cols_x, "n_features"), with = FALSE] + context$result_extra = best[, !c(cols_x, cols_y), with = FALSE] + context$result_y = unlist(best[, cols_y, with = FALSE]) + + context$instance$objective$store_models = callback$state$store_models + } + ) +} + + +load_callback_one_se_rule = function() { + callback_batch_tuning("mlr3tuning.one_se_rule", + label = "One Standard Error Rule Callback", + man = "mlr3tuning::mlr3tuning.one_se_rule", + + on_optimization_begin = function(callback, context) { + if ("selected_features" %nin% context$instance$objective$learner$properties) { + stopf("Learner '%s' does not support `$selected_features()`", context$instance$objective$learner$id) + } + callback$state$store_models = context$instance$objective$store_models + context$instance$objective$store_models = TRUE + }, + + on_eval_before_archive = function(callback, context) { + res = context$benchmark_result$aggregate(msr("selected_features")) + set(context$aggregated_performance, j = "n_features", value = res$selected_features) + if (!callback$state$store_models) { + context$benchmark_result$discard(models = TRUE) + } + }, + + on_tuning_result_begin = function(callback, context) { + archive = context$instance$archive + data = as.data.table(archive) + + # standard error + y = data[[archive$cols_y]] + se = sd(y) / sqrt(length(y)) + + if (se == 0) { + # select smallest future set when all scores are the same + best = data[which.min(get("n_features"))] + } else { + # select smallest future set within one standard error of the best + best_y = context$result_y + best = data[y > best_y - se & y < best_y + se, ][which.min(get("n_features"))] + } + + cols_x = context$instance$archive$cols_x + cols_y = context$instance$archive$cols_y + + context$result_xdt = best[, c(cols_x, "n_features"), with = FALSE] + context$result_extra = best[, !c(cols_x, cols_y), with = FALSE] + context$result_y = unlist(best[, cols_y, with = FALSE]) + + context$instance$objective$store_models = callback$state$store_models + } + ) +} diff --git a/R/zzz.R b/R/zzz.R index 8f9d68d61..4d53b84c1 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -6,6 +6,7 @@ #' @import bbotk #' @importFrom R6 R6Class #' @importFrom utils tail +#' @importFrom stats sd "_PACKAGE" .onLoad = function(libname, pkgname) { @@ -19,9 +20,11 @@ x$add("mlr3tuning.async_measures", load_callback_async_measures) x$add("mlr3tuning.async_mlflow", load_callback_async_mlflow) x$add("mlr3tuning.async_save_logs", load_callback_async_save_logs) + x$add("mlr3tuning.async_one_se_rule", load_callback_async_one_se_rule) x$add("mlr3tuning.backup", load_callback_backup) x$add("mlr3tuning.default_configuration", load_callback_default_configuration) x$add("mlr3tuning.measures", load_callback_measures) + x$add("mlr3tuning.one_se_rule", load_callback_one_se_rule) assign("lg", lgr::get_logger("bbotk"), envir = parent.env(environment())) if (Sys.getenv("IN_PKGDOWN") == "true") { diff --git a/man/CallbackAsyncTuning.Rd b/man/CallbackAsyncTuning.Rd index 748b16bc5..8b36f1766 100644 --- a/man/CallbackAsyncTuning.Rd +++ b/man/CallbackAsyncTuning.Rd @@ -18,15 +18,19 @@ For more information on tuning callbacks see \code{\link[=callback_async_tuning] \describe{ \item{\code{on_eval_after_xs}}{(\verb{function()})\cr Stage called after xs is passed. -Called in \code{ObjectiveTuning$eval()}.} +Called in \code{ObjectiveTuningAsync$eval()}.} \item{\code{on_eval_after_resample}}{(\verb{function()})\cr Stage called after hyperparameter configurations are evaluated. -Called in \code{ObjectiveTuning$eval()}.} +Called in \code{ObjectiveTuningAsync$eval()}.} \item{\code{on_eval_before_archive}}{(\verb{function()})\cr Stage called before performance values are written to the archive. -Called in \code{ObjectiveTuning$eval()}.} +Called in \code{ObjectiveTuningAsync$eval()}.} + +\item{\code{on_tuning_result_begin}}{(\verb{function()})\cr +Stage called before the results are written. +Called in \verb{TuningInstance*$assign_result()}.} } \if{html}{\out{}} } diff --git a/man/CallbackBatchTuning.Rd b/man/CallbackBatchTuning.Rd index beab90fdb..c09643b86 100644 --- a/man/CallbackBatchTuning.Rd +++ b/man/CallbackBatchTuning.Rd @@ -26,15 +26,19 @@ callback_batch_tuning("mlr3tuning.backup", \describe{ \item{\code{on_eval_after_design}}{(\verb{function()})\cr Stage called after design is created. -Called in \code{ObjectiveTuning$eval_many()}.} +Called in \code{ObjectiveTuningBatch$eval_many()}.} \item{\code{on_eval_after_benchmark}}{(\verb{function()})\cr Stage called after hyperparameter configurations are evaluated. -Called in \code{ObjectiveTuning$eval_many()}.} +Called in \code{ObjectiveTuningBatch$eval_many()}.} \item{\code{on_eval_before_archive}}{(\verb{function()})\cr Stage called before performance values are written to the archive. -Called in \code{ObjectiveTuning$eval_many()}.} +Called in \code{ObjectiveTuningBatch$eval_many()}.} + +\item{\code{on_tuning_result_begin}}{(\verb{function()})\cr +Stage called before the results are written. +Called in \verb{TuningInstance*$assign_result()}.} } \if{html}{\out{}} } diff --git a/man/ContextAsyncTuning.Rd b/man/ContextAsyncTuning.Rd index eefde2415..fc14f1e48 100644 --- a/man/ContextAsyncTuning.Rd +++ b/man/ContextAsyncTuning.Rd @@ -28,6 +28,9 @@ The resample result of the hyperparameter configuration currently evaluated.} Aggregated performance scores and training time of the evaluated hyperparameter configuration. This list is passed to the archive. A callback can add additional elements which are also written to the archive.} + +\item{\code{result_learner_param_vals}}{(list())\cr +The learner parameter values passed to \code{instance$assign_result()}.} } \if{html}{\out{}} } diff --git a/man/ContextBatchTuning.Rd b/man/ContextBatchTuning.Rd index 33c1bbba0..d13a26cfb 100644 --- a/man/ContextBatchTuning.Rd +++ b/man/ContextBatchTuning.Rd @@ -29,6 +29,9 @@ The benchmark result of the latest batch.} Aggregated performance scores and training time of the latest batch. This data table is passed to the archive. A callback can add additional columns which are also written to the archive.} + +\item{\code{result_learner_param_vals}}{(list())\cr +The learner parameter values passed to \code{instance$assign_result()}.} } \if{html}{\out{}} } diff --git a/man/assert_async_tuning_callback.Rd b/man/assert_async_tuning_callback.Rd new file mode 100644 index 000000000..fa91b1676 --- /dev/null +++ b/man/assert_async_tuning_callback.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/CallbackAsyncTuning.R +\name{assert_async_tuning_callback} +\alias{assert_async_tuning_callback} +\alias{assert_async_tuning_callbacks} +\title{Assertions for Callbacks} +\usage{ +assert_async_tuning_callback(callback, null_ok = FALSE) + +assert_async_tuning_callbacks(callbacks) +} +\arguments{ +\item{callback}{(\link{CallbackAsyncTuning}).} + +\item{null_ok}{(\code{logical(1)})\cr +If \code{TRUE}, \code{NULL} is allowed.} + +\item{callbacks}{(list of \link{CallbackAsyncTuning}).} +} +\value{ +[CallbackAsyncTuning | List of \link{CallbackAsyncTuning}s. +} +\description{ +Assertions for \link{CallbackAsyncTuning} class. +} diff --git a/man/assert_batch_tuning_callback.Rd b/man/assert_batch_tuning_callback.Rd new file mode 100644 index 000000000..959d43c2a --- /dev/null +++ b/man/assert_batch_tuning_callback.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/CallbackBatchTuning.R +\name{assert_batch_tuning_callback} +\alias{assert_batch_tuning_callback} +\alias{assert_batch_tuning_callbacks} +\title{Assertions for Callbacks} +\usage{ +assert_batch_tuning_callback(callback, null_ok = FALSE) + +assert_batch_tuning_callbacks(callbacks) +} +\arguments{ +\item{callback}{(\link{CallbackBatchTuning}).} + +\item{null_ok}{(\code{logical(1)})\cr +If \code{TRUE}, \code{NULL} is allowed.} + +\item{callbacks}{(list of \link{CallbackBatchTuning}).} +} +\value{ +[CallbackBatchTuning | List of \link{CallbackBatchTuning}s. +} +\description{ +Assertions for \link{CallbackBatchTuning} class. +} diff --git a/man/callback_async_tuning.Rd b/man/callback_async_tuning.Rd index f8bbfd00a..839688a7c 100644 --- a/man/callback_async_tuning.Rd +++ b/man/callback_async_tuning.Rd @@ -16,6 +16,9 @@ callback_async_tuning( on_eval_before_archive = NULL, on_optimizer_after_eval = NULL, on_worker_end = NULL, + on_tuning_result_begin = NULL, + on_result_begin = NULL, + on_result_end = NULL, on_result = NULL, on_optimization_end = NULL ) @@ -33,37 +36,70 @@ The referenced help package can be opened via method \verb{$help()}.} \item{on_optimization_begin}{(\verb{function()})\cr Stage called at the beginning of the optimization. -Called in \code{Optimizer$optimize()}.} +Called in \code{Optimizer$optimize()}. +The functions must have two arguments named \code{callback} and \code{context}.} \item{on_worker_begin}{(\verb{function()})\cr Stage called at the beginning of the optimization on the worker. -Called in the worker loop.} +Called in the worker loop. +The functions must have two arguments named \code{callback} and \code{context}.} \item{on_optimizer_before_eval}{(\verb{function()})\cr Stage called after the optimizer proposes points. -Called in \code{OptimInstance$.eval_point()}.} +Called in \code{OptimInstance$.eval_point()}. +The functions must have two arguments named \code{callback} and \code{context}. +The argument of \code{instance$.eval_point(xs)} and \code{xs_trafoed} and \code{extra} are available in the \code{context}. +Or \code{xs} and \code{xs_trafoed} of \code{instance$.eval_queue()} are available in the \code{context}.} \item{on_eval_after_xs}{(\verb{function()})\cr -Stage called after xs is passed. -Called in \code{ObjectiveTuning$eval()}.} +Stage called after xs is passed to the objective. +Called in \code{ObjectiveTuningAsync$eval()}. +The functions must have two arguments named \code{callback} and \code{context}. +The argument of \verb{$.eval(xs)} is available in the \code{context}.} \item{on_eval_after_resample}{(\verb{function()})\cr Stage called after a hyperparameter configuration is evaluated. -Called in \code{ObjectiveTuning$eval()}.} +Called in \code{ObjectiveTuningAsync$eval()}. +The functions must have two arguments named \code{callback} and \code{context}. +The \code{resample_result} is available in the `context} \item{on_eval_before_archive}{(\verb{function()})\cr Stage called before performance values are written to the archive. -Called in \code{ObjectiveTuning$eval()}.} +Called in \code{ObjectiveTuningAsync$eval()}. +The functions must have two arguments named \code{callback} and \code{context}. +The \code{aggregated_performance} is available in \code{context}.} \item{on_optimizer_after_eval}{(\verb{function()})\cr Stage called after points are evaluated. -Called in \code{OptimInstance$.eval_point()}.} +Called in \code{OptimInstance$.eval_point()}. +The functions must have two arguments named \code{callback} and \code{context}.} \item{on_worker_end}{(\verb{function()})\cr Stage called at the end of the optimization on the worker. -Called in the worker loop.} +Called in the worker loop. +The functions must have two arguments named \code{callback} and \code{context}.} + +\item{on_tuning_result_begin}{(\verb{function()})\cr +Stage called at the beginning of the result writing. +Called in \verb{TuningInstance*$assign_result()}. +The functions must have two arguments named \code{callback} and \code{context}. +The arguments of \verb{$assign_result(xdt, y, learner_param_vals, extra)} are available in \code{context}.} + +\item{on_result_begin}{(\verb{function()})\cr +Stage called at the beginning of the result writing. +Called in \code{OptimInstance$assign_result()}. +The functions must have two arguments named \code{callback} and \code{context}. +The arguments of \verb{$.assign_result(xdt, y, extra)} are available in the \code{context}.} + +\item{on_result_end}{(\verb{function()})\cr +Stage called after the result is written. +Called in \code{OptimInstance$assign_result()}. +The functions must have two arguments named \code{callback} and \code{context}. +The final result \code{instance$result} is available in the \code{context}.} \item{on_result}{(\verb{function()})\cr +Deprecated. +Use \code{on_result_end} instead. Stage called after the result is written. Called in \code{OptimInstance$assign_result()}.} @@ -93,7 +129,9 @@ The stages are prefixed with \verb{on_*}. End Optimization on Worker - on_worker_end End Worker - - on_result + - on_tuning_result_begin + - on_result_begin + - on_result_end - on_optimization_end End Tuning }\if{html}{\out{}} diff --git a/man/callback_batch_tuning.Rd b/man/callback_batch_tuning.Rd index aa83499fc..4d84e52c0 100644 --- a/man/callback_batch_tuning.Rd +++ b/man/callback_batch_tuning.Rd @@ -14,6 +14,9 @@ callback_batch_tuning( on_eval_after_benchmark = NULL, on_eval_before_archive = NULL, on_optimizer_after_eval = NULL, + on_tuning_result_begin = NULL, + on_result_begin = NULL, + on_result_end = NULL, on_result = NULL, on_optimization_end = NULL ) @@ -31,38 +34,69 @@ The referenced help package can be opened via method \verb{$help()}.} \item{on_optimization_begin}{(\verb{function()})\cr Stage called at the beginning of the optimization. -Called in \code{Optimizer$optimize()}.} +Called in \code{Optimizer$optimize()}. +The functions must have two arguments named \code{callback} and \code{context}.} \item{on_optimizer_before_eval}{(\verb{function()})\cr Stage called after the optimizer proposes points. -Called in \code{OptimInstance$eval_batch()}.} +Called in \code{OptimInstance$eval_batch()}. +The functions must have two arguments named \code{callback} and \code{context}. +The argument of \verb{$eval_batch(xdt)} is available in \code{context}.} \item{on_eval_after_design}{(\verb{function()})\cr Stage called after the design is created. -Called in \code{ObjectiveTuning$eval_many()}. -The context available is \link{ContextBatchTuning}.} +Called in \code{ObjectiveTuningBatch$eval_many()}. +The functions must have two arguments named \code{callback} and \code{context}. +The arguments of \verb{$eval_many(xss, resampling)} are available in \code{context}. +Additionally, the \code{design} is available in \code{context}.} \item{on_eval_after_benchmark}{(\verb{function()})\cr Stage called after hyperparameter configurations are evaluated. -Called in \code{ObjectiveTuning$eval_many()}. -The context available is \link{ContextBatchTuning}.} +Called in \code{ObjectiveTuningBatch$eval_many()}. +The functions must have two arguments named \code{callback} and \code{context}. +The \code{benchmark_result} is available in \code{context}.} \item{on_eval_before_archive}{(\verb{function()})\cr Stage called before performance values are written to the archive. -Called in \code{ObjectiveTuning$eval_many()}. -The context available is \link{ContextBatchTuning}.} +Called in \code{ObjectiveTuningBatch$eval_many()}. +The functions must have two arguments named \code{callback} and \code{context}. +The \code{aggregated_performance} is available in \code{context}.} \item{on_optimizer_after_eval}{(\verb{function()})\cr Stage called after points are evaluated. -Called in \code{OptimInstance$eval_batch()}.} +Called in \code{OptimInstance$eval_batch()}. +The functions must have two arguments named \code{callback} and \code{context}. +The new configurations and performances in \code{instance$archive} are available in \code{context}.} + +\item{on_tuning_result_begin}{(\verb{function()})\cr +Stage called at the beginning of the result writing. +Called in \code{TuningInstanceBatch$assign_result()}. +The functions must have two arguments named \code{callback} and \code{context}. +The arguments of \verb{$assign_result(xdt, y, learner_param_vals, extra)} are available in \code{context}.} + +\item{on_result_begin}{(\verb{function()})\cr +Stage called at the beginning of the result writing. +Called in \code{OptimInstance$assign_result()}. +The functions must have two arguments named \code{callback} and \code{context}. +The arguments of \verb{$assign_result(xdt, y, extra)} are available in \code{context}.} + +\item{on_result_end}{(\verb{function()})\cr +Stage called after the result is written. +Called in \code{OptimInstance$assign_result()}. +The functions must have two arguments named \code{callback} and \code{context}. +The final result \code{instance$result} is available in \code{context}.} \item{on_result}{(\verb{function()})\cr +Deprecated. +Use \code{on_result_end} instead. Stage called after the result is written. -Called in \code{OptimInstance$assign_result()}.} +Called in \code{OptimInstance$assign_result()}. +The functions must have two arguments named \code{callback} and \code{context}.} \item{on_optimization_end}{(\verb{function()})\cr Stage called at the end of the optimization. -Called in \code{Optimizer$optimize()}.} +Called in \code{Optimizer$optimize()}. +The functions must have two arguments named \code{callback} and \code{context}.} } \description{ Function to create a \link{CallbackBatchTuning}. @@ -82,7 +116,9 @@ The stages are prefixed with \verb{on_*}. End Evaluation - on_optimizer_after_eval End Tuner Batch - - on_result + - on_tuning_result_begin + - on_result_begin + - on_result_end - on_optimization_end End Tuning }\if{html}{\out{}} diff --git a/man/mlr3tuning.one_se_rule.Rd b/man/mlr3tuning.one_se_rule.Rd new file mode 100644 index 000000000..80d254b78 --- /dev/null +++ b/man/mlr3tuning.one_se_rule.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/mlr_callbacks.R +\name{mlr3tuning.one_se_rule} +\alias{mlr3tuning.one_se_rule} +\title{One Standard Error Rule Callback} +\source{ +Kuhn, Max, Johnson, Kjell (2013). +\dQuote{Applied Predictive Modeling.} +In chapter Over-Fitting and Model Tuning, 61--92. +Springer New York, New York, NY. +ISBN 978-1-4614-6849-3. +} +\description{ +The one standard error rule takes the number of features into account when selecting the best hyperparameter configuration. +Many learners support internal feature selection, which can be accessed via \verb{$selected_features()}. +The callback selects the hyperparameter configuration with the smallest feature set within one standard error of the best performing configuration. +If there are multiple such hyperparameter configurations with the same number of features, the first one is selected. +} +\examples{ +clbk("mlr3tuning.one_se_rule") + +# Run optimization on the pima data set with the callback +instance = tune( + tuner = tnr("random_search", batch_size = 15), + task = tsk("pima"), + learner = lrn("classif.rpart", cp = to_tune(1e-04, 1e-1, logscale = TRUE)), + resampling = rsmp("cv", folds = 3), + measures = msr("classif.ce"), + term_evals = 30, + callbacks = clbk("mlr3tuning.one_se_rule") +) + +# Hyperparameter configuration with the smallest feature set within one standard error of the best +instance$result +} diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml index 76845758d..853976fee 100644 --- a/pkgdown/_pkgdown.yml +++ b/pkgdown/_pkgdown.yml @@ -73,6 +73,10 @@ reference: - starts_with("Context") - starts_with("callback") - starts_with("mlr3tuning.") + - assert_async_tuning_callback + - assert_async_tuning_callbacks + - assert_batch_tuning_callback + - assert_batch_tuning_callbacks - title: Converters contents: - starts_with("as_") diff --git a/tests/testthat/test_CallbackAsyncTuning.R b/tests/testthat/test_CallbackAsyncTuning.R new file mode 100644 index 000000000..7b9b7ac63 --- /dev/null +++ b/tests/testthat/test_CallbackAsyncTuning.R @@ -0,0 +1,341 @@ +# stages in $optimize() -------------------------------------------------------- + +test_that("on_optimization_begin works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + callback = callback_async_tuning(id = "test", + on_optimization_begin = function(callback, context) { + context$instance$terminator$param_set$values$n_evals = 20 + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(instance$terminator$param_set$values$n_evals, 20) +}) + +test_that("on_optimization_end works", { + callback = callback_async_tuning(id = "test", + on_optimization_end = function(callback, context) { + context$instance$terminator$param_set$values$n_evals = 20 + } + ) + + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(instance$terminator$param_set$values$n_evals, 20) +}) + +# stager in worker_loop() ------------------------------------------------------ + +test_that("on_worker_begin works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + callback = callback_async_tuning(id = "test", + on_worker_begin = function(callback, context) { + instance = context$instance + mlr3misc::get_private(instance)$.eval_point(list(minsplit = 1)) + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_subset(1, instance$archive$data$minsplit) +}) + +test_that("on_worker_end works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + callback = callback_async_tuning(id = "test", + on_worker_end = function(callback, context) { + instance = context$instance + mlr3misc::get_private(instance)$.eval_point(list(minsplit = 1)) + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_subset(1, instance$archive$data$minsplit) +}) + +# stages in $.eval_point() ----------------------------------------------------- + +test_that("on_optimizer_before_eval and on_optimizer_after_eval works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + callback = callback_async_tuning(id = "test", + on_optimizer_before_eval = function(callback, context) { + context$xs = list(minsplit = 1) + context$xs_trafoed = list(minsplit = 0) + }, + + on_optimizer_after_eval = function(callback, context) { + context$ys = list(classif.ce = 0) + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_equal(unique(instance$archive$data$minsplit), 1) + expect_equal(unique(instance$archive$data$classif.ce), 0) +}) + +# stages in $eval() ------------------------------------------------------------ + +test_that("on_eval_after_xs works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + callback = callback_async_tuning(id = "test", + on_eval_after_xs = function(callback, context) { + context$xs_learner$minsplit = 1 + } + ) + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_equal(instance$archive$benchmark_result$resample_result(1)$learner$param_set$values$minsplit, 1) +}) + +test_that("on_eval_after_resample works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + callback = callback_async_tuning(id = "test", + on_eval_after_resample = function(callback, context) { + callback$state$extra_performance = context$resample_result$aggregate(msr("classif.acc")) + }, + + on_eval_before_archive = function(callback, context) { + context$aggregated_performance$classif.acc = callback$state$extra_performance + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_names(names(instance$archive$data), must.include = c("classif.ce", "classif.acc")) +}) + +# stages in $assign_result() in TuningInstanceAsyncSingleCrit ------------------ + +test_that("on_tuning_result_begin in TuningInstanceSingleCrit works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + callback = callback_async_tuning(id = "test", + on_tuning_result_begin = function(callback, context) { + context$result_xdt = data.table(minsplit = 1) + context$result_y = c(classif.ce = 0.7) + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(instance$result$minsplit, 1) + expect_equal(instance$result$classif.ce, 0.7) +}) + +test_that("on_result_end in TuningInstanceSingleCrit works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + rush::rush_plan(n_workers = 2) + callback = callback_async_tuning(id = "test", + on_result_end = function(callback, context) { + context$result$classif.ce = 0.7 + } + ) + + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(instance$result$classif.ce, 0.7) +}) + +test_that("on_result in TuningInstanceSingleCrit works", { + expect_warning({callback = callback_async_tuning(id = "test", + on_result = function(callback, context) { + context$result$classif.ce = 0.7 + } + )}, "deprecated") + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(instance$result$classif.ce, 0.7) +}) + +# stages in $assign_result() in TuningInstanceBatchMultiCrit ------------------- + +test_that("on_tuning_result_begin in TuningInstanceBatchMultiCrit works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + + callback = callback_async_tuning(id = "test", + on_tuning_result_begin = function(callback, context) { + context$result_xdt = data.table(minsplit = 1) + context$result_ydt = data.table(classif.ce = 0.7, classif.acc = 0.8) + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msrs(c("classif.ce", "classif.acc")), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(instance$result$minsplit, 1) + expect_equal(instance$result$classif.ce, 0.7) +}) + +test_that("on_result_end in TuningInstanceBatchMultiCrit works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + + callback = callback_async_tuning(id = "test", + on_result_end = function(callback, context) { + set(context$result, j = "classif.ce", value = 0.7) + } + ) + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msrs(c("classif.ce", "classif.acc")), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(unique(instance$result$classif.ce), 0.7) +}) + +test_that("on_result in TuningInstanceBatchMultiCrit works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + + expect_warning({callback = callback_async_tuning(id = "test", + on_result = function(callback, context) { + set(context$result, j = "classif.ce", value = 0.7) + } + )}, "deprecated") + + rush::rush_plan(n_workers = 2) + instance = tune( + tuner = tnr("async_random_search"), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msrs(c("classif.ce", "classif.acc")), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextAsyncTuning") + expect_equal(unique(instance$result$classif.ce), 0.7) +}) + diff --git a/tests/testthat/test_CallbackBatchTuning.R b/tests/testthat/test_CallbackBatchTuning.R new file mode 100644 index 000000000..750a09431 --- /dev/null +++ b/tests/testthat/test_CallbackBatchTuning.R @@ -0,0 +1,261 @@ +# stages in $optimize() -------------------------------------------------------- + +test_that("on_optimization_begin works", { + callback = callback_batch_tuning(id = "test", + on_optimization_begin = function(callback, context) { + context$instance$terminator$param_set$values$n_evals = 20 + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(instance$terminator$param_set$values$n_evals, 20) +}) + +test_that("on_optimization_end works", { + callback = callback_batch_tuning(id = "test", + on_optimization_end = function(callback, context) { + context$instance$terminator$param_set$values$n_evals = 20 + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(instance$terminator$param_set$values$n_evals, 20) +}) + +# stages in $eval_batch() ------------------------------------------------------ + +test_that("on_optimizer_after_eval works", { + callback = callback_batch_tuning(id = "test", + on_optimizer_before_eval = function(callback, context) { + set(context$xdt, j = "minsplit", value = 1) + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(unique(instance$archive$data$minsplit), 1) +}) + +test_that("on_optimizer_after_eval works", { + callback = callback_batch_tuning(id = "test", + on_optimizer_after_eval = function(callback, context) { + set(context$instance$archive$data, j = "classif.ce", value = 0.5) + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(unique(instance$archive$data$classif.ce), 0.5) +}) + +# stages in $eval_many() ------------------------------------------------------- + +test_that("on_eval_after_design works", { + callback = callback_batch_tuning(id = "test", + on_eval_after_design = function(callback, context) { + context$design$param_values[[1]][[1]] = list(list(minsplit = 1)) + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(instance$archive$benchmark_result$resample_result(1)$learner$param_set$values$minsplit, 1) +}) + +test_that("on_eval_after_benchmark and on_eval_before_archive works", { + callback = callback_batch_tuning(id = "test", + on_eval_after_benchmark = function(callback, context) { + callback$state$extra_performance = context$benchmark_result$aggregate(msr("classif.acc"))[, "classif.acc", with = FALSE] + }, + + on_eval_before_archive = function(callback, context) { + set(context$aggregated_performance, j = "classif.acc", value = callback$state$extra_performance) + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_names(names(instance$archive$data), must.include = c("classif.ce", "classif.acc")) +}) + +# stages in $assign_result() in TuningInstanceBatchSingleCrit ------------------ + +test_that("on_tuning_result_begin in TuningInstanceSingleCrit works", { + callback = callback_batch_tuning(id = "test", + on_tuning_result_begin = function(callback, context) { + context$result_xdt = data.table(minsplit = 1) + context$result_y = c(classif.ce = 0.7) + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(instance$result$minsplit, 1) + expect_equal(instance$result$classif.ce, 0.7) +}) + +test_that("on_result_end in TuningInstanceSingleCrit works", { + callback = callback_batch_tuning(id = "test", + on_result_end = function(callback, context) { + context$result$classif.ce = 0.7 + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(instance$result$classif.ce, 0.7) +}) + +test_that("on_result in TuningInstanceSingleCrit works", { + expect_warning({callback = callback_batch_tuning(id = "test", + on_result = function(callback, context) { + context$result$classif.ce = 0.7 + } + )}, "deprecated") + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msr("classif.ce"), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(instance$result$classif.ce, 0.7) +}) + +# stages in $assign_result() in TuningInstanceBatchMultiCrit ------------------- + +test_that("on_tuning_result_begin in TuningInstanceBatchMultiCrit works", { + callback = callback_batch_tuning(id = "test", + on_tuning_result_begin = function(callback, context) { + context$result_xdt = data.table(minsplit = 1) + context$result_ydt = data.table(classif.ce = 0.7, classif.acc = 0.8) + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msrs(c("classif.ce", "classif.acc")), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(instance$result$minsplit, 1) + expect_equal(instance$result$classif.ce, 0.7) +}) + +test_that("on_result_end in TuningInstanceBatchMultiCrit works", { + expect_warning({callback = callback_batch_tuning(id = "test", + on_result = function(callback, context) { + set(context$result, j = "classif.ce", value = 0.7) + } + )}, "deprecated") + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msrs(c("classif.ce", "classif.acc")), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(unique(instance$result$classif.ce), 0.7) +}) + +test_that("on_result in TuningInstanceBatchMultiCrit works", { + callback = callback_batch_tuning(id = "test", + on_result_end = function(callback, context) { + set(context$result, j = "classif.ce", value = 0.7) + } + ) + + instance = tune( + tuner = tnr("random_search", batch_size = 1), + task = tsk("pima"), + learner = lrn("classif.rpart", minsplit = to_tune(1, 10)), + resampling = rsmp ("holdout"), + measures = msrs(c("classif.ce", "classif.acc")), + term_evals = 2, + callbacks = callback) + + expect_class(instance$objective$context, "ContextBatchTuning") + expect_equal(unique(instance$result$classif.ce), 0.7) +}) + + diff --git a/tests/testthat/test_mlr_callbacks.R b/tests/testthat/test_mlr_callbacks.R index 7e71aeb32..ebe24f81d 100644 --- a/tests/testthat/test_mlr_callbacks.R +++ b/tests/testthat/test_mlr_callbacks.R @@ -412,4 +412,42 @@ test_that("async save logs callback works", { expect_data_table(instance$archive$data$log[[1]][[1]]) }) +# one se rule callback -------------------------------------------------------- +test_that("one se rule callback works", { + + instance = tune( + tuner = tnr("random_search", batch_size = 15), + task = tsk("pima"), + learner = lrn("classif.rpart", cp = to_tune(1e-04, 1e-1, logscale = TRUE)), + resampling = rsmp("cv", folds = 3), + measures = msr("classif.ce"), + term_evals = 30, + callbacks = clbk("mlr3tuning.one_se_rule") + ) + + expect_numeric(instance$archive$data$n_features) + expect_numeric(instance$result$n_features) +}) + +test_that("one se rule callback works", { + skip_on_cran() + skip_if_not_installed("rush") + flush_redis() + + rush::rush_plan(n_workers = 2) + instance = ti_async( + task = tsk("pima"), + learner = lrn("classif.rpart", cp = to_tune(1e-04, 1e-1)), + resampling = rsmp("cv", folds = 3), + measures = msr("classif.ce"), + terminator = trm("evals", n_evals = 5), + callbacks = clbk("mlr3tuning.async_one_se_rule") + ) + + tuner = tnr("async_random_search") + tuner$optimize(instance) + + expect_numeric(instance$archive$data$n_features) + expect_numeric(instance$result$n_features) +})