diff --git a/R/helper.R b/R/helper.R index a142566b..070c01a2 100644 --- a/R/helper.R +++ b/R/helper.R @@ -23,11 +23,22 @@ evaluate_default = function(inst) { xss = default_values(inst$objective$learner, inst$search_space, inst$objective$task) # parameters with exp transformation and log inverse transformation - has_logscale = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_logscale) # parameters with unknown inverse transformation - has_trafo = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_trafo) # parameter set with trafo - has_extra_trafo = get_private(inst$search_space)$.has_extra_trafo + if ("set_id" %in% names(ps())) { + # old paradox + has_logscale = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_logscale) + + has_trafo = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_trafo) + + has_extra_trafo = get_private(inst$search_space)$.has_extra_trafo + } else { + has_logscale = map_lgl(inst$search_space$params$.trafo, function(x) identical(x, exp)) + + has_trafo = map_lgl(inst$search_space$params$.trafo, function(x) !is.null(x) && !identical(x, exp)) + + has_extra_trafo = !is.null(inst$search_space$extra_trafo) + } if (any(has_trafo) || has_extra_trafo) { stop("Cannot evaluate default hyperparameter values. Search space contains transformation functions with unknown inverse function.") diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index a4503b02..d36d17f5 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -7,3 +7,10 @@ library(R6) lapply(list.files(system.file("testthat", package = "mlr3"), pattern = "^helper.*\\.[rR]", full.names = TRUE), source) lapply(list.files(system.file("testthat", package = "mlr3tuning"), pattern = "^helper.*\\.[rR]", full.names = TRUE), source) + +sortnames = function(x) { + if (!is.null(names(x))) { + x <- x[order(names(x), decreasing = TRUE)] + } + x +} diff --git a/tests/testthat/test_ArchiveTuning.R b/tests/testthat/test_ArchiveTuning.R index e3a82303..7bfd3a9e 100644 --- a/tests/testthat/test_ArchiveTuning.R +++ b/tests/testthat/test_ArchiveTuning.R @@ -69,12 +69,12 @@ test_that("ArchiveTuning access methods work", { # learner param values walk(instance$archive$data$uhash, function(uhash) { expect_list(instance$archive$learner_param_vals(uhash = uhash)) - expect_named(instance$archive$learner_param_vals(uhash = uhash), c("xval" ,"cp")) + expect_named(instance$archive$learner_param_vals(uhash = uhash), c("xval" ,"cp"), ignore.order = TRUE) }) walk(seq_row(instance$archive$data), function(i) { expect_list(instance$archive$learner_param_vals(i)) - expect_named(instance$archive$learner_param_vals(i), c("xval" ,"cp")) + expect_named(instance$archive$learner_param_vals(i), c("xval" ,"cp"), ignore.order = TRUE) }) # learners diff --git a/tests/testthat/test_AutoTuner.R b/tests/testthat/test_AutoTuner.R index 046d2d6d..7459d341 100644 --- a/tests/testthat/test_AutoTuner.R +++ b/tests/testthat/test_AutoTuner.R @@ -8,13 +8,14 @@ test_that("AutoTuner / train+predict", { expect_learner(at) at$train(task) expect_learner(at) - expect_equal(at$learner$param_set$values, list(xval = 0, cp = 0.2)) + + expect_equal(sortnames(at$learner$param_set$values), list(xval = 0, cp = 0.2)) inst = at$tuning_instance a = at$archive$data expect_data_table(a, nrows = 3L) r = at$tuning_result expect_equal(r$x_domain[[1]], list(cp = 0.2)) - expect_equal(r$learner_param_vals[[1]], list(xval = 0, cp = 0.2)) + expect_equal(sortnames(r$learner_param_vals[[1]]), list(xval = 0, cp = 0.2)) prd = at$predict(task) expect_prediction(prd) expect_s3_class(at$learner$model, "rpart") @@ -39,11 +40,12 @@ test_that("AutoTuner / resample", { rr = resample(tsk("iris"), at, r_outer, store_models = TRUE) + # check tuning results of all outer folds expect_equal(length(rr$learners), outer_folds) lapply(rr$learners, function(ll) { assert_r6(ll, "AutoTuner") - expect_equal(ll$learner$param_set$values, list(xval = 0, cp = 0.2)) + expect_equal(sortnames(ll$learner$param_set$values), list(xval = 0, cp = 0.2)) inst = ll$tuning_instance assert_r6(inst, "TuningInstanceSingleCrit") expect_data_table(inst$archive$data, nrows = inner_evals) diff --git a/tests/testthat/test_Tuner.R b/tests/testthat/test_Tuner.R index a8fccbb0..1a7a5602 100644 --- a/tests/testthat/test_Tuner.R +++ b/tests/testthat/test_Tuner.R @@ -51,7 +51,7 @@ test_that("we get a result when some subordinate params are not fulfilled", { }) test_that("print method workds", { - param_set = ParamSet$new(list(ParamLgl$new("p1"))) + param_set = ps(p1 = p_lgl()) param_set$values$p1 = TRUE param_classes = "ParamLgl" properties = "single-crit" @@ -70,7 +70,7 @@ test_that("print method workds", { }) test_that("optimize does not work in abstract class", { - param_set = ParamSet$new(list(ParamLgl$new("p1"))) + param_set = ps(p1 = p_lgl()) param_set$values$p1 = TRUE param_classes = "ParamDbl" properties = "single-crit" @@ -145,7 +145,7 @@ test_that("Tuner works with instantiated resampling", { }) test_that("Tuner active bindings work", { - param_set = ParamSet$new(list(ParamLgl$new("p1"))) + param_set = ps(p1 = p_lgl()) param_set$values$p1 = TRUE param_classes = "ParamLgl" properties = "single-crit" @@ -163,7 +163,7 @@ test_that("Tuner active bindings work", { expect_equal(tuner$properties, properties) expect_subset(packages, tuner$packages) - expect_error({tuner$param_set = ParamSet$new(list(ParamLgl$new("p2")))}, + expect_error({tuner$param_set = ps(p2 = p_lgl())}, regexp = "$param_set is read-only", fixed = TRUE) diff --git a/tests/testthat/test_TunerCmaes.R b/tests/testthat/test_TunerCmaes.R index ad8bf27c..8ed707e0 100644 --- a/tests/testthat/test_TunerCmaes.R +++ b/tests/testthat/test_TunerCmaes.R @@ -18,5 +18,5 @@ expect_tuner(tnr("cmaes")) ) expect_equal(instance$archive$n_evals, 10) - expect_named(instance$result_x_domain, c("cp", "minsplit", "minbucket")) + expect_named(instance$result_x_domain, c("cp", "minsplit", "minbucket"), ignore.order = TRUE) }) diff --git a/tests/testthat/test_TunerGenSA.R b/tests/testthat/test_TunerGenSA.R index 309736ae..56d9126f 100644 --- a/tests/testthat/test_TunerGenSA.R +++ b/tests/testthat/test_TunerGenSA.R @@ -15,12 +15,8 @@ test_that("TunerGenSA", { test_that("TunerGenSA with int params and trafo", { ps = ps( cp = p_dbl(lower = 0.001, upper = 0.1), - minsplit = p_dbl(lower = 1, upper = 10) + minsplit = p_dbl(lower = 1, upper = 10, trafo = function(x) as.integer(round(x))) ) - ps$trafo = function(x, param_set) { - x$minsplit = as.integer(round(x$minsplit)) - return(x) - } te = trm("evals", n_evals = 2) inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), te, ps) tt = TunerGenSA$new() diff --git a/tests/testthat/test_TunerGridSearch.R b/tests/testthat/test_TunerGridSearch.R index 4c7141d3..5c95c0fa 100644 --- a/tests/testthat/test_TunerGridSearch.R +++ b/tests/testthat/test_TunerGridSearch.R @@ -26,7 +26,7 @@ test_that("TunerGridSearch with TerminatorNone", { test_that("TunerGridSearch works with forward hotstart parameter", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) instance = tune( tuner = tnr( "grid_search", batch_size = 5, resolution = 5), @@ -40,12 +40,12 @@ test_that("TunerGridSearch works with forward hotstart parameter", { ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id) expect_equal(length(unique(ids)), 5) - expect_equal(unique(instance$archive$data$iter), c(1, 25, 50, 75, 100)) + expect_equal(unique(instance$archive$data$iter), c(1, 26, 51, 76, 101)) }) test_that("TunerGridSearch works with forward hotstart parameter", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) learner$properties[learner$properties %in% "hotstart_forward"] = "hotstart_backward" instance = tune( @@ -60,12 +60,12 @@ test_that("TunerGridSearch works with forward hotstart parameter", { ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id) expect_equal(length(unique(ids)), 5) - expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1)) + expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1)) }) test_that("TunerGridSearch works with forward and backward hotstart parameter", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) learner$properties = c(learner$properties, "hotstart_backward") instance = tune( @@ -80,5 +80,5 @@ test_that("TunerGridSearch works with forward and backward hotstart parameter", ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id) expect_equal(length(unique(ids)), 5) - expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1)) + expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1)) }) diff --git a/tests/testthat/test_TuningInstanceMultiCrit.R b/tests/testthat/test_TuningInstanceMultiCrit.R index 8fdd11aa..fd0ab341 100644 --- a/tests/testthat/test_TuningInstanceMultiCrit.R +++ b/tests/testthat/test_TuningInstanceMultiCrit.R @@ -143,7 +143,7 @@ test_that("TuningInstanceMultiCrit and empty search space works", { ) expect_data_table(instance$result) - expect_equal(instance$result$learner_param_vals[[1]], list(xval = 0, cp = 0.1)) + expect_equal(sortnames(instance$result$learner_param_vals[[1]]), list(xval = 0, cp = 0.1)) expect_equal(instance$result$x_domain[[1]], list()) # no constant @@ -182,5 +182,5 @@ test_that("assign_result works", { expect_equal(res$cp, c(0.1, 0.01)) expect_equal(res$classif.fpr, c(0.8, 0.7)) expect_equal(res$classif.tpr, c(0.3, 0.2)) - expect_equal(res$learner_param_vals[[1]], list(xval = 0, cp = 0.1)) + expect_equal(sortnames(res$learner_param_vals[[1]]), list(xval = 0, cp = 0.1)) }) diff --git a/tests/testthat/test_hotstart.R b/tests/testthat/test_hotstart.R index 5221c7fe..537d514b 100644 --- a/tests/testthat/test_hotstart.R +++ b/tests/testthat/test_hotstart.R @@ -1,6 +1,6 @@ test_that("hotstart works forwards", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) instance = tune( tuner = tnr("grid_search", batch_size = 5, resolution = 5), @@ -14,13 +14,13 @@ test_that("hotstart works forwards", { ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id) expect_equal(length(unique(ids)), 5) - expect_equal(unique(instance$archive$data$iter), c(1, 25, 50, 75, 100)) + expect_equal(unique(instance$archive$data$iter), c(1, 26, 51, 76, 101)) expect_null(instance$archive$data$resample_result) }) test_that("hotstart works backwards", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) learner$properties[learner$properties %in% "hotstart_forward"] = "hotstart_backward" instance = tune( @@ -35,13 +35,13 @@ test_that("hotstart works backwards", { ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id) expect_equal(length(unique(ids)), 5) - expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1)) + expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1)) expect_null(instance$archive$data$resample_result) }) test_that("hotstart works forwards and backwards", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) learner$properties = c(learner$properties, "hotstart_backward") instance = tune( @@ -56,7 +56,7 @@ test_that("hotstart works forwards and backwards", { ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id) expect_equal(length(unique(ids)), 5) - expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1)) + expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1)) expect_null(instance$archive$data$resample_result) }) @@ -79,7 +79,7 @@ test_that("hotstart flag is not set to TRUE if learners does not support hotstar test_that("models are discarded after storing them in the stack", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) instance = tune( tuner = tnr("grid_search", batch_size = 5, resolution = 5), @@ -102,7 +102,7 @@ test_that("models are discarded after storing them in the stack", { test_that("objects are cloned", { task = tsk("pima") - learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100)) + learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101)) instance = tune( tuner = tnr("grid_search", batch_size = 5, resolution = 5),