new paradox adaption

Author: mb706

R/helper.R

@@ -23,11 +23,22 @@ evaluate_default = function(inst) {
   xss = default_values(inst$objective$learner, inst$search_space, inst$objective$task)
 
   # parameters with exp transformation and log inverse transformation
-  has_logscale = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_logscale)
   # parameters with unknown inverse transformation
-  has_trafo = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_trafo)
   # parameter set with trafo
-  has_extra_trafo = get_private(inst$search_space)$.has_extra_trafo
+  if ("set_id" %in% names(ps())) {
+    # old paradox
+    has_logscale = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_logscale)
+
+    has_trafo = map_lgl(inst$search_space$params, function(param) get_private(param)$.has_trafo)
+
+    has_extra_trafo = get_private(inst$search_space)$.has_extra_trafo
+  } else {
+    has_logscale = map_lgl(inst$search_space$params$.trafo, function(x) identical(x, exp))
+
+    has_trafo = map_lgl(inst$search_space$params$.trafo, function(x) !is.null(x) && !identical(x, exp))
+
+    has_extra_trafo = !is.null(inst$search_space$extra_trafo)
+  }
 
   if (any(has_trafo) || has_extra_trafo) {
     stop("Cannot evaluate default hyperparameter values. Search space contains transformation functions with unknown inverse function.")

tests/testthat/helper.R

@@ -7,3 +7,10 @@ library(R6)
 
 lapply(list.files(system.file("testthat", package = "mlr3"), pattern = "^helper.*\\.[rR]", full.names = TRUE), source)
 lapply(list.files(system.file("testthat", package = "mlr3tuning"), pattern = "^helper.*\\.[rR]", full.names = TRUE), source)
+
+sortnames = function(x) {
+  if (!is.null(names(x))) {
+    x <- x[order(names(x), decreasing = TRUE)]
+  }
+  x
+}

tests/testthat/test_ArchiveTuning.R

@@ -69,12 +69,12 @@ test_that("ArchiveTuning access methods work", {
   # learner param values
   walk(instance$archive$data$uhash, function(uhash) {
     expect_list(instance$archive$learner_param_vals(uhash = uhash))
-    expect_named(instance$archive$learner_param_vals(uhash = uhash), c("xval" ,"cp"))
+    expect_named(instance$archive$learner_param_vals(uhash = uhash), c("xval" ,"cp"), ignore.order = TRUE)
   })
 
   walk(seq_row(instance$archive$data), function(i) {
     expect_list(instance$archive$learner_param_vals(i))
-    expect_named(instance$archive$learner_param_vals(i), c("xval" ,"cp"))
+    expect_named(instance$archive$learner_param_vals(i), c("xval" ,"cp"), ignore.order = TRUE)
   })
 
   # learners

tests/testthat/test_AutoTuner.R

@@ -8,13 +8,14 @@ test_that("AutoTuner / train+predict", {
   expect_learner(at)
   at$train(task)
   expect_learner(at)
-  expect_equal(at$learner$param_set$values, list(xval = 0, cp = 0.2))
+
+  expect_equal(sortnames(at$learner$param_set$values), list(xval = 0, cp = 0.2))
   inst = at$tuning_instance
   a = at$archive$data
   expect_data_table(a, nrows = 3L)
   r = at$tuning_result
   expect_equal(r$x_domain[[1]], list(cp = 0.2))
-  expect_equal(r$learner_param_vals[[1]], list(xval = 0, cp = 0.2))
+  expect_equal(sortnames(r$learner_param_vals[[1]]), list(xval = 0, cp = 0.2))
   prd = at$predict(task)
   expect_prediction(prd)
   expect_s3_class(at$learner$model, "rpart")
@@ -39,11 +40,12 @@ test_that("AutoTuner / resample", {
 
   rr = resample(tsk("iris"), at, r_outer, store_models = TRUE)
 
+
   # check tuning results of all outer folds
   expect_equal(length(rr$learners), outer_folds)
   lapply(rr$learners, function(ll) {
     assert_r6(ll, "AutoTuner")
-    expect_equal(ll$learner$param_set$values, list(xval = 0, cp = 0.2))
+    expect_equal(sortnames(ll$learner$param_set$values), list(xval = 0, cp = 0.2))
     inst = ll$tuning_instance
     assert_r6(inst, "TuningInstanceSingleCrit")
     expect_data_table(inst$archive$data, nrows = inner_evals)

tests/testthat/test_Tuner.R

@@ -51,7 +51,7 @@ test_that("we get a result when some subordinate params are not fulfilled", {
 })
 
 test_that("print method workds", {
-  param_set = ParamSet$new(list(ParamLgl$new("p1")))
+  param_set = ps(p1 = p_lgl())
   param_set$values$p1 = TRUE
   param_classes = "ParamLgl"
   properties = "single-crit"
@@ -70,7 +70,7 @@ test_that("print method workds", {
 })
 
 test_that("optimize does not work in abstract class", {
-  param_set = ParamSet$new(list(ParamLgl$new("p1")))
+  param_set = ps(p1 = p_lgl())
   param_set$values$p1 = TRUE
   param_classes = "ParamDbl"
   properties = "single-crit"
@@ -145,7 +145,7 @@ test_that("Tuner works with instantiated resampling", {
 })
 
 test_that("Tuner active bindings work", {
-  param_set = ParamSet$new(list(ParamLgl$new("p1")))
+  param_set = ps(p1 = p_lgl())
   param_set$values$p1 = TRUE
   param_classes = "ParamLgl"
   properties = "single-crit"
@@ -163,7 +163,7 @@ test_that("Tuner active bindings work", {
   expect_equal(tuner$properties, properties)
   expect_subset(packages, tuner$packages)
 
-  expect_error({tuner$param_set = ParamSet$new(list(ParamLgl$new("p2")))},
+  expect_error({tuner$param_set = ps(p2 = p_lgl())},
     regexp = "$param_set is read-only",
     fixed = TRUE)
 

tests/testthat/test_TunerCmaes.R

@@ -18,5 +18,5 @@ expect_tuner(tnr("cmaes"))
   )
 
   expect_equal(instance$archive$n_evals, 10)
-  expect_named(instance$result_x_domain, c("cp", "minsplit", "minbucket"))
+  expect_named(instance$result_x_domain, c("cp", "minsplit", "minbucket"), ignore.order = TRUE)
 })

tests/testthat/test_TunerGenSA.R

@@ -15,12 +15,8 @@ test_that("TunerGenSA", {
 test_that("TunerGenSA with int params and trafo", {
   ps = ps(
     cp = p_dbl(lower = 0.001, upper = 0.1),
-    minsplit = p_dbl(lower = 1, upper = 10)
+    minsplit = p_dbl(lower = 1, upper = 10, trafo = function(x) as.integer(round(x)))
   )
-  ps$trafo = function(x, param_set) {
-    x$minsplit = as.integer(round(x$minsplit))
-    return(x)
-  }
   te = trm("evals", n_evals = 2)
   inst = TuningInstanceSingleCrit$new(tsk("iris"), lrn("classif.rpart"), rsmp("holdout"), msr("classif.ce"), te, ps)
   tt = TunerGenSA$new()

tests/testthat/test_TunerGridSearch.R

@@ -26,7 +26,7 @@ test_that("TunerGridSearch with TerminatorNone", {
 
 test_that("TunerGridSearch works with forward hotstart parameter", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
 
   instance = tune(
     tuner = tnr( "grid_search", batch_size = 5, resolution = 5),
@@ -40,12 +40,12 @@ test_that("TunerGridSearch works with forward hotstart parameter", {
 
   ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id)
   expect_equal(length(unique(ids)), 5)
-  expect_equal(unique(instance$archive$data$iter), c(1, 25, 50, 75, 100))
+  expect_equal(unique(instance$archive$data$iter), c(1, 26, 51, 76, 101))
 })
 
 test_that("TunerGridSearch works with forward hotstart parameter", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
   learner$properties[learner$properties %in% "hotstart_forward"] = "hotstart_backward"
 
   instance = tune(
@@ -60,12 +60,12 @@ test_that("TunerGridSearch works with forward hotstart parameter", {
 
   ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id)
   expect_equal(length(unique(ids)), 5)
-  expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1))
+  expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1))
 })
 
 test_that("TunerGridSearch works with forward and backward hotstart parameter", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
   learner$properties = c(learner$properties, "hotstart_backward")
 
   instance = tune(
@@ -80,5 +80,5 @@ test_that("TunerGridSearch works with forward and backward hotstart parameter",
 
   ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id)
   expect_equal(length(unique(ids)), 5)
-  expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1))
+  expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1))
 })

tests/testthat/test_TuningInstanceMultiCrit.R

@@ -143,7 +143,7 @@ test_that("TuningInstanceMultiCrit and empty search space works", {
   )
 
   expect_data_table(instance$result)
-  expect_equal(instance$result$learner_param_vals[[1]], list(xval = 0, cp = 0.1))
+  expect_equal(sortnames(instance$result$learner_param_vals[[1]]), list(xval = 0, cp = 0.1))
   expect_equal(instance$result$x_domain[[1]], list())
 
   # no constant
@@ -182,5 +182,5 @@ test_that("assign_result works", {
   expect_equal(res$cp, c(0.1, 0.01))
   expect_equal(res$classif.fpr, c(0.8, 0.7))
   expect_equal(res$classif.tpr, c(0.3, 0.2))
-  expect_equal(res$learner_param_vals[[1]], list(xval = 0, cp = 0.1))
+  expect_equal(sortnames(res$learner_param_vals[[1]]), list(xval = 0, cp = 0.1))
 })

tests/testthat/test_hotstart.R

@@ -1,6 +1,6 @@
 test_that("hotstart works forwards", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
 
   instance = tune(
     tuner = tnr("grid_search", batch_size = 5, resolution = 5),
@@ -14,13 +14,13 @@ test_that("hotstart works forwards", {
 
   ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id)
   expect_equal(length(unique(ids)), 5)
-  expect_equal(unique(instance$archive$data$iter), c(1, 25, 50, 75, 100))
+  expect_equal(unique(instance$archive$data$iter), c(1, 26, 51, 76, 101))
   expect_null(instance$archive$data$resample_result)
 })
 
 test_that("hotstart works backwards", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
   learner$properties[learner$properties %in% "hotstart_forward"] = "hotstart_backward"
 
   instance = tune(
@@ -35,13 +35,13 @@ test_that("hotstart works backwards", {
 
   ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id)
   expect_equal(length(unique(ids)), 5)
-  expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1))
+  expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1))
   expect_null(instance$archive$data$resample_result)
 })
 
 test_that("hotstart works forwards and backwards", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
   learner$properties = c(learner$properties, "hotstart_backward")
 
   instance = tune(
@@ -56,7 +56,7 @@ test_that("hotstart works forwards and backwards", {
 
   ids = map(extract_benchmark_result_learners(instance$archive$benchmark_result), function(l) l$model$id)
   expect_equal(length(unique(ids)), 5)
-  expect_equal(unique(instance$archive$data$iter), c(100, 75, 50, 25, 1))
+  expect_equal(unique(instance$archive$data$iter), c(101, 76, 51, 26, 1))
   expect_null(instance$archive$data$resample_result)
 })
 
@@ -79,7 +79,7 @@ test_that("hotstart flag is not set to TRUE if learners does not support hotstar
 
 test_that("models are discarded after storing them in the stack", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
 
   instance = tune(
     tuner = tnr("grid_search", batch_size = 5, resolution = 5),
@@ -102,7 +102,7 @@ test_that("models are discarded after storing them in the stack", {
 
 test_that("objects are cloned", {
   task = tsk("pima")
-  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 100))
+  learner = lrn("classif.debug", x = to_tune(), iter = to_tune(1, 101))
 
   instance = tune(
     tuner = tnr("grid_search", batch_size = 5, resolution = 5),