Feat/inference

DESCRIPTION

@@ -49,6 +49,7 @@ Suggests:
     mlr3cluster,
     mlr3filters,
     mlr3fselect (>= 1.0.0),
+	mlr3inference,
     mlr3learners,
     mlr3tuning (>= 1.0.0),
     paradox,
@@ -64,15 +65,16 @@ Suggests:
     survminer,
     mlr3proba (>= 0.6.3)
 Remotes:
-    mlr-org/mlr3proba
+    mlr-org/mlr3proba,
+    mlr-org/mlr3inference
 Additional_repositories:
   https://mlr-org.r-universe.dev
 Config/testthat/edition: 3
 Config/testthat/parallel: true
 Encoding: UTF-8
 NeedsCompilation: no
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.2
 Collate:
     'BenchmarkResult.R'
     'Filter.R'

NEWS.md

@@ -1,6 +1,7 @@
 # mlr3viz (development version)
 
 - Add plot for `LearnerSurvCoxPH`.
+- Add plot for confidence intervals (`mlr3inference`)
 
 # mlr3viz 0.9.0
 

R/BenchmarkResult.R

@@ -12,6 +12,7 @@
 #'   Requires package \CRANpkg{precrec}.
 #' * `"prc"`: Precision recall curve.
 #'    See `"roc"`.
+#' * `"ci"`: Plot confidence intervals. Pass a `msr("ci", ...)` from the `mlr3inference` package as argument `measure`.
 #'
 #' @param object ([mlr3::BenchmarkResult]).
 #' @template param_type
@@ -45,6 +46,43 @@ autoplot.BenchmarkResult = function(object, type = "boxplot", measure = NULL, th
 
   task = object$tasks$task[[1L]]
   measure = mlr3::assert_measure(mlr3::as_measure(measure, task_type = task$task_type), task = task)
+
+  if (identical(type, "ci")) {
+    mlr3misc::require_namespaces("mlr3inference")
+
+    assert_class(measure, "MeasureAbstractCi")
+    mid = measure$id
+
+    tbl = object$aggregate(measure)
+
+    tmp = map(object$resamplings$resampling, function(x) {
+      list(class(x), x$param_set$values)
+    })
+
+    if (length(unique(tmp)) != 1) {
+      stopf("Plot of type 'ci' requires exactly one resampling method")
+    }
+
+    # static checker
+    .data = NULL
+    task_id = NULL
+    p = ggplot(tbl, aes(x = .data[["learner_id"]], y = .data[[mid]])) +
+      geom_point() +
+      geom_errorbar(aes(ymin = .data[[paste0(mid, ".lower")]], ymax = .data[[paste0(mid, ".upper")]]), width = 0.2) +
+      facet_wrap(vars(task_id), scales = "free_y") +
+      labs(
+      title = sprintf("Confidence Intervals for alpha = %s", measure$param_set$values$alpha),
+      x = "Learner",
+      y = paste0(measure$measure$id)
+      ) +
+      theme +
+      theme(
+        axis.text.x = element_text(angle = 45, hjust = 1),
+        axis.title.x = element_blank()
+      )
+    return(p)
+  }
+
   measure_id = measure$id
   tab = fortify(object, measure = measure)
   tab$nr = sprintf("%09d", tab$nr)

tests/testthat/test_BenchmarkResult.R

@@ -48,3 +48,19 @@ test_that("holdout roc plot (#54)", {
 
   expect_doppelganger("bmr_holdout_roc", p)
 })
+
+skip_if_not_installed("mlr3inference")
+skip_if_not_installed("rpart")
+
+test_that("CI plot", {
+  bmr = benchmark(benchmark_grid(tsks(c("mtcars", "boston_housing")),
+    lrns(c("regr.featureless", "regr.rpart")), rsmp("holdout")))
+
+  p = autoplot(bmr, "ci", msr("ci", "regr.mse"))
+  expect_true(is.ggplot(p))
+  expect_doppelganger("bmr_holdout_ci", p)
+
+  bmr = benchmark(benchmark_grid(tsk("iris"), lrn("classif.rpart"),
+    rsmps(c("holdout", "cv"))))
+  expect_error(autoplot(bmr, "ci", msr("ci", "classif.acc")), "one resampling method")
+})