set seed, better readme, better errors, better .Rprofile

Author: sebffischer

paper/.Rprofile

@@ -0,0 +1,3 @@
+if(!file.exists("/.dockerenv")) {
+  source("renv/activate.R")
+}

paper/README.md

@@ -82,6 +82,14 @@ Also note that it's important to have enough RAM, otherwise the benchmarks will
 However, there are many other factors, such as the exact hardware that make it generally difficult to reproduce the runtime results.
 
 To run the benchmarks locally, ensure that you are in the `paper` directory.
+There are three scripts:
+
+* `paper/benchmark/linux-gpu.R`, which creates the folder `paper/benchmark/registry-linux-gpu`
+* `paper/benchmark/linux-cpu.R`, which creates the folder `paper/benchmark/registry-linux-cpu`
+* `paper/benchmark/linux-gpu-optimizer.R`, which creates the folder `paper/benchmark/registry-linux-gpu-optimizer`
+
+If one of the folders already exists and you want to re-run the benchmarks, you need to delete or move the folder.
+
 To run the GPU benchmarks (using the CUDA docker image) on linux, run:
 
 ```bash
@@ -100,7 +108,7 @@ To run the benchmark that compares "ignite" with standard optimizers (using the
 Rscript benchmark/linux-gpu-optimizer.R
 ```
 
-The results are stored in:
+The postprocessd results are stored in:
 
 * `paper/benchmark/result-linux-gpu.rds`
 * `paper/benchmark/result-linux-cpu.rds`

paper/benchmark/benchmark.R

@@ -5,7 +5,8 @@ setup = function(reg_path, python_path, work_dir) {
   reg = makeExperimentRegistry(
     file.dir = reg_path,
     work.dir = work_dir,
-    packages = "checkmate"
+    packages = "checkmate",
+    seed = 123
   )
   reg$cluster.functions = makeClusterFunctionsInteractive()
 

paper/benchmark/linux-cpu.R

@@ -2,6 +2,8 @@ library(here)
 
 source(here("benchmark", "benchmark.R"))
 
+set.seed(42)
+
 # Change this when not running this in the docker image
 # Below is the correct python path for the CPU docker image.
 PYTHON_PATH = "/opt/venv/bin/python"

paper/benchmark/linux-gpu-optimizer.R

@@ -2,10 +2,16 @@ library(here)
 
 source(here("benchmark", "benchmark.R"))
 
+set.seed(43)
+
 # Change this when not running this in the docker image
 # Below is the correct python path for the CUDA docker image
 PYTHON_PATH = "/usr/bin/python3"
 
+if (!torch::cuda_is_available()) {
+  stop("Cuda is not available for R-torch, please use the correct docker image.")
+}
+
 problem_design = expand.grid(
   list(
     n = N,

paper/benchmark/linux-gpu.R

@@ -2,10 +2,16 @@ library(here)
 
 source(here("benchmark", "benchmark.R"))
 
+set.seed(44)
+
 # Change this when not running this in the docker image
 # Below is the correct python path for the CUDA docker image
 PYTHON_PATH = "/usr/bin/python3"
 
+if (!torch::cuda_is_available()) {
+  stop("Cuda is not available for R-torch, please use the correct docker image.")
+}
+
 problem_design = expand.grid(
   list(
     n = N,
@@ -21,7 +27,7 @@ problem_design = expand.grid(
 )
 
 if (dir.exists(here("benchmark", "registry-linux-gpu"))) {
-  stop("Registry already exists. Delete it to run the benchmark again.")
+  stop("Registry benchmark/registry-linux-gpu already exists. Delete it to run the benchmark again.")
 }
 
 setup(

paper/benchmark/time_rtorch.R

@@ -1,9 +1,6 @@
 time_rtorch = function(epochs, batch_size, n_layers, latent, n, p, device, jit, seed, optimizer, mlr3torch = FALSE) {
   library(mlr3torch)
   library(torch)
-  mlr3pipelines::po
-  mlr3torch::LearnerTorch
-  mlr3::lrn
   torch_set_num_threads(1)
   torch_manual_seed(seed)