bench-torch/bench.qmd at main · mlverse/bench-torch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
---
title: "bench"
format: html
---

```{r}
#| echo: false
execute_r <- function(path) {
  fs::path_ext(path) <- "R"
  processx::run(
    command = "Rscript",
    args = path,
    cleanup_tree = TRUE
  )
}

execute_py <- function(path) {
  fs::path_ext(path) <- "py"
  processx::run(
    command = reticulate::py_config()$python,
    args = path,
    cleanup_tree = TRUE
  )
}

parse_result <- function(out) {
  as.numeric(out$stdout)
}

run_bench <- function(name) {
  tibble::tibble(
    bench = fs::path_file(name),
    r = parse_result(execute_r(name)),
    py = parse_result(execute_py(name))
  )
}

library(tidyverse)
run_benchmarks <- function() {
  purrr::map_dfr(1:10, ~{
  purrr::map_dfr(
    fs::dir_ls(path = "bench", glob = "*.R"),
    run_bench
  )
}) %>%
  group_by(bench) %>%
  summarise(
    median_r = median(r),
    median_py = median(py),
    mean_r = mean(r),
    mean_py = mean(py),
    min_r = min(r),
    min_py = min(py),
    max_r = max(r),
    max_py = max(py),
    rel = median_r/median_py
  ) %>%
  knitr::kable()
}
```

### batch_size = 1000

```{r}
Sys.setenv("BATCH_SIZE" = 1000)
Sys.setenv("ITER" = 500)
run_benchmarks()
```

### batch_size = 32

```{r}
Sys.setenv("BATCH_SIZE" = 32)
Sys.setenv("ITER" = 2000)
run_benchmarks()
```

### batch_size = 1000 - CRAN

```{r}
Sys.setenv("BATCH_SIZE" = 1000)
Sys.setenv("ITER" = 500)
Sys.setenv("R_LIBS_USER" = tail(.libPaths(), 1))
run_benchmarks()
```


### batch_size = 32 - CRAN

```{r}
Sys.setenv("BATCH_SIZE" = 32)
Sys.setenv("ITER" = 2000)
Sys.setenv("R_LIBS_USER" = tail(.libPaths(), 1))
run_benchmarks()
```