differences for PR #18

Author: actions-user

basic-targets.md

@@ -279,10 +279,10 @@ tar_make()
 ▶ dispatched target penguins_csv_file
 ● completed target penguins_csv_file [0.001 seconds, 190 bytes]
 ▶ dispatched target penguins_data_raw
-● completed target penguins_data_raw [0.188 seconds, 10.403 kilobytes]
+● completed target penguins_data_raw [0.203 seconds, 10.403 kilobytes]
 ▶ dispatched target penguins_data
 ● completed target penguins_data [0.007 seconds, 1.609 kilobytes]
-▶ ended pipeline [0.341 seconds]
+▶ ended pipeline [0.356 seconds]
 ```
 
 Congratulations, you've run your first workflow with `targets`!

branch.md

@@ -81,8 +81,8 @@ tar_plan(
 ✔ skipped target penguins_data_raw
 ✔ skipped target penguins_data
 ▶ dispatched target combined_model
-● completed target combined_model [0.024 seconds, 11.201 kilobytes]
-▶ ended pipeline [0.273 seconds]
+● completed target combined_model [0.044 seconds, 11.201 kilobytes]
+▶ ended pipeline [0.302 seconds]
 ```
 
 Let's have a look at the model. We will use the `glance()` function from the `broom` package. Unlike base R `summary()`, this function returns output as a tibble (the tidyverse equivalent of a dataframe), which as we will see later is quite useful for downstream analyses.
@@ -156,12 +156,12 @@ tar_plan(
 ▶ dispatched target species_model
 ● completed target species_model [0.001 seconds, 15.439 kilobytes]
 ▶ dispatched target combined_summary
-● completed target combined_summary [0.006 seconds, 348 bytes]
+● completed target combined_summary [0.007 seconds, 348 bytes]
 ▶ dispatched target interaction_summary
-● completed target interaction_summary [0.003 seconds, 348 bytes]
+● completed target interaction_summary [0.004 seconds, 348 bytes]
 ▶ dispatched target species_summary
-● completed target species_summary [0.003 seconds, 347 bytes]
-▶ ended pipeline [0.28 seconds]
+● completed target species_summary [0.004 seconds, 347 bytes]
+▶ ended pipeline [0.307 seconds]
 ```
 
 Let's look at the summary of one of the models:
@@ -234,11 +234,11 @@ First, let's look at the messages provided by `tar_make()`.
 ▶ dispatched branch model_summaries_812e3af782bee03f
 ● completed branch model_summaries_812e3af782bee03f [0.006 seconds, 348 bytes]
 ▶ dispatched branch model_summaries_2b8108839427c135
-● completed branch model_summaries_2b8108839427c135 [0.003 seconds, 347 bytes]
+● completed branch model_summaries_2b8108839427c135 [0.004 seconds, 347 bytes]
 ▶ dispatched branch model_summaries_533cd9a636c3e05b
-● completed branch model_summaries_533cd9a636c3e05b [0.003 seconds, 348 bytes]
+● completed branch model_summaries_533cd9a636c3e05b [0.004 seconds, 348 bytes]
 ● completed pattern model_summaries 
-▶ ended pipeline [0.302 seconds]
+▶ ended pipeline [0.314 seconds]
 ```
 
 There is a series of smaller targets (branches) that are each named like model_summaries_812e3af782bee03f, then one overall `model_summaries` target.
@@ -369,13 +369,13 @@ tar_plan(
 ✔ skipped target penguins_data
 ✔ skipped target models
 ▶ dispatched branch model_summaries_812e3af782bee03f
-● completed branch model_summaries_812e3af782bee03f [0.012 seconds, 374 bytes]
+● completed branch model_summaries_812e3af782bee03f [0.013 seconds, 374 bytes]
 ▶ dispatched branch model_summaries_2b8108839427c135
-● completed branch model_summaries_2b8108839427c135 [0.007 seconds, 371 bytes]
+● completed branch model_summaries_2b8108839427c135 [0.008 seconds, 371 bytes]
 ▶ dispatched branch model_summaries_533cd9a636c3e05b
 ● completed branch model_summaries_533cd9a636c3e05b [0.004 seconds, 377 bytes]
 ● completed pattern model_summaries 
-▶ ended pipeline [0.281 seconds]
+▶ ended pipeline [0.301 seconds]
 ```
 
 And this time, when we load the `model_summaries`, we can tell which model corresponds to which row (you may need to scroll to the right to see it).

config.yaml

@@ -56,8 +56,8 @@ tar_plan(
 
 ``` output
 ▶ dispatched target some_data
-● completed target some_data [0 seconds, 64 bytes]
-▶ ended pipeline [0.089 seconds]
+● completed target some_data [0.001 seconds, 64 bytes]
+▶ ended pipeline [0.094 seconds]
 ```
 
 If we inspect the contents of `some_data` with `tar_read(some_data)`, it will contain the string `"Hello World"` as expected.
@@ -77,7 +77,7 @@ tar_plan(
 
 ``` output
 ✔ skipped target some_data
-✔ skipped pipeline [0.087 seconds]
+✔ skipped pipeline [0.089 seconds]
 ```
 
 The target `some_data` was skipped, even though the contents of the file changed.
@@ -101,7 +101,7 @@ tar_plan(
 ● completed target data_file [0.001 seconds, 26 bytes]
 ▶ dispatched target some_data
 ● completed target some_data [0 seconds, 78 bytes]
-▶ ended pipeline [0.109 seconds]
+▶ ended pipeline [0.111 seconds]
 ```
 
 This time we see that `targets` does successfully re-build `some_data` as expected.
@@ -186,10 +186,10 @@ tar_plan(
 ▶ dispatched target penguins_data_raw_file
 ● completed target penguins_data_raw_file [0.001 seconds, 53.098 kilobytes]
 ▶ dispatched target penguins_data_raw
-● completed target penguins_data_raw [0.099 seconds, 10.403 kilobytes]
+● completed target penguins_data_raw [0.096 seconds, 10.403 kilobytes]
 ▶ dispatched target penguins_data
-● completed target penguins_data [0.015 seconds, 1.495 kilobytes]
-▶ ended pipeline [0.369 seconds]
+● completed target penguins_data [0.014 seconds, 1.495 kilobytes]
+▶ ended pipeline [0.368 seconds]
 ```
 
 ::::::::::::::::::::::::::::::::::
@@ -262,14 +262,14 @@ tar_plan(
 
 ``` output
 ▶ dispatched target hello_file
-● completed target hello_file [0 seconds, 26 bytes]
+● completed target hello_file [0.001 seconds, 26 bytes]
 ▶ dispatched target hello
 ● completed target hello [0 seconds, 78 bytes]
 ▶ dispatched target hello_caps
-● completed target hello_caps [0.001 seconds, 78 bytes]
+● completed target hello_caps [0 seconds, 78 bytes]
 ▶ dispatched target hello_caps_out
 ● completed target hello_caps_out [0 seconds, 26 bytes]
-▶ ended pipeline [0.111 seconds]
+▶ ended pipeline [0.107 seconds]
 ```
 
 Take a look at `hello_caps.txt` in the `results` folder and verify it is as you expect.

files.md

@@ -0,0 +1,49 @@
+library(crew.cluster)
+library(targets)
+library(tarchetypes)
+library(palmerpenguins)
+library(broom)
+suppressPackageStartupMessages(library(tidyverse))
+
+source("R/packages.R")
+source("R/functions.R")
+
+tar_option_set(
+  controller = crew_controller_slurm(
+    workers = 3,
+    script_lines = "module load R",
+    slurm_memory_gigabytes_per_cpu = 1
+  )
+)
+
+tar_plan(
+  # Load raw data
+  tar_file_read(
+    penguins_data_raw,
+    path_to_file("penguins_raw.csv"),
+    read_csv(!!.x, show_col_types = FALSE)
+  ),
+  # Clean data
+  penguins_data = clean_penguin_data(penguins_data_raw),
+  # Build models
+  models = list(
+    combined_model = lm(
+      bill_depth_mm ~ bill_length_mm, data = penguins_data),
+    species_model = lm(
+      bill_depth_mm ~ bill_length_mm + species, data = penguins_data),
+    interaction_model = lm(
+      bill_depth_mm ~ bill_length_mm * species, data = penguins_data)
+  ),
+  # Get model summaries
+  tar_target(
+    model_summaries,
+    glance_with_mod_name_slow(models),
+    pattern = map(models)
+  ),
+  # Get model predictions
+  tar_target(
+    model_predictions,
+    augment_with_mod_name_slow(models),
+    pattern = map(models)
+  )
+)

files/plans/plan_slurm.R

@@ -0,0 +1,52 @@
+graphics_devices <- function(){
+  system2("lshw", c("-class", "display"), stdout=TRUE, stderr=FALSE)
+}
+
+library(crew)
+library(targets)
+library(tarchetypes)
+library(crew.cluster)
+
+source("R/packages.R")
+source("R/functions.R")
+
+tar_option_set(
+  controller = crew_controller_group(
+    crew_controller_slurm(
+      name = "cpu_worker",
+      workers = 1,
+      script_lines = "module load R",
+      slurm_memory_gigabytes_per_cpu = 1,
+      slurm_cpus_per_task = 1
+    ),
+
+    crew_controller_slurm(
+      name = "gpu_worker",
+      workers = 1,
+      script_lines = c(
+        "#SBATCH --partition=gpuq",
+        "#SBATCH --gres=gpu:1",
+        "module load R"
+      ),
+      slurm_memory_gigabytes_per_cpu = 1,
+      slurm_cpus_per_task = 1
+    )
+  )
+)
+
+tar_plan(
+  tar_target(
+    cpu_hardware,
+    graphics_devices(),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "cpu_worker")
+    )
+  ),
+  tar_target(
+    gpu_hardware,
+    graphics_devices(),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "gpu_worker")
+    )
+  )
+)

files/plans/plan_slurm_gpu.R

@@ -0,0 +1,39 @@
+library(crew)
+library(targets)
+library(tarchetypes)
+library(crew.cluster)
+
+source("R/packages.R")
+source("R/functions.R")
+
+small_memory <- crew_controller_slurm(
+  name = "small_memory",
+  script_lines = "module load R",
+  slurm_memory_gigabytes_per_cpu = 1
+)
+big_memory <- crew_controller_slurm(
+  name = "big_memory",
+  script_lines = "module load R",
+  slurm_memory_gigabytes_per_cpu = 2
+)
+
+tar_option_set(
+  controller = crew_controller_group(small_memory, big_memory)
+)
+
+list(
+  tar_target(
+    name = big_memory_task,
+    command = Sys.getenv("SLURM_MEM_PER_CPU"),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "big_memory")
+    )
+  ),
+  tar_target(
+    name = small_memory_task,
+    command = Sys.getenv("SLURM_MEM_PER_CPU"),
+    resources = tar_resources(
+      crew = tar_resources_crew(controller = "small_memory")
+    )
+  )
+)