Add resource specifications to all Snakemake rules

workflow/rules/imaris.smk

@@ -10,6 +10,10 @@ rule imaris_to_metadata:
             acq="{acq,[a-zA-Z0-9]*imaris[a-zA-Z0-9]*}",
             suffix="SPIM.json",
         ),
+    threads: 1
+    resources:
+        mem_mb=2000,
+        runtime=60,
     benchmark:
         bids(
             root="benchmarks",

workflow/rules/import.smk

@@ -17,6 +17,10 @@ rule extract_sample:
                 )
             )
         ),
+    threads: 1
+    resources:
+        mem_mb=4000,
+        runtime=60,
     group:
         "preproc"
     log:
@@ -50,6 +54,10 @@ rule blaze_to_metadata_gcs:
             acq="{acq,[a-zA-Z0-9]*blaze[a-zA-Z0-9]*}",
             suffix="SPIM.json",
         ),
+    threads: 1
+    resources:
+        mem_mb=2000,
+        runtime=60,
     benchmark:
         bids(
             root="benchmarks",
@@ -91,6 +99,10 @@ rule blaze_to_metadata:
                 suffix="SPIM.json",
             )
         ),
+    threads: 1
+    resources:
+        mem_mb=2000,
+        runtime=60,
     benchmark:
         bids(
             root="benchmarks",
@@ -129,6 +141,10 @@ rule copy_blaze_metadata:
             acq="{acq,[a-zA-Z0-9]*blaze[a-zA-Z0-9]*}",
             suffix="SPIM.json",
         ),
+    threads: 1
+    resources:
+        mem_mb=1000,
+        runtime=10,
     log:
         bids(
             root="logs",
@@ -158,6 +174,10 @@ rule prestitched_to_metadata:
             acq="{acq,[a-zA-Z0-9]*prestitched[a-zA-Z0-9]*}",
             suffix="SPIM.json",
         ),
+    threads: 1
+    resources:
+        mem_mb=2000,
+        runtime=60,
     benchmark:
         bids(
             root="benchmarks",
@@ -228,11 +248,12 @@ rule tif_to_zarr_gcs:
             acq="{acq}",
             suffix="log.txt",
         ),
-    group:
-        "preproc"
+    threads: int(config["total_mem_mb"] / 8000)  # this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 
     resources:
         mem_mb=config["total_mem_mb"],
-    threads: int(config["total_mem_mb"] / 8000)  #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 
+        runtime=360,
+    group:
+        "preproc"
     container:
         config["containers"]["spimprep"]
     script:
@@ -284,12 +305,13 @@ rule bioformats_to_zarr:
             acq="{acq}",
             suffix="log.txt",
         ),
-    group:
-        "preproc"
+    threads: 16
     resources:
-        mem_mb=config["total_mem_mb"],  #TODO update this, along with threads.. 
+        mem_mb=config["total_mem_mb"],  # TODO update this, along with threads.. 
+        runtime=360,
         disk_mb=1000000,  #1TB
-    threads: 16
+    group:
+        "preproc"
     script:
         "../scripts/bioformats_to_zarr.py"
 
@@ -340,12 +362,13 @@ rule concat_tiles:
             acq="{acq}",
             suffix="log.txt",
         ),
-    group:
-        "preproc"
+    threads: 32
     resources:
         mem_mb=config["total_mem_mb"],
+        runtime=240,
         disk_mb=1000000,  #1TB
-    threads: 32
+    group:
+        "preproc"
     container:
         None
     script:

workflow/rules/multiview_stitcher.smk

@@ -61,6 +61,7 @@ rule mvstitcher_registration:
     threads: config["total_cores"]
     resources:
         mem_mb=config["total_mem_mb"],
+        runtime=240,
     group:
         "preproc"
     container:
@@ -101,6 +102,10 @@ rule mvstitcher_reg_plots:
             desc="mvstitched{desc}",
             suffix="groupwiseqc.png",
         ),
+    threads: 1
+    resources:
+        mem_mb=4000,
+        runtime=30,
     group:
         "preproc"
     container:
@@ -170,6 +175,7 @@ rule mvstitcher_fusion:
     threads: 128
     resources:
         mem_mb=config["total_mem_mb"],
+        runtime=480,
     group:
         "preproc"
     container:

workflow/rules/ome_zarr.smk

@@ -27,9 +27,10 @@ rule zarr_to_ome_zarr:
         storage_provider_settings=workflow.storage_provider_settings,
     output:
         **get_output_ome_zarr("blaze"),
+    threads: config["total_cores"]
     resources:
         mem_mb=config["total_mem_mb"],
-    threads: config["total_cores"]
+        runtime=480,
     log:
         bids(
             root="logs",
@@ -67,6 +68,10 @@ rule tif_stacks_to_ome_zarr:
         storage_provider_settings=workflow.storage_provider_settings,
     output:
         **get_output_ome_zarr("prestitched"),
+    threads: config["total_cores"]
+    resources:
+        runtime=720,
+        mem_mb=config["total_mem_mb"],
     log:
         bids(
             root="logs",
@@ -80,10 +85,6 @@ rule tif_stacks_to_ome_zarr:
         config["containers"]["spimprep"]
     group:
         "preproc"
-    threads: config["total_cores"]
-    resources:
-        runtime=720,
-        mem_mb=config["total_mem_mb"],
     script:
         "../scripts/tif_stacks_to_ome_zarr.py"
 
@@ -126,9 +127,12 @@ rule ome_zarr_to_nii:
             stain="{stain}",
             suffix="log.txt",
         ),
+    threads: config["total_cores"]
+    resources:
+        mem_mb=16000,
+        runtime=120,
     group:
         "preproc"
-    threads: config["total_cores"]
     container:
         None
     #        config["containers"]["spimprep"]

workflow/rules/qc.smk

@@ -5,6 +5,10 @@ rule setup_qc_dir:
         readme_md=config["report"]["resources"]["readme_md"],
     output:
         readme_md=remote_file(Path(root) / "qc" / "README.md"),
+    threads: 1
+    resources:
+        mem_mb=1000,
+        runtime=10,
     log:
         "logs/setup_qc_dir_log.txt",
     shell:
@@ -57,6 +61,10 @@ rule generate_flatfield_qc:
             / "images"
             / "uncorr"
         ),
+    threads: 1
+    resources:
+        mem_mb=8000,
+        runtime=60,
     log:
         bids(
             root="logs",
@@ -103,6 +111,10 @@ rule generate_whole_slice_qc:
             / "images"
             / "whole"
         ),
+    threads: 1
+    resources:
+        mem_mb=8000,
+        runtime=60,
     log:
         bids(
             root="logs",
@@ -145,6 +157,10 @@ rule generate_volume_qc:
             / "sub-{subject}_sample-{sample}_acq-{acq}"
             / "volume_qc.html"
         ),
+    threads: 1
+    resources:
+        mem_mb=8000,
+        runtime=60,
     log:
         bids(
             root="logs",
@@ -172,6 +188,10 @@ rule generate_subject_qc:
             / "sub-{subject}_sample-{sample}_acq-{acq}"
             / "subject.html"
         ),
+    threads: 1
+    resources:
+        mem_mb=2000,
+        runtime=10,
     log:
         bids(
             root="logs",
@@ -193,6 +213,10 @@ rule generate_aggregate_qc:
         samples=samples,
     output:
         total_html=remote_file(Path(root) / "qc" / "qc_report.html"),
+    threads: 1
+    resources:
+        mem_mb=2000,
+        runtime=10,
     log:
         bids(
             root="logs",