From cb5e31b1ef0eff0740c11ed47e0998ae701260b2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 00:39:20 +0000 Subject: [PATCH 1/3] Initial plan From b481557fb5db0949a48f435f2d36f0aefdc2cb99 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 00:47:08 +0000 Subject: [PATCH 2/3] Add threads, mem_mb, and runtime to all Snakemake rules Co-authored-by: akhanf <11492701+akhanf@users.noreply.github.com> --- workflow/rules/imaris.smk | 4 +++ workflow/rules/import.smk | 41 +++++++++++++++++++++------ workflow/rules/multiview_stitcher.smk | 6 ++++ workflow/rules/ome_zarr.smk | 16 +++++++---- workflow/rules/qc.smk | 24 ++++++++++++++++ 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/workflow/rules/imaris.smk b/workflow/rules/imaris.smk index fab09c1..7a55513 100644 --- a/workflow/rules/imaris.smk +++ b/workflow/rules/imaris.smk @@ -10,6 +10,10 @@ rule imaris_to_metadata: acq="{acq,[a-zA-Z0-9]*imaris[a-zA-Z0-9]*}", suffix="SPIM.json", ), + threads: 1 + resources: + mem_mb=2000, + runtime=60, benchmark: bids( root="benchmarks", diff --git a/workflow/rules/import.smk b/workflow/rules/import.smk index 9fdf90a..9eb8dfb 100644 --- a/workflow/rules/import.smk +++ b/workflow/rules/import.smk @@ -17,6 +17,10 @@ rule extract_sample: ) ) ), + threads: 1 + resources: + mem_mb=4000, + runtime=60, group: "preproc" log: @@ -50,6 +54,10 @@ rule blaze_to_metadata_gcs: acq="{acq,[a-zA-Z0-9]*blaze[a-zA-Z0-9]*}", suffix="SPIM.json", ), + threads: 1 + resources: + mem_mb=2000, + runtime=60, benchmark: bids( root="benchmarks", @@ -91,6 +99,10 @@ rule blaze_to_metadata: suffix="SPIM.json", ) ), + threads: 1 + resources: + mem_mb=2000, + runtime=60, benchmark: bids( root="benchmarks", @@ -129,6 +141,10 @@ rule copy_blaze_metadata: acq="{acq,[a-zA-Z0-9]*blaze[a-zA-Z0-9]*}", suffix="SPIM.json", ), + threads: 1 + resources: + mem_mb=1000, + runtime=10, log: bids( root="logs", @@ -158,6 +174,10 @@ rule prestitched_to_metadata: acq="{acq,[a-zA-Z0-9]*prestitched[a-zA-Z0-9]*}", suffix="SPIM.json", ), + threads: 1 + resources: + mem_mb=2000, + runtime=60, benchmark: bids( root="benchmarks", @@ -228,11 +248,12 @@ rule tif_to_zarr_gcs: acq="{acq}", suffix="log.txt", ), - group: - "preproc" + threads: int(config["total_mem_mb"] / 8000) #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 resources: mem_mb=config["total_mem_mb"], - threads: int(config["total_mem_mb"] / 8000) #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 + runtime=360, + group: + "preproc" container: config["containers"]["spimprep"] script: @@ -284,12 +305,13 @@ rule bioformats_to_zarr: acq="{acq}", suffix="log.txt", ), - group: - "preproc" + threads: 16 resources: mem_mb=config["total_mem_mb"], #TODO update this, along with threads.. + runtime=360, disk_mb=1000000, #1TB - threads: 16 + group: + "preproc" script: "../scripts/bioformats_to_zarr.py" @@ -340,12 +362,13 @@ rule concat_tiles: acq="{acq}", suffix="log.txt", ), - group: - "preproc" + threads: 32 resources: mem_mb=config["total_mem_mb"], + runtime=240, disk_mb=1000000, #1TB - threads: 32 + group: + "preproc" container: None script: diff --git a/workflow/rules/multiview_stitcher.smk b/workflow/rules/multiview_stitcher.smk index e7457a6..3c77a6e 100644 --- a/workflow/rules/multiview_stitcher.smk +++ b/workflow/rules/multiview_stitcher.smk @@ -61,6 +61,7 @@ rule mvstitcher_registration: threads: config["total_cores"] resources: mem_mb=config["total_mem_mb"], + runtime=240, group: "preproc" container: @@ -101,6 +102,10 @@ rule mvstitcher_reg_plots: desc="mvstitched{desc}", suffix="groupwiseqc.png", ), + threads: 1 + resources: + mem_mb=4000, + runtime=30, group: "preproc" container: @@ -170,6 +175,7 @@ rule mvstitcher_fusion: threads: 128 resources: mem_mb=config["total_mem_mb"], + runtime=480, group: "preproc" container: diff --git a/workflow/rules/ome_zarr.smk b/workflow/rules/ome_zarr.smk index 2227cfb..3e727d3 100644 --- a/workflow/rules/ome_zarr.smk +++ b/workflow/rules/ome_zarr.smk @@ -27,9 +27,10 @@ rule zarr_to_ome_zarr: storage_provider_settings=workflow.storage_provider_settings, output: **get_output_ome_zarr("blaze"), + threads: config["total_cores"] resources: mem_mb=config["total_mem_mb"], - threads: config["total_cores"] + runtime=480, log: bids( root="logs", @@ -67,6 +68,10 @@ rule tif_stacks_to_ome_zarr: storage_provider_settings=workflow.storage_provider_settings, output: **get_output_ome_zarr("prestitched"), + threads: config["total_cores"] + resources: + runtime=720, + mem_mb=config["total_mem_mb"], log: bids( root="logs", @@ -80,10 +85,6 @@ rule tif_stacks_to_ome_zarr: config["containers"]["spimprep"] group: "preproc" - threads: config["total_cores"] - resources: - runtime=720, - mem_mb=config["total_mem_mb"], script: "../scripts/tif_stacks_to_ome_zarr.py" @@ -126,9 +127,12 @@ rule ome_zarr_to_nii: stain="{stain}", suffix="log.txt", ), + threads: config["total_cores"] + resources: + mem_mb=16000, + runtime=120, group: "preproc" - threads: config["total_cores"] container: None # config["containers"]["spimprep"] diff --git a/workflow/rules/qc.smk b/workflow/rules/qc.smk index 36815f5..4b29c4d 100644 --- a/workflow/rules/qc.smk +++ b/workflow/rules/qc.smk @@ -5,6 +5,10 @@ rule setup_qc_dir: readme_md=config["report"]["resources"]["readme_md"], output: readme_md=remote_file(Path(root) / "qc" / "README.md"), + threads: 1 + resources: + mem_mb=1000, + runtime=10, log: "logs/setup_qc_dir_log.txt", shell: @@ -57,6 +61,10 @@ rule generate_flatfield_qc: / "images" / "uncorr" ), + threads: 1 + resources: + mem_mb=8000, + runtime=60, log: bids( root="logs", @@ -103,6 +111,10 @@ rule generate_whole_slice_qc: / "images" / "whole" ), + threads: 1 + resources: + mem_mb=8000, + runtime=60, log: bids( root="logs", @@ -145,6 +157,10 @@ rule generate_volume_qc: / "sub-{subject}_sample-{sample}_acq-{acq}" / "volume_qc.html" ), + threads: 1 + resources: + mem_mb=8000, + runtime=60, log: bids( root="logs", @@ -172,6 +188,10 @@ rule generate_subject_qc: / "sub-{subject}_sample-{sample}_acq-{acq}" / "subject.html" ), + threads: 1 + resources: + mem_mb=2000, + runtime=10, log: bids( root="logs", @@ -193,6 +213,10 @@ rule generate_aggregate_qc: samples=samples, output: total_html=remote_file(Path(root) / "qc" / "qc_report.html"), + threads: 1 + resources: + mem_mb=2000, + runtime=10, log: bids( root="logs", From 7f97067f7ed5533673bae8450b6f4994f0f4bc89 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 00:48:36 +0000 Subject: [PATCH 3/3] Fix comment formatting in import.smk Co-authored-by: akhanf <11492701+akhanf@users.noreply.github.com> --- workflow/rules/import.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rules/import.smk b/workflow/rules/import.smk index 9eb8dfb..d9828bf 100644 --- a/workflow/rules/import.smk +++ b/workflow/rules/import.smk @@ -248,7 +248,7 @@ rule tif_to_zarr_gcs: acq="{acq}", suffix="log.txt", ), - threads: int(config["total_mem_mb"] / 8000) #this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 + threads: int(config["total_mem_mb"] / 8000) # this is memory-limited -- seems to need ~8000mb for each thread, so threads=total_mem_mb / 8000 resources: mem_mb=config["total_mem_mb"], runtime=360, @@ -307,7 +307,7 @@ rule bioformats_to_zarr: ), threads: 16 resources: - mem_mb=config["total_mem_mb"], #TODO update this, along with threads.. + mem_mb=config["total_mem_mb"], # TODO update this, along with threads.. runtime=360, disk_mb=1000000, #1TB group: