chore: clean up (#1632)

mathiasbio · web-flow · commit 68b975f6ea16 · 2025-11-20T09:28:21.000+01:00
#### Changed * changed slurm extra qos with slurm_qos argument #1632 * moved default resource allocation to snakemake command #1632 * increased memory of samtools fixmate #1632 * no rerun for rule all #1632 * increased head-job runtime to 7 days #1632 * add attempt mem bump to vep_somatic_research_sv #1632 #### Removed * exome argument panel bed callback function #1632 * removed -l flag in head-job sbatch script #1632
diff --git a/BALSAMIC/commands/config/case.py b/BALSAMIC/commands/config/case.py
@@ -151,6 +151,12 @@ def case_config(
 
     LOG.info(f"Starting configuring analysis case: {case_id}.")
 
+    if exome and not panel_bed:
+        raise click.BadParameter(
+            "If --exome is provided, --panel-bed must also be provided.",
+            param_hint=["--panel-bed"],
+        )
+
     LOG.info(f"Creating case analysis directory: {analysis_dir}/{case_id}.")
     Path(analysis_dir, case_id).mkdir(exist_ok=True)
 
diff --git a/BALSAMIC/commands/options.py b/BALSAMIC/commands/options.py
@@ -23,7 +23,6 @@
 from BALSAMIC.constants.paths import WORKFLOW_PROFILE, CACHE_PROFILE
 from BALSAMIC.utils.cli import (
     validate_cache_version,
-    validate_exome_option,
     validate_umi_min_reads,
 )
 
@@ -222,7 +221,6 @@
     is_flag=True,
     default=False,
     help="Assign exome parameters to TGA workflow",
-    callback=validate_exome_option,
 )
 
 OPTION_FASTQ_PATH = click.option(
diff --git a/BALSAMIC/constants/analysis.py b/BALSAMIC/constants/analysis.py
@@ -14,7 +14,7 @@ class LogFile:
 class SubmitSnakemake:
     """Constants for sbatch script running snakemake on cluster"""
 
-    MAX_RUN_HOURS: int = 120
+    MAX_RUN_HOURS: int = 168
 
 
 class SnakemakeDAG:
diff --git a/BALSAMIC/constants/workflow_profile/config.yaml b/BALSAMIC/constants/workflow_profile/config.yaml
@@ -6,12 +6,10 @@ max-status-checks-per-second: 1
 max-jobs-per-second: 1
 restart-times: 2
 
-default-resources:
-    runtime: 120
-    mem_mb: 4000
-    slurm_partition: "core"
-
 set-resources:
+  all:
+    runtime: 60
+    mem_mb: 4000
   post_process_tnscope_info_fields_wgs:
     runtime: 60
     mem_mb: min(230000, 5000 * attempt)
@@ -471,7 +469,7 @@ set-resources:
     mem_mb: min(230000, 25000 * attempt)
     runtime: 1440
   samtools_fixmate:
-    mem_mb: min(230000, 50000 * attempt)
+    mem_mb: min(230000, 80000 * attempt)
     runtime: 560
   bam_compress_tumor:
     mem_mb: min(230000, 8000 * attempt)
diff --git a/BALSAMIC/models/sbatchsubmitter.py b/BALSAMIC/models/sbatchsubmitter.py
@@ -65,7 +65,7 @@ def _build_sbatch_header(self) -> str:
           - optional partition if `self.headjob_partition` is set
         """
         lines = [
-            "#!/bin/bash -l",
+            "#!/bin/bash",
             f"#SBATCH --account={self.account}",
             f"#SBATCH --job-name=BALSAMIC_snakemake_submit.{self.case_id}.%j",
             f"#SBATCH --output={self.log_path}/BALSAMIC_snakemake_submit.{self.case_id}.%j.out",
diff --git a/BALSAMIC/models/snakemake.py b/BALSAMIC/models/snakemake.py
@@ -137,7 +137,7 @@ def get_command(self) -> str:
 
     def get_slurm_job_arguments(self) -> str:
         return (
-            f'--default-resources slurm_extra="--qos={self.qos}" '
+            f'--default-resources slurm_extra="--qos={self.qos}" runtime=120 mem_mb=4000 '
             f"slurm_partition={self.workflow_partition} slurm_account={self.account}"
         )
 
diff --git a/BALSAMIC/snakemake_rules/annotation/somatic_sv_annotation.rule b/BALSAMIC/snakemake_rules/annotation/somatic_sv_annotation.rule
@@ -12,7 +12,7 @@ rule vep_somatic_research_sv:
     benchmark:
         Path(benchmark_dir, "vep_somatic_research_sv." + config["analysis"]["case_id"] + ".svdb.tsv").as_posix()
     resources:
-        mem_mb = lambda wc: (140000 if config_model.analysis.sequencing_type == SequencingType.WGS else 35000)
+        mem_mb = lambda wc, attempt: min(230000, (140000 if config_model.analysis.sequencing_type == SequencingType.WGS else 35000) * attempt)
     singularity:
         Path(singularity_image, config["bioinfo_tools"].get("ensembl-vep") + ".sif").as_posix()
     params:
diff --git a/BALSAMIC/snakemake_rules/variant_calling/germline_wgs.rule b/BALSAMIC/snakemake_rules/variant_calling/germline_wgs.rule
@@ -10,8 +10,6 @@ rule sentieon_DNAscope_wgs:
         recal_table = bam_dir + "{sample_type}.recal_data.table"
     output:
         vcf = vcf_dir + "SNV.germline.{sample_type}.dnascope.vcf.gz",
-    resources:
-        mem_mb = lambda wc, input: min(max(1.1 * input.size_mb, 10000), 250000) # Cap at 250GB. Scale memory with input size (min 10GB)
     params:
         tmpdir = tempfile.mkdtemp(prefix=tmp_dir),
         pcr_model = params.common.pcr_model,
diff --git a/BALSAMIC/utils/cli.py b/BALSAMIC/utils/cli.py
@@ -572,15 +572,6 @@ def get_analysis_fastq_files_directory(case_dir: str, fastq_path: str) -> str:
     return Path(fastq_path).as_posix()
 
 
-def validate_exome_option(ctx: click.Context, _param: click.Parameter, exome: bool):
-    """Validate that a panel-bed has been supplied together with the exome option."""
-    if exome and not ctx.params.get("panel_bed"):
-        raise click.BadParameter(
-            "If --exome is provided, --panel-bed must also be provided."
-        )
-    return exome
-
-
 def validate_cache_version(
     _ctx: click.Context, _param: click.Parameter, version: str
 ) -> str:
diff --git a/BALSAMIC/workflows/balsamic.smk b/BALSAMIC/workflows/balsamic.smk
@@ -635,6 +635,7 @@ rule all:
         tmp_dir=tmp_dir,
         case_name=config["analysis"]["case_id"],
         status_file=Path(get_result_dir(config), "analysis_status.txt").as_posix(),
+    retries: 0
     message:
         "Finalizing analysis for {params.case_name}"
     run:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -6,11 +6,18 @@ Added:
 
 Changed:
 ^^^^^^^^
+* moved default resource allocation to snakemake command https://github.com/Clinical-Genomics/BALSAMIC/pull/1632
+* increased memory of samtools fixmate https://github.com/Clinical-Genomics/BALSAMIC/pull/1632
+* no rerun for rule all https://github.com/Clinical-Genomics/BALSAMIC/pull/1632
+* increased head-job runtime to 7 days https://github.com/Clinical-Genomics/BALSAMIC/pull/1632
 * improved information on failed job status https://github.com/Clinical-Genomics/BALSAMIC/pull/1636
 * scontrol replaced with sacct in jobstatus script https://github.com/Clinical-Genomics/BALSAMIC/pull/1636
+* add attempt mem bump to vep_somatic_research_sv https://github.com/Clinical-Genomics/BALSAMIC/pull/1632
 
 Removed:
 ^^^^^^^^
+* exome argument panel bed callback function https://github.com/Clinical-Genomics/BALSAMIC/pull/1632
+* removed -l flag in head-job sbatch script https://github.com/Clinical-Genomics/BALSAMIC/pull/1632
 
 Fixed:
 ^^^^^^
diff --git a/docs/install.rst b/docs/install.rst
@@ -25,7 +25,7 @@ Step 1. Installing BALSAMIC
 
 ::
 
-    conda create --name S_balsamic -c conda-forge python=3.11 pip "cython<3" pygraphviz wkhtmltopdf snakemake-executor-plugin-slurm
+    conda create --name S_balsamic -c conda-forge python=3.11 pip "cython<3" pygraphviz wkhtmltopdf snakemake-executor-plugin-slurm=1.7.0
 
 
 
diff --git a/tests/models/test_sbatchsubmitter.py b/tests/models/test_sbatchsubmitter.py
@@ -95,7 +95,7 @@ def submitter_with_partition(tmp_path, monkeypatch, logger, fake_exec):
 def test_build_sbatch_header_includes_partition(submitter_with_partition):
     s = submitter_with_partition
     header = s._build_sbatch_header()
-    assert header.startswith("#!/bin/bash -l")
+    assert header.startswith("#!/bin/bash")
     assert f"#SBATCH --account={s.account}" in header
     assert f"#SBATCH --job-name=BALSAMIC_snakemake_submit.{s.case_id}.%j" in header
     assert (
@@ -132,7 +132,7 @@ def test_build_sbatch_header_omits_partition_when_none(
     )
     header = s._build_sbatch_header()
     assert "#SBATCH --partition=" not in header
-    assert "#!/bin/bash -l" in header
+    assert "#!/bin/bash" in header
     assert "#SBATCH --time=2:00:00" in header
 
 
@@ -197,7 +197,7 @@ def test_create_sbatch_script_writes_once_and_contains_parts(submitter):
         assert isinstance(content, str) and content.strip()
 
         # Contains header + snakemake + status check + success snippet
-        assert "#!/bin/bash -l" in content
+        assert "#!/bin/bash" in content
         assert "snakemake --notemp -p --rerun-triggers mtime" in content
         assert "python " in content and "analysis_status.txt" in content
         assert "analysis_finished_successfully" in content
diff --git a/tests/models/test_snakemake_models.py b/tests/models/test_snakemake_models.py
@@ -198,8 +198,9 @@ def test_get_snakemake_cluster_options(
 
     # THEN the expected format should be returned
     assert snakemake_cluster_options == (
-        f"--slurm-logdir {session_tmp_path} --profile {reference_file.as_posix()} --default-resources "
-        f'slurm_extra="--qos={QOS.HIGH}" slurm_partition={Partition.CORE} slurm_account={ClusterAccount.DEVELOPMENT} '
+        f"--slurm-logdir {session_tmp_path} --profile {reference_file.as_posix()} "
+        f'--default-resources slurm_extra="--qos={QOS.HIGH}" runtime=120 mem_mb=4000 '
+        f"slurm_partition={Partition.CORE} slurm_account={ClusterAccount.DEVELOPMENT} "
         "--slurm-keep-successful-logs"
     )
 
@@ -226,6 +227,6 @@ def test_get_snakemake_command(
         f"--use-singularity --singularity-args '--cleanenv --bind {session_tmp_path.as_posix()}:/' --quiet "
         f"--slurm-logdir {session_tmp_path} "
         f"--profile {reference_file.as_posix()} "
-        f'--default-resources slurm_extra="--qos={QOS.HIGH}" slurm_partition={Partition.CORE} slurm_account={ClusterAccount.DEVELOPMENT} --slurm-keep-successful-logs '
+        f'--default-resources slurm_extra="--qos={QOS.HIGH}" runtime=120 mem_mb=4000 slurm_partition={Partition.CORE} slurm_account={ClusterAccount.DEVELOPMENT} --slurm-keep-successful-logs '
         "--cores 36"
     )
diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py
@@ -37,7 +37,6 @@
     get_sample_list,
     get_snakefile,
     validate_cache_version,
-    validate_exome_option,
     validate_umi_min_reads,
 )
 from BALSAMIC.utils.exc import BalsamicError, WorkflowRunError
@@ -994,23 +993,6 @@ def test_get_fastp_parameters(balsamic_model: ConfigModel):
     assert "--disable_quality_filtering" in fastp_params_tga["fastp_trim_adapter"]
 
 
-def test_validate_exome_option(panel_bed_file: str):
-    # GIVEN that a panel bedfile has been supplied and exome parameter set to true
-    ctx = click.Context(case_config)
-    ctx.params["panel_bed"] = panel_bed_file
-    # WHEN validating exome option
-    # THEN exome argument should be correctly set
-    assert validate_exome_option(ctx, click.Parameter, True) == True
-
-    # GIVEN that a panel bedfile has NOT been supplied and exome parameter set to true
-    ctx.params["panel_bed"] = None
-
-    # WHEN validating exome option
-    # THEN a bad parameter error should be raised
-    with pytest.raises(click.BadParameter):
-        validate_exome_option(ctx, click.Parameter, True)
-
-
 def test_validate_cache_version_develop():
     """Test develop cache version validation."""
 

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,6 @@`
`23`	`23`	`from BALSAMIC.constants.paths import WORKFLOW_PROFILE, CACHE_PROFILE`
`24`	`24`	`from BALSAMIC.utils.cli import (`
`25`	`25`	`validate_cache_version,`
`26`		`- validate_exome_option,`
`27`	`26`	`validate_umi_min_reads,`
`28`	`27`	`)`
`29`	`28`
`@@ -222,7 +221,6 @@`
`222`	`221`	`is_flag=True,`
`223`	`222`	`default=False,`
`224`	`223`	`help="Assign exome parameters to TGA workflow",`
`225`		`- callback=validate_exome_option,`
`226`	`224`	`)`
`227`	`225`
`228`	`226`	`OPTION_FASTQ_PATH = click.option(`
Original file line number	Diff line number	Diff line change
`@@ -137,7 +137,7 @@ def get_command(self) -> str:`
`137`	`137`
`138`	`138`	`def get_slurm_job_arguments(self) -> str:`
`139`	`139`	`return (`
`140`		`- f'--default-resources slurm_extra="--qos={self.qos}" '`
	`140`	`+ f'--default-resources slurm_extra="--qos={self.qos}" runtime=120 mem_mb=4000 '`
`141`	`141`	`f"slurm_partition={self.workflow_partition} slurm_account={self.account}"`
`142`	`142`	`)`
`143`	`143`