diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d5561651..2bec0cc32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Add nf-core template version badges to README ([#3396](https://github.com/nf-core/tools/pull/3396)) - Add Bluesky badge to readme ([#3475](https://github.com/nf-core/tools/pull/3475)) - Run awsfulltest after release, and with dev revision on PRs to master ([#3485](https://github.com/nf-core/tools/pull/3485)) +- Separate `PIPELINE_COMPLETION` in its own file ([#3486](https://github.com/nf-core/tools/pull/3486)) ### Linting @@ -16,35 +17,39 @@ - Fix: linting with comments after the input directive ([#3458](https://github.com/nf-core/tools/pull/3458)) - EDAM ontology fixes ([#3460](https://github.com/nf-core/tools/pull/3460)) - Fix default linting of nf-core components when `nf-core pipelines lint` is ran ([#3480](https://github.com/nf-core/tools/pull/3480)) +- Fix components linting in `nf-core subworkflows lint` with multiple workflows in a `main.nf` ([#3486](https://github.com/nf-core/tools/pull/3486)) ### Modules -- increase meta index for multiple input channels ([#3463](https://github.com/nf-core/tools/pull/3463)) +- Increase meta index for multiple input channels ([#3463](https://github.com/nf-core/tools/pull/3463)) - Configure the default module repository, branch, and path from environment variables. ([#3481](https://github.com/nf-core/tools/pull/3481)) ### Subworkflows +- Forbid more than one `workflow` block by `main.nf` ([#3486](https://github.com/nf-core/tools/pull/3486)) + ### General -- output passed to write_params_file as Path object ([#3435](https://github.com/nf-core/tools/pull/3435)) -- chore(deps): update python:3.12-slim docker digest to 69ce3ae ([#3433](https://github.com/nf-core/tools/pull/3433)) -- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.4 ([#3438](https://github.com/nf-core/tools/pull/3438)) -- format name/value with YAML syntax ([#3442](https://github.com/nf-core/tools/pull/3442)) -- chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.2.0 ([#3446](https://github.com/nf-core/tools/pull/3446)) -- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.5 ([#3445](https://github.com/nf-core/tools/pull/3445)) -- chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.15.0 ([#3447](https://github.com/nf-core/tools/pull/3447)) +- Output passed to write_params_file as Path object ([#3435](https://github.com/nf-core/tools/pull/3435)) +- Chore(deps): update python:3.12-slim docker digest to 69ce3ae ([#3433](https://github.com/nf-core/tools/pull/3433)) +- Chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.4 ([#3438](https://github.com/nf-core/tools/pull/3438)) +- Format name/value with YAML syntax ([#3442](https://github.com/nf-core/tools/pull/3442)) +- Chore(deps): update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.2.0 ([#3446](https://github.com/nf-core/tools/pull/3446)) +- Chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.5 ([#3445](https://github.com/nf-core/tools/pull/3445)) +- Chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.15.0 ([#3447](https://github.com/nf-core/tools/pull/3447)) - Update prettier to 3.5.0 ([#3448](https://github.com/nf-core/tools/pull/3448)) -- chore(deps): update python:3.12-slim docker digest to 34656cd ([#3450](https://github.com/nf-core/tools/pull/3450)) +- Chore(deps): update python:3.12-slim docker digest to 34656cd ([#3450](https://github.com/nf-core/tools/pull/3450)) - Remove Twitter from README ([#3454](https://github.com/nf-core/tools/pull/3454)) -- docs: fix contributing link in the main README ([#3459](https://github.com/nf-core/tools/pull/3459)) +- Docs: fix contributing link in the main README ([#3459](https://github.com/nf-core/tools/pull/3459)) - Cleanup: Removed Redundant if Condition ([#3468](https://github.com/nf-core/tools/pull/3468)) -- chore(deps): update gitpod/workspace-base docker digest to 7f35e40 ([#3473](https://github.com/nf-core/tools/pull/3473)) -- chore(deps): update python:3.12-slim docker digest to aaa3f8c ([#3474](https://github.com/nf-core/tools/pull/3474)) -- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.9 ([#3470](https://github.com/nf-core/tools/pull/3470)) -- chore(deps): update github actions ([#3488](https://github.com/nf-core/tools/pull/3488)) -- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.11.0 ([#3492](https://github.com/nf-core/tools/pull/3492)) -- chore(deps): update dependency textual to v2 ([#3471](https://github.com/nf-core/tools/pull/3471)) +- Chore(deps): update gitpod/workspace-base docker digest to 7f35e40 ([#3473](https://github.com/nf-core/tools/pull/3473)) +- Chore(deps): update python:3.12-slim docker digest to aaa3f8c ([#3474](https://github.com/nf-core/tools/pull/3474)) +- Chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.9 ([#3470](https://github.com/nf-core/tools/pull/3470)) +- Chore(deps): update github actions ([#3488](https://github.com/nf-core/tools/pull/3488)) +- Chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.11.0 ([#3492](https://github.com/nf-core/tools/pull/3492)) +- Chore(deps): update dependency textual to v2 ([#3471](https://github.com/nf-core/tools/pull/3471)) - Install subworkflows with modules from different remotes ([#3083](https://github.com/nf-core/tools/pull/3083)) +- Remove duplicated `pdiff` in `requirements.txt` ([#3486](https://github.com/nf-core/tools/pull/3486)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 70bdc274e..8dc55e4db 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -20,7 +20,7 @@ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' {%- if modules %} include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' -include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline_completion' {%- if igenomes %} include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_completion/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_completion/main.nf new file mode 100644 index 000000000..e15daaa83 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_completion/main.nf @@ -0,0 +1,86 @@ +// +// Subworkflow with for pipeline completion +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +{% if nf_schema %} +include { paramsSummaryMap } from 'plugin/nf-schema' +{% endif %} +{%- if email %} +include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +{%- endif %} +include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' +{%- if adaptivecard or slackreport %} +include { imNotification } from '../../nf-core/utils_nfcore_pipeline' +{%- endif %} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW FOR PIPELINE COMPLETION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow PIPELINE_COMPLETION { + + take: + {%- if email %} + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + {%- endif %} + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + {%- if adaptivecard or slackreport %} + hook_url // string: hook URL for notifications + {% endif %} + {%- if multiqc %} + multiqc_report // string: Path to MultiQC report + {% endif %} + + main: + {%- if nf_schema %} + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + {%- else %} + summary_params = [:] + {%- endif %} + + {%- if multiqc %} + def multiqc_reports = multiqc_report.toList() + {%- endif %} + + // + // Completion email and summary + // + workflow.onComplete { + {%- if email %} + if (email || email_on_fail) { + completionEmail( + summary_params, + email, + email_on_fail, + plaintext_email, + outdir, + monochrome_logs, + {% if multiqc %}multiqc_reports.getVal(),{% else %}[]{% endif %} + ) + } + {%- endif %} + + completionSummary(monochrome_logs) + + {%- if adaptivecard or slackreport %} + if (hook_url) { + imNotification(summary_params, hook_url) + } + {%- endif %} + } + + workflow.onError { + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + } +} diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_completion/meta.yml b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_completion/meta.yml new file mode 100644 index 000000000..e11aa9d47 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_completion/meta.yml @@ -0,0 +1,39 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "pipeline_completion" +description: Notify user of pipeline completion +keywords: + - notification + - email + - completion +components: [] +input: + - email: + type: string + description: Email address + - email_on_fail: + type: string + description: Email address sent on pipeline failure + - plaintext_email: + type: boolean + description: Send plain-text email instead of HTML + - outdir: + type: string + description: Path to output directory where results will be published + - monochrome_logs: + type: boolean + description: Disable ANSI colour codes in log output + - hook_url: + type: string + description: Hook URL for notifications + - multiqc_report: + type: string + description: Path to MultiQC report +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nf-core-bot" +maintainers: + - "@mirpedrol" diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 06faf35d7..cfb7e89f8 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -9,15 +9,7 @@ */ {% if nf_schema %}include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' -include { paramsSummaryMap } from 'plugin/nf-schema' include { samplesheetToList } from 'plugin/nf-schema'{% endif %} -{%- if email %} -include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' -{%- endif %} -include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -{%- if adaptivecard or slackreport %} -include { imNotification } from '../../nf-core/utils_nfcore_pipeline' -{%- endif %} include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' @@ -112,70 +104,6 @@ workflow PIPELINE_INITIALISATION { versions = ch_versions } -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - SUBWORKFLOW FOR PIPELINE COMPLETION -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -workflow PIPELINE_COMPLETION { - - take: - {%- if email %} - email // string: email address - email_on_fail // string: email address sent on pipeline failure - plaintext_email // boolean: Send plain-text email instead of HTML - {%- endif %} - outdir // path: Path to output directory where results will be published - monochrome_logs // boolean: Disable ANSI colour codes in log output - {%- if adaptivecard or slackreport %} - hook_url // string: hook URL for notifications{% endif %} - {%- if multiqc %} - multiqc_report // string: Path to MultiQC report{% endif %} - - main: - {%- if nf_schema %} - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - {%- else %} - summary_params = [:] - {%- endif %} - - {%- if multiqc %} - def multiqc_reports = multiqc_report.toList() - {%- endif %} - - // - // Completion email and summary - // - workflow.onComplete { - {%- if email %} - if (email || email_on_fail) { - completionEmail( - summary_params, - email, - email_on_fail, - plaintext_email, - outdir, - monochrome_logs, - {% if multiqc %}multiqc_reports.getVal(),{% else %}[]{% endif %} - ) - } - {%- endif %} - - completionSummary(monochrome_logs) - - {%- if adaptivecard or slackreport %} - if (hook_url) { - imNotification(summary_params, hook_url) - } - {%- endif %} - } - - workflow.onError { - log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" - } -} - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FUNCTIONS diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/meta.yml b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/meta.yml new file mode 100644 index 000000000..22740c986 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/meta.yml @@ -0,0 +1,48 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "pipeline_initialisation" +description: Initialise the pipeline parameters, validate them and prepare the samplesheet +keywords: + - initialisation + - validation + - parameters +components: [] +input: + - version: + type: boolean + description: Display version and exit + - validate_params: + type: boolean + description: Boolean whether to validate parameters against the schema at runtime + - monochrome_logs: + type: boolean + description: Do not use coloured log outputs + - nextflow_cli_args: + type: array + description: List of positional nextflow CLI args + - outdir: + type: string + description: The output directory where the results will be saved + - input: + type: string + description: Path to input samplesheet +output: + - samplesheet: + description: Channel with the processed output of the samplesheet + structure: + - meta: + type: map + description: Metadata map + - fastq: + type: File + description: Fastq file + pattern: "*.fastq" + - versions: + description: Channel containing software versions file + structure: + - versions.yml: + type: file + description: File containing versions of the software used +authors: + - "@mirpedrol" +maintainers: + - "@mirpedrol" diff --git a/nf_core/pipelines/create/create.py b/nf_core/pipelines/create/create.py index 1800a5f10..3237e89b4 100644 --- a/nf_core/pipelines/create/create.py +++ b/nf_core/pipelines/create/create.py @@ -314,6 +314,9 @@ def render_template(self) -> None: rename_files: Dict[str, str] = { "workflows/pipeline.nf": f"workflows/{short_name}.nf", "subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf": f"subworkflows/local/utils_nfcore_{short_name}_pipeline/main.nf", + "subworkflows/local/utils_nfcore_pipeline_pipeline/meta.yml": f"subworkflows/local/utils_nfcore_{short_name}_pipeline/meta.yml", + "subworkflows/local/utils_nfcore_pipeline_completion/main.nf": f"subworkflows/local/utils_nfcore_{short_name}_pipeline_completion/main.nf", + "subworkflows/local/utils_nfcore_pipeline_completion/meta.yml": f"subworkflows/local/utils_nfcore_{short_name}_pipeline_completion/meta.yml", } # Set the paths to skip according to customization diff --git a/nf_core/subworkflows/lint/main_nf.py b/nf_core/subworkflows/lint/main_nf.py index 3ad3f3486..e3a70a21b 100644 --- a/nf_core/subworkflows/lint/main_nf.py +++ b/nf_core/subworkflows/lint/main_nf.py @@ -47,11 +47,22 @@ def main_nf(_, subworkflow: NFCoreComponent) -> Tuple[List[str], List[str]]: # Perform section-specific linting state = "subworkflow" subworkflow_lines = [] + subworkflow_nb = 0 workflow_lines = [] main_lines = [] + open_brackets = close_brackets = 0 for line in lines: + # Keep track of open and close brackets + if "{" in line: + open_brackets += line.count("{") + if "}" in line: + close_brackets += line.count("}") + if open_brackets - close_brackets == 0: + state = "subworkflow" + if re.search(r"^\s*workflow\s*\w*\s*{", line) and state == "subworkflow": state = "workflow" + subworkflow_nb += 1 if re.search(r"take\s*:", line) and state in ["workflow"]: state = "take" continue @@ -80,6 +91,16 @@ def main_nf(_, subworkflow: NFCoreComponent) -> Tuple[List[str], List[str]]: else: subworkflow.passed.append(("main_nf_script_outputs", "Workflow 'emit' block found", subworkflow.main_nf)) + # Check for only one workflow per file + if subworkflow_nb > 1: + subworkflow.warned.append( + ("main_nf_script_outputs", "More than one 'workflow' block found", subworkflow.main_nf) + ) + elif subworkflow_nb == 0: + subworkflow.failed.append(("main_nf_script_outputs", "No 'workflow' block found", subworkflow.main_nf)) + else: + subworkflow.passed.append(("main_nf_script_outputs", "One 'workflow' block found", subworkflow.main_nf)) + # Check the subworkflow include statements included_components = check_subworkflow_section(subworkflow, subworkflow_lines) diff --git a/requirements-dev.txt b/requirements-dev.txt index 04c6372d7..cbf11b680 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,12 +3,9 @@ myst_parser pytest-cov pytest-datafiles responses -ruff Sphinx sphinx-rtd-theme textual-dev==1.5.1 -types-PyYAML -types-requests types-jsonschema types-Markdown types-PyYAML diff --git a/requirements.txt b/requirements.txt index 32311598a..b3ed70f68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ jsonschema>=4.0 markdown>=3.3 packaging pillow -pdiff pre-commit prompt_toolkit<=3.0.48 pydantic>=2.2.1