diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..68f01d2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,49 @@ +name: Bug report +description: Report something that is broken or incorrect +labels: bug +body: + - type: markdown + attributes: + value: | + Before you post this issue, please check the documentation: + + - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) + - [nf-core/biodivpipeline pipeline documentation](https://nf-co.re/biodivpipeline/usage) + - type: textarea + id: description + attributes: + label: Description of the bug + description: A clear and concise description of what the bug is. + validations: + required: true + + - type: textarea + id: command_used + attributes: + label: Command used and terminal output + description: Steps to reproduce the behaviour. Please paste the command you used to launch the pipeline and the output from your terminal. + render: console + placeholder: | + $ nextflow run ... + + Some output where something broke + + - type: textarea + id: files + attributes: + label: Relevant files + description: | + Please drag and drop the relevant files here. Create a `.zip` archive if the extension is not allowed. + Your verbose log file `.nextflow.log` is often useful _(this is a hidden file in the directory where you launched the pipeline)_ as well as custom Nextflow configuration files. + + - type: textarea + id: system + attributes: + label: System information + description: | + * Nextflow version _(eg. 23.04.0)_ + * Hardware _(eg. HPC, Desktop, Cloud)_ + * Executor _(eg. slurm, local, awsbatch)_ + * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ + * OS _(eg. CentOS Linux, macOS, Linux Mint)_ + * Version of nf-core/biodivpipeline _(eg. 1.1, 1.5, 1.8.2)_ diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..bed11b0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +contact_links: + - name: Join nf-core + url: https://nf-co.re/join + about: Please join the nf-core community here + - name: "Slack #biodivpipeline channel" + url: https://nfcore.slack.com/channels/biodivpipeline + about: Discussion about the nf-core/biodivpipeline pipeline diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..b417a10 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,11 @@ +name: Feature request +description: Suggest an idea for the nf-core/biodivpipeline pipeline +labels: enhancement +body: + - type: textarea + id: description + attributes: + label: Description of feature + description: Please describe your suggestion for a new feature. It might help to describe a problem or use case, plus any alternatives that you have considered. + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..101f103 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ + + +## PR checklist + +- [ ] This comment contains a description of changes (with reason). +- [ ] If you've fixed a bug or added code that should be tested, add tests! +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/biodivpipeline/tree/master/docs/CONTRIBUTING.md) +- [ ] If necessary, also make a PR on the nf-core/biodivpipeline _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] Make sure your code lints (`nf-core pipelines lint`). +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). +- [ ] Usage Documentation in `docs/usage.md` is updated. +- [ ] Output Documentation in `docs/output.md` is updated. +- [ ] `CHANGELOG.md` is updated. +- [ ] `README.md` is updated (including new tool citations and authors/contributors). diff --git a/.github/actions/get-shards/action.yml b/.github/actions/get-shards/action.yml new file mode 100644 index 0000000..e2833ee --- /dev/null +++ b/.github/actions/get-shards/action.yml @@ -0,0 +1,69 @@ +name: "Get number of shards" +description: "Get the number of nf-test shards for the current CI job" +inputs: + max_shards: + description: "Maximum number of shards allowed" + required: true + paths: + description: "Component paths to test" + required: false + tags: + description: "Tags to pass as argument for nf-test --tag parameter" + required: false +outputs: + shard: + description: "Array of shard numbers" + value: ${{ steps.shards.outputs.shard }} + total_shards: + description: "Total number of shards" + value: ${{ steps.shards.outputs.total_shards }} +runs: + using: "composite" + steps: + - name: Install nf-test + uses: nf-core/setup-nf-test@4069fbbaabe94c08faba4ad261bfa88225ba133f # v2 + with: + version: ${{ env.NFT_VER }} + - name: Get number of shards + id: shards + shell: bash + run: | + # Run nf-test with dynamic parameter + nftest_output=$(nf-test test \ + --profile +docker \ + $(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \ + --dry-run \ + --ci \ + --changed-since HEAD^) || { + echo "nf-test command failed with exit code $?" + echo "Full output: $nftest_output" + exit 1 + } + echo "nf-test dry-run output: $nftest_output" + + # Default values for shard and total_shards + shard="[]" + total_shards=0 + + # Check if there are related tests + if echo "$nftest_output" | grep -q 'No tests to execute'; then + echo "No related tests found." + else + # Extract the number of related tests + number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p') + if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then + shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} )) + shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .) + total_shards="$shards_to_run" + else + echo "Unexpected output format. Falling back to default values." + fi + fi + + # Write to GitHub Actions outputs + echo "shard=$shard" >> $GITHUB_OUTPUT + echo "total_shards=$total_shards" >> $GITHUB_OUTPUT + + # Debugging output + echo "Final shard array: $shard" + echo "Total number of shards: $total_shards" diff --git a/.github/actions/nf-test/action.yml b/.github/actions/nf-test/action.yml new file mode 100644 index 0000000..ad686e8 --- /dev/null +++ b/.github/actions/nf-test/action.yml @@ -0,0 +1,111 @@ +name: "nf-test Action" +description: "Runs nf-test with common setup steps" +inputs: + profile: + description: "Profile to use" + required: true + shard: + description: "Shard number for this CI job" + required: true + total_shards: + description: "Total number of test shards(NOT the total number of matrix jobs)" + required: true + paths: + description: "Test paths" + required: true + tags: + description: "Tags to pass as argument for nf-test --tag parameter" + required: false +runs: + using: "composite" + steps: + - name: Setup Nextflow + uses: nf-core/setup-nextflow@b4ec1bc7c16a94435159de94a05253542fddf6ef # v3 + with: + version: "${{ env.NXF_VERSION }}" + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.14" + + - name: Install nf-test + uses: nf-core/setup-nf-test@4069fbbaabe94c08faba4ad261bfa88225ba133f # v2 + with: + version: "${{ env.NFT_VER }}" + install-pdiff: true + + - name: Setup apptainer + if: contains(inputs.profile, 'singularity') + uses: eWaterCycle/setup-apptainer@3f706d898c9db585b1d741b4692e66755f3a1b40 # v2 + + - name: Set up Singularity + if: contains(inputs.profile, 'singularity') + shell: bash + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Conda setup + if: contains(inputs.profile, 'conda') + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4 + with: + auto-update-conda: true + conda-solver: libmamba + channels: conda-forge + channel-priority: strict + conda-remove-defaults: true + + - name: Run nf-test + shell: bash + env: + NFT_WORKDIR: ${{ env.NFT_WORKDIR }} + run: | + nf-test test \ + --profile=+${{ inputs.profile }} \ + $(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \ + --ci \ + --changed-since HEAD^ \ + --verbose \ + --tap=test.tap \ + --shard ${{ inputs.shard }}/${{ inputs.total_shards }} + + # Save the absolute path of the test.tap file to the output + echo "tap_file_path=$(realpath test.tap)" >> $GITHUB_OUTPUT + + - name: Generate test summary + if: always() + shell: bash + run: | + # Add header if it doesn't exist (using a token file to track this) + if [ ! -f ".summary_header" ]; then + echo "# 🚀 nf-test results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Status | Test Name | Profile | Shard |" >> $GITHUB_STEP_SUMMARY + echo "|:------:|-----------|---------|-------|" >> $GITHUB_STEP_SUMMARY + touch .summary_header + fi + + if [ -f test.tap ]; then + while IFS= read -r line; do + if [[ $line =~ ^ok ]]; then + test_name="${line#ok }" + # Remove the test number from the beginning + test_name="${test_name#* }" + echo "| ✅ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + elif [[ $line =~ ^not\ ok ]]; then + test_name="${line#not ok }" + # Remove the test number from the beginning + test_name="${test_name#* }" + echo "| ❌ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + fi + done < test.tap + else + echo "| ⚠ | No test results found | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY + fi + + - name: Clean up + if: always() + shell: bash + run: | + sudo rm -rf /home/ubuntu/tests/ diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml new file mode 100644 index 0000000..77431ef --- /dev/null +++ b/.github/workflows/branch.yml @@ -0,0 +1,46 @@ +name: nf-core branch protection +# This workflow is triggered on PRs to `main`/`master` branch on the repository +# It fails when someone tries to make a PR against the nf-core `main`/`master` branch instead of `dev` +on: + pull_request_target: + branches: + - main + - master + +jobs: + test: + runs-on: ubuntu-latest + steps: + # PRs to the nf-core repo main/master branch are only ok if coming from the nf-core repo `dev` or any `patch` branches + - name: Check PRs + if: github.repository == 'nf-core/biodivpipeline' + run: | + { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/biodivpipeline ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + + # If the above check failed, post a comment on the PR explaining the failure + # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0 # v3 + with: + message: | + ## This PR is against the `${{github.event.pull_request.base.ref}}` branch :x: + + * Do not close this PR + * Click _Edit_ and change the `base` to `dev` + * This CI test will remain failed until you push a new commit + + --- + + Hi @${{ github.event.pull_request.user.login }}, + + It looks like this pull-request is has been made against the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) ${{github.event.pull_request.base.ref}} branch. + The ${{github.event.pull_request.base.ref}} branch on nf-core repositories should always contain code from the latest release. + Because of this, PRs to ${{github.event.pull_request.base.ref}} are only allowed if they come from the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) `dev` branch. + + You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page. + Note that even after this, the test will continue to show as failing until you push a new commit. + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 0000000..8738ffc --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,76 @@ +name: nf-core linting +# This workflow is triggered on pushes and PRs to the repository. +# It runs the `nf-core pipelines lint` and markdown lint tests to ensure +# that the code meets the nf-core guidelines. +on: + pull_request: + release: + types: [published] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Install Nextflow + uses: nf-core/setup-nextflow@b4ec1bc7c16a94435159de94a05253542fddf6ef # v3 + + - name: Run prek + uses: j178/prek-action@6ad80277337ad479fe43bd70701c3f7f8aa74db3 # v2 + + nf-core: + runs-on: ubuntu-latest + steps: + - name: Check out pipeline code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Install Nextflow + uses: nf-core/setup-nextflow@b4ec1bc7c16a94435159de94a05253542fddf6ef # v3 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.14" + architecture: "x64" + + - name: Setup uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: read .nf-core.yml + uses: pietrobolcato/action-read-yaml@9f13718d61111b69f30ab4ac683e67a56d254e1d # 1.1.0 + id: read_yml + with: + config: ${{ github.workspace }}/.nf-core.yml + + - name: Install dependencies + run: uv tool install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + + - name: Run nf-core pipelines lint + if: ${{ github.base_ref != 'master' || github.base_ref != 'main' }} + env: + GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} + run: nf-core -l lint_log.txt pipelines lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + + - name: Run nf-core pipelines lint --release + if: ${{ github.base_ref == 'master' || github.base_ref == 'main' }} + env: + GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} + run: nf-core -l lint_log.txt pipelines lint --release --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + + - name: Save PR number + if: ${{ always() }} + run: echo ${{ github.event.pull_request.number }} > PR_number.txt + + - name: Upload linting log file artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: linting-logs + path: | + lint_log.txt + lint_results.md + PR_number.txt diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 0000000..efd72d6 --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,144 @@ +name: Run nf-test +on: + pull_request: + paths-ignore: + - "docs/**" + - "**/meta.yml" + - "**/*.md" + - "**/*.png" + - "**/*.svg" + release: + types: [published] + workflow_dispatch: + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NFT_VER: "0.9.4" + NFT_WORKDIR: "~" + NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + +jobs: + nf-test-changes: + name: nf-test-changes + runs-on: # use self-hosted runners + - runs-on=${{ github.run_id }}-nf-test-changes + - runner=4cpu-linux-x64 + outputs: + shard: ${{ steps.set-shards.outputs.shard }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} + steps: + - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner + run: | + ls -la ./ + rm -rf ./* || true + rm -rf ./.??* || true + ls -la ./ + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: get number of shards + id: set-shards + uses: ./.github/actions/get-shards + env: + NFT_VER: ${{ env.NFT_VER }} + with: + max_shards: 7 + + - name: debug + run: | + echo ${{ steps.set-shards.outputs.shard }} + echo ${{ steps.set-shards.outputs.total_shards }} + + nf-test: + name: "${{ matrix.profile }} | ${{ matrix.NXF_VER }} | ${{ matrix.shard }}/${{ needs.nf-test-changes.outputs.total_shards }}" + needs: [nf-test-changes] + if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} + runs-on: # use self-hosted runners + - runs-on=${{ github.run_id }}-nf-test + - runner=4cpu-linux-x64 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }} + profile: [conda, docker, singularity] + isMain: + - ${{ github.base_ref == 'master' || github.base_ref == 'main' }} + # Exclude conda and singularity on dev + exclude: + - isMain: false + profile: "conda" + - isMain: false + profile: "singularity" + NXF_VER: + - "25.10.4" + - "latest-everything" + env: + NXF_ANSI_LOG: false + TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Run nf-test + id: run_nf_test + uses: ./.github/actions/nf-test + continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }} + env: + NFT_WORKDIR: ${{ env.NFT_WORKDIR }} + NXF_VERSION: ${{ matrix.NXF_VER }} + with: + profile: ${{ matrix.profile }} + shard: ${{ matrix.shard }} + total_shards: ${{ env.TOTAL_SHARDS }} + + - name: Report test status + if: ${{ always() }} + run: | + if [[ "${{ steps.run_nf_test.outcome }}" == "failure" ]]; then + echo "::error::Test with ${{ matrix.NXF_VER }} failed" + # Add to workflow summary + echo "## ❌ Test failed: ${{ matrix.profile }} | ${{ matrix.NXF_VER }} | Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }}" >> $GITHUB_STEP_SUMMARY + if [[ "${{ matrix.NXF_VER }}" == "latest-everything" ]]; then + echo "::warning::Test with latest-everything failed but will not cause workflow failure. Please check if the error is expected or if it needs fixing." + fi + if [[ "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then + exit 1 + fi + fi + + confirm-pass: + needs: [nf-test] + if: always() + runs-on: # use self-hosted runners + - runs-on=${{ github.run_id }}-confirm-pass + - runner=2cpu-linux-x64 + steps: + - name: One or more tests failed (excluding latest-everything) + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: One or more tests cancelled + if: ${{ contains(needs.*.result, 'cancelled') }} + run: exit 1 + + - name: All tests ok + if: ${{ contains(needs.*.result, 'success') }} + run: exit 0 + + - name: debug-print + if: always() + run: | + echo "::group::DEBUG: `needs` Contents" + echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" + echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" + echo "::endgroup::" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f562229 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Nextflow +.nextflow* +work/ +results/ +.lineage/ +null/ + +# Data (too large for git — download separately) +Belege_aus_D.csv + +# OS +.DS_Store + +# Python +*.pyc +__pycache__/ +.venv/ +*.egg-info/ + +# Testing +testing/ +testing* + +# IDE +.vscode/ + +# Docker +*.tar diff --git a/.nf-core.yml b/.nf-core.yml new file mode 100644 index 0000000..084a29d --- /dev/null +++ b/.nf-core.yml @@ -0,0 +1,27 @@ +repository_type: pipeline +nf_core_version: 4.0.2 +lint: + actions_awsfulltest: false + actions_awstest: false + actions_nf_test: false + actions_schema_validation: false + files_unchanged: false + files_exist: + - .github/workflows/branch.yml + - .github/workflows/ci.yml + - .github/workflows/linting.yml + - .github/workflows/linting_comment.yml + pipeline_todos: false + multiqc_config: false + rocrate_readme_sync: false + modules_json: false +template: + org: nf-core + name: biodivpipeline + description: Modular nf-core workflow for FAIR biodiversity data processing + author: "" + version: 1.0.0dev + force: true + outdir: nf-core-biodivpipeline + is_nfcore: true + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f51e1a2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.1.0" + hooks: + - id: prettier + additional_dependencies: + - prettier@3.8.3 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + exclude: | + (?x)^( + .*ro-crate-metadata.json$| + modules/(?!local/).*| + subworkflows/(?!local/).*| + .*\.snap$ + )$ + - id: end-of-file-fixer + exclude: | + (?x)^( + .*ro-crate-metadata.json$| + modules/(?!local/).*| + subworkflows/(?!local/).*| + .*\.snap$ + )$ + - repo: https://github.com/seqeralabs/nf-lint-pre-commit + rev: v0.3.0 + hooks: + - id: nextflow-lint + files: '\.nf$|nextflow\.config$' + args: ["-output", "json"] diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..63cde50 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,14 @@ +email_template.html +.nextflow* +work/ +data/ +results/ +.DS_Store +testing/ +testing* +*.pyc +bin/ +.nf-test/ +ro-crate-metadata.json +modules/nf-core/ +subworkflows/nf-core/ diff --git a/.prettierrc.yml b/.prettierrc.yml new file mode 100644 index 0000000..07dbd8b --- /dev/null +++ b/.prettierrc.yml @@ -0,0 +1,6 @@ +printWidth: 120 +tabWidth: 4 +overrides: + - files: "*.{md,yml,yaml,html,css,scss,js,cff}" + options: + tabWidth: 2 diff --git a/CITATIONS.md b/CITATIONS.md new file mode 100644 index 0000000..473e11b --- /dev/null +++ b/CITATIONS.md @@ -0,0 +1,27 @@ +# nf-core/biodivpipeline: Citations + +## [nf-core](https://pubmed.ncbi.nlm.nih.gov/32055031/) + +> Ewels PA, Peltzer A, Fillinger S, Patel H, Alneberg J, Wilm A, Garcia MU, Di Tommaso P, Nahnsen S. The nf-core framework for community-curated bioinformatics pipelines. Nat Biotechnol. 2020 Mar;38(3):276-278. doi: 10.1038/s41587-020-0439-x. PubMed PMID: 32055031. + +## [Nextflow](https://pubmed.ncbi.nlm.nih.gov/28398311/) + +> Di Tommaso P, Chatzou M, Floden EW, Barja PP, Palumbo E, Notredame C. Nextflow enables reproducible computational workflows. Nat Biotechnol. 2017 Apr 11;35(4):316-319. doi: 10.1038/nbt.3820. PubMed PMID: 28398311. + +## Software packaging/containerisation tools + +- [Bioconda](https://pubmed.ncbi.nlm.nih.gov/29967506/) + + > GrĂŒning B, Dale R, Sjödin A, Chapman BA, Rowe J, Tomkins-Tinch CH, Valieris R, Köster J; Bioconda Team. Bioconda: sustainable and comprehensive software distribution for the life sciences. Nat Methods. 2018 Jul;15(7):475-476. doi: 10.1038/s41592-018-0046-7. PubMed PMID: 29967506. + +- [BioContainers](https://pubmed.ncbi.nlm.nih.gov/28379341/) + + > da Veiga Leprevost F, GrĂŒning B, Aflitos SA, Röst HL, Uszkoreit J, Barsnes H, Vaudel M, Moreno P, Gatto L, Weber J, Bai M, Jimenez RC, Sachsenberg T, Pfeuffer J, Alvarez RV, Griss J, Nesvizhskii AI, Perez-Riverol Y. BioContainers: an open-source and community-driven framework for software standardization. Bioinformatics. 2017 Aug 15;33(16):2580-2582. doi: 10.1093/bioinformatics/btx192. PubMed PMID: 28379341; PubMed Central PMCID: PMC5870671. + +- [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241) + + > Merkel, D. (2014). Docker: lightweight linux containers for consistent development and deployment. Linux Journal, 2014(239), 2. doi: 10.5555/2600239.2600241. + +- [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/) + + > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0f0c660 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) The nf-core/biodivpipeline team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 044dda7..7015f26 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,54 @@ # BiodivPipeline -A Modular nf-core Workflow for FAIR Biodiversity Data Processing + +A modular nf-core / Nextflow pipeline scaffold that orchestrates pluggable modules to transform biodiversity CSV records into FAIR-compliant RDF outputs. + +**Framework:** [nf-core](https://nf-co.re) template v4.0.2 + +## Quick start + +**Prerequisites:** Java 11+, [Nextflow](https://www.nextflow.io/) >= 24.x, [Docker](https://www.docker.com/) (optional) + +```bash +# Run with the bundled mock dataset +nextflow run main.nf -profile test --outdir results + +# Run with your own input +nextflow run main.nf -profile docker \ + --input path/to/biodiv.csv \ + --outdir results +``` + +### Parameters + +| Parameter | Required | Description | +| ------------------ | -------- | ------------------------------------------------------------------------------------------- | +| `--input` | Yes | Path to input CSV file (biodiversity records) | +| `--mapping_schema` | No | Path to RDF mapping schema (JSON-LD or Turtle). Defaults to `assets/default_mapping.jsonld` | +| `--outdir` | Yes | Path to output directory | +| `-profile` | Yes | Execution profile: `docker`, `singularity`, `test` | + +## Output + +Results are published under `--outdir`. See [docs/output.md](docs/output.md) for the directory layout. + +## Project structure + +``` +BiodivPipeline/ +├── main.nf # Pipeline entry point +├── workflows/biodivpipeline.nf # Workflow DAG +├── modules/local/ # Pipeline modules (one per work package) +├── conf/ # Nextflow config profiles +├── assets/ # Static files (default mapping schema) +├── test_data/ # Mock dataset for the test profile +├── subworkflows/ # nf-core shared utilities +└── docs/ # Documentation +``` + +## Documentation + +- [Usage](docs/usage.md) — how to run, parameters, profiles +- [Output](docs/output.md) — output directory reference +- [Contributing](docs/CONTRIBUTING.md) — git conventions, branch model, module structure + +Built on the [nf-core](https://nf-co.re) framework — see [`CITATIONS.md`](CITATIONS.md). diff --git a/assets/default_mapping.jsonld b/assets/default_mapping.jsonld new file mode 100644 index 0000000..491dcfb --- /dev/null +++ b/assets/default_mapping.jsonld @@ -0,0 +1,7 @@ +{ + "@context": {}, + "@type": "MappingSchema", + "name": "Placeholder mapping schema", + "description": "Empty placeholder. Real mapping schema is owned by the RDF transform module team.", + "mappings": [] +} diff --git a/conf/base.config b/conf/base.config new file mode 100644 index 0000000..ae6a179 --- /dev/null +++ b/conf/base.config @@ -0,0 +1,66 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + nf-core/biodivpipeline Nextflow base config file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + A 'blank slate' config file, appropriate for general use on most high performance + compute environments. Assumes that all software is installed and available on + the PATH. Runs in `local` mode - all jobs will be run on the logged in environment. +---------------------------------------------------------------------------------------- +*/ + +process { + + // TODO nf-core: Check the defaults for all processes + cpus = { 1 * task.attempt } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } + + errorStrategy = { task.exitStatus in ((130..145) + 104 + (175..177)) ? 'retry' : 'finish' } + maxRetries = 1 + maxErrors = '-1' + + // Process-specific resource requirements + // NOTE - Please try and reuse the labels below as much as possible. + // These labels are used and recognised by default in DSL2 files hosted on nf-core/modules. + // If possible, it would be nice to keep the same label naming convention when + // adding in your local modules too. + // TODO nf-core: Customise requirements for specific processes. + // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors + withLabel:process_single { + cpus = { 1 } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } + } + withLabel:process_low { + cpus = { 2 * task.attempt } + memory = { 12.GB * task.attempt } + time = { 4.h * task.attempt } + } + withLabel:process_medium { + cpus = { 6 * task.attempt } + memory = { 36.GB * task.attempt } + time = { 8.h * task.attempt } + } + withLabel:process_high { + cpus = { 12 * task.attempt } + memory = { 72.GB * task.attempt } + time = { 16.h * task.attempt } + } + withLabel:process_long { + time = { 20.h * task.attempt } + } + withLabel:process_high_memory { + memory = { 200.GB * task.attempt } + } + withLabel:error_ignore { + errorStrategy = 'ignore' + } + withLabel:error_retry { + errorStrategy = 'retry' + maxRetries = 2 + } + withLabel: process_gpu { + ext.use_gpu = { workflow.profile.contains('gpu') } + accelerator = { workflow.profile.contains('gpu') ? 1 : null } + } +} diff --git a/conf/modules.config b/conf/modules.config new file mode 100644 index 0000000..f87b312 --- /dev/null +++ b/conf/modules.config @@ -0,0 +1,77 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Config file for defining DSL2 per module options and publishing paths +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Available keys to override module options: + ext.args = Additional arguments appended to command in module. + ext.args2 = Second set of arguments appended to command in module (multi-tool modules). + ext.args3 = Third set of arguments appended to command in module (multi-tool modules). + ext.prefix = File name prefix for output files. +---------------------------------------------------------------------------------------- +*/ + +process { + + publishDir = [ + path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + + withName: COLUMN_STANDARDISE { + publishDir = [ + path: { "${params.outdir}/column_standardisation" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: BIODIV_ANNOTATE { + publishDir = [ + path: { "${params.outdir}/annotation" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: TAXONOMY_CLASSIFY { + publishDir = [ + path: { "${params.outdir}/taxonomy" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: OUTLIER_DETECT { + publishDir = [ + path: { "${params.outdir}/quality" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: PROVISIONAL_CONCEPTS { + publishDir = [ + path: { "${params.outdir}/provisional_concepts" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: RDF_TRANSFORM { + publishDir = [ + path: { "${params.outdir}/rdf" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: COLLECT_REPORTS { + publishDir = [ + path: { "${params.outdir}/reports" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + +} diff --git a/conf/test.config b/conf/test.config new file mode 100644 index 0000000..be323f1 --- /dev/null +++ b/conf/test.config @@ -0,0 +1,22 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Use as: nextflow run main.nf -profile test,docker --outdir results +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources for CI + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + + // Input data + input = "${projectDir}/test_data/sample.csv" + mapping_schema = "${projectDir}/assets/default_mapping.jsonld" + outdir = "${projectDir}/results" +} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 0000000..00b282d --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,128 @@ +# BiodivPipeline: Contributing + +## Branch model + +| Branch | Purpose | +|---|---| +| `master` | Stable releases only. Never commit directly. | +| `dev` | Integration branch. All feature branches merge here via PR. | +| `wp1-*`, `wp2-*`, ... | Feature branches per work package | + +``` +master ← dev ← wp1-nf-core-pipeline + ← wp2-annotator-service + ← wp3-taxonomy-classifier + ... +``` + +## Git conventions + +### Commit messages + +Use [Conventional Commits](https://www.conventionalcommits.org/): + +``` +(): + +[optional body] +``` + +Types: `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, `ci`, `build` + +Examples: +``` +feat(pipeline): wire 7 modules into biodivpipeline workflow DAG +docs: rewrite README for biodiversity pipeline +fix(schema): remove samplesheet validation from nextflow_schema.json +chore: clean .gitignore and remove stray Nextflow artifacts +``` + +### Branch naming + +``` +wp- +``` + +Examples: `wp1-nf-core-pipeline`, `wp2-annotator-service`, `wp4-outlier-detection` + +## Adding a new module + +Each module lives in `modules/local//main.nf`. Follow this interface: + +### Module structure + +```groovy +process MODULE_NAME { + tag "$input_file" + label 'process_single' + + input: + path input_file + + output: + path "output_file.ext", emit: output_name + path "versions.yml", emit: versions + + script: + """ + # Your processing logic here + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + tool_name: \$(tool --version) + END_VERSIONS + """ +} +``` + +### Wiring into the workflow + +1. Add the module import to `workflows/biodivpipeline.nf` +2. Call the process and connect its inputs/outputs to the DAG +3. Mix versions into `ch_versions`: `ch_versions = ch_versions.mix(MODULE_NAME.out.versions)` +4. Add a `publishDir` entry in `conf/modules.config` + +### Publishing config + +In `conf/modules.config`: + +```groovy +withName: MODULE_NAME { + publishDir = [ + path: { "${params.outdir}/directory_name" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] +} +``` + +## Testing + +Run the pipeline with the test profile to verify changes: + +```bash +nextflow run main.nf -profile test,docker --outdir results +``` + +The test profile uses `test_data/sample.csv` and `assets/default_mapping.jsonld`. + +All modules should complete successfully: + +``` +NFCORE_BIODIVPIPELINE:BIODIVPIPELINE:COLUMN_STANDARDISE +NFCORE_BIODIVPIPELINE:BIODIVPIPELINE:BIODIV_ANNOTATE +NFCORE_BIODIVPIPELINE:BIODIVPIPELINE:TAXONOMY_CLASSIFY +NFCORE_BIODIVPIPELINE:BIODIVPIPELINE:OUTLIER_DETECT +NFCORE_BIODIVPIPELINE:BIODIVPIPELINE:PROVISIONAL_CONCEPTS +NFCORE_BIODIVPIPELINE:BIODIVPIPELINE:RDF_TRANSFORM +NFCORE_BIODIVPIPELINE:BIODIVPIPELINE:COLLECT_REPORTS +``` + +## Pull request process + +1. Create a feature branch from `dev` +2. Make changes and test locally +3. Push and open a PR against `dev` +4. Ensure the pipeline test passes +5. Request review from at least one team member +6. Squash-merge or rebase-merge (no merge commits) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..34c442c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,16 @@ +# BiodivPipeline Documentation + +- [Usage](usage.md) — how to run the pipeline, parameters, input format, and execution profiles +- [Output](output.md) — description of the output directory layout +- [Contributing](CONTRIBUTING.md) — git conventions, branch model, and module interface + +For the high-level overview and quick start, see the [main README](../README.md). + +## Technical stack + +| Component | Version | +| ---------------- | ----------------------------- | +| Nextflow | >= 24.x (tested with 26.04.0) | +| nf-core template | 4.0.2 | +| Java | 11+ (tested with OpenJDK 21) | +| Docker | >= 20.x (tested with 29.3.0) | diff --git a/docs/linting_decisions.md b/docs/linting_decisions.md new file mode 100644 index 0000000..dfd6e7e --- /dev/null +++ b/docs/linting_decisions.md @@ -0,0 +1,24 @@ +# BiodivPipeline — Lint Configuration Decisions + +## Context + +nf-core lint is designed for production pipelines with GitHub Actions CI, AWS test runs, and nf-core registry releases. We are not using CI for now and are not publishing to the nf-core registry, so most of these tests don't apply. + +## Disabled Rules + +| Rule | Reason | +|------|--------| +| `actions_awsfulltest` | No AWS infrastructure (no cloud) | +| `actions_awstest` | No AWS infrastructure | +| `actions_nf_test` | Not using CI for now | +| `actions_schema_validation` | Not using CI for now | +| `files_unchanged` | Template files are intentionally customised for this project | +| `files_exist` (partial) | 4 GitHub Actions workflow files excluded; all other required files still enforced | +| `pipeline_todos` | TODOs are intentional during development — re-enable before final submission | +| `multiqc_config` | Pipeline doesn't use MultiQC; custom per-step quality reports used instead | +| `rocrate_readme_sync` | Not publishing to nf-core registry | +| `modules_json` | All modules are written locally (`modules/local/`), none imported from nf-core registry | + +## Note on `files_exist` + +`branch.yml`, `ci.yml`, `linting.yml`, and `linting_comment.yml` have no dedicated lint rule of their own, so they can only be excluded via the `files_exist` list. All other required files (README, nextflow.config, main.nf, Dockerfile, CHANGELOG, docs/, conf/base.config) remain enforced. \ No newline at end of file diff --git a/docs/output.md b/docs/output.md new file mode 100644 index 0000000..aa45ba1 --- /dev/null +++ b/docs/output.md @@ -0,0 +1,34 @@ +# BiodivPipeline: Output + +All output files are written to the directory specified by `--outdir`. Paths below are relative to that directory. + +## Directory structure + +``` +results/ +├── column_standardisation/ +├── annotation/ +├── taxonomy/ +├── quality/ +├── provisional_concepts/ +├── rdf/ +├── reports/ +└── pipeline_info/ +``` + +Each module publishes its outputs to its own subdirectory. The file contents and schemas inside those subdirectories are owned by the module teams and are described in each module's interface spec. + +## Pipeline information + +Directory: `pipeline_info/` + +Standard Nextflow execution artifacts. + +| File | Description | +| -------------------------------------- | -------------------------------------------------------------------- | +| `execution_report_*.html` | Nextflow execution report with resource usage per process | +| `execution_timeline_*.html` | Timeline visualisation of process execution | +| `execution_trace_*.txt` | Tab-delimited trace of every task (CPU, memory, duration, exit code) | +| `pipeline_dag_*.html` | DAG visualisation of the workflow | +| `params_*.json` | Parameters used for this run | +| `biodivpipeline_software_versions.yml` | Versions of all software used in the pipeline | diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..eebe2f7 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,100 @@ +# BiodivPipeline: Usage + +## Prerequisites + +| Requirement | Minimum | Tested | +| ------------------------------------ | ------- | --------------- | +| Java | 11+ | OpenJDK 21.0.11 | +| [Nextflow](https://www.nextflow.io/) | >= 24.x | 26.04.0 | +| [Docker](https://www.docker.com/) | >= 20.x | 29.3.0 | + +## Input data + +The pipeline expects a CSV file with biodiversity specimen records and a header row. Column names do not need to follow any specific standard — the first pipeline module is responsible for mapping headers to a target schema. + +A small mock dataset is provided under `test_data/sample.csv` for use with `-profile test`. + +## Running the pipeline + +### Quick test run + +```bash +nextflow run main.nf -profile test,docker --outdir results +``` + +The `test` profile points to `test_data/sample.csv` and `assets/default_mapping.jsonld` so no other parameters are needed. + +### Custom input run + +```bash +nextflow run main.nf -profile docker \ + --input path/to/your.csv \ + --outdir results +``` + +### Custom mapping schema + +```bash +nextflow run main.nf -profile docker \ + --input data/my_records.csv \ + --mapping_schema schemas/my_mapping.jsonld \ + --outdir results +``` + +## Parameters + +| Parameter | Required | Default | Description | +| ------------------ | -------- | ------------------------------- | ---------------------------------------------------------------------------------- | +| `--input` | Yes | — | Path to input CSV file containing biodiversity records | +| `--mapping_schema` | No | `assets/default_mapping.jsonld` | Path to RDF mapping schema (JSON-LD or Turtle format) | +| `--outdir` | Yes | — | Directory for pipeline output files | +| `-profile` | Yes | — | Execution profile: `docker`, `singularity`, `test`, or comma-separated combination | + +Parameters can also be supplied via a YAML file: + +```bash +nextflow run main.nf -profile docker -params-file params.yaml +``` + +```yaml +# params.yaml +input: "./data.csv" +mapping_schema: "./assets/default_mapping.jsonld" +outdir: "./results" +``` + +## Execution profiles + +Multiple profiles can be combined: `-profile test,docker`. They load in order, so later profiles override earlier ones. + +| Profile | Description | +| ------------- | ------------------------------------------------------------------ | +| `test` | Uses the bundled mock CSV in `test_data/`. No other params needed. | +| `docker` | Run module containers with Docker | +| `singularity` | Run module containers with Singularity | +| `podman` | Run module containers with Podman | +| `conda` | Use Conda environments (not recommended for reproducibility) | + +## Resuming a run + +Nextflow caches intermediate results in the `work/` directory. To resume from where a previous run stopped: + +```bash +nextflow run main.nf -profile docker --input data.csv --outdir results -resume +``` + +## Running in the background + +```bash +nextflow run main.nf -profile docker --input data.csv --outdir results -bg > pipeline.log 2>&1 +``` + +Or use `screen` / `tmux` for a detached session. + +## Nextflow memory + +If Nextflow requests too much JVM memory, set: + +```bash +export NXF_OPTS='-Xms1g -Xmx4g' +``` diff --git a/main.nf b/main.nf new file mode 100644 index 0000000..7c89eda --- /dev/null +++ b/main.nf @@ -0,0 +1,106 @@ +#!/usr/bin/env nextflow +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + nf-core/biodivpipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + A modular nf-core workflow for FAIR biodiversity data processing. + Transforms raw CSV biodiversity records into quality-annotated, + taxonomically resolved RDF triples. +---------------------------------------------------------------------------------------- +*/ + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { BIODIVPIPELINE } from './workflows/biodivpipeline' +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_biodivpipeline_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_biodivpipeline_pipeline' + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + NAMED WORKFLOWS FOR PIPELINE +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// +// WORKFLOW: Run main analysis pipeline +// +workflow NFCORE_BIODIVPIPELINE { + + take: + ch_input // channel: path to input CSV + + main: + + // Resolve the mapping schema — use provided or fall back to default stub + def ch_schema = params.mapping_schema + ? channel.fromPath(params.mapping_schema, checkIfExists: true) + : channel.fromPath("${projectDir}/assets/default_mapping.jsonld", checkIfExists: true) + + // + // WORKFLOW: Run pipeline + // + BIODIVPIPELINE ( + ch_input, + ch_schema, + params.outdir, + ) + + emit: + rdf_turtle = BIODIVPIPELINE.out.rdf_turtle + summary = BIODIVPIPELINE.out.summary +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + RUN MAIN WORKFLOW +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow { + + main: + // + // SUBWORKFLOW: Run initialisation tasks + // + PIPELINE_INITIALISATION ( + params.version, + params.validate_params, + params.monochrome_logs, + args, + params.outdir, + params.input, + params.help, + params.help_full, + params.show_hidden + ) + + // Input channel comes from the initialisation subworkflow + // (validates params and creates the channel from --input path) + def ch_input = PIPELINE_INITIALISATION.out.input + + // + // WORKFLOW: Run main workflow + // + NFCORE_BIODIVPIPELINE ( ch_input ) + + // + // SUBWORKFLOW: Run completion tasks + // + PIPELINE_COMPLETION ( + params.email, + params.email_on_fail, + params.plaintext_email, + params.outdir, + params.monochrome_logs, + ) +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + THE END +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ diff --git a/modules.json b/modules.json new file mode 100644 index 0000000..98f7432 --- /dev/null +++ b/modules.json @@ -0,0 +1,30 @@ +{ + "name": "nf-core/biodivpipeline", + "homePage": "https://github.com/nf-core/biodivpipeline", + "repos": { + "https://github.com/nf-core/modules.git": { + "modules": { + "nf-core": {} + }, + "subworkflows": { + "nf-core": { + "utils_nextflow_pipeline": { + "branch": "master", + "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d", + "installed_by": ["subworkflows"] + }, + "utils_nfcore_pipeline": { + "branch": "master", + "git_sha": "a3fb7351b1fdb2b1de282b765816bbea190e86a8", + "installed_by": ["subworkflows"] + }, + "utils_nfschema_plugin": { + "branch": "master", + "git_sha": "fdc08b8b1ae74f56686ce21f7ea11ad11990ce57", + "installed_by": ["subworkflows"] + } + } + } + } + } +} diff --git a/modules/local/biodiv_annotate/main.nf b/modules/local/biodiv_annotate/main.nf new file mode 100644 index 0000000..3c35507 --- /dev/null +++ b/modules/local/biodiv_annotate/main.nf @@ -0,0 +1,28 @@ +process BIODIV_ANNOTATE { + tag "annotate" + label 'process_low' + + container 'biodivpipeline/placeholder:0.1' // TODO: replace with module team's container + + input: + path csv + + output: + path "annotated.jsonld", emit: annotations + path "unresolved_terms.csv", emit: unresolved + path "versions.yml", emit: versions + + script: + """ + # Stub — module team replaces this script with the real implementation. + # Emits minimal valid outputs matching the channel contract. + + echo '{"@context": {}, "@graph": []}' > annotated.jsonld + : > unresolved_terms.csv + + cat <<-VERSIONS > versions.yml + "${task.process}": + stub: "0.0.0" + VERSIONS + """ +} diff --git a/modules/local/collect_reports/main.nf b/modules/local/collect_reports/main.nf new file mode 100644 index 0000000..a81d6b5 --- /dev/null +++ b/modules/local/collect_reports/main.nf @@ -0,0 +1,21 @@ +process COLLECT_REPORTS { + tag "reports" + label 'process_single' + + container 'biodivpipeline/placeholder:0.1' // TODO: replace with module team's container + + input: + path quality_report + path rdf_report + path column_mapping + path provisional_concepts + + output: + path "pipeline_summary.json", emit: summary + + script: + """ + # Stub — aggregator. Real implementation merges per-module reports into a single summary. + echo '{}' > pipeline_summary.json + """ +} diff --git a/modules/local/column_standardise/main.nf b/modules/local/column_standardise/main.nf new file mode 100644 index 0000000..ab2f473 --- /dev/null +++ b/modules/local/column_standardise/main.nf @@ -0,0 +1,28 @@ +process COLUMN_STANDARDISE { + tag "column_std" + label 'process_single' + + container 'biodivpipeline/placeholder:0.1' // TODO: replace with module team's container + + input: + path csv + + output: + path "standardised.csv", emit: csv + path "column_mapping.json", emit: mapping + path "versions.yml", emit: versions + + script: + """ + # Stub — module team replaces this script with the real implementation. + # Emits minimal valid outputs matching the channel contract. + + cp ${csv} standardised.csv + echo '{}' > column_mapping.json + + cat <<-VERSIONS > versions.yml + "${task.process}": + stub: "0.0.0" + VERSIONS + """ +} diff --git a/modules/local/outlier_detect/main.nf b/modules/local/outlier_detect/main.nf new file mode 100644 index 0000000..6a18fa5 --- /dev/null +++ b/modules/local/outlier_detect/main.nf @@ -0,0 +1,28 @@ +process OUTLIER_DETECT { + tag "outlier" + label 'process_low' + + container 'biodivpipeline/placeholder:0.1' // TODO: replace with module team's container + + input: + path csv + + output: + path "quality_report.json", emit: report + path "flagged_records.csv", emit: flagged + path "versions.yml", emit: versions + + script: + """ + # Stub — module team replaces this script with the real implementation. + # Emits minimal valid outputs matching the channel contract. + + echo '{}' > quality_report.json + : > flagged_records.csv + + cat <<-VERSIONS > versions.yml + "${task.process}": + stub: "0.0.0" + VERSIONS + """ +} diff --git a/modules/local/provisional_concepts/main.nf b/modules/local/provisional_concepts/main.nf new file mode 100644 index 0000000..22e368f --- /dev/null +++ b/modules/local/provisional_concepts/main.nf @@ -0,0 +1,26 @@ +process PROVISIONAL_CONCEPTS { + tag "concepts" + label 'process_single' + + container 'biodivpipeline/placeholder:0.1' // TODO: replace with module team's container + + input: + path unresolved_csv + + output: + path "provisional_concepts.json", emit: concepts + path "versions.yml", emit: versions + + script: + """ + # Stub — module team replaces this script with the real implementation. + # Emits minimal valid outputs matching the channel contract. + + echo '{}' > provisional_concepts.json + + cat <<-VERSIONS > versions.yml + "${task.process}": + stub: "0.0.0" + VERSIONS + """ +} diff --git a/modules/local/rdf_transform/main.nf b/modules/local/rdf_transform/main.nf new file mode 100644 index 0000000..b8ee1b0 --- /dev/null +++ b/modules/local/rdf_transform/main.nf @@ -0,0 +1,33 @@ +process RDF_TRANSFORM { + tag "rdf" + label 'process_low' + + container 'biodivpipeline/placeholder:0.1' // TODO: replace with module team's container + + input: + path csv + path annotations + path taxonomy + path mapping_schema + + output: + path "output.ttl", emit: rdf_turtle + path "output.jsonld", emit: rdf_jsonld + path "rdf_report.json", emit: report + path "versions.yml", emit: versions + + script: + """ + # Stub — module team replaces this script with the real implementation. + # Emits minimal valid outputs matching the channel contract. + + : > output.ttl + echo '{"@context": {}, "@graph": []}' > output.jsonld + echo '{}' > rdf_report.json + + cat <<-VERSIONS > versions.yml + "${task.process}": + stub: "0.0.0" + VERSIONS + """ +} diff --git a/modules/local/taxonomy_classify/main.nf b/modules/local/taxonomy_classify/main.nf new file mode 100644 index 0000000..5cfc3fb --- /dev/null +++ b/modules/local/taxonomy_classify/main.nf @@ -0,0 +1,26 @@ +process TAXONOMY_CLASSIFY { + tag "taxonomy" + label 'process_low' + + container 'biodivpipeline/placeholder:0.1' // TODO: replace with module team's container + + input: + path csv + + output: + path "taxonomy_resolved.csv", emit: resolved + path "versions.yml", emit: versions + + script: + """ + # Stub — module team replaces this script with the real implementation. + # Emits minimal valid outputs matching the channel contract. + + cp ${csv} taxonomy_resolved.csv + + cat <<-VERSIONS > versions.yml + "${task.process}": + stub: "0.0.0" + VERSIONS + """ +} diff --git a/nextflow.config b/nextflow.config new file mode 100644 index 0000000..539d905 --- /dev/null +++ b/nextflow.config @@ -0,0 +1,249 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + nf-core/biodivpipeline Nextflow config file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Default config options for all compute environments +---------------------------------------------------------------------------------------- +*/ + +// Global default params, used in configs +params { + + // Input options + input = null // Path to input CSV (biodiversity records) + mapping_schema = null // Path to RDF mapping schema (JSON-LD/Turtle). Falls back to assets/default_mapping.jsonld + + // Boilerplate options + outdir = null + publish_dir_mode = 'copy' + email = null + email_on_fail = null + plaintext_email = false + monochrome_logs = false + help = false + help_full = false + show_hidden = false + version = false + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' + trace_report_suffix = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') + + // Config options + config_profile_name = null + config_profile_description = null + + custom_config_version = 'master' + custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" + config_profile_contact = null + config_profile_url = null + + // Schema validation default options + validate_params = true +} + +// Backwards compatibility for publishDir syntax +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + +// Load base.config by default for all pipelines +includeConfig 'conf/base.config' + +profiles { + debug { + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false + nextflow.enable.configProcessNamesValidation = true + } + conda { + conda.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + conda.channels = ['conda-forge', 'bioconda'] + apptainer.enabled = false + } + mamba { + conda.enabled = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + } + docker { + docker.enabled = true + conda.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' + } + arm64 { + process.arch = 'arm64' + // TODO https://github.com/nf-core/modules/issues/6694 + // For now if you're using arm64 you have to use wave for the sake of the maintainers + // wave profile + apptainer.ociAutoPull = true + singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' + } + emulate_amd64 { + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + conda.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + } + podman { + podman.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + } + shifter { + shifter.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + } + charliecloud { + charliecloud.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + apptainer.enabled = false + } + apptainer { + apptainer.enabled = true + apptainer.autoMounts = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + wave { + apptainer.ociAutoPull = true + singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' + } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + apptainer.runOptions = '--nv' + singularity.runOptions = '--nv' + } + test { includeConfig 'conf/test.config' } + +} + +// Load nf-core custom profiles from different institutions + +// If params.custom_config_base is set AND either the NXF_OFFLINE environment variable is not set or params.custom_config_base is a local path, the nfcore_custom.config file from the specified base path is included. +// Load nf-core/biodivpipeline custom profiles from different institutions. +includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" + + +// Load nf-core/biodivpipeline custom profiles from different institutions. +// TODO nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs +// includeConfig params.custom_config_base && (!System.getenv('NXF_OFFLINE') || !params.custom_config_base.startsWith('http')) ? "${params.custom_config_base}/pipeline/biodivpipeline.config" : "/dev/null" + +// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled +// Set to your registry if you have a mirror of containers +apptainer.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' +charliecloud.registry = 'quay.io' + + + +// Export these variables to prevent local Python/R libraries from conflicting with those in the container +// The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. +// See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. + +env { + PYTHONNOUSERSITE = 1 + R_PROFILE_USER = "/.Rprofile" + R_ENVIRON_USER = "/.Renviron" + JULIA_DEPOT_PATH = "/usr/local/share/julia" +} + +// Set bash options +process.shell = [ + "bash", + "-C", // No clobber - prevent output redirection from overwriting files. + "-e", // Exit if a tool returns a non-zero status/exit code + "-u", // Treat unset variables and parameters as an error + "-o", // Returns the status of the last command to exit.. + "pipefail" // ..with a non-zero status or zero if all successfully execute +] + +// Disable process selector warnings by default. Use debug profile to enable warnings. +nextflow.enable.configProcessNamesValidation = false + +timeline { + enabled = true + file = "${params.outdir}/pipeline_info/execution_timeline_${params.trace_report_suffix}.html" +} +report { + enabled = true + file = "${params.outdir}/pipeline_info/execution_report_${params.trace_report_suffix}.html" +} +trace { + enabled = true + file = "${params.outdir}/pipeline_info/execution_trace_${params.trace_report_suffix}.txt" +} +dag { + enabled = true + file = "${params.outdir}/pipeline_info/pipeline_dag_${params.trace_report_suffix}.html" +} + +manifest { + name = 'nf-core/biodivpipeline' + contributors = [] + homePage = 'https://github.com/nf-core/biodivpipeline' + description = """Modular nf-core workflow for FAIR biodiversity data processing""" + mainScript = 'main.nf' + defaultBranch = 'master' + nextflowVersion = '!>=25.10.4' + version = '1.0.0dev' + doi = '' +} + +// Nextflow plugins +plugins { + id 'nf-schema@2.5.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} + +validation { + defaultIgnoreParams = [] + monochromeLogs = params.monochrome_logs +} +// Load modules.config for DSL2 module specific options +includeConfig 'conf/modules.config' diff --git a/nextflow_schema.json b/nextflow_schema.json new file mode 100644 index 0000000..67f724a --- /dev/null +++ b/nextflow_schema.json @@ -0,0 +1,235 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/nf-core/biodivpipeline/master/nextflow_schema.json", + "title": "nf-core/biodivpipeline pipeline parameters", + "description": "Modular nf-core workflow for FAIR biodiversity data processing", + "type": "object", + "$defs": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": [ + "input", + "outdir" + ], + "properties": { + "input": { + "type": "string", + "format": "file-path", + "exists": true, + "mimetype": "text/csv", + "pattern": "^\\S+\\.csv$", + "description": "Path to input CSV file containing biodiversity records.", + "fa_icon": "fas fa-file-csv" + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "email": { + "type": "string", + "description": "Email address for completion summary.", + "fa_icon": "fas fa-envelope", + "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.", + "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$" + }, + "multiqc_title": { + "type": "string", + "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", + "fa_icon": "fas fa-file-signature" + }, + "mapping_schema": { + "type": "string", + "format": "file-path", + "description": "Path to RDF mapping schema file (JSON-LD or Turtle). If not provided, a default stub mapping is used.", + "fa_icon": "fas fa-project-diagram" + } + } + }, + "reference_genome_options": { + "title": "Reference genome options", + "type": "object", + "fa_icon": "fas fa-dna", + "description": "Reference genome related files and options required for the workflow.", + "properties": {} + }, + "institutional_config_options": { + "title": "Institutional config options", + "type": "object", + "fa_icon": "fas fa-university", + "description": "Parameters used to describe centralised config profiles. These should not be edited.", + "help_text": "The centralised nf-core configuration profiles use a handful of pipeline parameters to describe themselves. This information is then printed to the Nextflow log when you run a pipeline. You should not need to change these values when you run a pipeline.", + "properties": { + "custom_config_version": { + "type": "string", + "description": "Git commit id for Institutional configs.", + "default": "master", + "hidden": true, + "fa_icon": "fas fa-users-cog" + }, + "custom_config_base": { + "type": "string", + "description": "Base directory for Institutional configs.", + "default": "https://raw.githubusercontent.com/nf-core/configs/master", + "hidden": true, + "help_text": "If you're running offline, Nextflow will not be able to fetch the institutional config files from the internet. If you don't need them, then this is not a problem. If you do need them, you should download the files from the repo and tell Nextflow where to find them with this parameter.", + "fa_icon": "fas fa-users-cog" + }, + "config_profile_name": { + "type": "string", + "description": "Institutional config name.", + "hidden": true, + "fa_icon": "fas fa-users-cog" + }, + "config_profile_description": { + "type": "string", + "description": "Institutional config description.", + "hidden": true, + "fa_icon": "fas fa-users-cog" + }, + "config_profile_contact": { + "type": "string", + "description": "Institutional config contact information.", + "hidden": true, + "fa_icon": "fas fa-users-cog" + }, + "config_profile_url": { + "type": "string", + "description": "Institutional config URL link.", + "hidden": true, + "fa_icon": "fas fa-users-cog" + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], + "hidden": true + }, + "email_on_fail": { + "type": "string", + "description": "Email address for completion summary, only when pipeline fails.", + "fa_icon": "fas fa-exclamation-triangle", + "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$", + "help_text": "An email address to send a summary email to when the pipeline is completed - ONLY sent if the pipeline does not exit successfully.", + "hidden": true + }, + "plaintext_email": { + "type": "boolean", + "description": "Send plain-text email instead of HTML.", + "fa_icon": "fas fa-remove-format", + "hidden": true + }, + "max_multiqc_email_size": { + "type": "string", + "description": "File size limit when attaching MultiQC reports to summary emails.", + "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$", + "default": "25.MB", + "fa_icon": "fas fa-file-upload", + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Do not use coloured log outputs.", + "fa_icon": "fas fa-palette", + "hidden": true + }, + "multiqc_config": { + "type": "string", + "format": "file-path", + "description": "Custom config file to supply to MultiQC.", + "fa_icon": "fas fa-cog", + "hidden": true + }, + "multiqc_logo": { + "type": "string", + "description": "Custom logo file to supply to MultiQC. File name must also be set in the MultiQC config file", + "fa_icon": "fas fa-image", + "hidden": true + }, + "multiqc_methods_description": { + "type": "string", + "description": "Custom MultiQC yaml file containing HTML including a methods description.", + "fa_icon": "fas fa-cog" + }, + "validate_params": { + "type": "boolean", + "description": "Boolean whether to validate parameters against the schema at runtime", + "default": true, + "fa_icon": "fas fa-check-square", + "hidden": true + }, + "pipelines_testdata_base_path": { + "type": "string", + "fa_icon": "far fa-check-circle", + "description": "Base URL or local path to location of pipeline test dataset files", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/", + "hidden": true + }, + "trace_report_suffix": { + "type": "string", + "fa_icon": "far calendar", + "description": "Suffix to add to the trace report filename. Default is the date and time in the format yyyy-MM-dd_HH-mm-ss.", + "hidden": true + }, + "help": { + "type": [ + "boolean", + "string" + ], + "description": "Display the help message." + }, + "help_full": { + "type": "boolean", + "description": "Display the full detailed help message." + }, + "show_hidden": { + "type": "boolean", + "description": "Display hidden parameters in the help message (only works when --help or --help_full are provided)." + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/input_output_options" + }, + { + "$ref": "#/$defs/reference_genome_options" + }, + { + "$ref": "#/$defs/institutional_config_options" + }, + { + "$ref": "#/$defs/generic_options" + } + ] +} diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 0000000..e2394ac --- /dev/null +++ b/nf-test.config @@ -0,0 +1,37 @@ +config { + // location for all nf-test tests + testsDir = "." + + // nf-test directory including temporary files for each test + workDir = System.getenv("NFT_WORKDIR") ?: ".nf-test" + + // location of an optional nextflow.config file specific for executing tests + configFile = "tests/nextflow.config" + + // ignore tests coming from the nf-core/modules repo + ignore = [ + 'modules/nf-core/**/tests/*', + 'subworkflows/nf-core/**/tests/*', + ] + + // run all test with defined profile(s) from the main nextflow.config + profile = "test" + + // list of filenames or patterns that should be trigger a full test run + triggers = [ + '.github/actions/nf-test/action.yml', + '.github/workflows/nf-test.yml', + 'bin/*', + 'conf/test.config', + 'nextflow.config', + 'nextflow_schema.json', + 'nf-test.config', + 'tests/.nftignore', + 'tests/nextflow.config', + ] + + // load the necessary plugins + plugins { + load "nft-utils@0.0.3" + } +} diff --git a/subworkflows/local/utils_nfcore_biodivpipeline_pipeline/main.nf b/subworkflows/local/utils_nfcore_biodivpipeline_pipeline/main.nf new file mode 100644 index 0000000..b1de5a1 --- /dev/null +++ b/subworkflows/local/utils_nfcore_biodivpipeline_pipeline/main.nf @@ -0,0 +1,173 @@ +// +// Subworkflow with functionality specific to the nf-core/biodivpipeline pipeline +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { paramsHelp } from 'plugin/nf-schema' +include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW TO INITIALISE PIPELINE +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow PIPELINE_INITIALISATION { + + take: + version // boolean: Display version and exit + validate_params // boolean: Boolean whether to validate parameters against the schema at runtime + monochrome_logs // boolean: Do not use coloured log outputs + nextflow_cli_args // array: List of positional nextflow CLI args + outdir // string: The output directory where the results will be saved + input // string: Path to input samplesheet + help // boolean: Display help message and exit + help_full // boolean: Show the full help message + show_hidden // boolean: Show hidden parameters in the help message + + main: + + ch_versions = channel.empty() + + // + // Print version and exit if required and dump pipeline parameters to JSON file + // + UTILS_NEXTFLOW_PIPELINE ( + version, + true, + outdir, + workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 + ) + + // + // Validate parameters and generate parameter summary to stdout + // + + def before_text = "" + def after_text = "" + before_text = """ +-\033[2m----------------------------------------------------\033[0m- + \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m +\033[0;34m ___ __ __ __ ___ \033[0;32m/,-._.--~\'\033[0m +\033[0;34m |\\ | |__ __ / ` / \\ |__) |__ \033[0;33m} {\033[0m +\033[0;34m | \\| | \\__, \\__/ | \\ |___ \033[0;32m\\`-._,-`-,\033[0m + \033[0;32m`._,._,\'\033[0m +\033[0;35m nf-core/biodivpipeline ${workflow.manifest.version}\033[0m +-\033[2m----------------------------------------------------\033[0m- +""" + after_text = """${workflow.manifest.doi ? "\n* The pipeline\n" : ""}${workflow.manifest.doi.tokenize(",").collect { doi -> " https://doi.org/${doi.trim().replace('https://doi.org/','')}"}.join("\n")}${workflow.manifest.doi ? "\n" : ""} +* The nf-core framework + https://doi.org/10.1038/s41587-020-0439-x + +* Software dependencies + https://github.com/nf-core/biodivpipeline/blob/master/CITATIONS.md +""" + if (monochrome_logs) { + before_text = before_text.replaceAll(/\033\[[0-9;]*m/, '') + } + + command = "nextflow run ${workflow.manifest.name} -profile --input data.csv --outdir " + + UTILS_NFSCHEMA_PLUGIN ( + workflow, + validate_params, + null, + help, + help_full, + show_hidden, + before_text, + after_text, + command + ) + + // + // Check config provided to the pipeline + // + UTILS_NFCORE_PIPELINE ( + nextflow_cli_args + ) + + // + // Custom validation for pipeline parameters + // + validateInputParameters() + + // + // Create channel from input CSV file + // Our input IS the data file itself, not a samplesheet pointing to files + // + channel + .fromPath(input, checkIfExists: true) + .set { ch_input } + + emit: + input = ch_input + versions = ch_versions +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW FOR PIPELINE COMPLETION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow PIPELINE_COMPLETION { + + take: + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + + main: + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + + // + // Completion email and summary + // + workflow.onComplete { + if (email || email_on_fail) { + completionEmail( + summary_params, + email, + email_on_fail, + plaintext_email, + outdir, + monochrome_logs, + [], + ) + } + + completionSummary(monochrome_logs) + + } + + workflow.onError { + log.error "Pipeline failed. Please refer to troubleshooting docs for common issues: https://nf-co.re/docs/running/troubleshooting" + } +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ +// +// Check and validate pipeline parameters +// +def validateInputParameters() { + if (!params.input) { + error("Please provide an input CSV file using --input") + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf new file mode 100644 index 0000000..d6e593e --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -0,0 +1,126 @@ +// +// Subworkflow with functionality that may be useful for any Nextflow pipeline +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW DEFINITION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow UTILS_NEXTFLOW_PIPELINE { + take: + print_version // boolean: print version + dump_parameters // boolean: dump parameters + outdir // path: base directory used to publish pipeline results + check_conda_channels // boolean: check conda channels + + main: + + // + // Print workflow version and exit on --version + // + if (print_version) { + log.info("${workflow.manifest.name} ${getWorkflowVersion()}") + System.exit(0) + } + + // + // Dump pipeline parameters to a JSON file + // + if (dump_parameters && outdir) { + dumpParametersToJSON(outdir) + } + + // + // When running with Conda, warn if channels have not been set-up appropriately + // + if (check_conda_channels) { + checkCondaChannels() + } + + emit: + dummy_emit = true +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// +// Generate version string +// +def getWorkflowVersion() { + def version_string = "" as String + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Dump pipeline parameters to a JSON file +// +def dumpParametersToJSON(outdir) { + def timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") + def jsonStr = groovy.json.JsonOutput.toJson(params) + temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) + + nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() +} + +// +// When running with -profile conda, warn if channels have not been set-up appropriately +// +def checkCondaChannels() { + def parser = new org.yaml.snakeyaml.Yaml() + def channels = [] + try { + def config = parser.load("conda config --show channels".execute().text) + channels = config.channels + } + catch (NullPointerException e) { + log.debug(e) + log.warn("Could not verify conda channel configuration.") + return null + } + catch (IOException e) { + log.debug(e) + log.warn("Could not verify conda channel configuration.") + return null + } + + // Check that all channels are present + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean + + // Check that they are in the right order + def channel_priority_violation = required_channels_in_order != channels.findAll { ch -> ch in required_channels_in_order } + + if (channels_missing | channel_priority_violation) { + log.warn """\ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + There is a problem with your Conda configuration! + You will need to set-up the conda-forge and bioconda channels correctly. + Please refer to https://bioconda.github.io/ + The observed channel order is + ${channels} + but the following channel order is required: + ${required_channels_in_order} + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + """.stripIndent(true) + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml new file mode 100644 index 0000000..e5c3a0a --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml @@ -0,0 +1,38 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NEXTFLOW_PIPELINE" +description: Subworkflow with functionality that may be useful for any Nextflow pipeline +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - print_version: + type: boolean + description: | + Print the version of the pipeline and exit + - dump_parameters: + type: boolean + description: | + Dump the parameters of the pipeline to a JSON file + - output_directory: + type: directory + description: Path to output dir to write JSON file to. + pattern: "results/" + - check_conda_channel: + type: boolean + description: | + Check if the conda channel priority is correct. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" + - "@drpatelh" +maintainers: + - "@adamrtalbot" + - "@drpatelh" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000..68718e4 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -0,0 +1,54 @@ + +nextflow_function { + + name "Test Functions" + script "subworkflows/nf-core/utils_nextflow_pipeline/main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Test Function getWorkflowVersion") { + + function "getWorkflowVersion" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dumpParametersToJSON") { + + function "dumpParametersToJSON" + + when { + function { + """ + // define inputs of the function here. Example: + input[0] = "$outputDir" + """.stripIndent() + } + } + + then { + assertAll( + { assert function.success } + ) + } + } + + test("Test Function checkCondaChannels") { + + function "checkCondaChannels" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000..e3f0baf --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,20 @@ +{ + "Test Function getWorkflowVersion": { + "content": [ + "v9.9.9" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:05.308243" + }, + "Test Function checkCondaChannels": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:12.425833" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000..02dbf09 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,113 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NEXTFLOW_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + workflow "UTILS_NEXTFLOW_PIPELINE" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Should run no inputs") { + + when { + workflow { + """ + print_version = false + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should print version") { + + when { + workflow { + """ + print_version = true + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + expect { + with(workflow) { + assert success + assert "nextflow_workflow v9.9.9" in stdout + } + } + } + } + + test("Should dump params") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = 'results' + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should not create params JSON if no output directory") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = null + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config new file mode 100644 index 0000000..a09572e --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf new file mode 100644 index 0000000..afca543 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -0,0 +1,355 @@ +// +// Subworkflow with utility functions specific to the nf-core pipeline template +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW DEFINITION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow UTILS_NFCORE_PIPELINE { + take: + nextflow_cli_args + + main: + valid_config = checkConfigProvided() + checkProfileProvided(nextflow_cli_args) + + emit: + valid_config = valid_config +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// +// Warn if a -profile or Nextflow config has not been provided to run the pipeline +// +def checkConfigProvided() { + def valid_config = true as Boolean + if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { + log.warn( + "[${workflow.manifest.name}] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + "Please refer to the quick start section and usage docs for the pipeline.\n " + ) + valid_config = false + } + return valid_config +} + +// +// Exit pipeline if --profile contains spaces +// +def checkProfileProvided(nextflow_cli_args) { + if (workflow.profile.endsWith(',')) { + error( + "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) + } + if (nextflow_cli_args[0]) { + log.warn( + "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) + } +} + +// +// Generate workflow version string +// +def getWorkflowVersion() { + def version_string = "" as String + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Get software versions for pipeline +// +def processVersionsFromYAML(yaml_file) { + def yaml = new org.yaml.snakeyaml.Yaml() + def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } + return yaml.dumpAsMap(versions).trim() +} + +// +// Get workflow version for pipeline +// +def workflowVersionToYAML() { + return """ + Workflow: + ${workflow.manifest.name}: ${getWorkflowVersion()} + Nextflow: ${workflow.nextflow.version} + """.stripIndent().trim() +} + +// +// Get channel of software versions used in pipeline in YAML format +// +def softwareVersionsToYAML(ch_versions) { + return ch_versions.unique().map { version -> processVersionsFromYAML(version) }.unique().mix(channel.of(workflowVersionToYAML())) +} + +// +// Get workflow summary for MultiQC +// +def paramsSummaryMultiqc(summary_params) { + def summary_section = '' + summary_params + .keySet() + .each { group -> + def group_params = summary_params.get(group) + // This gets the parameters of that particular group + if (group_params) { + summary_section += "

${group}

\n" + summary_section += "
\n" + group_params + .keySet() + .sort() + .each { param -> + summary_section += "
${param}
${group_params.get(param) ?: 'N/A'}
\n" + } + summary_section += "
\n" + } + } + + def yaml_file_text = "id: '${workflow.manifest.name.replace('/', '-')}-summary'\n" as String + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + + return yaml_file_text +} + +// +// ANSII colours used for terminal logging +// +def logColours(monochrome_logs=true) { + def colorcodes = [:] as Map + + // Reset / Meta + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" + + // Regular Colors + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + + // Bold + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + + // Underline + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + + // High Intensity + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + + // Bold High Intensity + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + + return colorcodes +} + +// Return a single report from an object that may be a Path or List +// +def getSingleReport(multiqc_reports) { + if (multiqc_reports instanceof Path) { + return multiqc_reports + } else if (multiqc_reports instanceof List) { + if (multiqc_reports.size() == 0) { + log.warn("[${workflow.manifest.name}] No reports found from process 'MULTIQC'") + return null + } else if (multiqc_reports.size() == 1) { + return multiqc_reports.first() + } else { + log.warn("[${workflow.manifest.name}] Found multiple reports from process 'MULTIQC', will use only one") + return multiqc_reports.first() + } + } else { + return null + } +} + +// +// Construct and send completion email +// +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { + + // Set up the e-mail variables + def subject = "[${workflow.manifest.name}] Successful: ${workflow.runName}" + if (!workflow.success) { + subject = "[${workflow.manifest.name}] FAILED: ${workflow.runName}" + } + + def summary = [:] + summary_params + .keySet() + .sort() + .each { group -> + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['Date Started'] = workflow.start + misc_fields['Date Completed'] = workflow.complete + misc_fields['Pipeline script file path'] = workflow.scriptFile + misc_fields['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) { + misc_fields['Pipeline repository Git URL'] = workflow.repository + } + if (workflow.commitId) { + misc_fields['Pipeline repository Git Commit'] = workflow.commitId + } + if (workflow.revision) { + misc_fields['Pipeline Git branch/tag'] = workflow.revision + } + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build + misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + def email_fields = [:] + email_fields['version'] = getWorkflowVersion() + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary << misc_fields + + // On success try attach the multiqc report + def mqc_report = getSingleReport(multiqc_report) + + // Check if we are only sending emails on failure + def email_address = email + if (!email && email_on_fail && !workflow.success) { + email_address = email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("${workflow.projectDir}/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("${workflow.projectDir}/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as MemoryUnit + def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] + def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + def colors = logColours(monochrome_logs) as Map + if (email_address) { + try { + if (plaintext_email) { + new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') + } + // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } + ['sendmail', '-t'].execute() << sendmail_html + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") + } + catch (Exception msg) { + log.debug(msg.toString()) + log.debug("Trying with mail instead of sendmail") + // Catch failures and try with plaintext + def mail_cmd = ['mail', '-s', subject, '--content-type=text/html', email_address] + mail_cmd.execute() << email_html + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (mail)-") + } + } + + // Write summary e-mail HTML to a file + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") + output_hf.withWriter { w -> w << email_html } + nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html") + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") + output_tf.withWriter { w -> w << email_txt } + nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt") + output_tf.delete() +} + +// +// Print pipeline summary on completion +// +def completionSummary(monochrome_logs=true) { + def colors = logColours(monochrome_logs) as Map + if (workflow.success) { + if (workflow.stats.ignoredCount == 0) { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Pipeline completed successfully${colors.reset}-") + } + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-") + } + } + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml new file mode 100644 index 0000000..d08d243 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFCORE_PIPELINE" +description: Subworkflow with utility functions specific to the nf-core pipeline template +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - nextflow_cli_args: + type: list + description: | + Nextflow CLI positional arguments +output: + - success: + type: boolean + description: | + Dummy output to indicate success +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000..f117040 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -0,0 +1,126 @@ + +nextflow_function { + + name "Test Functions" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Test Function checkConfigProvided") { + + function "checkConfigProvided" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function checkProfileProvided") { + + function "checkProfileProvided" + + when { + function { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function without logColours") { + + function "logColours" + + when { + function { + """ + input[0] = true + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function with logColours") { + function "logColours" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function getSingleReport with a single file") { + function "getSingleReport" + + when { + function { + """ + input[0] = file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.contains("test.tsv") } + ) + } + } + + test("Test Function getSingleReport with multiple files") { + function "getSingleReport" + + when { + function { + """ + input[0] = [ + file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/network.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/expression.tsv', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.contains("test.tsv") }, + { assert !function.result.contains("network.tsv") }, + { assert !function.result.contains("expression.tsv") } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000..02c6701 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,136 @@ +{ + "Test Function checkProfileProvided": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:03.360873" + }, + "Test Function checkConfigProvided": { + "content": [ + true + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:59.729647" + }, + "Test Function without logColours": { + "content": [ + { + "reset": "", + "bold": "", + "dim": "", + "underlined": "", + "blink": "", + "reverse": "", + "hidden": "", + "black": "", + "red": "", + "green": "", + "yellow": "", + "blue": "", + "purple": "", + "cyan": "", + "white": "", + "bblack": "", + "bred": "", + "bgreen": "", + "byellow": "", + "bblue": "", + "bpurple": "", + "bcyan": "", + "bwhite": "", + "ublack": "", + "ured": "", + "ugreen": "", + "uyellow": "", + "ublue": "", + "upurple": "", + "ucyan": "", + "uwhite": "", + "iblack": "", + "ired": "", + "igreen": "", + "iyellow": "", + "iblue": "", + "ipurple": "", + "icyan": "", + "iwhite": "", + "biblack": "", + "bired": "", + "bigreen": "", + "biyellow": "", + "biblue": "", + "bipurple": "", + "bicyan": "", + "biwhite": "" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:17.969323" + }, + "Test Function with logColours": { + "content": [ + { + "reset": "\u001b[0m", + "bold": "\u001b[1m", + "dim": "\u001b[2m", + "underlined": "\u001b[4m", + "blink": "\u001b[5m", + "reverse": "\u001b[7m", + "hidden": "\u001b[8m", + "black": "\u001b[0;30m", + "red": "\u001b[0;31m", + "green": "\u001b[0;32m", + "yellow": "\u001b[0;33m", + "blue": "\u001b[0;34m", + "purple": "\u001b[0;35m", + "cyan": "\u001b[0;36m", + "white": "\u001b[0;37m", + "bblack": "\u001b[1;30m", + "bred": "\u001b[1;31m", + "bgreen": "\u001b[1;32m", + "byellow": "\u001b[1;33m", + "bblue": "\u001b[1;34m", + "bpurple": "\u001b[1;35m", + "bcyan": "\u001b[1;36m", + "bwhite": "\u001b[1;37m", + "ublack": "\u001b[4;30m", + "ured": "\u001b[4;31m", + "ugreen": "\u001b[4;32m", + "uyellow": "\u001b[4;33m", + "ublue": "\u001b[4;34m", + "upurple": "\u001b[4;35m", + "ucyan": "\u001b[4;36m", + "uwhite": "\u001b[4;37m", + "iblack": "\u001b[0;90m", + "ired": "\u001b[0;91m", + "igreen": "\u001b[0;92m", + "iyellow": "\u001b[0;93m", + "iblue": "\u001b[0;94m", + "ipurple": "\u001b[0;95m", + "icyan": "\u001b[0;96m", + "iwhite": "\u001b[0;97m", + "biblack": "\u001b[1;90m", + "bired": "\u001b[1;91m", + "bigreen": "\u001b[1;92m", + "biyellow": "\u001b[1;93m", + "biblue": "\u001b[1;94m", + "bipurple": "\u001b[1;95m", + "bicyan": "\u001b[1;96m", + "biwhite": "\u001b[1;97m" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:21.714424" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test new file mode 100644 index 0000000..8940d32 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap new file mode 100644 index 0000000..859d103 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000..8940d32 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap new file mode 100644 index 0000000..859d103 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config new file mode 100644 index 0000000..d0a926b --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 0000000..1df8b76 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,73 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' +include { paramsHelp } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines + help // boolean: show help message + help_full // boolean: show full help message + show_hidden // boolean: show hidden parameters in help message + before_text // string: text to show before the help message and parameters summary + after_text // string: text to show after the help message and parameters summary + command // string: an example command of the pipeline + + main: + + if(help || help_full) { + help_options = [ + beforeText: before_text, + afterText: after_text, + command: command, + showHidden: show_hidden, + fullHelp: help_full, + ] + if(parameters_schema) { + help_options << [parametersSchema: parameters_schema] + } + log.info paramsHelp( + help_options, + (params.help instanceof String && params.help != "true") ? params.help : "", + ) + exit 0 + } + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + + summary_options = [:] + if(parameters_schema) { + summary_options << [parametersSchema: parameters_schema] + } + log.info before_text + log.info paramsSummaryLog(summary_options, input_workflow) + log.info after_text + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + validateOptions = [:] + if(parameters_schema) { + validateOptions << [parametersSchema: parameters_schema] + } + validateParameters(validateOptions) + } + + emit: + dummy_emit = true +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 0000000..f7d9f02 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - input_workflow: + type: object + description: | + The workflow object of the used pipeline. + This object contains meta data used to create the params summary log + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. + - parameters_schema: + type: string + description: | + Path to the parameters JSON schema. + This has to be the same as the schema given to the `validation.parametersSchema` config + option. When this input is empty it will automatically use the configured schema or + "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way + for meta pipelines. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test new file mode 100644 index 0000000..c977917 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -0,0 +1,173 @@ +nextflow_workflow { + + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" + script "../main.nf" + workflow "UTILS_NFSCHEMA_PLUGIN" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/utils_nfschema_plugin" + tag "plugin/nf-schema" + + config "./nextflow.config" + + test("Should run nothing") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "" + input[3] = false + input[4] = false + input[5] = false + input[6] = "" + input[7] = "" + input[8] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params") { + + when { + + params { + test_data = '' + outdir = null + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "" + input[3] = false + input[4] = false + input[5] = false + input[6] = "" + input[7] = "" + input[8] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should run nothing - custom schema") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + input[3] = false + input[4] = false + input[5] = false + input[6] = "" + input[7] = "" + input[8] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params - custom schema") { + + when { + + params { + test_data = '' + outdir = null + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + input[3] = false + input[4] = false + input[5] = false + input[6] = "" + input[7] = "" + input[8] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should create a help message") { + + when { + + params { + test_data = '' + outdir = null + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + input[3] = true + input[4] = false + input[5] = false + input[6] = "Before" + input[7] = "After" + input[8] = "nextflow run test/test" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 0000000..f6537cc --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.6.1" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json new file mode 100644 index 0000000..331e0d2 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", + "title": ". pipeline parameters", + "description": "", + "type": "object", + "$defs": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["outdir"], + "properties": { + "validate_params": { + "type": "boolean", + "description": "Validate parameters?", + "default": true, + "hidden": true + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "test_data_base": { + "type": "string", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/modules", + "description": "Base for test data directory", + "hidden": true + }, + "test_data": { + "type": "string", + "description": "Fake test data param", + "hidden": true + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "help": { + "type": "boolean", + "description": "Display help text.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "logo": { + "type": "boolean", + "default": true, + "description": "Display nf-core logo in console output.", + "fa_icon": "fas fa-image", + "hidden": true + }, + "singularity_pull_docker_container": { + "type": "boolean", + "description": "Pull Singularity container from Docker?", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Use monochrome_logs", + "hidden": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/input_output_options" + }, + { + "$ref": "#/$defs/generic_options" + } + ] +} diff --git a/test_data/sample.csv b/test_data/sample.csv new file mode 100644 index 0000000..96c5459 --- /dev/null +++ b/test_data/sample.csv @@ -0,0 +1,110 @@ +ï»ż"HerbariumID","Bild","DB","Family","FullNameCache","Anmerkungen","Sammlerteam","Sammelnummer","CollectionDateBegin","CollectionDateEnd","Country","Locality","TitelEtikett","Expeditionsangabe","ShowOnMap","Latitude","Longitude","FundortUNdOeko","NameCache","Genus","Identifier","Barcode","StableURI" +"B100064379","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100064379/manifest.json","JACQ","BORAGINACEAE","Echium vulgare L.","MTB: 8434/1","Willing,R. & Willing,E.","15898 D","2001-08-07","","Germany","Lkr. Bad-Tölz-Wolfratshausen, W Vorderriß","","","https://www.openstreetmap.org/?mlat=47.5428&mlon=11.3556#map=15/47.5428/11.3556",47.542781829833984,11.355560302734375,"","Echium vulgare","Echium","E.Willing","B 10 0064379","https://herbarium.bgbm.org/object/B100064379" +"B100094552","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100094552/manifest.json","JACQ","ASTERACEAE","Hieracium L.","","unclear","","","","Germany","","","","","","","","Hieracium","Hieracium","","B 10 0094552","https://herbarium.bgbm.org/object/B100094552" +"B100132913","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100132913/manifest.json","BGBM","CHENOPODIACEAE","Atriplex laciniata L.","GBOL, Blattmaterial entnommen; Georeferenzierung nachtrĂ€glich","Kuhbier,M.H.","s. n.","1995-08-21","","Germany","Germany: Niedersachsen, Memmert, auf dem Sandstrand westl. des Hauses. 21.08.1995, Leg.: M. H. Kuhbier s. n. ex herb. / ded. : ex Herbarium BREM.","","","https://www.openstreetmap.org/?mlat=53.6386&mlon=6.86639#map=15/53.6386/6.86639",53.63861083984375,6.866390228271484,"Germany: Niedersachsen, Memmert, auf dem Sandstrand westl. des Hauses.","Atriplex laciniata","Atriplex","R. Hand","B 10 0132913","https://herbarium.bgbm.org/object/B100132913" +"B100198478","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100198478/manifest.json","HERB","Cupressaceae","Juniperus communis","Label data transcribed during a Herbonauten mission","R. Gross","","1926-09-01","1926-09-30","Germany","Germany: Berlin. Karlshorst. im Walde. 1926-09-01 - 1926-09-30, Leg.: R. Gross.","","","https://www.openstreetmap.org/?mlat=52.4739&mlon=13.5136#map=15/52.4739/13.5136",52.47394561767578,13.513612747192383,"","Juniperus communis","Juniperus","","B 10 0198478","http://herbarium.bgbm.org/object/B100198478" +"B100263237","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100263237/manifest.json","BGBM","COMPOSITAE","Serratula tinctoria L.","","BornmĂŒller,J.F.N.","","1882-07","","Germany","Germany: Potsdam in sicris. 07.1882, Leg.: J. F. N. BornmĂŒller s.n.","e flora marihica","","","","","Germany: Potsdam in sicris.","Serratula tinctoria","Serratula","","B 10 0263237","https://herbarium.bgbm.org/object/B100263237" +"B100325691","","JACQ","SCROPHULARIACEAE","Veronica austriaca subsp. dentata (F. W. Schmidt) Watzl","cult. In horto Diersch","BornmĂŒller,J.","s.n.","1892-05-16","","Germany","Weimar.","","","","","","","Veronica austriaca subsp. dentata","Veronica","B. M. Rojas AndrĂ©s (SALA) 2016","B 10 0325691","https://herbarium.bgbm.org/object/B100325691" +"B100340892","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100340892/manifest.json","BGBM","ROSACEAE","Potentilla argentea L.","","Ristow,M.","476/08","2008-05-29","","Germany","Germany: Brandenburg. MĂ€rkische Schweiz, E nahe der Bahn ca 1 km N Bahnhof MĂŒncheberg, Mtb 3450/41. Ackerrand. Alt.: 50m. 29.05.2008, Leg.: M. Ristow 476/08.","","","https://www.openstreetmap.org/?mlat=52.5339&mlon=14.0944#map=15/52.5339/14.0944",52.53388977050781,14.094440460205078,"Germany: Brandenburg. MĂ€rkische Schweiz, E nahe der Bahn ca 1 km N Bahnhof MĂŒncheberg, Mtb 3450/41. Ackerrand.","Potentilla argentea","Potentilla","","B 10 0340892","https://herbarium.bgbm.org/object/B100340892" +"B100379250","","JACQ","POACEAE","Triticum monococcum subsp. monococcum L.","Herbar Roman Schulz acc.1945","Schulz,O. & Schulz,R.","s.n.","1897-08-11","","Germany","Berlin, kultiviert","","","","","","","","Triticum","M. W. van Slageren 2017-02-23","B 10 0379250","https://herbarium.bgbm.org/object/B100379250" +"B100463293","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100463293/manifest.json","JACQ","AMARANTHACEAE","Amaranthus powellii S. Watson","GBOL750, Silikaprobe genommen, nur 1 Exemplar beprobt","Ciongwa,P.","PC 67","2013-09-08","","Germany","Niedersachsen, Northeim, Ortslage Northeim, Alt.: 130 m.","","","https://www.openstreetmap.org/?mlat=51.7081&mlon=9.99556#map=15/51.7081/9.99556",51.70806121826172,9.995559692382812,"","Amaranthus powellii","Amaranthus","T. Raus, R. Hand & M. Ristow","B 10 0463293","https://herbarium.bgbm.org/object/B100463293" +"B100505084","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100505084/manifest.json","HERB","Potamogetonaceae","Potamogeton pusillus","Label data transcribed during a Herbonauten mission","Herrenkohl","863","","","Germany","Germany: Nordrhein-Westfalen. GrĂ€ben der Schottheide bei Cleve. Leg.: Herrenkohl 863.","","","https://www.openstreetmap.org/?mlat=51.7723&mlon=6.06548#map=15/51.7723/6.06548",51.77230453491211,6.065483093261719,"","Potamogeton pusillus","Potamogeton","","B 10 0505084","http://herbarium.bgbm.org/object/B100505084" +"B100535271","","JACQ","POACEAE","Triticum aestivum subsp. spelta (L.) Thell.","Sheet 1 of 2 B 10 0537542","Hohenacker","118","","","Germany","cult.","","","","","","Rother Sommerspelz. Rother Sommerdinkel.","Triticum aestivum subsp. spelta","Triticum","M. W. van Slageren 24.1.2017","B 10 0535271","https://herbarium.bgbm.org/object/B100535271" +"B100553784","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100553784/manifest.json","BGBM","CYPERACEAE","Carex divulsa Stokes","GBOL1146, Silikaprobe genommen, nur 1 Exemplar beprobt","Mause,R.","76","2013-07-02","","Germany","Germany: Nordrhein-Westfalen. Bonn, Nussallee. 02.07.2013, Leg.: R. Mause 76.","","","https://www.openstreetmap.org/?mlat=50.7258&mlon=7.09028#map=15/50.7258/7.09028",50.725830078125,7.090280055999756,"Germany: Nordrhein-Westfalen. Bonn, Nussallee.","Carex divulsa","Carex","R. Mause","B 10 0553784","https://herbarium.bgbm.org/object/B100553784" +"B100582909","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100582909/manifest.json","JACQ","CYPERACEAE","Carex praecox Schreb.","MTB: 4239/2/1/4","Willing,R. & Willing,E.","28469 D","2014-04-29","","Germany","Sachsen-Anhalt, SO Törten","","","https://www.openstreetmap.org/?mlat=51.7847&mlon=12.2744#map=15/51.7847/12.2744",51.78472137451172,12.274439811706543,"","Carex praecox","Carex","E.Willing","B 10 0582909","https://herbarium.bgbm.org/object/B100582909" +"B100586917","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100586917/manifest.json","JACQ","AMARANTHACEAE","Amaranthus powellii S. Watson","MTB: 4139/3/4/2","Willing,R. & Willing,E.","29581 D","2014-08-31","","Germany","Sachsen-Anhalt, Dessau-SĂŒd","","","https://www.openstreetmap.org/?mlat=51.8175&mlon=12.2428#map=15/51.8175/12.2428",51.817501068115234,12.242779731750488,"","Amaranthus powellii","Amaranthus","E.Willing","B 10 0586917","https://herbarium.bgbm.org/object/B100586917" +"B100612470","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100612470/manifest.json","BGBM","CARYOPHYLLACEAE","Lychnis flos-cuculi L. subsp. flos-cuculi","","Zippel,E.","13191","2013-06-21","","Germany","Germany: Sachsen. SĂ€chsische Schweiz-Osterzgebirge, Altenberg, B170, Schwarzwasserwiese zwischen Zinnwald-Georgenfeld - Altenberg. Borstgrasrasen. Alt.: 785 m. 21.06.2013, Leg.: E. Zippel 13191.","","","https://www.openstreetmap.org/?mlat=50.7594&mlon=13.7531#map=15/50.7594/13.7531",50.75944900512695,13.753060340881348,"Germany: Sachsen. SĂ€chsische Schweiz-Osterzgebirge, Altenberg, B170, Schwarzwasserwiese zwischen Zinnwald-Georgenfeld - Altenberg. Borstgrasrasen.","Lychnis flos-cuculi subsp. flos-cuculi","Lychnis","E. Zippel","B 10 0612470","https://herbarium.bgbm.org/object/B100612470" +"B100628509","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100628509/manifest.json","BGBM","COMPOSITAE","Galinsoga parviflora Cav.","GBOL2180, Silikaprobe genommen","Testroet,P.","s.n.","2014-07-24","","Germany","Germany: Nordrhein-Westfalen. Meckenheim, Stadtgebiet. an einer Kreuzung. 24.07.2014, Leg.: P. Testroet s.n. ex herb. / ded. : herb. Philip Testroet.","","","https://www.openstreetmap.org/?mlat=50.6306&mlon=7.02556#map=15/50.6306/7.02556",50.630550384521484,7.025559902191162,"Germany: Nordrhein-Westfalen. Meckenheim, Stadtgebiet. an einer Kreuzung.","Galinsoga parviflora","Galinsoga","P. Testroet","B 10 0628509","https://herbarium.bgbm.org/object/B100628509" +"B100655266","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100655266/manifest.json","HERB","Chenopodiaceae","Polycnemum arvense","Label data transcribed during a Herbonauten mission","","","","","Germany","Germany: Sachsen-Anhalt. Cröllwitz nach Lieskau zu.","","","https://www.openstreetmap.org/?mlat=51.5114&mlon=11.9221#map=15/51.5114/11.9221",51.51139831542969,11.922100067138672,"","Polycnemum arvense","Polycnemum","","B 10 0655266","http://herbarium.bgbm.org/object/B100655266" +"B100682266","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100682266/manifest.json","JACQ","ASTERACEAE","Hieracium murorum L.","MTB: 4433/2/4","Willing,R. & Willing,E.","29350 D","2014-08-11","","Germany","Sachsen-Anhalt, SO Wippra","","","https://www.openstreetmap.org/?mlat=51.5683&mlon=11.2967#map=15/51.5683/11.2967",51.568328857421875,11.296669960021973,"","Hieracium murorum","Hieracium","G. Gottschlich","B 10 0682266","https://herbarium.bgbm.org/object/B100682266" +"B100699347","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100699347/manifest.json","HERB","Chenopodiaceae","Chenopodium polyspermum","Label data transcribed during a Herbonauten mission","Fritz Hans","","1972-09-06","1972-09-06","Germany","Germany: Brandenburg. Karlshof, Krs. Bad Freienwalde, Vorgarten. 1972-09-06, Leg.: Fritz Hans.","","","https://www.openstreetmap.org/?mlat=52.7473&mlon=14.2615#map=15/52.7473/14.2615",52.74729919433594,14.261500358581543,"","Chenopodium polyspermum","Chenopodium","M. Ristow","B 10 0699347","http://herbarium.bgbm.org/object/B100699347" +"B100705423","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100705423/manifest.json","BGBM","LEGUMINOSAE-PAPILIONOIDEAE","Ononis repens subsp. procurrens (Wallr.) Bonnier & Layens","GBOL4172, Silikaprobe genommen, Aufsammlung von einem Individuum","Hand,R. & Niederbichler,C.","RH 7787","2016-07-30","","Germany","Germany: Bayern. Reichersbeuern, am Bahnhof. gemĂ€htes GrĂŒnland. Alt.: 723m. 30.07.2016, Leg.: R. Hand & C. Niederbichler RH 7787.","","","https://www.openstreetmap.org/?mlat=47.7744&mlon=11.6353#map=15/47.7744/11.6353",47.77444076538086,11.635279655456543,"Germany: Bayern. Reichersbeuern, am Bahnhof. gemĂ€htes GrĂŒnland.","Ononis repens subsp. procurrens","Ononis","R. Hand","B 10 0705423","https://herbarium.bgbm.org/object/B100705423" +"B100733887","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100733887/manifest.json","JACQ","FABACEAE","Trifolium medium L.","","Willing,E.","30379 D","2016-07-22","","Germany","Sachsen-Anhalt, LK Wittenberg, OSO Goltewitz, GK 4530896 /5739474 Photo","","","https://www.openstreetmap.org/?mlat=51.7883&mlon=12.4461#map=15/51.7883/12.4461",51.788330078125,12.446109771728516,"","Trifolium medium","Trifolium","E. Willing","B 10 0733887","https://herbarium.bgbm.org/object/B100733887" +"B100741855","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100741855/manifest.json","BGBM","CRUCIFERAE","Capsella bursa-pastoris (L.) Medik.","GBOL, Silikaprobe genommen","Mause,R.","2016-25","2016-05-26","","Germany","Germany: Nordrhein-Westfalen. sĂŒdlich Lommersdorf. Fettwiese. 26.05.2016, Leg.: R. Mause 2016-25. ex herb. / ded. : herb. RenĂ© Mause.","","","https://www.openstreetmap.org/?mlat=50.4061&mlon=6.73722#map=15/50.4061/6.73722",50.40610885620117,6.73721981048584,"Germany: Nordrhein-Westfalen. sĂŒdlich Lommersdorf. Fettwiese.","Capsella bursa-pastoris","Capsella","R. Mause","B 10 0741855","https://herbarium.bgbm.org/object/B100741855" +"B100761641","https://image.bgbm.org/images/internal/HerbarThumbs/B100761641_1700","VVis","","Campanula patula L.","Dr. Phil. Wirtgen Herbar. plant. critic., select. hybrid. Florae Rhenanae (Edit. nov.) (1753) 163. Herbar A. Ludwig IMAGE 2025 Mus. Bot. Berol. BG Botanischer Garten & BM Botanisches Museum Berlin","H. Andres","191","","","Germany",", Bergisches Land, Odental (Burg Strauweiler) bei Altenberg; c. 85 m. s. m. VÂł Z5.","","","","","","","Campanula patula","Campanula","","B 10 0761641","https://herbarium.bgbm.org/object/B100761641" +"B100766578","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100766578/manifest.json","HERB","BETULACEAE","Betula aurata","Label data transcribed during a Herbonauten mission","A. Ludwig","22","1931-08-06","1931-08-06","Germany","Germany: Nordrhein-Westfalen. Kr. Siegen: im Hauberg zwischen Rödgen und Wilnsdorf. 1931-08-06, Leg.: A. Ludwig 22.","","","https://www.openstreetmap.org/?mlat=50.8315&mlon=8.08525#map=15/50.8315/8.08525",50.8315315246582,8.085250854492188,"","Betula aurata","Betula","","B 10 0766578","http://herbarium.bgbm.org/object/B100766578" +"B100768790","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B100768790/manifest.json","JACQ","ASTERACEAE","Senecio cacaliaster subsp. hercynicus (Herborg) Oberpr.","","Oberprieler,C. & Hölzle,K.","497-6","1988-07-08","","Germany","Bayern, Landkreis OberallgĂ€u, Eschacher Wald, Weg vom Parkplatz ""Eschacher Weiher"" zur großen Schwedenschanze auf dem Ursersberg, sĂŒdgewandte Hochstaudenflur auf dem Ursersberg, Alt.: 1130m.","","","https://www.openstreetmap.org/?mlat=47.7242&mlon=10.2069#map=15/47.7242/10.2069",47.72417068481445,10.206939697265625,"","Senecio cacaliaster subsp. hercynicus","Senecio","Ch. Oberprieler","B 10 0768790","https://herbarium.bgbm.org/object/B100768790" +"B101001398","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101001398/manifest.json","HERB","Balsaminaceae","Impatiens sp.","Label data transcribed during a Herbonauten mission","H. Ristow","","1948-06-01","1948-06-30","Germany","Germany: Berlin. Spreeufer, FĂŒrstenbrunn. 1948-06-01 - 1948-06-30, Leg.: H. Ristow.","","","https://www.openstreetmap.org/?mlat=52.5287&mlon=13.2705#map=15/52.5287/13.2705",52.52872848510742,13.270454406738281,"","Impatiens sp.","Impatiens","","B 10 1001398","http://herbarium.bgbm.org/object/B101001398" +"B101012560","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101012560/manifest.json","HERB","Potamogetonaceae","Potamogeton lucens","Label data transcribed during a Herbonauten mission","O. et R. Schulz","","1895-07-09","1895-07-09","Germany","Germany: Brandenburg. AngermĂŒnde: Im Paarsteiner See in der NĂ€he des Werders. 1895-07-09, Leg.: O. et R. Schulz.","","","https://www.openstreetmap.org/?mlat=52.9574&mlon=13.9814#map=15/52.9574/13.9814",52.95737838745117,13.981390953063965,"","Potamogeton lucens","Potamogeton","","B 10 1012560","http://herbarium.bgbm.org/object/B101012560" +"B101023411","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101023411/manifest.json","BGBM","ORCHIDACEAE","Traunsteinera globosa (L.) Rchb.","GBOL, Silikaprobe genommen; beprobtes Exemplar mit * markiert","Hand,R. & Berghofer,M.","RH 9073","2018-07-07","","Germany","Germany: Bayern. Bad Reichenhall, Predigtstuhl, zwischen Seilbahnstation und Gipfel. subalpine Matte. Alt.: 1561 m. 07.07.2018, Leg.: R. Hand & M. Berghofer RH 9073.","","","https://www.openstreetmap.org/?mlat=47.6964&mlon=12.8775#map=15/47.6964/12.8775",47.696388244628906,12.8774995803833,"Germany: Bayern. Bad Reichenhall, Predigtstuhl, zwischen Seilbahnstation und Gipfel. subalpine Matte.","Traunsteinera globosa","Traunsteinera","R. Hand","B 10 1023411","https://herbarium.bgbm.org/object/B101023411" +"B101027679","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101027679/manifest.json","HERB","Cupressaceae","Thuja occidentalis","Label data transcribed during a Herbonauten mission","O. et R. Schulz","","1893-07-01","1893-07-31","Germany","Germany: Brandenburg. Chorin i/Mark: Auf dem Kirchhofe des Dorfes SenftenhĂŒtte angepflanzt. 1893-07-01 - 1893-07-31, Leg.: O. et R. Schulz.","","","https://www.openstreetmap.org/?mlat=52.9324&mlon=13.8587#map=15/52.9324/13.8587",52.932350158691406,13.85872745513916,"","Thuja occidentalis","Thuja","","B 10 1027679","http://herbarium.bgbm.org/object/B101027679" +"B101057305","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101057305/manifest.json","JACQ","SOLANACEAE","Solanum decipiens Opiz","PRN2021-013; MTB: 4140-1/2/2; GK rechts/hoch: 4528177/5750591","Willing,E.","30640 D","2019-09-12","","Germany","LK Wittenberg; NW Buro","","","https://www.openstreetmap.org/?mlat=51.8886&mlon=12.4075#map=15/51.8886/12.4075",51.88861083984375,12.407500267028809,"","Solanum decipiens","Solanum","E.Willing","B 10 1057305","https://herbarium.bgbm.org/object/B101057305" +"B101066692","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101066692/manifest.json","HERB","ACANTHACEAE","Acanthus longifolius","Label data transcribed during a Herbonauten mission","O. et R. Schulz","","1898-08-29","1898-08-29","Germany","Germany: Berlin. Berlin. 1898-08-29, Leg.: O. et R. Schulz.","","","https://www.openstreetmap.org/?mlat=52.5164&mlon=13.3899#map=15/52.5164/13.3899",52.51642990112305,13.389930725097656,"","Acanthus longifolius","Acanthus","","B 10 1066692","http://herbarium.bgbm.org/object/B101066692" +"B101077282","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101077282/manifest.json","JACQ","CRASSULACEAE","KalanchoĂ« crenata (Andrews) Haw.","Bogen 1/2","Anonymous collector","s.n.","1979-01","","Germany","Cultivated in Botanic Garden Berlin; provided by: collector; originally collected: Uganda: 2: Kigezi, 60 km nach Kabale in Richtung Kisoro, leg.: Pfennig,H. 1258, 31.8.1977","","","","","","BlĂŒten gelb, BlĂ€tter geöhrt","KalanchoĂ« crenata","KalanchoĂ«","E. Raadts (B)","B 10 1077282","https://herbarium.bgbm.org/object/B101077282" +"B101132213","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101132213/manifest.json","JACQ","BETULACEAE","Betula pubescens Ehrh.","PRN2020-021","Knoph,J.-G.","s.n.","1981-06-25","","Germany","Restseen nahe Deixlfurther See","","","","","","","Betula pubescens","Betula","J.-G. Knoph","B 10 1132213","https://herbarium.bgbm.org/object/B101132213" +"B101143892","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101143892/manifest.json","JACQ","CRASSULACEAE","KalanchoĂ« delagoĂ«nsis Eckl. & Zeyh.","Gartenherbar","Cubr,M.","52601","2024-02-08","","Germany","Cultivated in Botanic Garden Berlin; provided by: Frankfurt, Palmengarten der Stadt Frankfurt; originally collected: Madagaskar, TulĂ©ar (Prov.), Manakaravavy, Ampaniky, 280 m, leg.: Anonymous s.n., s.d.","","","","","","BlĂŒtenfarbe: rosarot, Filamente: rotviolett, Antheren: hellgelb, Narbe: weiß, Griffel: weiß, BlĂ€tter hellbraun, d.braun gestrichelt, Sproß rosa","KalanchoĂ« delagoĂ«nsis","KalanchoĂ«","S. Bernhard (B) 2025-12-08","B 10 1143892","https://herbarium.bgbm.org/object/B101143892" +"B101157822","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101157822/manifest.json","JACQ","CARYOPHYLLACEAE","Dianthus deltoides L.","Herbar Melzheimer (Volker Melzheimer, 1939 – 2024 und Gertraud Melzheimer)","Melzheimer,V. & Melzheimer,G.","s.n.","1989-09-12","","Germany","Deutschland; Hessen:Wetter OT Amönau: am Treisbach","","","https://www.openstreetmap.org/?mlat=50.9089&mlon=8.69056#map=15/50.9089/8.69056",50.90888977050781,8.690560340881348,"","Dianthus deltoides","Dianthus","orig.","B 10 1157822","https://herbarium.bgbm.org/object/B101157822" +"B101171447","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101171447/manifest.json","JACQ","ROSACEAE","Sanguisorba minor subsp. balearica (Bourg. ex Nyman) Muñoz Garm. & C. Navarro","PRN2021-013; MTB: 4130/4/1/1; GK rechts/hoch: 4414035/5746261","Willing,E. & Willing,R.","31912 D","2020-09-28","","Germany","SO Darlingerode","","","https://www.openstreetmap.org/?mlat=51.8436&mlon=10.7508#map=15/51.8436/10.7508",51.84360885620117,10.750829696655273,"","Sanguisorba minor subsp. balearica","Sanguisorba","E.Willing","B 10 1171447","https://herbarium.bgbm.org/object/B101171447" +"B101172459","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101172459/manifest.json","JACQ","AMARANTHACEAE","Amaranthus retroflexus L.","PRN2021-013; MTB: 4239/2/1/1","Willing,E.","30593 D","2019-08-12","","Germany","Dessau, SO Törten","","","https://www.openstreetmap.org/?mlat=51.7919&mlon=12.2656#map=15/51.7919/12.2656",51.79193878173828,12.265560150146484,"","Amaranthus retroflexus","Amaranthus","E.Willing","B 10 1172459","https://herbarium.bgbm.org/object/B101172459" +"B101185579","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101185579/manifest.json","JACQ","CRASSULACEAE","KalanchoĂ« lanceolata (Forssk.) Pers.","Aus Samen von Bally Nr. 11918, Somalia, Samen erhalten vom Jardin Bot. ""Les CĂšdres"", Aussaat am 10.3.1967, Samen dieser Pfl. nochmals ausgesĂ€t 1971 ?, eingelegt am 29.1.1972 ?; Chromosomen 2n= 34, 1.2.1972; Nr. 73/1, Knospen vom 29.1.72; Blatt-Material fĂŒ","Anonymous collector","s.n.","1972-01-29","","Germany","Cultivated in Botanic Garden Berlin; provided by: Jardin Bot. ""Les CĂšdres""; originally collected: Somalia: leg.: Bally,P.R.O. 11918, s.d.","","","","","","","KalanchoĂ« lanceolata","KalanchoĂ«","Anonymous","B 10 1185579","https://herbarium.bgbm.org/object/B101185579" +"B101199939","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101199939/manifest.json","HERB","BERBERIDACEAE","Berberis vulgaris L.","Label data transcribed during a Herbonauten mission","[BornmĂŒller]","","1945-01-01","1945-12-31","Germany","Germany: ThĂŒringen. Weimar cult. Belvedere. 1945, Leg.: [BornmĂŒller].","","","https://www.openstreetmap.org/?mlat=50.9478&mlon=11.3495#map=15/50.9478/11.3495",50.947776794433594,11.349498748779297,"","Berberis vulgaris L.","Berberis","","B 10 1199939","http://herbarium.bgbm.org/object/B101199939" +"B101213143","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101213143/manifest.json","HERB","BETULACEAE","indet.","Label data transcribed during a Herbonauten mission","J. B[ornmĂŒller]","","1945-05-30","1945-05-30","Germany","Germany: ThĂŒringen. W[eimar]. 1945-05-30, Leg.: J. B[ornmĂŒller].","","","https://www.openstreetmap.org/?mlat=50.9805&mlon=11.3276#map=15/50.9805/11.3276",50.980491638183594,11.327569007873535,"","indet.","indet.","","B 10 1213143","http://herbarium.bgbm.org/object/B101213143" +"B101231411","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101231411/manifest.json","JACQ","RHAMNACEAE","Rhamnus cathartica L.","PRN2022-045; Herbarium Hans-Christian KlĂ€ge","KlĂ€ge,H.-C.","54","2013-05-29","","Germany","KĂŒhnauer Heide, NW-Teil","","","","","","","Rhamnus cathartica","Rhamnus","anonymous","B 10 1231411","https://herbarium.bgbm.org/object/B101231411" +"B101252985","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101252985/manifest.json","JACQ","ROSACEAE","Potentilla erecta (L.) Raeusch.","PRN2022-045; Herbarium Hans-Christian KlĂ€ge","KlĂ€ge,H.-C.","22","1969-09-10","","Germany","Landkreis Dahme-Spreewald, Bergen-Weißacker Moor; MTB 4248NW4","","","","","","","Potentilla erecta","Potentilla","H. Illig","B 10 1252985","https://herbarium.bgbm.org/object/B101252985" +"B101273417","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101273417/manifest.json","JACQ","POLYGONACEAE","Rumex patientia L.","PRN2024-003; Pictures of the same individual or population as the preserved specimen(s). Photographer: Eckhard Willing","Willing,E. & Willing,R.","37573 D","2023-06-08","","Germany","S Schönhausen","","","https://www.openstreetmap.org/?mlat=52.5628&mlon=12.0306#map=15/52.5628/12.0306",52.56277847290039,12.030559539794922,"","Rumex patientia","Rumex","E. Willing","B 10 1273417","https://herbarium.bgbm.org/object/B101273417" +"B101288043","https://image.bgbm.org/images/internal/HerbarThumbs/B101288043_1700","VVis","","Pulmonaria officinalis var. obscura DĂŒmortier.","Borraginaceae Desvaux 1935 Herbar H. Herold IMAGE 2024 Mus. bot. Berol. BGBM Botanischer Garten & Botanisches Museum Berlin MUSEUM BOTANICUM BEROLINENSE Mus. Bot. Berol.","Herold,H.","","1966-03-08","","Germany","Bayern, Krumbach, Schwaben, Thannhausen, am Schloßberg","","","","","","","Pulmonaria officinalis var. obscura","Pulmonaria","","B 10 1288043","https://herbarium.bgbm.org/object/B101288043" +"B101303154","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101303154/manifest.json","JACQ","RUBIACEAE","Galium album Mill.","PRN2024-003","Willing,E. & Willing,R.","37714 D","2023-06-09","","Germany","JL, Kliezenieck","","","https://www.openstreetmap.org/?mlat=52.4714&mlon=12.0169#map=15/52.4714/12.0169",52.47138977050781,12.016940116882324,"","Galium album","Galium","E. Willing","B 10 1303154","https://herbarium.bgbm.org/object/B101303154" +"B101304167","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101304167/manifest.json","JACQ","ASTERACEAE","Tripleurospermum perforatum (MĂ©rat) M. LaĂ­nz","PRN2024-003","Willing,E. & Willing,R.","35792 D","2022-09-01","","Germany","LK Harz, SO Wegeleben","","","https://www.openstreetmap.org/?mlat=51.8642&mlon=11.1842#map=15/51.8642/11.1842",51.86417007446289,11.18416976928711,"","Tripleurospermum perforatum","Tripleurospermum","E. Willing","B 10 1304167","https://herbarium.bgbm.org/object/B101304167" +"B101306337","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101306337/manifest.json","JACQ","HYPERICACEAE","Hypericum perforatum L.","Herbarium Gregor Steinbrecher; PRN2024-006; Herbarbeleg aus der Lehrveranstaltung Exkursionen zur Vegetationsökologie TU Berlin, Institut fĂŒr Ökologie, Dr. Birgit Seitz","Steinbrecher,Gregor","s.n.","2019-06-16","","Germany","Teufelsmoor","","","","","","","Hypericum perforatum","Hypericum","G. Steinbrecher","B 10 1306337","https://herbarium.bgbm.org/object/B101306337" +"B101309350","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101309350/manifest.json","JACQ","APIACEAE","Anthriscus sylvestris (L.) Hoffm.","PRN2024-006; Herbarbeleg aus der Lehrveranstaltung Exkursionen zur Vegetationsökologie TU Berlin, Institut fĂŒr Ökologie, Dr. Birgit Seitz; Herbarium Valentin Fischer","Fischer,Valentin","s.n.","2015-05-31","","Germany","Lebus, Odertal Niederung","","","","","","","Anthriscus sylvestris","Anthriscus","V. Fischer","B 10 1309350","https://herbarium.bgbm.org/object/B101309350" +"B101311664","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B101311664/manifest.json","JACQ","SALICACEAE","Salix silesiaca Willd.","Herbarium Fabian Streich; PRN2024-006; Herbarbeleg aus der Lehrveranstaltung Exkursionen zur Vegetationsökologie TU Berlin, Institut fĂŒr Ökologie, Dr. Birgit Seitz","Streich,Fabian","s.n.","2020-07-23","","Germany","NSG Odertal Frankfurt-Lebus mit Pontischen HĂ€ngen","","","","","","","Salix silesiaca","Salix","F. Streich","B 10 1311664","https://herbarium.bgbm.org/object/B101311664" +"B101322018","https://image.bgbm.org/images/internal/HerbarThumbs/B101322018_1700","VVis","","Campanula rotundifolia L.","IMAGE 2025 BG BM Botanischer Garten & Botanisches Museum Berlin HERBARIUM WILLING Mus. Bot. Berol. Mus. bot. Berol. Nr. Dat. Fam. Campanulaceae Art Ort Anm. MTB 7046/3, 2414-2440 leg.","EisenblĂ€tter,R., Willing,E.","2419 D","1995-08-10","","Germany","Bayern, Freyung-Grafenau, N Spiegelau, S-Abhang Bocksberg","","","https://www.openstreetmap.org/?mlat=58.925&mlon=13.3667#map=15/58.925/13.3667",58.92499923706055,13.366666793823242,"","Campanula rotundifolia","Campanula","","B 10 1322018","https://herbarium.bgbm.org/object/B101322018" +"B180012427","","BGBM","PINACEAE","Pinus balfouriana S.Watson","(aristata)","","","1896-08-25","","Germany","Germany: Wörlitz. 25.08.[18]96.","","","","","","Germany: Wörlitz.","Pinus balfouriana","Pinus","","B 18 0012427","https://herbarium.bgbm.org/object/B180012427" +"B180015736","","BGBM","COMPOSITAE","Cirsium oleraceum (L.) Scop.","","DĂŒrbye,T., Henneken,I. & Wiechert","DÜR 2591","1993-08-12","","Germany","Germany: Brandenburg, Havelland (Kreis), Falkensee, Nieder-Neuendorfer Kanal. Kanalrand. 12.8.1993, Leg.: T. DĂŒrbye, I. Henneken & Wiechert DÜR 2591.","","","","","","Germany: Brandenburg, Havelland (Kreis), Falkensee, Nieder-Neuendorfer Kanal. Kanalrand.","Cirsium oleraceum","Cirsium","T. Raus","B 18 0015736","https://herbarium.bgbm.org/object/B180015736" +"B180018140","","BGBM","UMBELLIFERAE","Heracleum sphondylium L.","","Schwarz","s.n.","","","Germany","Germany: Niedersachsen, Solling. Alt.: 240 m. Leg.: Schwarz s.n.","","","","","","Germany: Niedersachsen, Solling.","Heracleum sphondylium","Heracleum","","B 18 0018140","https://herbarium.bgbm.org/object/B180018140" +"B180020723","","BGBM","CARYOPHYLLACEAE","Dianthus superbus L.","","Auhagen","","1987-07-03","","Germany","Germany: Berlin, Spandauer Forst, Eiskeller. 03.07.1987, Leg.: Auhagen s.n.","","","","","","Germany: Berlin, Spandauer Forst, Eiskeller.","Dianthus superbus","Dianthus","","B 18 0020723","https://herbarium.bgbm.org/object/B180020723" +"B200127101","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B200127101/manifest.json","JACQ","EQUISETACEAE","Equisetum palustre L.","MTB: 3144/3","Willing,R. & Willing,E.","1641 D","1994-10-02","","Germany","Brandenburg, S Beetz","","","https://www.openstreetmap.org/?mlat=52.8083&mlon=13.0167#map=15/52.8083/13.0167",52.80833053588867,13.016670227050781,"","Equisetum palustre","Equisetum","E.Willing","B 20 0127101","https://herbarium.bgbm.org/object/B200127101" +"B200173340","https://image.bgbm.org/images/internal/HerbarThumbs/B200173340_1700","VVis","","Asplenium ruta-muraria","IMAGE 2025 Mus. Bot. Berol. BG Botanischer Garten & BM Botanisches Museum Berlin","","","2001-08-22","","Germany","ThĂŒringen, MĂŒhlhausen, Divi Blasii - Kirche untere Mauer Frontseite","","","","","","","Asplenium ruta-muraria","Asplenium","","B 20 0173340","https://herbarium.bgbm.org/object/B200173340" +"B200184258","https://image.bgbm.org/images/internal/HerbarThumbs/B200184258_1700","VVis","","Woodsia ilvensis","Herb. D. E. Meyer No... 919... Museum botanicum Berolinense eingel. Herb. D. E. Meyer No... 978... Museum botanicum Berolinense (Kassel) Mus. Bot. Berol. Mus. bot. Berol. BG BM Botanischer Garten & Botanisches Museum Berlin","Meyer,D.","919, 978","1958-06-04","","Germany","Berlin, , Hort. Bot. Berol. (von Kassel)","","","","","","Spor. Spont.","Woodsia ilvensis","Woodsia","D. Meyer.","B 20 0184258","https://herbarium.bgbm.org/object/B200184258" +"B200226315","https://image.bgbm.org/images/internal/HerbarThumbs/B200226315_1700","VVis","","Athyrium Filix-femina L.","Mus. Bot. Berol. IMAGE 2025 [MULTIPLE BARCODES] Herbarium des Instituts fĂŒr Spezielle Botanik der Humboldt-UniversitĂ€t zu Berlin Flora Dahme Grenzgraben det. Bm 430/62 10 EX BHU Flora von Brandenburg Herberium M. Schmattorsch Mus. bot. Berol. BGBM Botanischer Garten & Botanisches Museum Berlin","Schmattorsch","","1950-10-04","","Germany","Brandenburg, , Dahme Grenzgraben bis Jg. 15 des KĂ€mmerforst gegen Osten","","","","","","sehr schlank imnissinvoll","Athyrium Filix-femina","Athyrium","Herbarium M. Schmattorsch","B 20 0226315","https://herbarium.bgbm.org/object/B200226315" +"B300312284","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B300312284/manifest.json","HERB","keine","Rhynchostegium riparioides","Label data transcribed during a Herbonauten mission","Frahm","","1970-10-30","1970-10-30","Germany","Germany: Hamburg. NSG Heuckenlock bei Stillhorn/SĂŒderelbe; an der Basis von Pop. nigra im Hochwasserbereich; Schwarzpappelauenwald; ungewöhnl., robuste, rund bebl, etwas sparrige fo. 1970-10-30, Leg.: Frahm.","","","https://www.openstreetmap.org/?mlat=53.4732&mlon=10.0394#map=15/53.4732/10.0394",53.473201751708984,10.039400100708008,"","Rhynchostegium riparioides","Rhynchostegium","","B 30 0312284","http://herbarium.bgbm.org/object/B300312284" +"B300316594","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B300316594/manifest.json","HERB","!indet.","Pottia intermedia","Label data transcribed during a Herbonauten mission","Frahm","","2008-03-27","2008-03-27","Germany","Germany: Rheinland-Pfalz. Wöllstatt. 2008-03-27, Leg.: Frahm.","","","https://www.openstreetmap.org/?mlat=49.8064&mlon=7.9441#map=15/49.8064/7.9441",49.806400299072266,7.9440999031066895,"","Pottia intermedia","Pottia","","B 30 0316594","http://herbarium.bgbm.org/object/B300316594" +"B300322072","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B300322072/manifest.json","HERB","Orthotrichaceae","Orthotrichum stramineum","Label data transcribed during a Herbonauten mission","Fr.","","1965-07-23","1965-07-23","Germany","Germany: Niedersachsen. Kr. Soltau, am Waldrand im NSG LĂŒneburger Heide. 1965-07-23, Leg.: Fr.","","","https://www.openstreetmap.org/?mlat=53.1205&mlon=9.86708#map=15/53.1205/9.86708",53.12049865722656,9.867079734802246,"","Orthotrichum stramineum","Orthotrichum","Fr.","B 30 0322072","http://herbarium.bgbm.org/object/B300322072" +"B300326472","","HERB","Fontinalaceae","Fontinalis antipyretica Hedw.","Label data transcribed during a Herbonauten mission","W. Heimhold","863","1974-07-04","1974-07-04","Germany","Germany: Niedersachsen. Langelsheim/Harz, flutend in der Innerste, alt. 180 m. 1974-07-04, Leg.: W. Heimhold 863.","","","https://www.openstreetmap.org/?mlat=51.9332&mlon=10.3318#map=15/51.9332/10.3318",51.933231353759766,10.331783294677734,"","Fontinalis antipyretica Hedw.","Fontinalis","Frahm","B 30 0326472","http://herbarium.bgbm.org/object/B300326472" +"B300330323","","HERB","Mniaceae","Cinclidium stygium Sw.","Label data transcribed during a Herbonauten mission","N. Jensen","","1933-08-15","1933-08-15","Germany","Germany: Schleswig-Holstein. Kr. Rendsburg, Moor bei Wennbek. 1933-08-15, Leg.: N. Jensen.","","","https://www.openstreetmap.org/?mlat=54.1896&mlon=9.89147#map=15/54.1896/9.89147",54.1895637512207,9.891474723815918,"","Cinclidium stygium Sw.","Cinclidium","N. Jensen","B 30 0330323","http://herbarium.bgbm.org/object/B300330323" +"B300333050","","HERB","Brachytheciaceae","Brachythecium plumosum (Hedw.) Schimp.","Label data transcribed during a Herbonauten mission","Frahm","","1975-05-03","1975-05-03","Germany","Germany: Nordrhein-Westfalen. Kr. Wuppertal, Beyenberg [Beyenburg], offen exponierte Grauwackenfelsen an der Wupper. 1975-05-03, Leg.: Frahm.","","","https://www.openstreetmap.org/?mlat=51.2462&mlon=7.29689#map=15/51.2462/7.29689",51.246212005615234,7.296891689300537,"","Brachythecium plumosum (Hedw.) Schimp.","Brachythecium","DĂŒll","B 30 0333050","http://herbarium.bgbm.org/object/B300333050" +"B300401125","","HERB","Gymnomitriaceae","Marsupella emarginata (Ehrh.) Dumort.","Label data transcribed during a Herbonauten mission","Fr. [Frahm]","","1964-10-07","1964-10-07","Germany","Germany: Niedersachsen. Kreis: Zellerfeld, an Steinen unterhalb des Wasserspiegels eines Quellbaches der Radau östl. Torhaus, 800 m ĂŒb. NN. 1964-10-07, Leg.: Fr. [Frahm].","","","https://www.openstreetmap.org/?mlat=51.7998&mlon=10.5406#map=15/51.7998/10.5406",51.799808502197266,10.540631294250488,"","Marsupella emarginata (Ehrh.) Dumort.","Marsupella","Fr. [Frahm]","B 30 0401125","http://herbarium.bgbm.org/object/B300401125" +"B31084302","https://iiif.jacq.org/b/?manifest=https://herbarium.bgbm.org/object/B31084302/manifest.json","BGBM","HYPNACEAE","Hypnum purum Hedw.","","","","","","Germany","Germany.","","","","","","Germany.","Hypnum purum","Hypnum","","B 31 0843 02","https://herbarium.bgbm.org/object/B31084302" +"B700002459","","JACQ","DISCINACEAE","Gyromitra esculenta (Pers.) Fr.","MTB 3847","Benkert,D.","s.n.","1975-04-28","","Germany","Landkreis Dahme-Spreewald, nordwestl. Egsdorf","","","https://www.openstreetmap.org/?mlat=52.1333&mlon=13.5833#map=15/52.1333/13.5833",52.133331298828125,13.583330154418945,"","Gyromitra esculenta","Gyromitra","D. Benkert","B 70 0002459","https://herbarium.bgbm.org/object/B700002459" +"B700003645","","JACQ","PEZIZACEAE","Pachyella babingtonii (Berk. & Broome) Boud.","","Benkert,D.","s.n.","1995-09-23","","Germany","Landkreis Potsdam-Mittelmark, nördlich Rottstock (OT von Ziesar), Quellgebiet der ""Gesundbrunnen""","","","","","","","Pachyella babingtonii","Pachyella","D. Benkert","B 70 0003645","https://herbarium.bgbm.org/object/B700003645" +"B700005016","","JACQ","ASCOBOLACEAE","Ascobolus sacchariferus Brumm.","MTB 3546","Marx,H.","s.n.","1991-03-02","","Germany","Bezirk Treptow, Baumschulenweg, Königsheide","","","https://www.openstreetmap.org/?mlat=52.4667&mlon=13.4833#map=15/52.4667/13.4833",52.466670989990234,13.483329772949219,"","Ascobolus sacchariferus","Ascobolus","D. Benkert","B 70 0005016","https://herbarium.bgbm.org/object/B700005016" +"B700008206","","JACQ","PYRONEMATACEAE","Octospora coccinea (P. Crouan & H. Crouan) Brumm.","MTB 3553","Benkert,D.","s.n.","1993-10-07","","Germany","Landkreis MĂ€rkisch-Oderland, sĂŒdlich Lebus","","","https://www.openstreetmap.org/?mlat=52.4167&mlon=14.5333#map=15/52.4167/14.5333",52.41667175292969,14.533329963684082,"","Octospora coccinea","Octospora","D. Benkert","B 70 0008206","https://herbarium.bgbm.org/object/B700008206" +"B700011207","","JACQ","PEZIZACEAE","Peziza saniosa Schrad.","MTB 4626/2","Benkert,D.","s.n.","1981-09-09","","Germany","Landkreis Eichsfeld, bei Heilbad Heiligenstadt, an der Elisabethquelle","","","https://www.openstreetmap.org/?mlat=51.3667&mlon=10.1333#map=15/51.3667/10.1333",51.366668701171875,10.133330345153809,"","Peziza saniosa","Peziza","D. Benkert","B 70 0011207","https://herbarium.bgbm.org/object/B700011207" +"BGT0001296","","BGBM","ROSACEAE","Prunus mahaleb L.","","Ristow,M.","419/08","2008-05-10","","Germany","Germany: Brandenburg. Havelland, Burgwall Kapellberg, E ehemaliger Siedlung Knoblauch SE Etzin, Mtb 3443/34. Trockenrasen. Alt.: 50-60m. 10.05.2008, Leg.: M. Ristow 419/08.","","","https://www.openstreetmap.org/?mlat=52.51&mlon=12.8778#map=15/52.51/12.8778",52.5099983215332,12.877779960632324,"Germany: Brandenburg. Havelland, Burgwall Kapellberg, E ehemaliger Siedlung Knoblauch SE Etzin, Mtb 3443/34. Trockenrasen.","Prunus mahaleb","Prunus","","B GT 0001296","https://herbarium.bgbm.org/object/BGT0001296" +"BGT0005855","","BGBM","ROSACEAE","Rosa agrestis Savi","","Ristow,M. & Lauterbach,D.","909/09","2009-08-12","","Germany","Germany: Brandenburg. Sperenberg, auf dem westlichsten GipshĂŒgel, Mtb 3846/31. brachliegender Trockenrasen. Alt.: 60m. 12.08.2009, Leg.: M. Ristow & D. Lauterbach 909/09.","","","https://www.openstreetmap.org/?mlat=52.1386&mlon=13.3722#map=15/52.1386/13.3722",52.13861083984375,13.372220039367676,"Germany: Brandenburg. Sperenberg, auf dem westlichsten GipshĂŒgel, Mtb 3846/31. brachliegender Trockenrasen.","Rosa agrestis","Rosa","M. Ristow","B GT 0005855","https://herbarium.bgbm.org/object/BGT0005855" +"BGT0009340","","BGBM","GRAMINEAE","Festuca psammophila (Hack. ex Celak.) Fritsch","","Ismail,S. & Duwe,V.","SI14008","2014-06-02","","Germany","Germany: Brandenburg. Wernsdorf, Wegrand. 02.06.2014, Leg.: S. Ismail & V. Duwe SI14008.","","","https://www.openstreetmap.org/?mlat=52.3739&mlon=13.7086#map=15/52.3739/13.7086",52.3738899230957,13.708609580993652,"Germany: Brandenburg. Wernsdorf, Wegrand.","Festuca psammophila","Festuca","","B GT 0009340","https://herbarium.bgbm.org/object/BGT0009340" +"BGT0010431","","BGBM","DIPSACACEAE","Scabiosa canescens Waldst. & Kit.","","Ismail,S. & Duwe,V.","SI 14 013","2014-07-02","","Germany","Germany: Schwaben. Augsburger Stadtwald, NSG Königsbrunnerheide. 02.07.2014, Leg.: S. Ismail & V. Duwe SI 14 013.","","","https://www.openstreetmap.org/?mlat=48.2722&mlon=10.9078#map=15/48.2722/10.9078",48.272220611572266,10.907779693603516,"Germany: Schwaben. Augsburger Stadtwald, NSG Königsbrunnerheide.","Scabiosa canescens","Scabiosa","","B GT 0010431","https://herbarium.bgbm.org/object/BGT0010431" +"BGT0011502","","BGBM","LAMINARIACEAE","Laminaria digitata (Hudson) J.V.Lamouroux","","Wagner,A.","n.n.","2007-09-11","","Germany","Germany: Schleswig-Holstein. Helgoland. 11.09.2007, Leg.: A. Wagner n.n.","","","https://www.openstreetmap.org/?mlat=54.1667&mlon=7.88333#map=15/54.1667/7.88333",54.16667175292969,7.88332986831665,"Germany: Schleswig-Holstein. Helgoland.","Laminaria digitata","Laminaria","A. Wagner","B GT 0011502","https://herbarium.bgbm.org/object/BGT0011502" +"BGT0012622","","BGBM","ORCHIDACEAE","Dactylorhiza majalis (Rchb.) Hunt & Summerh.","","Lakmann,Dr., Duwe,V. & Wanke,R.","GL1504/ VD15020","2015-05-30","","Germany","Germany: Nordrhein-Westfalen. NSG Barbruch. Alt.: 88 m. 30.05.2015, Leg.: Dr. Lakmann, V. Duwe & R. Wanke GL1504/ VD15020.","","","","","","Germany: Nordrhein-Westfalen. NSG Barbruch.","Dactylorhiza majalis","Dactylorhiza","","B GT 0012622","https://herbarium.bgbm.org/object/BGT0012622" +"BGT0014362","","JACQ","ROSACEAE","Sorbus latifolia (Lam.) Pers.","GBOL3843, Silikaprobe genommen, nur 1 Exemplar beprobt; Arbeitsname ""Edelmannswald""; Tormaria-Sippe mit 2n = 68; Voucher in B: B 10 0612654","Meyer,N.","NM060","","","Germany","Edelmannswald, Wegrand gegenĂŒber S. aff. croceocarpa, TK 6125/13.","","","https://www.openstreetmap.org/?mlat=49.8719&mlon=9.86861#map=15/49.8719/9.86861",49.87194061279297,9.868610382080078,"","Sorbus latifolia","Sorbus","N. Meyer","B GT 0014362","https://herbarium.bgbm.org/object/BGT0014362" +"BGT0017393","","BGBM","IRIDACEAE","Crocus tommasinianus Herb.","GBOL, Silikaprobe genommen; Stinsenpflanze","Ciongwa,P.","PC 105","2016-03-01","","Germany","Germany: Niedersachsen. Northeim, Alter Friedhof. Parkanlage. Alt.: 130m. 01.03.2016, Leg.: P. Ciongwa PC 105. ex herb. / ded. : herb. Peter Ciongwa.","","","https://www.openstreetmap.org/?mlat=51.7061&mlon=9.99472#map=15/51.7061/9.99472",51.70610809326172,9.994720458984375,"Germany: Niedersachsen. Northeim, Alter Friedhof. Parkanlage.","Crocus tommasinianus","Crocus","P. Ciongwa","B GT 0017393","https://herbarium.bgbm.org/object/BGT0017393" +"BGT0018479","","BGBM","COMPOSITAE","Galinsoga parviflora Cav.","GBOL, Silikaprobe genommen","Buttler,K.P.","36686","2016-07-25","","Germany","Germany: Hessen. Frankfurt-Schwanheim, S-Seite des Höchster Wegs c. 270 m SSE der Anlegestelle der Höchster FĂ€hre . Gartenland. 25.07.2016, Leg.: K. P. Buttler 36686. ex herb. / ded. : herb. Karl Peter Buttler.","","","https://www.openstreetmap.org/?mlat=50.0847&mlon=8.55889#map=15/50.0847/8.55889",50.084720611572266,8.558890342712402,"Germany: Hessen. Frankfurt-Schwanheim, S-Seite des Höchster Wegs c. 270 m SSE der Anlegestelle der Höchster FĂ€hre . Gartenland.","Galinsoga parviflora","Galinsoga","K. P. Buttler","B GT 0018479","https://herbarium.bgbm.org/object/BGT0018479" +"BGT0021377","","BGBM","LABIATAE","Stachys alpina L.","GBOL, Silikaprobe genommen","Ciongwa,P.","PC 248","2017-06-14","","Germany","Germany: Niedersachsen. Northeim, Wieter NOM, 1 km NE Sudheim. Laubwald, Kalk. Alt.: 260m. 14.06.2017, Leg.: P. Ciongwa PC 248. ex herb. / ded. : herb. Peter Ciongwa.","","","https://www.openstreetmap.org/?mlat=51.1111&mlon=10#map=15/51.1111/10",51.11111068725586,10.0,"Germany: Niedersachsen. Northeim, Wieter NOM, 1 km NE Sudheim. Laubwald, Kalk.","Stachys alpina","Stachys","P. Ciongwa","B GT 0021377","https://herbarium.bgbm.org/object/BGT0021377" +"BGT0024809","","BGBM","CRASSULACEAE","Sempervivum tectorum L.","GBOL, Silikaprobe genommen; beprobtes Exemplar mit * markiert; indigene Moseltal-Population","Hand,R.","8948","2018-05-24","","Germany","Germany: Rheinland-Pfalz. Treis-Karden, Nordrand von Karden, Beginn des Buchsbaumweges zum Klickerterhof. an Schieferfelsen. Alt.: 106 m. 24.05.2018, Leg.: R. Hand 8948.","","","https://www.openstreetmap.org/?mlat=50.185&mlon=7.30167#map=15/50.185/7.30167",50.185001373291016,7.301670074462891,"Germany: Rheinland-Pfalz. Treis-Karden, Nordrand von Karden, Beginn des Buchsbaumweges zum Klickerterhof. an Schieferfelsen.","Sempervivum tectorum","Sempervivum","R. Hand","B GT 0024809","https://herbarium.bgbm.org/object/BGT0024809" +"JACQID1060010","","JACQ","ASTERACEAE","Achillea setacea Waldst. & Kit.","MTB: 3741/3","Willing,R. & Willing,E.","19835 D","2002-08-03","","Germany","Brandenburg, W Ragösen","","","https://www.openstreetmap.org/?mlat=52.245&mlon=12.5619#map=15/52.245/12.5619",52.244998931884766,12.56194019317627,"","Achillea setacea","Achillea","E.Willing","JACQ-ID 1060010","https://herbarium.bgbm.org/object/JACQID1060010" +"JACQID1061159","","JACQ","APIACEAE","Angelica sylvestris L.","MTB: 8426/1","Willing,R. & Willing,E.","9233 D","1999-08-10","","Germany","Bayern, SO Kalzhofen","","","https://www.openstreetmap.org/?mlat=47.56&mlon=10.0369#map=15/47.56/10.0369",47.560001373291016,10.03693962097168,"","Angelica sylvestris","Angelica","E.Willing","JACQ-ID 1061159","https://herbarium.bgbm.org/object/JACQID1061159" +"JACQID1062275","","JACQ","ASTERACEAE","Asteraceae Bercht. & J. Presl","MTB: 6844/2","Willing,R. & Willing,E.","1973 D","1995-08-05","","Germany","Bayern, 1,0 km SO Sommerau","","","https://www.openstreetmap.org/?mlat=49.15&mlon=13.1167#map=15/49.15/13.1167",49.150001525878906,13.116669654846191,"","Asteraceae","Asteraceae","E.Willing","JACQ-ID 1062275","https://herbarium.bgbm.org/object/JACQID1062275" +"JACQID1063318","","JACQ","POACEAE","Calamagrostis epigejos (L.) Roth","MTB: 4138/3/2/3","Willing,R. & Willing,E.","26128 D","2009-07-30","","Germany","Sachsen-Anhalt, NNW Kleinzerbst","","","https://www.openstreetmap.org/?mlat=51.8331&mlon=12.0453#map=15/51.8331/12.0453",51.83306121826172,12.045280456542969,"","Calamagrostis epigejos","Calamagrostis","E.Willing","JACQ-ID 1063318","https://herbarium.bgbm.org/object/JACQID1063318" +"JACQID1064511","","JACQ","CYPERACEAE","Carex hirta L.","MTB: 4138/4/1/4","Willing,R. & Willing,E.","23660 D","2008-05-14","","Germany","Sachsen-Anhalt, NW Chörau","","","https://www.openstreetmap.org/?mlat=51.8261&mlon=12.1094#map=15/51.8261/12.1094",51.82611083984375,12.109439849853516,"","Carex hirta","Carex","E.Willing","JACQ-ID 1064511","https://herbarium.bgbm.org/object/JACQID1064511" +"JACQID1065683","","JACQ","ASTERACEAE","Crepis capillaris (L.) Wallr.","MTB: 4045/4","Willing,R. & Willing,E.","3019 D","1995-10-11","","Germany","Brandenburg, SO Nonnendorf","","","https://www.openstreetmap.org/?mlat=51.9&mlon=13.2583#map=15/51.9/13.2583",51.900001525878906,13.258330345153809,"","Crepis capillaris","Crepis","E.Willing","JACQ-ID 1065683","https://herbarium.bgbm.org/object/JACQID1065683" +"JACQID1066761","","JACQ","ASTERACEAE","Chrysanthemum vulgare (L.) Bernh.","MTB: 3843/4","Willing,R. & Willing,E.","763 D","1994-07-10","","Germany","Brandenburg, 1,5 km SO Buchholz","","","https://www.openstreetmap.org/?mlat=52.1417&mlon=12.9333#map=15/52.1417/12.9333",52.14167022705078,12.933329582214355,"","Chrysanthemum vulgare","Chrysanthemum","E.Willing","JACQ-ID 1066761","https://herbarium.bgbm.org/object/JACQID1066761" +"JACQID1067871","","JACQ","POACEAE","Danthonia decumbens (L.) DC.","MTB: 4040/2/2/4","Willing,R. & Willing,E.","27314 D","2011-08-18","","Germany","Sachsen-Anhalt, SSO Göritz","","","https://www.openstreetmap.org/?mlat=51.9786&mlon=12.4761#map=15/51.9786/12.4761",51.97861099243164,12.476110458374023,"","Danthonia decumbens","Danthonia","E.Willing","JACQ-ID 1067871","https://herbarium.bgbm.org/object/JACQID1067871" +"JACQID1069019","","JACQ","ONAGRACEAE","Epilobium parviflorum Schreb.","MTB: 4138/1/1/1","Willing,R. & Willing,E.","25104 D","2008-09-11","","Germany","Sachsen-Anhalt, W Steckby","","","https://www.openstreetmap.org/?mlat=51.8922&mlon=12.0139#map=15/51.8922/12.0139",51.89221954345703,12.013890266418457,"","Epilobium parviflorum","Epilobium","E.Willing","JACQ-ID 1069019","https://herbarium.bgbm.org/object/JACQID1069019" +"JACQID1070380","","JACQ","GERANIACEAE","Erodium cicutarium (L.) L'HĂ©r.","MTB: 3542/3","Willing,R. & Willing,E.","19266 D","2002-07-13","","Germany","Brandenburg, W Jeserig","","","https://www.openstreetmap.org/?mlat=52.4083&mlon=12.6783#map=15/52.4083/12.6783",52.408329010009766,12.678330421447754,"","Erodium cicutarium","Erodium","E.Willing","JACQ-ID 1070380","https://herbarium.bgbm.org/object/JACQID1070380" +"JACQID1071480","","JACQ","POACEAE","Festuca gigantea (L.) Vill.","MTB: 8332/4","Willing,R. & Willing,E.","14501 D","2001-07-30","","Germany","Bayern, W Grafenaschau","","","https://www.openstreetmap.org/?mlat=47.6475&mlon=11.1094#map=15/47.6475/11.1094",47.647499084472656,11.109439849853516,"","Festuca gigantea","Festuca","E.Willing","JACQ-ID 1071480","https://herbarium.bgbm.org/object/JACQID1071480" +"JACQID1072616","","JACQ","POACEAE","Holcus lanatus L.","MTB: 4346/3","Willing,R. & Willing,E.","12763 D","2001-06-16","","Germany","Brandenburg, SO Schilda","","","https://www.openstreetmap.org/?mlat=51.6017&mlon=13.4142#map=15/51.6017/13.4142",51.60166931152344,13.414170265197754,"","Holcus lanatus","Holcus","E.Willing","JACQ-ID 1072616","https://herbarium.bgbm.org/object/JACQID1072616" +"JACQID1073848","","JACQ","HYPERICACEAE","Hypericum perforatum L.","MTB: 6843/3","Willing,R. & Willing,E.","1904 D","1995-08-04","","Germany","Bayern, 0,75 km W Höllenstein","","","https://www.openstreetmap.org/?mlat=49.125&mlon=12.8667#map=15/49.125/12.8667",49.125,12.866669654846191,"","Hypericum perforatum","Hypericum","E.Willing","JACQ-ID 1073848","https://herbarium.bgbm.org/object/JACQID1073848" +"JACQID1075148","","JACQ","ASTERACEAE","Lapsana communis L.","MTB: 8226/4","Willing,R. & Willing,E.","10639 D","1999-08-19","","Germany","Baden-WĂŒrttemberg, SO Winterstetten","","","https://www.openstreetmap.org/?mlat=47.7464&mlon=10.1144#map=15/47.7464/10.1144",47.74639129638672,10.114439964294434,"","Lapsana communis","Lapsana","E.Willing","JACQ-ID 1075148","https://herbarium.bgbm.org/object/JACQID1075148" +"JACQID1076226","","JACQ","POACEAE","Lolium L.","MTB: 8330/4","Willing,R. & Willing,E.","13465 D","2001-07-24","","Germany","Bayern, NW Halbloch","","","https://www.openstreetmap.org/?mlat=47.6422&mlon=10.8169#map=15/47.6422/10.8169",47.64221954345703,10.816940307617188,"","Lolium","Lolium","E.Willing","JACQ-ID 1076226","https://herbarium.bgbm.org/object/JACQID1076226" +"JACQID1077294","","JACQ","FABACEAE","Medicago lupulina L.","MTB: 3744/3","Willing,R. & Willing,E.","17921 D","2002-06-15","","Germany","Brandenburg, NW Zauchwitz","","","https://www.openstreetmap.org/?mlat=52.2289&mlon=13.0336#map=15/52.2289/13.0336",52.22888946533203,13.033610343933105,"","Medicago lupulina","Medicago","E.Willing","JACQ-ID 1077294","https://herbarium.bgbm.org/object/JACQID1077294" +"JACQID1078399","","JACQ","BORAGINACEAE","Myosotis palustris (L.) Hill","MTB: 8325/4","Willing,R. & Willing,E.","10449 D","1999-08-18","","Germany","Bayern, S Wolfertshofen","","","https://www.openstreetmap.org/?mlat=47.6367&mlon=9.92056#map=15/47.6367/9.92056",47.63666915893555,9.920559883117676,"","Myosotis palustris","Myosotis","E.Willing","JACQ-ID 1078399","https://herbarium.bgbm.org/object/JACQID1078399" +"JACQID1079515","","JACQ","CARYOPHYLLACEAE","Petrorhagia prolifera (L.) P. W. Ball & Heywood","MTB: 4138/4/3/4","Willing,R. & Willing,E.","22783 D","2007-08-24","","Germany","Sachsen-Anhalt, W Mosigkau","","","https://www.openstreetmap.org/?mlat=51.8028&mlon=12.1128#map=15/51.8028/12.1128",51.80278015136719,12.11277961730957,"","Petrorhagia prolifera","Petrorhagia","E.Willing","JACQ-ID 1079515","https://herbarium.bgbm.org/object/JACQID1079515" +"JACQID1080582","","JACQ","POACEAE","Poa annua L.","MTB: 3745/2","Willing,R. & Willing,E.","12149 D","2000-08-27","","Germany","Brandenburg, SW Kerzendorf","","","https://www.openstreetmap.org/?mlat=52.2694&mlon=13.2681#map=15/52.2694/13.2681",52.269439697265625,13.268059730529785,"","Poa annua","Poa","E.Willing","JACQ-ID 1080582","https://herbarium.bgbm.org/object/JACQID1080582" +"JACQID1081687","","JACQ","POLYGONACEAE","Polygonum L.","MTB: 3843/3","Willing,R. & Willing,E.","1260 D","1994-07-24","","Germany","Brandenburg, 0,9 km W Niebel","","","https://www.openstreetmap.org/?mlat=52.1333&mlon=12.9083#map=15/52.1333/12.9083",52.133331298828125,12.908329963684082,"","Polygonum","Polygonum","E.Willing","JACQ-ID 1081687","https://herbarium.bgbm.org/object/JACQID1081687" +"JACQID1082907","","JACQ","CRASSULACEAE","Sedum acre L.","MTB: 3445/3","Willing,R. & Willing,E.","4688 D","1998-06-04","","Germany","Berlin, Haveluver, Burgwallgraben","","","https://www.openstreetmap.org/?mlat=52.5231&mlon=13.2036#map=15/52.5231/13.2036",52.5230598449707,13.20361042022705,"","Sedum acre","Sedum","E.Willing","JACQ-ID 1082907","https://herbarium.bgbm.org/object/JACQID1082907" +"JACQID1084093","","JACQ","ASTERACEAE","Senecio vernalis Waldst. & Kit.","MTB: 3742/1","Willing,R. & Willing,E.","12478 D","2001-05-26","","Germany","Brandenburg, S Lehnin","","","https://www.openstreetmap.org/?mlat=52.2989&mlon=12.7378#map=15/52.2989/12.7378",52.29888916015625,12.73777961730957,"","Senecio vernalis","Senecio","E.Willing","JACQ-ID 1084093","https://herbarium.bgbm.org/object/JACQID1084093" +"JACQID1085170","","JACQ","SOLANACEAE","Solanum nigrum L.","MTB: 3541/3","Willing,R. & Willing,E.","19363 D","2002-07-13","","Germany","Brandenburg, SO Brandenburg","","","https://www.openstreetmap.org/?mlat=52.405&mlon=12.5822#map=15/52.405/12.5822",52.404998779296875,12.582220077514648,"","Solanum nigrum","Solanum","E.Willing","JACQ-ID 1085170","https://herbarium.bgbm.org/object/JACQID1085170" +"JACQID1086220","","JACQ","ASTERACEAE","Taraxacum erythrospermum Andrz. ex Besser","MTB: 3044/3","Willing,R. & Willing,E.","4331 D","1998-05-16","","Germany","Brandenburg, O Vielitz","","","https://www.openstreetmap.org/?mlat=52.9319&mlon=13.0244#map=15/52.9319/13.0244",52.93193817138672,13.024439811706543,"","Taraxacum erythrospermum","Taraxacum","E.Willing","JACQ-ID 1086220","https://herbarium.bgbm.org/object/JACQID1086220" +"JACQID1087278","","JACQ","FABACEAE","Trifolium dubium Sibth.","MTB: 4138/1/3/2","Willing,R. & Willing,E.","24161 D","2008-05-21","","Germany","Sachsen-Anhalt, NW Aken","","","https://www.openstreetmap.org/?mlat=51.8628&mlon=12.0211#map=15/51.8628/12.0211",51.8627815246582,12.021109580993652,"","Trifolium dubium","Trifolium","E.Willing","JACQ-ID 1087278","https://herbarium.bgbm.org/object/JACQID1087278" +"JACQID1088400","","JACQ","VALERIANACEAE","Valeriana officinalis L.","MTB: 8331/4","Willing,R. & Willing,E.","14814 D","2001-08-01","","Germany","Bayern, W Altenau","","","https://www.openstreetmap.org/?mlat=47.6483&mlon=10.9831#map=15/47.6483/10.9831",47.64833068847656,10.983059883117676,"","Valeriana officinalis","Valeriana","E.Willing","JACQ-ID 1088400","https://herbarium.bgbm.org/object/JACQID1088400" +"JACQID1089470","","JACQ","FABACEAE","Vicia cracca L.","MTB: 4138/1/4/2","Willing,R. & Willing,E.","25534 D","2009-06-04","","Germany","Sachsen-Anhalt, S Steutz","","","https://www.openstreetmap.org/?mlat=51.8744&mlon=12.075#map=15/51.8744/12.075",51.87443923950195,12.074999809265137,"","Vicia cracca","Vicia","E.Willing","JACQ-ID 1089470","https://herbarium.bgbm.org/object/JACQID1089470" diff --git a/tests/.nftignore b/tests/.nftignore new file mode 100644 index 0000000..e128a12 --- /dev/null +++ b/tests/.nftignore @@ -0,0 +1,12 @@ +.DS_Store +multiqc/multiqc_data/fastqc_top_overrepresented_sequences_table.txt +multiqc/multiqc_data/multiqc.parquet +multiqc/multiqc_data/multiqc.log +multiqc/multiqc_data/multiqc_data.json +multiqc/multiqc_data/multiqc_sources.txt +multiqc/multiqc_data/multiqc_software_versions.txt +multiqc/multiqc_data/llms-full.txt +multiqc/multiqc_plots/{svg,pdf,png}/*.{svg,pdf,png} +multiqc/multiqc_report.html +fastqc/*_fastqc.{html,zip} +pipeline_info/*.{html,json,txt,yml} diff --git a/tests/default.nf.test b/tests/default.nf.test new file mode 100644 index 0000000..8c68fe4 --- /dev/null +++ b/tests/default.nf.test @@ -0,0 +1,33 @@ +nextflow_pipeline { + + name "Test pipeline" + script "../main.nf" + tag "pipeline" + + test("-profile test") { + + when { + params { + outdir = "$outputDir" + } + } + + then { + // stable_path: All files + folders in ${params.outdir}/ with a stable path (including file name) + def stable_path = getAllFilesFromDir(params.outdir, relative: true, includeDir: true, ignore: ['pipeline_info/*.{html,json,txt}']) + // stable_content: All files in ${params.outdir}/ with stable content + def stable_content = getAllFilesFromDir(params.outdir, ignoreFile: 'tests/.nftignore') + assert workflow.success + assertAll( + { assert snapshot( + // pipeline versions.yml file for multiqc from which Nextflow version is removed because we test pipelines on multiple Nextflow versions + removeNextflowVersion("$outputDir/pipeline_info/nf_core_biodivpipeline_software_mqc_versions.yml"), + // All stable path name, with a relative path + stable_path, + // All files with stable contents + stable_content + ).match() } + ) + } + } +} diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 0000000..180dc90 --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,14 @@ +/* +======================================================================================== + Nextflow config file for running nf-test tests +======================================================================================== +*/ + +// TODO nf-core: Specify any additional parameters here +// Or any resources requirements +params { + modules_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/' + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/biodivpipeline/' +} + +aws.client.anonymous = true // fixes S3 access issues on self-hosted runners diff --git a/workflows/biodivpipeline.nf b/workflows/biodivpipeline.nf new file mode 100644 index 0000000..13ed113 --- /dev/null +++ b/workflows/biodivpipeline.nf @@ -0,0 +1,89 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ +include { COLUMN_STANDARDISE } from '../modules/local/column_standardise/main' +include { BIODIV_ANNOTATE } from '../modules/local/biodiv_annotate/main' +include { TAXONOMY_CLASSIFY } from '../modules/local/taxonomy_classify/main' +include { OUTLIER_DETECT } from '../modules/local/outlier_detect/main' +include { PROVISIONAL_CONCEPTS } from '../modules/local/provisional_concepts/main' +include { RDF_TRANSFORM } from '../modules/local/rdf_transform/main' +include { COLLECT_REPORTS } from '../modules/local/collect_reports/main' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + RUN MAIN WORKFLOW +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow BIODIVPIPELINE { + + take: + ch_input // channel: path to input CSV file + ch_schema // channel: path to RDF mapping schema (JSON-LD or Turtle) + outdir + + main: + + def ch_versions = channel.empty() + + COLUMN_STANDARDISE( ch_input ) + ch_versions = ch_versions.mix(COLUMN_STANDARDISE.out.versions) + + def ch_standardised = COLUMN_STANDARDISE.out.csv + + BIODIV_ANNOTATE( ch_standardised ) + ch_versions = ch_versions.mix(BIODIV_ANNOTATE.out.versions) + + TAXONOMY_CLASSIFY( ch_standardised ) + ch_versions = ch_versions.mix(TAXONOMY_CLASSIFY.out.versions) + + OUTLIER_DETECT( ch_standardised ) + ch_versions = ch_versions.mix(OUTLIER_DETECT.out.versions) + + PROVISIONAL_CONCEPTS( BIODIV_ANNOTATE.out.unresolved ) + ch_versions = ch_versions.mix(PROVISIONAL_CONCEPTS.out.versions) + + RDF_TRANSFORM( + ch_standardised, + BIODIV_ANNOTATE.out.annotations, + TAXONOMY_CLASSIFY.out.resolved, + ch_schema + ) + ch_versions = ch_versions.mix(RDF_TRANSFORM.out.versions) + + COLLECT_REPORTS( + OUTLIER_DETECT.out.report, + RDF_TRANSFORM.out.report, + COLUMN_STANDARDISE.out.mapping, + PROVISIONAL_CONCEPTS.out.concepts + ) + + // + // Collate software versions + // + softwareVersionsToYAML(ch_versions) + .collectFile( + storeDir: "${outdir}/pipeline_info", + name: 'biodivpipeline_software_versions.yml', + sort: true, + newLine: true + ) + + emit: + rdf_turtle = RDF_TRANSFORM.out.rdf_turtle // channel: path to Turtle RDF output + rdf_jsonld = RDF_TRANSFORM.out.rdf_jsonld // channel: path to JSON-LD RDF output + quality = OUTLIER_DETECT.out.report // channel: path to quality report + flagged = OUTLIER_DETECT.out.flagged // channel: path to flagged records + taxonomy = TAXONOMY_CLASSIFY.out.resolved // channel: path to resolved taxonomy + summary = COLLECT_REPORTS.out.summary // channel: path to pipeline summary + versions = ch_versions // channel: [ path(versions.yml) ] +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + THE END +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/