Merge pull request #135 from tkchafin/dev

Precomputed BUSCO [WIP]

Author: tkchafin

.github/workflows/ci.yml

@@ -41,3 +41,10 @@ jobs:
         # Remember that you can parallelise this by using strategy.matrix
         run: |
           nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+
+      - name: Run pipeline with test data and precomputed BUSCOs
+        # You can customise CI pipeline run tests as required
+        # For example: adding multiple test runs with different parameters
+        # Remember that you can parallelise this by using strategy.matrix
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test_nobusco,docker --outdir ./results

.github/workflows/linting.yml

@@ -99,7 +99,7 @@ jobs:
 
       - name: Upload linting log file artifact
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: linting-logs
           path: |

.github/workflows/sanger_test.yml

@@ -26,7 +26,7 @@ jobs:
               "use_work_dir_as_temp": true,
             }
           profiles: test,sanger,singularity,cleanup
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: Tower debug log file
           path: |

.github/workflows/sanger_test_full.yml

@@ -34,7 +34,7 @@ jobs:
               "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ env.REVISION }}",
             }
           profiles: test_full,sanger,singularity,cleanup
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: Tower debug log file
           path: |

CHANGELOG.md

@@ -14,6 +14,7 @@ The pipeline is now considered to be a complete and suitable replacement for the
 - Updated the Blastn settings to allow 7 days runtime at most, since that
   covers 99.7% of the jobs.
 - Allow database inputs to be optionally compressed (`.tar.gz`)
+- Allow `BUSCO` run outputs to be optionally pre-computed and provided with `--busco_output`
 
 ### Software dependencies
 
@@ -22,11 +23,19 @@ Note, since the pipeline is using Nextflow DSL2, each process will be run with i
 | Dependency  | Old version       | New version     |
 | ----------- | ----------------- | --------------- |
 | blast       | 2.14.1 and 2.15.0 | only 2.15.0     |
-| blobtoolkit | 4.3.9             | 4.4.0           |
+| blobtoolkit | 4.3.9             | 4.4.4           |
 | busco       | 5.5.0             | 5.7.1           |
 | multiqc     | 1.20 and 1.21     | 1.20 and 1.25.1 |
 | samtools    | 1.18 and 1.19.2   | 1.20 and 1.21   |
 
+### Parameters
+
+| Old parameter | New parameter  |
+| ------------- | -------------- |
+|               | --busco-output |
+
+> **NB:** Parameter has been **updated** if both old and new parameter information is present. </br> **NB:** Parameter has been **added** if just the new parameter information is present. </br> **NB:** Parameter has been **removed** if new parameter information isn't present.
+
 ## [[0.6.0](https://github.com/sanger-tol/blobtoolkit/releases/tag/0.6.0)] – Bellsprout – [2024-09-13]
 
 The pipeline has now been validated for draft (unpublished) assemblies.

bin/generate_config.py

@@ -47,6 +47,7 @@ def parse_args(args=None):
     parser.add_argument("--blastx", help="Path to the blastx database", required=True)
     parser.add_argument("--blastn", help="Path to the blastn database", required=True)
     parser.add_argument("--taxdump", help="Path to the taxonomy database", required=True)
+    parser.add_argument("--busco_output", action="append", help="Path to BUSCO output directory", required=False)
     parser.add_argument("--version", action="version", version="%(prog)s 2.0")
     return parser.parse_args(args)
 
@@ -121,20 +122,33 @@ def get_classification(taxon_info: TaxonInfo) -> typing.Dict[str, str]:
     return {r: ancestors[r] for r in RANKS if r in ancestors}
 
 
-def get_odb(taxon_info: TaxonInfo, lineage_tax_ids: str, requested_buscos: typing.Optional[str]) -> typing.List[str]:
+def get_odb(
+    taxon_info: TaxonInfo,
+    lineage_tax_ids: str,
+    requested_buscos: typing.Optional[str],
+    pre_computed_buscos: typing.List[str],
+) -> typing.List[str]:
     # Read the mapping between the BUSCO lineages and their taxon_id
     with open(lineage_tax_ids) as file_in:
         lineage_tax_ids_dict: typing.Dict[int, str] = {}
         for line in file_in:
             arr = line.split()
             lineage_tax_ids_dict[int(arr[0])] = arr[1] + "_odb10"
 
-    if requested_buscos:
+    valid_odbs = set(lineage_tax_ids_dict.values())
+
+    if pre_computed_buscos:
+        # Use pre-computed BUSCO lineages if available
+        odb_arr = pre_computed_buscos
+        for odb in odb_arr:
+            if odb not in valid_odbs:
+                print(f"Invalid pre-computed BUSCO lineage: {odb}", file=sys.stderr)
+                sys.exit(1)
+    elif requested_buscos:
         odb_arr = requested_buscos.split(",")
-        valid_odbs = set(lineage_tax_ids_dict.values())
         for odb in odb_arr:
             if odb not in valid_odbs:
-                print(f"Invalid BUSCO lineage: {odb}", file=sys.stderr)
+                print(f"Invalid requested BUSCO lineage: {odb}", file=sys.stderr)
                 sys.exit(1)
     else:
         # Do the intersection to find the ancestors that have a BUSCO lineage
@@ -327,7 +341,9 @@ def main(args=None):
 
     taxon_info = fetch_taxon_info(args.taxon_query)
     classification = get_classification(taxon_info)
-    odb_arr = get_odb(taxon_info, args.lineage_tax_ids, args.busco)
+
+    precomputed_busco = [os.path.basename(path).replace("run_", "") for path in (args.busco_output or [])]
+    odb_arr = get_odb(taxon_info, args.lineage_tax_ids, args.busco, precomputed_busco)
     taxon_id = adjust_taxon_id(args.nt, taxon_info)
 
     if sequence_report:

conf/test_nobusco.config

@@ -0,0 +1,47 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run sanger-tol/blobtoolkit -profile test,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'Test profile'
+    config_profile_description = 'Minimal aligned test dataset to check pipeline function'
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus   = 2
+    max_memory = '6.GB'
+    max_time   = '6.h'
+
+    // Input test data
+    // Specify the paths to your test data
+    // Give any required params for the test so that command line flags are not needed
+    input     = "${projectDir}/assets/test/samplesheet_s3.csv"
+
+    // Fasta references
+    fasta     = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/assembly/release/mMelMel3.1_paternal_haplotype/GCA_922984935.2.subset.phiXspike.fasta.gz"
+    accession = "GCA_922984935.2"
+    taxon     = "Meles meles"
+
+    // Databases
+    taxdump = "https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/new_taxdump/new_taxdump.tar.gz"
+    busco   = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/resources/blobtoolkit.GCA_922984935.2.2023-08-03.tar.gz"
+    blastp  = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/resources/mMelMel3.1.buscogenes.dmnd.tar.gz"
+    blastx  = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/resources/mMelMel3.1.buscoregions.dmnd.tar.gz"
+    blastn  = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/resources/nt_mMelMel3.1.tar.gz"
+
+    // Precomputed BUSCO outputs
+    // busco_output_noArchaea.tar.gz deliberately leaves out archaea_odb10 to test the pipeline's detection and filling of missing lineages
+    // Switch to *_busco_output.tar.gz for fully precomputed BUSCOs
+    busco_output = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/resources/GCA_922984935.2_busco_output_noArchaea.tar.gz"
+    //busco_output = "https://tolit.cog.sanger.ac.uk/test-data/Meles_meles/resources/GCA_922984935.2_busco_output.tar.gz"
+
+    // Need to be set to avoid overfilling /tmp
+    use_work_dir_as_temp = true
+}

docs/usage.md

@@ -54,6 +54,26 @@ An [example samplesheet](assets/test/samplesheet.csv) has been provided with the
 The pipeline can also accept a samplesheet generated by the [nf-core/fetchngs](https://nf-co.re/fetchngs) pipeline (tested with version 1.11.0).
 The pipeline then needs the `--fetchngs_samplesheet true` option _and_ `--align true`, since the data files would all be unaligned.
 
+### Support for pre-computed `BUSCO` outputs
+
+The pipeline may be optionally run with a set of pre-computed [`BUSCO`](https://busco.ezlab.org) runs, provided using the `--busco_output` parameter. These can be provided as either a directory path, or a `.tar.gz` compressed archive. The contents should be each `run_` output directory (directly from `BUSCO`) named as `run_[odb_dabasase_name]`:
+
+```
+GCA_922984935.2_busco_output/
+├── run_archaea_odb10
+├── run_bacteria_odb10
+├── run_carnivora_odb10
+├── run_eukaryota_odb10
+├── run_eutheria_odb10
+├── run_laurasiatheria_odb10
+├── run_mammalia_odb10
+├── run_metazoa_odb10
+├── run_tetrapoda_odb10
+└── run_vertebrata_odb10
+```
+
+The pipeline minimally requires outputs for the 'basal' lineages (archaea, eukaryota, and bacteria) -- any of these which are not present in the pre-computed outputs will be automatically detected and run.
+
 ## Database parameters
 
 Configure access to your local databases with the `--busco`, `--blastp`, `--blastx`, `--blastn`, and `--taxdump` parameters.
@@ -272,7 +292,7 @@ List of tools for any given dataset can be fetched from the API, for example htt
 
 | Dependency        | Snakemake | Nextflow |
 | ----------------- | --------- | -------- |
-| blobtoolkit       | 4.3.2     | 4.4.0    |
+| blobtoolkit       | 4.3.2     | 4.4.4    |
 | blast             | 2.12.0    | 2.14.1   |
 | blobtk            | 0.5.0     | 0.5.1    |
 | busco             | 5.3.2     | 5.5.0    |

modules/local/blobtoolkit/chunk.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_CHUNK {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_CHUNK module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "docker.io/genomehubs/blobtoolkit:4.4.0"
+    container "docker.io/genomehubs/blobtoolkit:4.4.4"
 
     input:
     tuple val(meta) , path(fasta)

modules/local/blobtoolkit/countbuscos.nf

@@ -5,7 +5,7 @@ process BLOBTOOLKIT_COUNTBUSCOS {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         exit 1, "BLOBTOOLKIT_COUNTBUSCOS module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
-    container "docker.io/genomehubs/blobtoolkit:4.4.0"
+    container "docker.io/genomehubs/blobtoolkit:4.4.4"
 
     input:
     tuple val(meta), path(table, stageAs: 'dir??/*')