Merge pull request #147 from sanger-tol/dev

Dev

Author: weaglesBio

.nf-core.yml

@@ -48,4 +48,4 @@ template:
     - seqera_platform
     - multiqc
     - rocrate
-  version: 1.6.0
+  version: 1.6.1

CHANGELOG.md

@@ -3,6 +3,30 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [[1.6.1](https://github.com/sanger-tol/curationpretext/releases/tag/1.6.1)] - UNSC Trafalgar (H1) - [2025-03-13]
+
+## Added and Fixed
+
+- Update to add the `--run_ultra` parameter
+  - Enum of ["yes", "no", "force"]
+    - When run as 'yes', ultra resolution maps will be generated if the genome is > 4.Gb.
+    - When run as 'ultra', an ultra resolution map will be generated regardless of genome size.
+- Update to `pretextmap` to a version that supports `--ultraRes`
+- Update to add the `--snapshot_order` now supported by `pretextsnapshot`
+  - This is a `.txt` with a scaffold name per line.
+- Update the `repeat_density` subworkflow from local to `sanger-tol`
+- Update to config file to support the above fixes.
+- Addition of example params file `assets/example_params_file.yaml`
+
+### Software Dependencies
+
+Note, since the pipeline is using Nextflow DSL2, each process will be run with its own Biocontainer. This means that on occasion it is entirely possible for the pipeline to be using different versions of the same tool. However, the overall software dependency changes compared to the last release have been listed below for reference.
+
+| Module            | Old Version | New Versions |
+| ----------------- | ----------- | ------------ |
+| `PRETEXTMAP`      | 0.1.9       | 0.2.4        |
+| `PRETEXTSNAPSHOT` | 0.0.5       | 0.0.7        |
+
 ## [[1.6.0](https://github.com/sanger-tol/curationpretext/releases/tag/1.6.0)] - UNSC Trafalgar - [2025-02-19]
 
 ## Added and Fixed

README.md

@@ -77,6 +77,15 @@ Currently, the pipeline uses the following flags:
 - `--cram_chunk_size`
   - The number of records in a cram file should be chunked into, defaults to 10000.
 
+- `--run_hires`
+  - A boolean to run the pipeline in hires mode, i.e., generate hires resolution maps. Default is `true`
+
+- `--run_ultra`
+  - A string argument to run the pipeline in ultra resolution mode, i.e., generate ultra resolution maps. Options are: `yes`, `force`, `no`. Default is `yes`, this runs ultra resolution maps is the genome file is > 4.Gb.
+
+- `--snapshot_order`
+  - A path to a `genome`, `sizes` or `fai` file containing the scaffolds in the order required for the output snapshot png file.
+
 Now, you can run the pipeline using:
 
 ```bash

assets/example_params_file.yaml

@@ -0,0 +1,14 @@
+sample: "CurationPretextTest_MIN"
+teloseq: "TTA"
+aligner: "bwamem2"
+all_output: false
+skip_tracks: "NONE"
+run_hires: false
+run_ultra: "force"
+split_telomere: true
+input: "https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData/assembly/draft/grTriPseu1.fa"
+reads:
+  - "https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData/genomic_data/pacbio/seqkitPacbio50000.fasta.gz"
+cram:
+  - "https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData/genomic_data/hic-arima/SUBSET-1000.cram"
+  - "https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData/genomic_data/hic-arima/SUBSET-2000.cram"

assets/schema_input.json

@@ -47,7 +47,7 @@ process {
 
     withName: ".*:ALIGN_CRAM:CRAMALIGN_BWAMEM2ALIGNHIC" {
         cpus    = { 16   * 1 }
-        memory  = { 1.GB * ( reference.size() < 2e9 ? 80 : Math.ceil( ( reference.size() / 1e+9 ) * 30 ) * Math.ceil( task.attempt * 1 ) ) }
+        memory  = { 1.GB * ( reference.size() < 2e9 ? 80 : Math.ceil( ( reference.size() / 1e+9 ) * 30 ) * task.attempt ) }
     }
 
     withName: ".*:ALIGN_CRAM:CRAMALIGN_MINIMAP2ALIGNHIC" {
@@ -69,14 +69,20 @@ process {
     }
 
     withName: ".*:CREATE_MAPS_STDRD:PRETEXTMAP" {
-        cpus    = { 8        * 1 }
+        cpus    = { 6        * 1 }
         memory  = { 3.GB     * task.attempt }
         time    = { 1.h      * ( ( fasta.size() < 4e9 ? 24 : 48 ) * task.attempt ) }
     }
 
     withName: ".*:CREATE_MAPS_HIRES:PRETEXTMAP" {
-        cpus    = {  8       * task.attempt }
-        memory  = {  20.GB   * Math.ceil( task.attempt * 2.6 ) }
+        cpus    = {  6       * task.attempt }
+        memory  = {  20.GB   * task.attempt }
+        time    = {  1.h     * ( ( fasta.size() < 4e9 ? 24 : 48 ) * Math.ceil( task.attempt * 1 ) ) }
+    }
+
+    withName: ".*:CREATE_MAPS_ULTRA:PRETEXTMAP" {
+        cpus    = {  6       * task.attempt }
+        memory  = {  40.GB   * task.attempt }
         time    = {  1.h     * ( ( fasta.size() < 4e9 ? 24 : 48 ) * Math.ceil( task.attempt * 1 ) ) }
     }
 

conf/base.config

@@ -15,7 +15,7 @@ process {
     //
     // NOTE: GROUP THE PUBLISHDIR DIRECTIVES
     //
-    withName: 'PRETEXT_INGEST_SNDRD|PRETEXT_INGEST_HIRES' {
+    withName: 'PRETEXT_INGEST_SNDRD|PRETEXT_INGEST_HIRES|PRETEXT_INGEST_ULTRA' {
         publishDir = [
             [
                 path:       { "${params.outdir}/pretext_maps_processed" },
@@ -29,6 +29,12 @@ process {
                 mode:       params.publish_dir_mode,
                 saveAs:     { filename -> filename.equals('versions.yml') ? null : filename },
             ],
+            [
+                path:       { "${params.outdir}/pretext_maps_processed" },
+                pattern:    "*ultra.pretext",
+                mode:       params.publish_dir_mode,
+                saveAs:     { filename -> filename.equals('versions.yml') ? null : filename },
+            ],
         ]
     }
 
@@ -40,7 +46,7 @@ process {
                 saveAs:     { filename -> filename.equals('versions.yml') ? null : filename },
             ]
         ]
-        ext.args        = { "--sequences '=full' --resolution 1440 --memory ${task.memory}" }
+        ext.args        = { (order_file ? "" : "--sequences '=full'") + " --resolution 1440 --memory ${task.memory}" }
         ext.prefix      = { "${meta.id}_normal" }
     }
 
@@ -55,7 +61,7 @@ process {
         ]
     }
 
-    withName: ".*:REPEAT_DENSITY:UCSC_BEDGRAPHTOBIGWIG" {
+    withName: ".*:REPEAT_DENSITY:FEATURE_DENSITY:UCSC_BEDGRAPHTOBIGWIG" {
         ext.prefix      = { "${meta.id}_repeat_density" }
     }
 
@@ -84,7 +90,6 @@ process {
     }
 
     withName: 'GAWK_RENAME_IDS' {
-        ext.args2 = "'{ gsub(/\\./, \"0\"); print}'"
         ext.suffix = '_renamed.bed'
     }
 
@@ -170,9 +175,9 @@ process {
         ext.prefix      = { "${meta.id}_hr" }
     }
 
-    withName: "PRETEXTSNAPSHOT" {
-        ext.args        = { "--sequences '=full' --resolution 1440 --memory ${task.memory}" }
-        ext.prefix      = { "${meta.id}_normal" }
+    withName: "PRETEXT_INGEST_ULTRA" {
+        ext.args        = { "--textureBuffer 1G" }
+        ext.prefix      = { "${meta.id}_ultra" }
     }
 
     //
@@ -238,6 +243,11 @@ process {
         ext.prefix      = { "${meta.id}_hr_pi" }
     }
 
+    withName: ".*:CREATE_MAPS_ULTRA:PRETEXTMAP" {
+        ext.args        = { "${meta.map_order.equals("length") ? "--sortby length": "--sortby nosort" } --ultraRes --mapq ${params.multi_mapping}" }
+        ext.prefix      = { "${meta.id}_ultra_pi" }
+    }
+
     withName: ".*:ALIGN_CRAM:CRAMALIGN_BWAMEM2ALIGNHIC" {
         ext.args        = ""
         ext.args2       = "-F0xB00 -nt"

conf/modules.config

@@ -32,5 +32,6 @@ params {
     all_output      = false
     skip_tracks     = "NONE"
     run_hires       = false
+    run_ultra       = "no"
     split_telomere  = true
 }

conf/test.config

@@ -29,6 +29,7 @@ params {
     all_output      = true
     skip_tracks     = "NONE"
     run_hires       = true
+    run_ultra       = "force"
     split_telomere  = true
 
 }

conf/test_full.config

@@ -73,4 +73,10 @@ process {
         memory = { 40.GB * task.attempt }
         time   = { 20.h  * task.attempt }
     }
+
+    withName: CRAMALIGN_BWAMEM2ALIGNHIC {
+        cpus    = { 16       * 1            }
+        memory  = { 100.GB   * task.attempt }
+        time    = { 40.h     * task.attempt }
+    }
 }