Merge pull request #415 from sanger-tol/dp24_fofn_function

weaglesBio · web-flow · commit a49fcf85c0b8 · 2025-06-20T11:54:06.000+01:00
Dp24 fofn function
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,16 +16,21 @@ Our 7th release for sanger-tol/treeval.
 - Addition of `run_hires` flag (boolean) to control use of the hires pretext modules.
 - Removal of unused modules such as `avgcov`, the average coverage module.
 - Addition of the `mode` parameter, to replace the depreciating `-entry` flag.
+  - Mode lists are include lists, listing the processes needed per run.
 - Removing entry points has significantly simplified the pipeline; `mode` now controls a set of `steps` for pipeline execution.
 - Addition of specified reads.
   - Longreads and HiC are now specified in the input yaml file. Details in the usage document.
-  - Longread and HiC data can optionally take a fofn (file of file names) where each line contains one file.
-- Corrected input values by @DLBPointon
+  - Longread and HiC data can optionally take a fofn (file of file names) where each line contains one file, see `YAML_INPUT` function `fn_get_validated_channel`.
+  - NOTE: in cases where the same file is in both the fofn and a supplied list, the pipeline will exit with an error which will be detailed in the nextflow.log file.
+- Corrected input values.
 - SummaryStats Code has been removed, this will be replaced by a plugin in the future!
 - Replace 5 modules with GAWK instead of cat | sed pattern modules.
 - Simplification of the `YAML_INPUT` subworkflow.
 - Removal of `GrabFiles` and replacement with the newer `.resolve()`.
 - Cleaning up of `it -> it[0]` into the more correct `_meta, file -> file` pattern.
+- Moved GAWK commands into their own awk script files, this cleans up modules.config.
+  - Added `my_abs` function to the `GAWK_REFORMAT_INTERSECT` to caluclate difference between start and end.
+- Added `[TreeVal:Error]` and `[TreeVal:Info]` into intentional print statements to make it easier to see on the CLI and to search for in logs.
 
 ### Parameters
 
diff --git a/assets/github_testing/TreeValTinyFullTest.yaml b/assets/github_testing/TreeValTinyFullTest.yaml
@@ -20,7 +20,7 @@ hic_data:
 kmer_profile:
   # kmer_length will act as input for kmer_read_cov fastk and as the name of folder in profile_dir
   kmer_length: 31
-  dir: /home/runner/work/treeval/treeval/TreeValTinyData/
+  profile: /home/runner/work/treeval/treeval/TreeValTinyData/
 alignment:
   genesets:
     - /home/runner/work/treeval/treeval/TreeValTinyData/gene_alignment_data/fungi/csv_data/LaetiporusSulphureus.gfLaeSulp1-data.csv
diff --git a/assets/local_testing/grTriPseu1.yaml b/assets/local_testing/grTriPseu1.yaml
@@ -0,0 +1,36 @@
+assembly:
+  assem_level: scaffold
+  assem_version: 1
+  sample_id: grTriPseu1
+  latin_name: to_provide_taxonomic_rank
+  defined_class: nematode
+  project_id: DTOL
+reference_file: /nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/assembly/draft/grTriPseu1.fa
+map_order: length
+assem_reads:
+  read_type: hifi
+  read_data:
+    - /nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/assets/local_testing/grTriPseu1_reads.fofn
+  supplementary_data: path
+hic_data:
+  hic_cram:
+    - /nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/assets/local_testing/grTriPseu1_hic.fofn
+  hic_aligner: minimap2
+kmer_profile:
+  # kmer_length will act as input for kmer_read_cov fastk and as the name of folder in profile_dir
+  kmer_length: 31
+  profile: /lustre/scratch122/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/
+alignment:
+  genesets:
+    - /nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/gene_alignment_data/fungi/csv_data/LaetiporusSulphureus.gfLaeSulp1-data.csv
+self_comp:
+  motif_len: 0
+intron:
+  size: "50k"
+telomere:
+  teloseq: TTAGGG
+synteny:
+  - /nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/synteny/fungi/LaetiporusSulphureus.fasta
+busco:
+  lineages_path: /nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/busco/subset/
+  lineage: fungi_odb10
diff --git a/assets/local_testing/grTriPseu1_hic.fofn b/assets/local_testing/grTriPseu1_hic.fofn
@@ -0,0 +1,2 @@
+/nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/genomic_data/hic-arima/SUBSET-1000.cram
+/nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/genomic_data/hic-arima/SUBSET-2000.cram
diff --git a/assets/local_testing/grTriPseu1_reads.fofn b/assets/local_testing/grTriPseu1_reads.fofn
@@ -0,0 +1 @@
+/nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/genomic_data/pacbio/seqkitPacbio50000.fasta.gz
diff --git a/assets/local_testing/nxOscDF5033.yaml b/assets/local_testing/nxOscDF5033.yaml
diff --git a/assets/local_testing/nxOscDF5033_hic.fofn b/assets/local_testing/nxOscDF5033_hic.fofn
diff --git a/assets/local_testing/nxOscDF5033_reads.fofn b/assets/local_testing/nxOscDF5033_reads.fofn
diff --git a/assets/local_testing/nxOscSUBSET.yaml b/assets/local_testing/nxOscSUBSET.yaml
@@ -5,23 +5,24 @@ assembly:
   latin_name: to_provide_taxonomic_rank
   defined_class: nematode
   project_id: DTOL
-reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta
+reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta
 map_order: length
 assem_reads:
   read_type: hifi
   read_data:
-    - /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/genomic_data/pacbio/subset.fasta.gz
+    - /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/genomic_data/pacbio/subset.fasta.gz
   supplementary_data: path
 hic_data:
-  hic_cram: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/
-  hic_aligner: minimap2
+  hic_cram:
+    - /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/genomic_data/hic-arima2/SUBSET-1000.cram
+    - /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/genomic_data/hic-arima2/SUBSET-2000.cram
 kmer_profile:
   # kmer_length will act as input for kmer_read_cov fastk and as the name of folder in profile_dir
   kmer_length: 31
-  dir: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/
+  dir: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/genomic_data/nxOscSpes1/pacbio/
 alignment:
   genesets:
-    - /lustre/scratch123/tol/resources/treeval/gene_alignment_data/nematode/csv_data/OscheiusTipulae.ASM1342590v1-data.csv
+    - /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/gene_set/nematode/csv_data/CaenorhabditisElegans.WBcel235-data.csv
 self_comp:
   motif_len: 0
 intron:
diff --git a/assets/schema_input.json b/assets/schema_input.json
@@ -48,11 +48,15 @@
             "properties": {
                 "read_type": {
                     "type": "string",
-                    "errorMessage": "Long read data type"
+                    "errorMessage": "Long read data type",
+                    "enum": ["hifi", "clr", "ont", "illumina"]
                 },
                 "read_data": {
                     "type": "array",
-                    "errorMessage": "Paths to folder containing fasta.gz files or FOFN of files"
+                    "errorMessage": "Paths to fasta.gz files and/or FOFN of fasta.gz files",
+                    "pattern": "(^\\S+\\.(fa|fasta|fna).gz)$|(^\\S+\\.fofn)$",
+                    "format": "file-path",
+                    "exists": true
                 },
                 "supplementary_data": {
                     "type": "string",
@@ -65,11 +69,15 @@
             "properties": {
                 "hic_cram": {
                     "type": "array",
-                    "errorMessage": "Paths to folder containing cram files of FOFN of cram"
+                    "errorMessage": "Paths to an array of cram files and/or FOFN of cram files",
+                    "pattern": "(^\\S+\\.cram)$|(^\\S+\\.fofn)$",
+                    "format": "file-path",
+                    "exists": true
                 },
                 "hic_aligner": {
                     "type": "string",
-                    "errorMessage": "HiC Aligner - minimap2"
+                    "errorMessage": "HiC Aligner - minimap2 (default), bwamem2",
+                    "enum": ["minimap2", "bwamem2"]
                 }
             }
         },
@@ -80,9 +88,9 @@
                     "type": "integer",
                     "errorMessage": "K-mer length"
                 },
-                "dir": {
+                "profile": {
                     "type": "string",
-                    "errorMessage": "Path to folder containing fasta.gz files"
+                    "errorMessage": "Path to *.hist file"
                 }
             }
         },
@@ -91,7 +99,8 @@
             "properties": {
                 "genesets": {
                     "type": "array",
-                    "errorMessage": "Path for gene alignment file."
+                    "errorMessage": "Path for gene alignment file.",
+                    "pattern": "^\\S+\\.csv$"
                 }
             }
         },
@@ -123,7 +132,8 @@
             }
         },
         "synteny": {
-            "type": "array"
+            "type": "array",
+            "pattern": "^\\S+\\.(fa|fasta|fna)$"
         },
         "busco": {
             "type": "object",
diff --git a/bin/gawk_gap_length.awk b/bin/gawk_gap_length.awk
@@ -0,0 +1,7 @@
+## Calculate the length of a gap
+## Date: 17/06/2025
+
+BEGIN { OFS = "\t" }
+{
+    print $0, sqrt(($3-$2)*($3-$2))
+}
diff --git a/bin/gawk_map_telo.awk b/bin/gawk_map_telo.awk
@@ -0,0 +1,11 @@
+## Format the telo map file
+## Date: 17/06/2025
+
+BEGIN {
+    OFS = "\t"
+}
+
+{
+    sub(/^>/, "", $1)
+    print $1, $4, $5, $6
+}
diff --git a/bin/gawk_reformat_intersect.awk b/bin/gawk_reformat_intersect.awk
@@ -0,0 +1,11 @@
+## Reformat the intercept file
+## Date: 17/06/2025
+
+function my_abs(x) {
+    return x < 0 ? -x : x
+}
+
+{
+    gsub(/\./, "0")
+    printf "%s\t%.0f\n", $0, my_abs($3 - $2)
+}
diff --git a/bin/gawk_replace_dots.awk b/bin/gawk_replace_dots.awk
@@ -0,0 +1,7 @@
+## Replace dots in ID
+## Date: 17/06/2025
+
+{
+    gsub(/\./, "0")
+    print
+}
diff --git a/bin/gawk_upper_sequence.awk b/bin/gawk_upper_sequence.awk
@@ -0,0 +1,11 @@
+## For lines !starting with '>' ensure they are upper characters
+## Date: 17/06/2025
+
+/^>/ {
+    print
+    next
+}
+
+{
+    print toupper($0)
+}
diff --git a/bin/generate_cram_csv.sh b/bin/generate_cram_csv.sh
@@ -15,7 +15,7 @@ chunk_cram() {
     realcram=$(readlink -f ${cram})
     realcrai=$(readlink -f ${cram}.crai)
     local rgline=$(samtools view -H "${realcram}" | grep "@RG" | sed 's/\t/\\t/g' | sed "s/'//g")
-    local ncontainers=$(cat "${realcrai}" | wc -l)
+    local ncontainers=$(zcat "${realcrai}" | wc -l)
     local base=$(basename "${realcram}" .cram)
     local from=0
     local to=10000
diff --git a/conf/modules.config b/conf/modules.config
@@ -74,7 +74,7 @@ process {
     // NOTE: GAWK module derivatives - More to come
     //
     withName: 'GAWK_UPPER_SEQUENCE' {
-        ext.args2  = { "'/^>/ {print; next} {print toupper(\$0)}'" } // Convert sequence to uppercase on lines starting with '>'
+        ext.args2 = { "'/^>/ {print; next} {print toupper(\$0)}'" } // Convert sequence to uppercase on lines starting with '>'
         ext.prefix = { "${meta.id}_UPPERCASED" }
         ext.suffix = 'fasta'
     }
@@ -86,31 +86,27 @@ process {
     }
 
     withName: 'GAWK_MAP_TELO' {
-        ext.args2    = { "-v OFS=\"\t\" 'BEGIN { sub(/^>/, \"\"); print \$1, \$4, \$5, \$6 }'" }
+        ext.args2 = { "-v OFS=\"\t\" 'BEGIN { sub(/^>/, \"\"); print \$1, \$4, \$5, \$6 }'" }
         ext.prefix   = { "${meta.id}_map_telo" }
         ext.suffix   = 'bed'
     }
 
     withName: 'GAWK_GAP_LENGTH' {
-        ext.args2  = { "'BEGIN { OFS = \"\t\" } { print \$0, abs(\$3 - \$2) }'" }
         ext.prefix = { "${meta.id}.gap" }
         ext.suffix = 'bedgraph'
     }
 
     withName: 'GAWK_REPLACE_DOTS' {
-        ext.args2  = { "' gsub(/\\./, \"0\"); print '"}
         ext.prefix = { "${meta.id}_nodot" }
         ext.suffix = 'bed'
     }
 
     withName: 'GAWK_RENAME_IDS' {
-        ext.args2  = { "' gsub(/\\./, \"0\"); print '" }
         ext.prefix = { "${meta.id}_renamed" }
         ext.suffix = 'bed'
     }
 
     withName: 'GAWK_REFORMAT_INTERSECT' {
-        ext.args2   = { "' gsub(/\\./, \"0\"); printf \"%s\t%.0f\n\", \$0, abs(\$3 - \$2) '" }
         ext.prefix  = { "${meta.id}_repeat_mk" }
         ext.suffix  = 'bed'
     }
diff --git a/conf/test.config b/conf/test.config
@@ -13,13 +13,20 @@
 ----------------------------------------------------------------------------------------
 */
 
-cleanup = true
+
+process {
+    resourceLimits = [
+        cpus: 4,
+        memory: '10.GB',
+        time: '1.h'
+    ]
+}
 
 params {
     config_profile_name        = "Test profile"
     config_profile_description = "Minimal test dataset to check pipeline function"
 
     // Input data
-    input       = "${projectDir}/assets/local_testing/nxOscDF5033.yaml"
+    input       = "${projectDir}/assets/local_testing/grTriPseu1.yaml"
     outdir      = "TinyTest"
 }
diff --git a/conf/test_full.config b/conf/test_full.config
@@ -28,6 +28,6 @@ params {
     config_profile_description = "FULL test dataset to check pipeline function, using a current full local dataset"
 
     // Input data
-    input       = "${projectDir}/assets/local_testing/nxOscDF5033.yaml"
-    outdir      = "SmallTest"
+    input       = "${projectDir}/assets/local_testing/odCymConc1_FULL_TEST.yaml"
+    outdir      = "SmallTest-odCymConc1"
 }
diff --git a/docs/usage.md b/docs/usage.md
@@ -441,6 +441,31 @@ work                # Directory containing the nextflow working files
 # Other nextflow hidden files, eg. history of pipeline runs and old logs.
 ```
 
+### Mode
+
+The TreeVal pipeline now contains a command line option for `--mode` which replaces the now depreciated `-entry` parameter.
+
+This enum param expects only one of ["FULL", "RAPID", "RAPID-TOL", "JBROWSE"].
+
+FULL will run all subworkflows shows in all_steps_list.
+
+RAPID and RAPID-TOL will run all subworkflows in rapid_include_list. Although there is no obvious difference, RAPID-TOL includes a check later in the pipeline to stop the generation of Juicer files which are no longer in use at Sanger. The logic is:
+
+```
+if workflow != RAPID_TOL and param.juicer == false, then run juicer subsetting
+
+This qualifies as: if (false && false) { run juicer } or if workflow is RAPID_TOL AND juicer == false { run juicer }
+```
+
+JBROWSE, generates the data which can be ingested by JBROWSE. Useful when RAPID has been used in a previous run and you now need "the rest" of the data. This runs all subworkflows notes in jbrowse_include_list.
+
+```
+all_steps_list          = ["insilico_digest", "gene_alignment", "repeat_density", "gap_finder", "selfcomp", "synteny", "read_coverage", "telo_finder", "busco", "kmer", "hic_mapping", "NONE"]
+
+jbrowse_include_list    = ["insilico_digest", "gene_alignment", "selfcomp", "synteny", "busco", "kmer"]
+rapid_include_list      = ["repeat_density", "gap_finder", "read_coverage", "telo_finder", "hic_mapping", "kmer"]
+```
+
 ### Updating the pipeline
 
 When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline:
diff --git a/subworkflows/local/gap_finder/main.nf b/subworkflows/local/gap_finder/main.nf
@@ -27,7 +27,7 @@ workflow GAP_FINDER {
     //
     GAWK_GAP_LENGTH (
         SEQTK_CUTN.out.bed,
-        [],
+        file("${projectDir}/bin/gawk_gap_length.awk"),
         false
     )
     ch_versions     = ch_versions.mix( GAWK_GAP_LENGTH.out.versions )
diff --git a/subworkflows/local/repeat_density/main.nf b/subworkflows/local/repeat_density/main.nf
diff --git a/subworkflows/local/telo_finder/main.nf b/subworkflows/local/telo_finder/main.nf
diff --git a/subworkflows/local/yaml_input/main.nf b/subworkflows/local/yaml_input/main.nf
diff --git a/workflows/treeval.nf b/workflows/treeval.nf

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+/nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/genomic_data/hic-arima/SUBSET-1000.cram`
	`2`	`+/nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/genomic_data/hic-arima/SUBSET-2000.cram`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+/nfs/treeoflife-01/teams/tola/users/dp24/new/new-new/treeval/TreeValTinyData/genomic_data/pacbio/seqkitPacbio50000.fasta.gz`