Release v0.2.0 (#19)

Author: leahkemp

README.md

@@ -1,3 +1,4 @@
+
 # Pipeface
 
 ## Overview
@@ -25,11 +26,13 @@ snp_indel_calling{{"SNP/indel variant calling"}}
 snp_indel_phasing{{"SNP/indel phasing"}}
 snp_indel_annotation{{"SNP/indel annotation (optional - hg38 only)"}}
 haplotagging{{"Haplotagging bams"}}
+generate_meth_probs{{"Generate site methylation probabilities (pacbio data only)"}}
 sv_calling{{"Structural variant calling"}}
 
 input_data-.->merging-.->alignment-.->snp_indel_calling-.->snp_indel_phasing-.->haplotagging-.->sv_calling
 alignment-.->depth
 alignment-.->haplotagging
+haplotagging-.->generate_meth_probs
 snp_indel_phasing-.->snp_indel_annotation
 
 ```
@@ -51,24 +54,24 @@ merging_m1{{"Description: merge runs <br><br> Main tools: GNU coreutils <br><br>
 merging_m2{{"Description: merge runs <br><br> Main tools: Samtools <br><br> Commands: samtools merge"}}
 
 alignment_s1{{"Description: alignment, sorting <br><br> Main tools: Minimap2 and Samtools <br><br> Commands: minimap2 and samtools sort"}}
-alignment_s2{{"Description: alignment, sorting <br><br> Main tools: Minimap2 and Samtools <br><br> Commands: minimap2 and samtools sort"}}
-alignment_s3{{"Description: bam to fastq conversion, alignment, sorting <br><br> Main tools: Minimap2 and Samtools <br><br> Commands: minimap2 and samtools sort"}}
+alignment_s2{{"Description: bam to fastq conversion, alignment, sorting <br><br> Main tools: Minimap2 and Samtools <br><br> Commands: minimap2 and samtools sort"}}
+alignment_s3{{"Description: alignment, sorting <br><br> Main tools: Minimap2 and Samtools <br><br> Commands: minimap2 and samtools sort"}}
 alignment_s4{{"Description: bam to fastq conversion, alignment, sorting <br><br> Main tools: Minimap2 and Samtools <br><br> Commands: minimap2 and samtools sort"}}
 
-depth_s1{{"Description: calculate alignment depth <br><br> Main tools: Samtools <br><br> Commands: samtools depth"}}
-depth_s2{{"Description: calculate alignment depth <br><br> Main tools: Samtools <br><br> Commands: samtools depth"}}
-depth_s3{{"Description: calculate alignment depth <br><br> Main tools: Samtools <br><br> Commands: samtools depth"}}
-depth_s4{{"Description: calculate alignment depth <br><br> Main tools: Samtools <br><br> Commands: samtools depth"}}
+depth_s1{{"Description: calculate alignment depth <br><br> Main tools: mosdepth <br><br> Commands: mosdepth depth"}}
+depth_s2{{"Description: calculate alignment depth <br><br> Main tools: mosdepth <br><br> Commands: mosdepth depth"}}
+depth_s3{{"Description: calculate alignment depth <br><br> Main tools: mosdepth <br><br> Commands: mosdepth depth"}}
+depth_s4{{"Description: calculate alignment depth <br><br> Main tools: mosdepth <br><br> Commands: mosdepth depth"}}
 
 snp_indel_calling_s1{{"Description: SNP/indel variant calling <br><br> Main tools: Clair3 or DeepVariant <br><br> Commands: run_clair3.sh or run_deepvariant"}}
 snp_indel_calling_s2{{"Description: SNP/indel variant calling <br><br> Main tools: Clair3 or DeepVariant <br><br> Commands: run_clair3.sh or run_deepvariant"}}
 snp_indel_calling_s3{{"Description: SNP/indel variant calling <br><br> Main tools: Clair3 or DeepVariant <br><br> Commands: run_clair3.sh or run_deepvariant"}}
 snp_indel_calling_s4{{"Description: SNP/indel variant calling <br><br> Main tools: Clair3 or DeepVariant <br><br> Commands: run_clair3.sh or run_deepvariant"}}
 
-snp_indel_phasing_s1{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase"}}
-snp_indel_phasing_s2{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase"}}
-snp_indel_phasing_s3{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase"}}
-snp_indel_phasing_s4{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase"}}
+snp_indel_phasing_s1{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase and whatshap stats"}}
+snp_indel_phasing_s2{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase and whatshap stats"}}
+snp_indel_phasing_s3{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase and whatshap stats"}}
+snp_indel_phasing_s4{{"Description: SNP/indel phasing <br><br> Main tools: WhatsHap <br><br> Commands: whatshap phase and whatshap stats"}}
 
 snp_indel_annotation_s1{{"Description: SNP/indel annotation (optional - hg38 only)" <br><br> Main tools: ensembl-vep <br><br> Commands: vep}}
 snp_indel_annotation_s2{{"Description: SNP/indel annotation (optional - hg38 only)" <br><br> Main tools: ensembl-vep <br><br> Commands: vep}}
@@ -80,18 +83,19 @@ haplotagging_s2{{"Description: haplotagging bams <br><br> Main tools: WhatsHap <
 haplotagging_s3{{"Description: haplotagging bams <br><br> Main tools: WhatsHap <br><br> Commands: whatshap haplotag"}}
 haplotagging_s4{{"Description: haplotagging bams <br><br> Main tools: WhatsHap <br><br> Commands: whatshap haplotag"}}
 
+generate_meth_probs_s2{{"Description: Generate site methylation probabilities <br><br> Main tools: pb-CpG-tools <br><br> Commands: aligned_bam_to_cpg_scores"}}
+
 sv_calling_s1{{"Description: structural variant calling <br><br> Main tools: Sniffles2 and/or cuteSV <br><br> Commands: sniffles and/or cuteSV"}}
 sv_calling_s2{{"Description: structural variant calling <br><br> Main tools: Sniffles2 and/or cuteSV <br><br> Commands: sniffles and/or cuteSV"}}
 sv_calling_s3{{"Description: structural variant calling <br><br> Main tools: Sniffles2 and/or cuteSV <br><br> Commands: sniffles and/or cuteSV"}}
 sv_calling_s4{{"Description: structural variant calling <br><br> Main tools: Sniffles2 and/or cuteSV <br><br> Commands: sniffles and/or cuteSV"}}
 
 ont_data_f1-.->merging_m1-.->alignment_s1-.->snp_indel_calling_s1-.->snp_indel_phasing_s1-.->haplotagging_s1-.->sv_calling_s1
 ont_data_f2-.->merging_m1
-ont_data_f5-.->alignment_s2-.->snp_indel_calling_s2-.->snp_indel_phasing_s2-.->haplotagging_s2-.->sv_calling_s2
-ont_data_f6-.->alignment_s3-.->snp_indel_calling_s3-.->snp_indel_phasing_s3-.->haplotagging_s3-.->sv_calling_s3
-
-pacbio_data_f3-.->merging_m2-.->alignment_s4-.->snp_indel_calling_s4-.->snp_indel_phasing_s4-.->haplotagging_s4-.->sv_calling_s4
+pacbio_data_f3-.->merging_m2-.->alignment_s2-.->snp_indel_calling_s2-.->snp_indel_phasing_s2-.->haplotagging_s2-.->sv_calling_s2
 pacbio_data_f4-.->merging_m2
+ont_data_f5-.->alignment_s3-.->snp_indel_calling_s3-.->snp_indel_phasing_s3-.->haplotagging_s3-.->sv_calling_s3
+ont_data_f6-.->alignment_s4-.->snp_indel_calling_s4-.->snp_indel_phasing_s4-.->haplotagging_s4-.->sv_calling_s4
 
 alignment_s1-.->depth_s1
 alignment_s2-.->depth_s2
@@ -103,6 +107,8 @@ alignment_s2-.->haplotagging_s2
 alignment_s3-.->haplotagging_s3
 alignment_s4-.->haplotagging_s4
 
+haplotagging_s2-.->generate_meth_probs_s2
+
 snp_indel_phasing_s1-.->snp_indel_annotation_s1
 snp_indel_phasing_s2-.->snp_indel_annotation_s2
 snp_indel_phasing_s3-.->snp_indel_annotation_s3
@@ -112,8 +118,8 @@ snp_indel_phasing_s4-.->snp_indel_annotation_s4
 
 ## Main analyses
 
-- ONT and pacbio HiFi data
-- WGS and targeted
+- ONT and/or pacbio HiFi data
+- WGS and/or targeted
 - hg38 or hs1 reference genome
 
 ## Main tools
@@ -123,6 +129,8 @@ snp_indel_phasing_s4-.->snp_indel_annotation_s4
 - [WhatsHap](https://github.com/whatshap/whatshap)
 - [Sniffles2](https://github.com/fritzsedlazeck/Sniffles) and/or [cuteSV](https://github.com/tjiangHIT/cuteSV)
 - [Samtools](https://github.com/samtools/samtools)
+- [mosdepth](https://github.com/brentp/mosdepth)
+- [pb-CpG-tools](https://github.com/PacificBiosciences/pb-CpG-tools)
 - [ensembl-vep](https://github.com/Ensembl/ensembl-vep)
 
 ## Main input files
@@ -132,7 +140,6 @@ snp_indel_phasing_s4-.->snp_indel_annotation_s4
 - ONT/pacbio HiFi FASTQ (gzipped or uncompressed) or unaligned BAM
 - Indexed reference genome
 - Clair3 models (if running Clair3)
-- [DeepVariant GPU 1.6.1 docker container](https://hub.docker.com/layers/google/deepvariant/1.6.1-gpu/images/sha256-7929c55106d3739daa18d52802913c43af4ca2879db29656056f59005d1d46cb?context=explore) pulled via singularity (if running DeepVariant)
 
 ### Optional
 
@@ -142,19 +149,24 @@ snp_indel_phasing_s4-.->snp_indel_annotation_s4
 ## Main output files
 
 - Aligned, sorted and haplotagged bam
+- Depth per chromosome (and per region in the case of targeted sequencing) (optional)
 - Clair3 or DeepVariant phased SNP/indel VCF file
 - Clair3 or DeepVariant phased and annotated SNP/indel VCF file (optional - hg38 only)
+- Bed and bigwig site methylation probabilities for complete read set and separate haplotypes (pacbio only)
 - Phased Sniffles2 and/or un-phased cuteSV SV VCF file
 
+> **_Note:_** Running DeepVariant on ONT data assumes r10 data
+
 ## Assumptions
 
 - Running pipeline on Australia's [National Computational Infrastructure (NCI)](https://nci.org.au/)
 - Access to if89 project on [National Computational Infrastructure (NCI)](https://nci.org.au/)
 - Access to xy86 project on [National Computational Infrastructure (NCI)](https://nci.org.au/) (if running variant annotation)
 - Access to pipeline dependencies:
-    - [Nextflow and it's java dependency](https://nf-co.re/docs/usage/installation). Validated to run on:
-        - Nextflow 24.04.1
-        - Java 17.0.2
+    - [Nextflow 24.04.1 and it's Java 17.0.2 dependency](https://nf-co.re/docs/usage/installation)
+    - [DeepVariant GPU 1.6.1 docker container](https://hub.docker.com/layers/google/deepvariant/1.6.1-gpu/images/sha256-7929c55106d3739daa18d52802913c43af4ca2879db29656056f59005d1d46cb?context=explore) pulled via singularity (if running DeepVariant)
+    - [mosdepth 0.3.9 binary](https://github.com/brentp/mosdepth/releases/tag/v0.3.9) (if running depth calculation)
+    - [pb-CpG-tools 2.3.2 binary](https://github.com/PacificBiosciences/pb-CpG-tools/releases/tag/v2.3.2) (if processing pacbio data)
 
 *[See the list of software and their versions used by this version of pipeface](./docs/software_versions.txt) as well as the [list of variant databases and their versions](./docs/database_versions.txt) if variant annotation is carried out (assuming the default [nextflow_pipeface.config](./config/nextflow_pipeface.config) file is used).*
 

config/nextflow_pipeface.config

@@ -48,12 +48,11 @@ process {
         module = 'minimap2/2.28:samtools/1.19'
     }
 
-    withName: depth {
+    withName: mosdepth {
         queue = 'normal'
         cpus = '8'
-        time = '4h'
+        time = '2h'
         memory = '32GB'
-        module = 'samtools/1.19'
     }
 
     withName: clair3 {
@@ -71,13 +70,13 @@ process {
         time = '8h'
         memory = '192GB'
         disk = '80GB'
-        module = 'singularity'
+        module = 'singularity:bcftools/1.12:htslib/1.16'
     }
 
     withName: vep_snv {
         queue = 'normal'
         cpus = '32'
-        time = '10h'
+        time = '24h'
         memory = '128GB'
         module = 'singularity:htslib/1.16:ensemblorg/ensembl-vep/release_112.0'
     }
@@ -90,6 +89,13 @@ process {
         module = 'whatshap/2.3:htslib/1.16:samtools/1.19'
     }
 
+    withName: pbcpgtools {
+        queue = 'normal'
+        cpus = '48'
+        time = '2h'
+        memory = '192GB'
+    }
+
     withName: sniffles {
         queue = 'normal'
         cpus = '4'
@@ -106,11 +112,5 @@ process {
         module = 'cuteSV/1.0.13:htslib/1.16'
     }
 
-    withName: 'publish_settings|publish_bam_header|publish_depth|publish_whatshap_phase|publish_whatshap_haplotag|publish_sniffles|publish_cutesv' {
-        queue = 'normal'
-        cpus = '1'
-        time = '20m'
-        memory = '4GB'
-    }
-
 }
+

config/parameters_pipeface.json

@@ -2,13 +2,18 @@
 {
 
     "in_data": "",
+    "in_data_format": "",
     "ref": "",
     "ref_index": "",
     "tandem_repeat": "",
     "snp_indel_caller": "",
     "sv_caller": "",
     "annotate": "",
+    "calculate_depth": "",
     "outdir": "",
-    "deepvariant_container": ""
+    "deepvariant_container": "",
+    "mosdepth_binary": "",
+    "pbcpgtools_binary": ""
 
 }
+

docs/run_on_nci.md

@@ -10,6 +10,8 @@
       - [ONT](#ont)
       - [Pacbio HiFi revio](#pacbio-hifi-revio)
     - [DeepVariant container (if running DeepVariant)](#deepvariant-container-if-running-deepvariant)
+    - [mosdepth binary (if running depth calculation)](#mosdepth-binary-if-running-depth-calculation)
+    - [pb-CpG-tools binary (if processing pacbio data)](#pb-cpg-tools-binary-if-processing-pacbio-data)
   - [3. Modify in\_data.csv](#3-modify-in_datacsv)
   - [4. Modify nextflow\_pipeface.config](#4-modify-nextflow_pipefaceconfig)
   - [5. Modify parameters\_pipeface.json](#5-modify-parameters_pipefacejson)
@@ -27,10 +29,10 @@ cd pipeface
 
 ## 2. Get pipeline inputs
 
-*Please keep in mind that, while hs1 is new, smancy and exciting, hg38 is still the latest GRCh assembly and is better annotated by most projects.*
-
 ### Reference genome
 
+> **_Note:_** SNP/indel variant annotation is only available for hg38
+
 #### hg38
 
 Get a copy of the hg38 reference genome
@@ -119,13 +121,33 @@ tar -xvf hifi_revio.tar.gz
 
 ### DeepVariant container (if running DeepVariant)
 
+> **_Note:_** Running DeepVariant on ONT data assumes r10 data
+
 Get a local copy of the DeepVariant GPU container v1.6.1 (singularity image file)
 
 ```bash
 module load singularity
 singularity pull deepvariant_1.6.1-gpu.sif docker://google/deepvariant:deeptrio-1.6.1-gpu
 ```
 
+### mosdepth binary (if running depth calculation)
+
+Get a local copy of the mosdepth v0.3.9 binary
+
+```bash
+wget https://github.com/brentp/mosdepth/releases/download/v0.3.9/mosdepth -O mosdepth_0.3.9
+chmod +x mosdepth_0.3.9
+```
+
+### pb-CpG-tools binary (if processing pacbio data)
+
+Get a local copy of the pb-CpG-tools v2.3.2 binary
+
+```bash
+wget https://github.com/PacificBiosciences/pb-CpG-tools/releases/download/v2.3.2/pb-CpG-tools-v2.3.2-x86_64-unknown-linux-gnu.tar.gz
+tar -xzf pb-CpG-tools-v2.3.2-x86_64-unknown-linux-gnu.tar.gz
+```
+
 ## 3. Modify in_data.csv
 
 Specify the sample ID, file path to the data, data type, file path to regions of interest bed file (optional) and file path to clair3 model (if running Clair3) for each data to be analysed. Eg:
@@ -171,7 +193,13 @@ Modify access to project specific directories. Eg:
 Specify the path to `in_data.csv`. Eg:
 
 ```json
-    "input": "./config/in_data.csv",
+    "in_data": "./config/in_data.csv",
+```
+
+Specify the input data format ('ubam_fastq'). Eg:
+
+```json
+    "in_data_format": "ubam_fastq",
 ```
 
 Specify the path to the reference genome and it's index. Eg:
@@ -188,7 +216,7 @@ Specify the path to the reference genome and it's index. Eg:
     "ref_index": "./hs1.fa.fai",
 ```
 
-Optionally specify the path to the tandem repeat bed file. Eg:
+Optionally specify the path to the tandem repeat bed file. Set to 'NONE' if not required. Eg:
 
 ```json
     "tandem_repeat": "./*.trf.bed",
@@ -202,6 +230,7 @@ Optionally specify the path to the tandem repeat bed file. Eg:
 
 Specify the SNP/indel caller to use ('clair3' or 'deepvariant'). Eg:
 
+
 ```json
     "snp_indel_caller": "clair3",
 ```
@@ -212,6 +241,8 @@ Specify the SNP/indel caller to use ('clair3' or 'deepvariant'). Eg:
     "snp_indel_caller": "deepvariant",
 ```
 
+> **_Note:_** Running DeepVariant on ONT data assumes r10 data
+
 Specify the SV caller to use ('sniffles', 'cutesv' or 'both'). Eg:
 
 ```json
@@ -242,18 +273,62 @@ Specify whether variant annotation should be carried out ('yes' or 'no'). Eg:
     "annotate": "yes",
 ```
 
+> **_Note:_** SNP/indel variant annotation is only available for hg38
+
+Specify whether alignment depth should be calculated ('yes' or 'no'). Eg:
+
+```json
+    "calculate_depth": "no",
+```
+
+*OR*
+
+```json
+    "calculate_depth": "yes",
+```
+
 Specify the directory in which to write the pipeline outputs (please provide a full path). Eg:
 
 ```json
     "outdir": "/g/data/ox63/results"
 ```
 
-Specify the path to DeepVariant GPU container v1.6.1 (singularity image file). Eg:
+Specify the path to the DeepVariant GPU container v1.6.1 (singularity image file) (if running DeepVariant). Eg:
 
 ```json
     "deepvariant_container": "./deepvariant_1.6.1-gpu.sif"
 ```
 
+*OR*
+
+```json
+    "deepvariant_container": "NONE"
+```
+
+Specify the path to the mosdepth binary (if running depth calculation). Eg:
+
+```json
+    "mosdepth_binary": "./mosdepth_0.3.9"
+```
+
+*OR*
+
+```json
+    "mosdepth_binary": "NONE"
+```
+
+Specify the path to the pb-CpG-tools binary (if processing pacbio data). Eg:
+
+```json
+    "pbcpgtools_binary": "./pb-CpG-tools-v2.3.2-x86_64-unknown-linux-gnu/"
+```
+
+*OR*
+
+```json
+    "pbcpgtools_binary": "NONE"
+```
+
 ## 6. Get pipeline dependencies
 
 You may use the centrally installed nextflow environmental module available on NCI to access the nextflow and java dependencies

docs/software_versions.txt

@@ -1,8 +1,10 @@
 Samtools: 1.19
 Minimap2: 2.28-r1209
+mosdepth: 0.3.9
 Clair3: 1.0.9
 DeepVariant 1.6.1
 WhatsHap: 2.3
+pb-CpG-tools: 2.3.2
 Sniffles2: 2.3.3
 cuteSV: 1.0.13
 GNU coreutils: 8.30