Merge pull request #30 from CaroAMN/dev

Updating Container Images + Improved File staging + MultiQC

Author: CaroAMN

README.md

@@ -35,26 +35,33 @@
 2. Channel Alignment
 3. Iterative Stitching
 
-**Analysis**
+**ARA Registration**
 
 4. ARA Registration subworkflow (optional)
 5. Cell Nuclei Quantification
 
+**Full**
+
+1. Preprocessing
+2. Nuclei quantification
+
 ## Pipeline Summary
 
-The pipeline consists of two major stages, the `preprocessing`stage and the `analysis`stage.
+The pipeline consists of two major workflows `preprocessing` and the `full` workflow. The `ara-regsitration` is an optional subworkflow that works only for whole mouse brain samples.
 
 ### Preprocessing
 
 Preprocessing is performed on raw 2D single-channel 16-bit `.tif` images produced by a light sheet microscope. Three individual steps are perfomed :
 
-- Measuring and adjustemnts for intensities
-- Image channel alignemnt for at least two different channels
-- Image tile stitching to recustruct the full image for each channel and z-slice
+- **Intensity adjustments** to correct for the Gaussian shape of the lightsheet and intensity differences between adjacent tiles
+- **Image channel alignment** using a 2D rigid approach or a nonlinear 3D approach using Elastix.
+- **Image tile stitching** via an iterative 2D stitching approach by calculating z displacements and xy translations using phase correlation and SIFT.
+
+### Full
 
-### Analysis
+Quantification of cell-nuclei is performed using a 3D-Unet. It is performed on the nuclear channel only, assuming that the corresponding image file names contain the pattern `C1`.
 
-Analysis is performed using a 3D-Unet to qunatify the amount of cell-nuclei in the given sample. The quantification is performed on the nuclear channel only, assuming that the corresponding image file names contain the pattern `C1`.
+### ARA Registration
 
 Optionally registration to the Allen Refernce Atlas (ARA) for functional brain region annotation can be perfomed before segmentation.
 This includes the following two steps:

conf/modules.config

@@ -18,10 +18,6 @@ process {
         saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
     ]
 
-    withName: FASTQC {
-        ext.args = '--quiet'
-    }
-
     withName: 'MULTIQC' {
         ext.args   = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' }
         publishDir = [
@@ -30,14 +26,12 @@ process {
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
-    /*
-    withName: 'MAT2JSON' {
+
+    withName: 'STAGEFILES' {
         publishDir = [
-            path: { "${params.outdir}/${meta.id}/mat2json" },
-            mode: params.publish_dir_mode
+            enabled: false
         ]
     }
-    */
 
     withName: 'NUMORPHINTENSITY' {
         publishDir = [

docs/output.md

@@ -19,6 +19,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d
 - [NumorphRegister](#numorphregister) - Performs registartion to the Allen Reference Atlas (ARA)
 - [Numorph3DUnet](#numorph3dunet) - Perfomrs cell-nuclei segmentation and quantification
 - [Mat2JSON] (#mat3json) - Converts `.mat`files to JSON
+- [MultiQC](#MultiQC) Aggregate report describing workflow run and tools used from the whole pipeline.
 - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution
 
 ### NumorphIntensity
@@ -129,6 +130,19 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d
 
   **Mat2JSON** converts a given `.mat`file into a `CSV` if the data is stored as a table datastructure or a `JSON` for other nested datastructures.
 
+### MultiQC
+
+<details markdown="1">
+<summary>Output files</summary>
+
+- `multiqc/`
+  - `multiqc_report.html`: a standalone HTML file that can be viewed in your web browser.
+  - `multiqc_data/`: directory containing parsed statistics from the different tools used in the pipeline.
+
+</details>
+
+**MultiQC** collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see <http://multiqc.info>.
+
 ### Pipeline information
 
 <details markdown="1">

docs/usage.md

@@ -51,12 +51,11 @@ The individual parameters are explained [here](###Analysisspecificparameters)
 
 ### Analysis specific parameters
 
-This section descripbes every parameter that can be set in the `parameter.csv`. In order that the pipeline runs correctly all named parameters need to be present in the parameter file and its recommended to use the provided parameter file (link). Every parameter has a default value that will be set if not otherwise defined in the `parameter.csv`.
+This section descripbes every parameter that can be set in the `parameter.csv`. In order for the pipeline to run correctly all named parameters need to be present in the parameter file and its recommended to use the provided parameter file (link). Every parameter has a default value that will be set if not otherwise defined in the `parameter.csv`.
 
 | Parameter                   | Description                                                                                                                                                                                                                                     |
 | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `darkfield_intensity`       | 1xn_channels; Constant darkfield intensity value (i.e. average intensity of image with nothing present). **Default: 101**                                                                                                                       |
-| `img_directory`             |                                                                                                                                                                                                                                                 |
 | `single_sheet`              | true, false; Whether a single sheet was used for acquisition                                                                                                                                                                                    |
 | `ls_width`                  | 1xn_channels interger. Light sheet width setting for UltraMicroscope II as percentage. **Default: 50**                                                                                                                                          |
 | `laser_y_displacement`      | [-0.5,0.5]; Displacement of light-sheet along y axis. Value of 0.5 means light-sheet center is positioned at the top of the image. **Default: 0**                                                                                               |
@@ -164,7 +163,7 @@ This section descripbes every parameter that can be set in the `parameter.csv`.
 The typical command for running the pipeline is as follows:
 
 ```bash
-nextflow run nf-core/lsmquant --input ./samplesheet.csv --outdir ./results -profile docker
+nextflow run nf-core/lsmquant --input ./samplesheet.csv --outdir ./results -profile docker -work-dir ./work
 ```
 
 This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles.
@@ -178,6 +177,8 @@ work                # Directory containing the nextflow working files
 # Other nextflow hidden files, eg. history of pipeline runs and old logs.
 ```
 
+For this pipeline it is recommended to specify the location of the work directory as well with `-work-dir`. The directory will contain any nextflow working files which includes all in- and output files. The work directory will be larger than the input sample size. If you don't specify a location, the work directory will be created in the location from where the pipeline got started.
+
 If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file.
 
 Pipeline settings can be provided in a `yaml` or `json` file via `-params-file <file>`.

main.nf

@@ -35,6 +35,9 @@ workflow NFCORE_LSMQUANT {
 
     LSMQUANT(samplesheet)
 
+    emit:
+    multiqc_report = LSMQUANT.out.multiqc_report
+
 }
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -73,6 +76,7 @@ workflow {
         params.outdir,
         params.monochrome_logs,
         params.hook_url,
+        NFCORE_LSMQUANT.out.multiqc_report,
     )
 }
 

modules/local/mat2json/main.nf

@@ -3,7 +3,7 @@ process MAT2JSON {
     tag "$meta.id - $matfile.baseName"
     label 'process_single'
 
-    container 'carolinschwitalla/mat2json:latest'
+    container 'nf-core/mat2json:1.0.0'
 
     input:
     tuple val(meta), path(matfile)

modules/local/numorph3dunet/main.nf

@@ -3,7 +3,7 @@ process NUMORPH3DUNET {
     label 'process_gpu'
 
 
-    container "carolinschwitalla/numorph-3dunet:latest"
+    container "nf-core/numorph-3dunet:1.0.0"
 
 
     input:
@@ -35,7 +35,7 @@ process NUMORPH3DUNET {
     mkdir -p ./images
 
     # move images to images directory
-    mv ${img_directory} ./images/
+    ln -sr ${img_directory} ./images/
 
     numorph_3dunet.predict \\
         -i images/ \\

modules/local/numorphalign/main.nf

@@ -3,7 +3,7 @@ process NUMORPHALIGN {
     label 'process_high_long'
 
 
-    container "carolinschwitalla/numorph_preprocessing:0.9.0"
+    container "nf-core/numorph_preprocessing:1.0.0"
 
     input:
     tuple val(meta), path(img_directory),  path(parameter_file)
@@ -32,18 +32,20 @@ process NUMORPHALIGN {
     """
     mkdir -p results/samples/
     mkdir -p results/variables/
+    mkdir -p ./images
 
-    mv ${adj_params_mat} results/variables
-    mv ${path_table_mat} results/variables
-    mv ${thresholds_mat} results/variables
+    ln -sr ${img_directory} ./images
+    ln -sr ${adj_params_mat} results/variables
+    ln -sr ${path_table_mat} results/variables
+    ln -sr ${thresholds_mat} results/variables
 
     # resolve symlinks and paths
-    img_directory=\$(readlink -f ${img_directory})
+    img_dir=\$(readlink -f ./images)
     parameter_file=\$(readlink -f ${parameter_file})
     results_dir=\$(readlink -f ./results)
     NM_variables=\$(readlink -f ${NM_variables})
 
-    numorph_preprocessing 'input_dir' \$img_directory 'output_dir' \$results_dir 'parameter_file' \$parameter_file 'sample_name' ${meta.id} 'stage' 'align' 'NM_variables' \$NM_variables
+    numorph_preprocessing 'input_dir' \$img_dir 'output_dir' \$results_dir 'parameter_file' \$parameter_file 'sample_name' ${meta.id} 'stage' 'align' 'NM_variables' \$NM_variables
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/local/numorphintensity/main.nf

@@ -3,7 +3,7 @@ process NUMORPHINTENSITY {
     label 'process_high_long'
 
 
-    container "carolinschwitalla/numorph_preprocessing:0.9.0"
+    container "nf-core/numorph_preprocessing:1.0.0"
 
 
     input:
@@ -26,14 +26,17 @@ process NUMORPHINTENSITY {
 
     """
     mkdir -p ./results
+    mkdir -p ./images
+
+    ln -sr ${img_directory} ./images
 
     # resolve symlinks and paths
-    img_directory=\$(readlink -f ${img_directory})
+    img_dir=\$(readlink -f ./images)
     parameter_file=\$(readlink -f ${parameter_file})
     results_dir=\$(readlink -f ./results)
 
 
-    numorph_preprocessing 'input_dir' \$img_directory 'output_dir' \$results_dir 'parameter_file' \$parameter_file 'sample_name' ${meta.id} 'stage' 'intensity'
+    numorph_preprocessing 'input_dir' \$img_dir 'output_dir' \$results_dir 'parameter_file' \$parameter_file 'sample_name' ${meta.id} 'stage' 'intensity'
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/local/numorphregister/main.nf

@@ -2,17 +2,15 @@ process NUMORPHREGISTER {
     tag "$meta.id"
     label 'process_high_long'
 
-    container "carolinschwitalla/numorph_analyze:latest"
+    container "nf-core/numorph_analyze:1.0.1"
 
     input:
     tuple val(meta), path(resampled_directory), path(parameter_file)
-    path NM_variables
-
 
     output:
-    path "results/variables/reg_params.mat"      , emit: reg_params_mat
-    path "results/variables/*_mask.mat"          , emit: reg_mask
-    path "results/NM_variables.mat"              , emit: NM_variables
+    path "results/*results.mat"                 , emit: res_mat
+    path "results/variables/*"                  , emit: variables
+    path "results/NM_variables.mat"             , emit: NM_variables
     path "results/registered/*"                 , emit: registered
     path "versions.yml"                         , emit: versions
 
@@ -23,22 +21,21 @@ process NUMORPHREGISTER {
     script:
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
-    def nm_variables = NM_variables ? "${NM_variables}" : ""
 
     """
-    mkdir -p \$PWD/results/variables/
-    mkdir -p \$PWD/results/resampled/
-    mkdir -p \$PWD/results/registered/
+    mkdir -p results/variables/
+    mkdir -p results/resampled/
+    mkdir -p results/registered/
+
 
-    mv $resampled_directory \$PWD/results/resampled
+    ln -sr ${resampled_directory} results/resampled
 
     #resolve symlinks and paths
     resampled_directory=\$(readlink -f ./results/resampled/)
     parameter_file=\$(readlink -f ${parameter_file})
-    NM_variables=\$(readlink -f ${NM_variables})
     results_dir=\$(readlink -f ./results)
 
-    numorph_analyze 'input_dir' \$resampled_directory 'output_dir' \$results_dir 'parameter_file' \$parameter_file 'sample_name' ${meta.id} 'stage' 'register' 'NM_variables' \$NM_variables 'use_processed_images' 'resampled'
+    numorph_analyze 'input_dir' \$resampled_directory 'output_dir' \$results_dir 'parameter_file' \$parameter_file 'sample_name' ${prefix} 'stage' 'register' 'NM_variables' '' 'use_processed_images' 'resampled'
 
 
     cat <<-END_VERSIONS > versions.yml
@@ -56,9 +53,10 @@ process NUMORPHREGISTER {
     mkdir -p results/variables
 
     touch results/variables/reg_params.mat
-    touch results/variables/${meta.id}_mask.mat
+    touch results/variables/${prefix}_mask.mat
     touch results/NM_variables.mat
-    touch results/registered/${meta.id}_registered.tif
+    touch results/${prefix}_results.mat
+    touch results/registered/${prefix}_registered.tif
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":