Refactored code

Author: nkongenelly

bin/get_metadata.py

@@ -14,12 +14,10 @@
 
 
 class RunfolderInfo:
-    def __init__(self, runfolder, demultiplexer_outdir, demultiplexer):
+    def __init__(self, runfolder, demultiplexer_outdir):
         self.runfolder = runfolder
-        self.demultiplexer = demultiplexer
         self.run_info = self.read_run_info()
         self.run_parameters = self.read_run_parameters()
-        self.stats_json = self.read_stats_json(demultiplexer_outdir, demultiplexer)
         self.description_and_identifier = OrderedDict()
         self.run_parameters_tags = {
             "RunId": "Run ID",
@@ -82,38 +80,11 @@ def find_flowcell_type_novaseqx(self):
             return None
         return {"Flowcell type": flowcell_type}
 
-    def read_stats_json(self, demultiplexer_outdir, demultiplexer):
-        stats_path = "Reports" if demultiplexer == "bclconvert" else "Stats/Stats.json"
-        stats_json_path = os.path.join(self.runfolder, demultiplexer_outdir, stats_path)
-        if os.path.exists(stats_json_path):
-            if demultiplexer == "bclconvert":
-                # Bclconvert produces multiple statistical output files
-                files = glob.glob(stats_json_path + "/*.csv")
-                bclconvert_data = {}
-                for file in files:
-                    with open(file) as csvfile:
-                        reader = csv.reader(csvfile)
-                        file_name = re.sub(r".*/|\.csv", "", file)
-                        bclconvert_data[file_name] = [row for row in reader]
-                return bclconvert_data
-            else:
-                with open(stats_json_path) as f:
-                    return json.load(f)
-        else:
-            return None
-
-    def get_demultiplexer_version(self, runfolder):
-        with open(os.path.join(runfolder, f"{self.demultiplexer}_version")) as f:
-            demultiplexer_str = f.read()
-        return demultiplexer_str.split("v")[1].strip()
-
     def get_software_version(self, runfolder):
-        pipeline_info_filename = (
-            "nf_core_pipeline_software_mqc_versions.yml"
-            if self.demultiplexer == "bcl2fastq"
-            else "nf_core_demultiplex_software_mqc_versions.yml"
-        )
-        with open(Path(runfolder) / "pipeline_info" / pipeline_info_filename) as f:
+        pipeline_dir = Path(runfolder) / "pipeline_info"
+        pipeline_info_filename = next(pipeline_dir.glob("*_software_mqc_versions.yml"))
+
+        with open(pipeline_info_filename) as f:
             return {
                 software: version
                 for software_dict in yaml.safe_load(f).values()
@@ -165,15 +136,6 @@ def get_info(self):
         return results
 
     def get_demultiplexing_info(self):
-        try:
-            return {
-                "Demultiplexing": {
-                    self.demultiplexer: self.get_demultiplexer_version(self.runfolder)
-                }
-            }
-        except FileNotFoundError:
-            pass
-
         try:
             return {"Demultiplexing": self.get_software_version(self.runfolder)}
         except FileNotFoundError:
@@ -187,25 +149,18 @@ def get_demultiplexing_info(self):
     parser.add_argument(
         "--runfolder", type=str, required=True, help="Path to runfolder"
     )
-    parser.add_argument(
-        "--demultiplexer",
-        type=str,
-        default="bcl2fastq",
-        help="Name of demultiplexer used",
-    )
     parser.add_argument(
         "--demultiplexer-outdir",
         type=str,
-        default="Unaligned",
+        default="Data/Intensities/BaseCalls",
         help="Path to demultiplexer output folder relative to the runfolder",
     )
 
     args = parser.parse_args()
     runfolder = args.runfolder
-    demultiplexer = args.demultiplexer
     demultiplexer_outdir = args.demultiplexer_outdir
 
-    runfolder_info = RunfolderInfo(runfolder, demultiplexer_outdir, demultiplexer)
+    runfolder_info = RunfolderInfo(runfolder, demultiplexer_outdir)
     info = runfolder_info.get_info()
 
     print(

config/nextflow_config/test_bclconvert.config

@@ -0,0 +1,19 @@
+/*
+========================================================================================
+    Nextflow config file for running minimal tests
+========================================================================================
+    Defines input files and everything required to run a fast and simple pipeline test.
+    Use as follows:
+        nextflow run main.nf -profile dev,test,singularity
+
+
+    This config takes inspiration from https://github.com/nf-core/rnaseq
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    run_folder = "$baseDir/test_data/230825_M04034_0043_000000000-L6NVV"
+    fastqscreen_databases = "$baseDir/test_data/Test_FastQ_Screen_Genomes"
+    checkqc_config = "$baseDir/test_data/checkqc_config.yaml"
+    config_dir = "$baseDir/test_data/test_config"
+}    

config/tool_config/bclconvert/multiqc_flowcell_config.yaml

@@ -4,6 +4,7 @@ run_modules:
     - fastqc
     - fastq_screen
     - bcl2fastq
+    - bclconvert
     - interop
     - custom_content
 

config/tool_config/bcl2fastq/multiqc_flowcell_config.yaml renamed to config/tool_config/multiqc_flowcell_config.yaml

@@ -49,7 +49,7 @@ def helpMessage() {
 
     Optional parameters:
         --result_dir                        Path to write results (default: results)
-        --demultiplexer_outdir              Folder name to check for fastq.gz files and demultiplexing stats (default: Unaligned)
+        --demultiplexer_outdir              Folder name to check for fastq.gz files and demultiplexing stats (default: Data/Intensities/BaseCalls)
         --checkqc_config                    Configuration file for CheckQC
         --assets_dir                        Location of project assests (default: "\$baseDir/assets").
         --config_dir                        Location of tool configuration files (default: "\$baseDir/config/tool_config").
@@ -64,11 +64,7 @@ def helpMessage() {
     """
 }
 
-if (params.help || !params.run_folder){
-    helpMessage()
-    exit 0
-}
-if (params.help || !params.demultiplexer){
+if (params.help || !params.run_folder || !params.demultiplexer){
     helpMessage()
     exit 0
 }
@@ -146,7 +142,7 @@ workflow CHECK_RUN_QUALITY {
 
         INTEROP_SUMMARY(run_folder)
         GET_QC_THRESHOLDS(run_folder)
-        GET_METADATA(run_folder, demultiplexer)
+        GET_METADATA(run_folder)
         project_and_reads = get_project_and_reads(params.run_folder)
         FASTQC(project_and_reads,
             params.config_dir)
@@ -163,8 +159,7 @@ workflow CHECK_RUN_QUALITY {
             GET_METADATA.out.collect(),
             demux_stats,
             params.assets_dir,
-            params.config_dir,
-            demultiplexer)
+            params.config_dir)
         MULTIQC_PER_PROJECT( params.run_folder,
             combine_results_by_project(
                 FASTQC.out.groupTuple(),
@@ -257,7 +252,6 @@ process GET_METADATA {
 
     input:
     path runfolder
-    val demultiplexer
 
     output:
     path 'sequencing_metadata_mqc.yaml'
@@ -270,7 +264,6 @@ process GET_METADATA {
     }
     """
     python ${params.script_dir}/get_metadata.py --runfolder $runfolder \\
-        --demultiplexer $demultiplexer \\
         $demultiplexer_outdir_section &> sequencing_metadata_mqc.yaml
     """
 }
@@ -305,13 +298,7 @@ process MULTIQC_PER_FLOWCELL {
     path demux_stats                // demux logs
     path assets                     // Staged copy of assets folder
     path config_dir                 // Staged copy of config folder
-    val demultiplexer               // Demultiplexer name
 
-    script:
-    // """
-    // echo $demux_stats
-    // echo demultiplexer: $demultiplexer
-    // """
     output:
     tuple path("*multiqc_report.html"), path("*_data.zip")
 
@@ -326,7 +313,7 @@ process MULTIQC_PER_FLOWCELL {
         --title "Flowcell report for \${RUNFOLDER}" \\
         --filename \${RUNFOLDER}_multiqc_report.html -z \\
         -c ${config_dir}/multiqc_main_config.yaml \\
-        -c ${config_dir}/${demultiplexer}/multiqc_flowcell_config.yaml \\
+        -c ${config_dir}/multiqc_flowcell_config.yaml \\
         ${threshold_parameter} \\
         .
     """

main.nf

@@ -62,5 +62,9 @@ profiles {
     test {
         includeConfig 'config/nextflow_config/test.config'
     }
+
+    test_bclconvert {
+        includeConfig 'config/nextflow_config/test_bclconvert.config'
+    }
 
 }

nextflow.config

@@ -0,0 +1 @@
+../../config/tool_config/multiqc_flowcell_config.yaml

test_data/test_config/bcl2fastq/multiqc_flowcell_config.yaml

@@ -9,30 +9,30 @@
 # Run pipeline in test mode, this is done once per test session
 @pytest.fixture(scope="session", autouse=True)
 def result_dir(request, tmpdir_factory):
-    demultiplexer = "bcl2fastq"
-    if hasattr(request, "param"):
-        demultiplexer = request.param
+    demultiplexer = request.param
 
     result_dir = tmpdir_factory.mktemp("results")
-    cmd = [
-        "nextflow",
-        "run",
-        "main.nf",
-        "-profile",
-        "dev,test,singularity",
-        "--demultiplexer",
-        demultiplexer,
-        "--result_dir",
-        result_dir,
-    ]
-    if demultiplexer == "bclconvert":
-        cmd = [*cmd, *["--run_folder", "test_data/230825_M04034_0043_000000000-L6NVV"]]
-
-    subprocess.run(cmd, check=True)
+    extra_profile = "test_bclconvert" if demultiplexer == "bclconvert" else "test"
+
+    subprocess.run(
+        [
+            "nextflow",
+            "run",
+            "main.nf",
+            "-profile",
+            f"dev,{extra_profile},singularity",
+            "--demultiplexer",
+            demultiplexer,
+            "--result_dir",
+            result_dir,
+        ],
+        check=True,
+    )
 
     yield result_dir
 
 
+@pytest.mark.parametrize("result_dir", ["bcl2fastq"], indirect=True)
 def test_results_dirs_exist(result_dir):
     flowcell_dir = os.path.join(result_dir, "flowcell_report")
     projects_dir = os.path.join(result_dir, "projects")
@@ -41,6 +41,7 @@ def test_results_dirs_exist(result_dir):
     assert os.path.isdir(projects_dir)
 
 
+@pytest.mark.parametrize("result_dir", ["bcl2fastq"], indirect=True)
 def test_project_dirs_exist(result_dir):
     projects_dir = os.path.join(result_dir, "projects")
     projects = ["Zymo", "Qiagen", "NoProject"]
@@ -49,6 +50,7 @@ def test_project_dirs_exist(result_dir):
         assert os.path.isdir(os.path.join(projects_dir, project))
 
 
+@pytest.mark.parametrize("result_dir", ["bcl2fastq"], indirect=True)
 def test_flowcell_report_exist(result_dir):
     flowcell_dir = os.path.join(result_dir, "flowcell_report")
     report_path = os.path.join(
@@ -58,6 +60,7 @@ def test_flowcell_report_exist(result_dir):
     assert os.path.isfile(report_path)
 
 
+@pytest.mark.parametrize("result_dir", ["bcl2fastq"], indirect=True)
 def test_project_reports_exist(result_dir):
     projects_dir = os.path.join(result_dir, "projects")
     projects = ["Zymo", "Qiagen", "NoProject"]
@@ -71,6 +74,7 @@ def test_project_reports_exist(result_dir):
         assert os.path.isfile(report_path)
 
 
+@pytest.mark.parametrize("result_dir", ["bcl2fastq"], indirect=True)
 def check_sections_in_report(report_path, sections):
     with open(report_path, "r") as html_file:
         parser = BeautifulSoup(html_file.read(), "lxml")
@@ -79,6 +83,7 @@ def check_sections_in_report(report_path, sections):
             assert len(hits) > 0
 
 
+@pytest.mark.parametrize("result_dir", ["bcl2fastq"], indirect=True)
 def test_all_sections_included_in_flowcell_report(result_dir):
     flowcell_dir = os.path.join(result_dir, "flowcell_report")
     report_path = os.path.join(
@@ -116,6 +121,7 @@ def test_all_sections_included_in_bclcovert_flowcell_report(result_dir):
     check_sections_in_report(report_path, sections)
 
 
+@pytest.mark.parametrize("result_dir", ["bcl2fastq"], indirect=True)
 def test_all_sections_included_in_project_reports(result_dir):
     projects_dir = os.path.join(result_dir, "projects")
     projects = ["Zymo", "Qiagen", "NoProject"]