Testing usig checkqc illumina parser for bclconvert

Author: nkongenelly

.github/workflows/unit_tests.yaml

@@ -13,10 +13,14 @@ jobs:
       with:
         python-version: '3.13.1'
 
+    # checkQC currently only accepts python version between 3.10 and 3.11 but 
+    # projman uses 3.13 thuis this is temporarily fixed here by cloning it
     - name: Install dependencies
       run: |
         python3 -m pip install --upgrade pip
         pip install -e . -r requirements_dev.txt
+        git clone https://github.com/Molmed/checkQC.git
+
     
     - name: Launch tests
       run: |

projman_filler/app.py

@@ -15,7 +15,7 @@
 from projman_filler.repositories.sample_results_repo import SampleResultRepo
 from projman_filler.repositories.flowcell_lane_results_repo import FlowcellLaneResultsRepo
 from projman_filler.repositories.flowcell_runfolder_repo import FlowcellRunfolderRepo
-
+from checkQC.qc_data import QCData
 
 class App(object):
 
@@ -59,39 +59,61 @@ def insert_flowcell_runfolder_into_db(self, runfolder, flowcell_name):
                                                run_date=runfolder_date)
         self.flowcell_runfolder_repo.add(flowcell_runfolder)
 
-    def insert_runfolder_into_db(self, runfolder, bcl2fastq_stats_dir, force=False, atac_seq_mode=False, olink_mode=False):
-        if olink_mode:
-            print("Olink mode activated. Will read lane-level statistics from InterOp files instead of bcl2fastq Stats.json.")
-            return self.insert_olink_runfolder_into_db(runfolder, force)
-
-        bcl2fastq_stats = Bcl2fastqRunStatsParser(os.path.join(runfolder, bcl2fastq_stats_dir))
-        flowcell_name = bcl2fastq_stats.get_flowcell_name()
-        reads_and_cycles = bcl2fastq_stats.get_reads_and_cycles()
-        conversion_results = bcl2fastq_stats.get_conversion_results()
-
-        # Check if flowcell exists and should be overriden
-        self.delete_existing_flowcell_from_db(flowcell_name, force)
-
-        # For atac-seq we run bcl2fastq with special parameters declaring
-        # that the second index should be interpreted as a non-index read.
-        # So we allow overriding the Interop list of non-index-reads with
-        # a custom list obtained from bcl2fastq stats. /ML 2021-09
-        non_index_reads = None
-        if atac_seq_mode:
-            print("ATAC-seq mode activated. Will re-map read numbers according to settings used by bcl2fastq.")
-            non_index_reads = bcl2fastq_stats.get_non_index_reads()
-
-        interop = InteropRunStatsParser(runfolder, non_index_reads)
-        lane_stats = calculate_lane_statistics(interop, flowcell_name, conversion_results)
-        self.flowcell_lane_results_repo.add(list(lane_stats))
-
-        samplesheet_file = os.path.join(runfolder, "SampleSheet.csv")
-        samplesheet = Samplesheet(samplesheet_file)
-
-        sample_stats = calculate_sample_statistics(flowcell_name, conversion_results, reads_and_cycles, samplesheet)
-        self.sample_results_repo.add(list(sample_stats))
-
-        self.insert_flowcell_runfolder_into_db(runfolder, flowcell_name)
+    def insert_runfolder_into_db(self, runfolder, bcl2fastq_stats_dir, demultiplexer, force=False, atac_seq_mode=False, olink_mode=False):
+        interop = InteropRunStatsParser(runfolder)
+        if demultiplexer == "bclconvert":
+
+            qc_data_constructor = getattr(QCData, f"from_{demultiplexer}")
+            qc_data = qc_data_constructor(
+                runfolder_path=runfolder,
+                parser_config={
+                    "reports_location": "Reports"
+                }
+            )
+            flowcell_lane_results, sample_results = \
+                interop.get_checkqc_interop_stats(qc_data, runfolder)
+            
+            flowcell_name = interop.get_flowcell_name()
+            # Check if flowcell exists and should be overriden
+            self.delete_existing_flowcell_from_db(flowcell_name, force)
+            
+            self.flowcell_lane_results_repo.add(flowcell_lane_results)
+            self.sample_results_repo.add(sample_results)
+
+            self.insert_flowcell_runfolder_into_db(runfolder, flowcell_name)
+
+        else:   
+            if olink_mode:
+                print("Olink mode activated. Will read lane-level statistics from InterOp files instead of bcl2fastq Stats.json.")
+                return self.insert_olink_runfolder_into_db(runfolder, force)
+
+            bcl2fastq_stats = Bcl2fastqRunStatsParser(os.path.join(runfolder, bcl2fastq_stats_dir))
+            flowcell_name = bcl2fastq_stats.get_flowcell_name()
+            reads_and_cycles = bcl2fastq_stats.get_reads_and_cycles()
+            conversion_results = bcl2fastq_stats.get_conversion_results()
+
+            # Check if flowcell exists and should be overriden
+            self.delete_existing_flowcell_from_db(flowcell_name, force)
+
+            # For atac-seq we run bcl2fastq with special parameters declaring
+            # that the second index should be interpreted as a non-index read.
+            # So we allow overriding the Interop list of non-index-reads with
+            # a custom list obtained from bcl2fastq stats. /ML 2021-09
+            non_index_reads = None
+            if atac_seq_mode:
+                print("ATAC-seq mode activated. Will re-map read numbers according to settings used by bcl2fastq.")
+                non_index_reads = bcl2fastq_stats.get_non_index_reads()
+
+            lane_stats = calculate_lane_statistics(interop, flowcell_name, conversion_results)
+            self.flowcell_lane_results_repo.add(list(lane_stats))
+
+            samplesheet_file = os.path.join(runfolder, "SampleSheet.csv")
+            samplesheet = Samplesheet(samplesheet_file)
+
+            sample_stats = calculate_sample_statistics(flowcell_name, conversion_results, reads_and_cycles, samplesheet)
+            self.sample_results_repo.add(list(sample_stats))
+
+            self.insert_flowcell_runfolder_into_db(runfolder, flowcell_name)
 
 
     def insert_olink_runfolder_into_db(self, runfolder, force=False):

projman_filler/cli.py

@@ -19,14 +19,15 @@
 @click.option('--atac-seq-mode', is_flag=True)
 @click.option('--olink-mode', is_flag=True)
 @click.option('-b', '--bcl2fastq-stats', default="Unaligned/Stats", type=click.Path())
+@click.option('-d', '--demultiplexer', default ="bcl2fastq", type=click.Choice(['bcl2fastq', 'bclconvert']))
 @click.argument('runfolder', type=click.Path())
-def main(runfolder, force, atac_seq_mode, olink_mode, bcl2fastq_stats, debug):
+def main(runfolder, force, atac_seq_mode, olink_mode, bcl2fastq_stats, demultiplexer, debug):
     """Console script for projman_filler."""
     print("projman_filler v{}".format(projman_filler_version))
     try:
         db_connection_string = os.environ["PROJMAN_DB"]
         app = App(db_connection_string, debug)
-        app.insert_runfolder_into_db(runfolder, bcl2fastq_stats, force=force, atac_seq_mode=atac_seq_mode, olink_mode=olink_mode)
+        app.insert_runfolder_into_db(runfolder, bcl2fastq_stats, demultiplexer, force=force, atac_seq_mode=atac_seq_mode, olink_mode=olink_mode)
     except FlowcellAlreadyInDb:
         print("Flowcell was already present in db.")
         sys.exit(1)

projman_filler/interop_run_stats_parser.py

@@ -4,6 +4,7 @@
 from math import isnan
 from projman_filler.run_stats_parser_interface import RunStatsParserInterface
 from projman_filler.lane import Lane
+from projman_filler.models.db_models import FlowcellLaneResult, SampleResult
 
 
 class InteropRunStatsParser(RunStatsParserInterface):
@@ -128,5 +129,67 @@ def _get_conversion_results(self) -> list:
 
             lanes.append(Lane(l, total_clusters_raw, total_clusters_pf))
         return lanes
-
-
+    
+    def get_checkqc_interop_stats(self, qc_data, runfolder):
+        """
+        Gets run stats from checkqc illumina parser and the rest from iterop
+
+        Params:
+        :qc_data (acheckQC QCData object): results from checkQC illumina parser 
+            for specified demultiplexer
+        :runfolder (str): Runfolder path string
+
+        Returns:
+        : tuple of flowcell_lane_result, sample_results to be added to respective DB
+        """
+        from checkQC.parsers.illumina import _read_interop_summary
+        flowcell_lane_results = []
+        sample_results = []
+        
+        run_summary, index_summary = _read_interop_summary(runfolder)
+        flowcell_id = self.get_flowcell_name()  #self._run_info.flowcell_id()
+
+        for lane_no, lane_dict in qc_data.sequencing_metrics.items():
+            for read_no, read_dict in lane_dict["reads"].items():
+                if read_dict["is_index"]:
+                    continue
+                interop_lane = run_summary.at(read_no-1).at(lane_no-1)
+                cycles = run_summary.at(read_no-1).read().total_cycles()
+                error_rate = run_summary.at(0).at(lane_no-1).error_rate().mean()  # or interop_lane.error_rate().mean() ?,
+                
+                flowcell_lane_results.append(
+                     FlowcellLaneResult(
+                        flowcell_id=flowcell_id,
+                        lane_num=lane_no,
+                        read_num=read_no,
+                        raw_density=interop_lane.density().mean(),
+                        pf_density=interop_lane.density_pf().mean(),
+                        error_rate= None if isnan(error_rate) else error_rate,
+                        raw_clusters=run_summary.at(0).at(lane_no-1).cluster_count().mean(),  #Not sure if this should be cluster_count,
+                        pf_clusters= lane_dict["total_cluster_pf"],
+                        cycles=cycles,
+                        pct_q30=float((interop_lane.percent_gt_q30()) / 100),
+                    )
+                 )
+
+                sample_reads = index_summary.at(lane_no-1).at(read_no-1)
+                lane_summary =  run_summary.at(read_no-1).at(lane_no-1)
+                sample_results.append(
+                    SampleResult(
+                        flowcell_id=flowcell_id,
+                        project_id=sample_reads.project_name(),
+                        sample_name=sample_reads.project_name(),
+                        tag_seq=f"{sample_reads.index1()}-{sample_reads.index2()}",
+                        lane_num=lane_no,
+                        read_num=read_no,
+                        cycles=cycles,
+                        pct_lane="",
+                        pf_clusters=sample_reads.cluster_count(),
+                        pct_q30=read_dict["percent_q30"],
+                        pct_tag_err="",  #  (float(mismatch_counts) / float(number_of_reads))*100
+                        library_name="",
+                        mean_q=None,
+                    )
+                )
+    
+        return flowcell_lane_results, sample_results

setup.py

@@ -16,7 +16,14 @@
     'SQLAlchemy~=2.0.40',
     'pymssql~=2.3.4',
     'pandas~=2.2.3',
-    'numpy~=2.2.4'
+    'numpy~=2.2.4',
+    # checkQC must be installed manually
+    # git clone https://github.com/Molmed/checkQC.git
+    # cd checkQC
+    # sed -i 's/python_requires=.*$/python_requires=">=3.10",/' setup.py
+    # sed -i 's/^interop~=.*$/interop==1.4.0/' requirements/prod
+    # pip install --no-deps -r requirements/dev .
+
 ]
 
 setup_requirements = [