Merge pull request #61 from remiolsen/master

Remi-Andre Olsen · web-flow · commit cdc197b92a0e · 2017-10-02T15:04:43.000+02:00
Finishing up bioinfo kpis
diff --git a/make_dashboards/templates/internal/js/dashboard.js b/make_dashboards/templates/internal/js/dashboard.js
@@ -76,11 +76,10 @@ $(function () {
             '<span style="color:#258F0B">'+pl['hiseq_sequencing_l']+'H</span>, '+
             '<span style="color:#4c85bc">'+pl['hiseqX_sequencing_l']+'X</span> lanes in progress';
         make_balance_plot('#rc_finished_balance', pl_l['initial_qc_lanes'],   pl['initial_qc_lanes'],   undefined, pl['initial_qc_lanes']+' lanes in progress');
-        console.log(pl['initial_qc_samples']);
         make_balance_plot('#rc_balance',          pl_l['initial_qc_samples'], pl['initial_qc_samples'], undefined, pl['initial_qc_samples']+' samples in progress');
         make_balance_plot('#lp_balance',          pl_l['library_prep'],       pl['library_prep'],       undefined, pl['library_prep']+' samples in progress');
-        make_balance_plot('#seq_balance',         pl_l['sequencing'],         [pl['miseq_sequencing_l'], pl['hiseq_sequencing_l'], pl['hiseqX_sequencing_l']],         undefined, sequencing_subtext);
         make_balance_plot('#bioinfo_balance',     pl_l['bioinformatics'],     pl['bioinformatics'],     undefined, pl['bioinformatics']+' lanes in progress');
+        make_balance_plot('#seq_balance',         pl_l['sequencing'],         [pl['miseq_sequencing_l'], pl['hiseq_sequencing_l'], pl['hiseqX_sequencing_l']],         undefined, sequencing_subtext);
 
         // Bottom row
         make_success_plot('#rc_success', suc['initial_qc']*100);
@@ -291,7 +290,6 @@ function make_balance_plot(target, aim, now, prev, subtext){
         }else{
             my_plotlines = Array();
             chroma_colors = chroma.scale(['#AF2323','#259F0B','#4c85cc']).colors(now.length);
-            pl = [];
             for (i in now){
                 my_plotlines.push({
                     name:'serie'+i,
diff --git a/update_kpis/kpiupdater/__init__.py b/update_kpis/kpiupdater/__init__.py
@@ -19,17 +19,19 @@ class ProjectViewsIter:
             ...,
             project_dates:      projectsDB/project/summary_dates[summary.key],
             project_samples:    projectsDB/project/samples[summary.key],
-            worksets:           worksetsDB/project/ws_proj[summary.key]
+            worksets:           worksetsDB/project/ws_proj[summary.key],
+            bioinfo_lanes       Bioinfo_analysisDB/genomics-dashboard/project_status_counts[summary.key]
         }
     """
-    def __init__(self, project_summary, project_samples, project_dates, worksets_proj, ofptype=None):
+    def __init__(self, project_summary, project_samples, project_dates, worksets_proj, bioinfo_samples, ofptype=None):
         self.proj_key = None
         self.value = None
         self.ofptype = ofptype
         self.summary_iter = iter(project_summary.rows)
         self.project_samples = project_samples
         self.project_dates = project_dates
         self.worksets_proj = worksets_proj
+        self.bioinfo_samples = bioinfo_samples
 
     def __iter__(self):
         return self
@@ -64,6 +66,13 @@ def next(self):
         except:
             pass
 
+        try:
+            self.value["bioinfo"] = []
+            for row in self.bioinfo_samples[[self.proj_key]:[self.proj_key, u'\ufff0']].rows:
+                self.value["bioinfo"].append((row["value"], row.key[1]))
+        except:
+            pass
+
         return (self.proj_key, self.value)
 
 def sequencing_success(num_days=30):
diff --git a/update_kpis/kpiupdater/kpi.py b/update_kpis/kpiupdater/kpi.py
@@ -28,6 +28,42 @@ def _is_ongoing(doc):
         return False
 
 
+def _agregate_status(statuses):
+        """
+        From https://github.com/SciLifeLab/genomics-status/blob/38e1b946b40cc4b25ff75621f4aa9ff63e0e7bb9/status/bioinfo_analysis.py#L195
+        Helper function, agregates status from the lower levels
+        """
+
+        # my guess here agregation is already done from flowcell status
+        # so this condition will most probably always be true
+        if len(set(statuses)) == 1:
+            status = statuses[0]
+        elif 'Sequencing' in statuses:
+            status = 'Sequencing'
+        elif 'Demultiplexing' in statuses:
+            status = 'Demulitplexing'
+        elif 'Transferring' in statuses:
+            status = 'Transferring'
+        elif 'New' in statuses:
+            status = 'New'
+        elif 'QC-ongoing' in statuses:
+            status = 'QC-ongoing'
+        elif 'QC-done' in statuses:
+            status = 'QC-done'
+        elif 'BP-ongoing' in statuses:
+            status = 'BP-ongoing'
+        elif 'BP-done' in statuses:
+            status = 'BP-done'
+        elif 'Failed' in statuses:
+            status = 'Failed'
+        elif 'Delivered' in statuses:
+            status = 'Delivered'
+        else:
+            pass
+            # unknown status, if happens it will fail
+        return status # may fail here, if somebody defined a new status without updating this function
+
+
 class KPIBase(object):
 
     def __init__(self):
@@ -105,8 +141,44 @@ def __call__(self, doc):
                         self.prep_passed += 1.0
 
     def summary(self):
-        return round(self.prep_passed / self.prep_finished, 2)
+        if self.prep_finished > 0:
+            return round(self.prep_passed / self.prep_finished, 2)
+        else:
+            return None
 
+class SuccessBioinfo(KPIBase):
+    """
+    Definition: Samples passed in final report/total samples
+    Approximation Passed run-lane-samples passed / total run-lane-samples
+    """
+    def __init__(self):
+        super(SuccessBioinfo, self).__init__()
+        self.failed = 0.0
+        self.total = 0.0
+
+    def __call__(self, doc):
+        details = doc.get("details",{})
+        ptype = details.get("type", "")
+        for sample, run_lane in doc.get("bioinfo", []):
+            if sample.get("sample_status","") == "Delivered" and ptype == "Production":
+                deliver_date = datetime.strptime(sample["datadelivered"], "%Y-%m-%d")
+                if deliver_date > self.start_date:
+                    self.total += 1.0
+                    isfail = False
+                    for key, value in sample.get("qc", {}).items():
+                        if value == "Fail":
+                            isfail = True
+                    for key, value in sample.get("bp", {}).items():
+                        if value == "Fail":
+                            isfail = True
+                    if isfail:
+                        self.failed += 1.0
+
+    def summary(self):
+        if self.total > 0:
+            return round(1-(self.failed / self.total),2)
+        else:
+            return None
 
 ### Projects
 
@@ -301,6 +373,56 @@ def __call__(self, doc):
 
             self.state += sum(osamples.values())
 
+class LoadBioinfoQueue(ProcessLoadBase):
+    """
+    Definition: total number of lanes with lane QC, but no action in bioinformatic checklist
+    Approximation: # project-run-lanes with status `Demultiplexing`, `New` or `Transferring`
+    """
+
+    def __call__(self, doc):
+        super(LoadBioinfoQueue, self).__call__(doc)
+        run_lanes = {}
+        ongoing = 0
+        if _is_ongoing(doc) and self.ptype == "Production":
+            for sample, run_lane in doc.get("bioinfo", []):
+                status = sample.get(u'sample_status', None)
+                if run_lane in run_lanes.keys():
+                    run_lanes[run_lane].append(status)
+                else:
+                    run_lanes[run_lane] = [status]
+
+            for run_lane, statuses in run_lanes.items():
+                lane_status =  _agregate_status(statuses)
+                if lane_status in [u'Demultiplexing', u'New', u'Transferring']:
+                    ongoing += 1
+
+        self.state += ongoing
+
+
+class LoadBioinfo(ProcessLoadBase):
+    """
+    Definition: Total number of lanes with an action taken in bioinformatic checklist, in projects without close date.
+    Approximation: # project-run-lanes with status `BP-ongoing`, `BP-done`, `QC-ongoing`, `QC-done`
+    """
+
+    def __call__(self, doc):
+        super(LoadBioinfo, self).__call__(doc)
+        run_lanes = {}
+        ongoing = 0
+        if _is_ongoing(doc) and self.ptype == "Production":
+            for sample, run_lane in doc.get("bioinfo", []):
+                status = sample.get(u'sample_status', None)
+                if run_lane in run_lanes.keys():
+                    run_lanes[run_lane].append(status)
+                else:
+                    run_lanes[run_lane] = [status]
+
+            for run_lane, statuses in run_lanes.items():
+                lane_status =  _agregate_status(statuses)
+                if lane_status in [u'BP-ongoing', u'BP-done', u'QC-ongoing', u'QC-done']:
+                    ongoing += 1
+
+        self.state += ongoing
 
 ### Turn-around times
 
diff --git a/update_kpis/tests/data/processload.yaml b/update_kpis/tests/data/processload.yaml
@@ -33,6 +33,9 @@ P0001:
                     sequencing: "some_object"
                 P0001_103:
                     library_status: "FAILED"
+    bioinfo:
+        - !!python/tuple [{user: "taca", sample_status: "Demultiplexing"}, 170925_D12345_001_ABCD_1]
+        - !!python/tuple [{qc: {basesq30: "Pass", fastq_screen: "Fail"}, sample_status: "QC-ongoing", user: "Frank Grimes"}, 170925_D12345_001_ABCD_2]
 P0002:
     open_date: "2015-12-12"
     no_samples: 2
diff --git a/update_kpis/tests/data/success.yaml b/update_kpis/tests/data/success.yaml
@@ -1,4 +1,10 @@
 P0001:
+    open_date: "2015-12-12"
+    no_samples: 3
+    details:
+        queued: "2015-12-24"
+        type: "Production"
+        sample_type: "Other"
     worksets:
         WS160116:
             date_run: "2016-01-16"
@@ -16,6 +22,9 @@ P0001:
                     rec_ctrl:
                         status: "FAILED"
                     library_status: "PASSED"
+    bioinfo:
+        - !!python/tuple [{qc: {basesq30: "Pass", fastq_screen: "Pass"}, datadelivered: "2017-10-01", sample_status: "Delivered", user: "Frank Grimes"}, 170925_D12345_001_ABCD_2]
+        - !!python/tuple [{qc: {basesq30: "Pass", fastq_screen: "Fail"}, datadelivered: "2017-10-01", sample_status: "Delivered", user: "Frank Grimes"}, 170925_D12345_001_ABCD_3] 
 P0002:
     worksets:
         WS160105:
diff --git a/update_kpis/tests/data/turnaround.yaml b/update_kpis/tests/data/turnaround.yaml
@@ -3,10 +3,11 @@ P0001:
     close_date: "2016-01-20"
     details:
         queued: "2016-01-12"
+        all_samples_sequenced: "2016-01-18"
         type: "Production"
         sample_type: "Finished Library"
     project_dates:
-        sequencing_start_date: "2016-01-15"
+        sequencing_start_date: "2016-01-15" 
 P0002:
     open_date: "2016-01-10"
     close_date: "2016-01-20"
diff --git a/update_kpis/tests/test_kpi.py b/update_kpis/tests/test_kpi.py
@@ -90,14 +90,21 @@ def test_initialqc(self):
         self.assertIsInstance(s_initqc.summary(), float)
 
     def test_libraryprep(self):
-        s_libprep =  SuccessLibraryPrep()
+        s_libprep = SuccessLibraryPrep()
         s_libprep.start_date = datetime(2016, 1, 1, 0, 0)
         for doc in self.p_iter:
             s_libprep(doc)
         self.assertEqual(3.0, s_libprep.prep_finished)
         self.assertEqual(2.0, s_libprep.prep_passed)
         self.assertIsInstance(s_libprep.summary(), float)
 
+    def test_bioinfo(self):
+        s_bioinfo = SuccessBioinfo()
+        s_bioinfo.start_date = datetime(2016, 1, 1, 0, 0)
+        for doc in self.p_iter:
+            s_bioinfo(doc)
+        self.assertEqual(0.5, s_bioinfo.summary())
+
 
 class TestProcessLoad(unittest.TestCase):
    
@@ -130,6 +137,18 @@ def test_libraryprep(self):
             pl_libprep(doc)
         self.assertEqual(1, pl_libprep.summary())
 
+    def test_bioinfo_lanes(self):
+        pl_bioinfo = LoadBioinfo()
+        for doc in self.p_iter:
+            pl_bioinfo(doc)
+        self.assertEqual(1, pl_bioinfo.summary())
+
+    def test_bioinfo_queue(self):
+        pl_bioinfoq = LoadBioinfoQueue()
+        for doc in self.p_iter:
+            pl_bioinfoq(doc)
+        self.assertEqual(1, pl_bioinfoq.summary())
+
 
 class TestTurnAroundTimes(unittest.TestCase):
 
@@ -167,6 +186,14 @@ def test_libraryprep(self):
         self.assertIsInstance(tat_libprep.summary(), float)
         self.assertIsInstance(tat_libprep_90th.summary(), float)
 
+    def test_bioinfo(self):
+        tat_bioinfo = TaTBioinformatics()
+        tat_bioinfo.start_date = self.start_date
+        for doc in self.p_iter:
+            tat_bioinfo(doc)
+        self.assertEqual([2], tat_bioinfo.state)
+        self.assertIsInstance(tat_bioinfo.summary(), float)
+
     def test_libprep_project(self):
         tat_libprep_proj = TaTLibprepProj()
         tat_libprep_proj.start_date = self.start_date
diff --git a/update_kpis/update_kpis.py b/update_kpis/update_kpis.py
@@ -47,12 +47,12 @@ def update_kpi(couch_user, password, couch_server):
     p_samples = projects_db.view('project/samples')
     p_dates = projects_db.view('project/summary_dates', group_level=1)
     w_proj = worksets_db.view('project/ws_proj')
-    #TODO bioinfo
-    #b_samples = bioinfo_db.view('latest_data/project_id')
+    b_samples = bioinfo_db.view('genomics_dashboard/run_lane_sample_status')
 
     kpis = {}
     kpis["s_initqc"] = SuccessInitialQC()
     kpis["s_libprep"] = SuccessLibraryPrep()
+    kpis["s_bioinfo"] = SuccessBioinfo()
     kpis["p_finlib"] = ProjectsFinishedLib()
     kpis["p_libprep"] = ProjectsLibraryPrep()
     kpis["p_inprod"] = ProjectsInProduction()
@@ -65,6 +65,8 @@ def update_kpi(couch_user, password, couch_server):
     kpis["pl_rclanes"] = LoadInitialQCLanes()
     kpis["pl_libprepq"] = LoadLibraryPrepQueue()
     kpis["pl_libprep"] = LoadLibraryPrep()
+    kpis["pl_bioinfoq"] = LoadBioinfoQueue()
+    kpis["pl_bioinfo"] = LoadBioinfo()
     kpis["t_libprep"] = TaTLibprep()
     kpis["t_initqc"] = TaTInitialQC()
     kpis["t_libproj"] = TaTLibprepProj()
@@ -79,7 +81,7 @@ def update_kpi(couch_user, password, couch_server):
     kpis["t_bioinfo_90th"] = TaTBioinfo_90th()
     
     logging.info("Generating KPIs")
-    for proj_key, doc in ProjectViewsIter(p_summary, p_samples, p_dates, w_proj):
+    for proj_key, doc in ProjectViewsIter(p_summary, p_samples, p_dates, w_proj, b_samples):
         logging.debug("Processing project: {}".format(proj_key))
         for kpiobj in kpis.values():
             try:
@@ -105,6 +107,8 @@ def update_kpi(couch_user, password, couch_server):
             "initial_qc_lanes": kpis["pl_rclanes"].summary(),
             "library_prep": kpis["pl_libprep"].summary(),
             "library_prep_queue": kpis["pl_libprepq"].summary(),
+            "bioinformatics_queue": kpis["pl_bioinfoq"].summary(),
+            "bioinformatics": kpis["pl_bioinfo"].summary(),
             "miseq_pooling_queue": pl_seq[0],
             "miseq_sequencing_queue_p": pl_seq[1],
             "miseq_sequencing_queue_l": pl_seq[2],
@@ -121,7 +125,8 @@ def update_kpi(couch_user, password, couch_server):
     out["success_rate"] = {
             "initial_qc": kpis["s_initqc"].summary(),
             "library_prep": kpis["s_libprep"].summary(),
-            "sequencing": sequencing_success()
+            "sequencing": sequencing_success(),
+            "bioinformatics": kpis["s_bioinfo"].summary()
     }
     out["turnaround_times"] = {
             "library_prep": kpis["t_libprep"].summary(),
diff --git a/version.txt b/version.txt
@@ -1 +1 @@
-0.1.5
+0.2.0
